Translate your whole site, with you approving every page

import { resolve } from 'node:path';
import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
import { MemorySaver } from '@langchain/langgraph';
import { ChatOpenRouter } from '@langchain/openrouter';
import { CompositeBackend, createDeepAgent, FilesystemBackend, StateBackend } from 'deepagents';
import { createMiddleware, modelRetryMiddleware, todoListMiddleware } from 'langchain';
import type { Language } from './language.ts';
import { type Run, siteTools } from './tools.ts';

const translate = ({
  name,
  code
}: Language) => `You translate a small business website into ${name} (${code}), with the
owner approving every document. The code has already copied every document that needs translating
into ${name}: each is a draft there, still in the main language, waiting for its words.

The glossary is /memories/glossary.md, your long-term memory: it is kept between runs, and loaded into
your agent memory at the start of each one. Under "## Keep as written" it lists the names never
translated, one per line ("- Acme"); under "## Rules", what the owner asked for, one per line.

Work in this order:
1. Plan your work with write_todos.
2. Call list_documents once.
3. If your agent memory holds the glossary, go to step 4. Otherwise read the first page and two or
   three others with read_document, and write /memories/glossary.md: a "## Keep as written" list of
   the business's name, its services' and products' names and its place names, then an empty
   "## Rules". Call ask_owner with the whole glossary, and ask the owner to approve it or say what to
   change. Update it with edit_file and ask again until the owner approves it.
4. For every document, one at a time, hand it to the translator subagent with the task tool: give its
   id, kind and title. Never ask the owner about a translation yourself: the translator's save shows
   it, and the owner decides there.
5. When a translator reports a rule the owner gave (a name to keep, a word to use), add it to
   /memories/glossary.md with edit_file: a name under "## Keep as written", anything else under
   "## Rules".
6. Finish with a short summary: how many documents you translated, and which ones the owner skipped.

Each ask_owner call asks ONE question. Write it in Markdown, with a blank line between paragraphs and
between list items.`;

const translator = ({
  name,
  code
}: Language) => `You translate one website document into ${name} (${code}).
Read /memories/glossary.md with read_file, then the document with read_document: its texts, each under
a key.

- Translate every text as a native speaker would write it for this business: natural, warm, plain.
  Keep the meaning, the facts and the tone. Never add or drop information.
- Keep every name under "## Keep as written" exactly as written, and follow every rule under "## Rules".
- Keep every word in braces, such as {year} or {business}, exactly as written: the site fills it in.
- Rich text is HTML: translate the words between the tags, and keep every tag, attribute and link
  exactly as it is.
- Leave a text as it is when it isn't words for a visitor (a code, a hashtag) or is already in ${name}.

Call check_translation with every key, fix what it names, and check again until it passes. Then call
save_translation ONCE with every key and the document's title. If the owner sends it back with notes,
revise to match, check again and save again. Answer in one line: what you translated, then any rule
the owner's notes gave, in their words.`;

/**
 * Each model call gets up to ten minutes: room for your longest page on a slow model, so the limit
 * only ends a call that hangs. A failed call is tried twice more; if it still fails, the agent is
 * told why and goes on, and that document stays to translate on the next run.
 */
const MODEL_CALL_SECONDS = 600;

const modelCalls = () => [
  modelRetryMiddleware({ maxRetries: 2, onFailure: 'continue' }),
  createMiddleware({
    name: 'ModelCallTimeLimit',
    wrapModelCall: (request, handler) =>
      handler({
        ...request,
        modelSettings: {
          ...request.modelSettings,
          signal: AbortSignal.timeout(MODEL_CALL_SECONDS * 1000)
        }
      })
  })
];

/** The deep agent: our SDK's operations as its tools, a translator per document, and the owner's approval before any draft is written. */
export function buildAgent(model: BaseChatModel, run: Run) {
  const tools = siteTools(run);
  return createDeepAgent({
    model,
    systemPrompt: translate(run.language),
    tools: [tools.listDocuments, tools.readDocument, tools.askOwner],
    middleware: [todoListMiddleware(), ...modelCalls()],
    subagents: [
      {
        name: 'translator',
        description: `Translates one document into ${run.language.name} and saves it as a draft, once the owner approves.`,
        systemPrompt: translator(run.language),
        tools: [tools.readDocument, tools.checkTranslation, tools.saveTranslation],
        middleware: modelCalls()
      }
    ],
    // Long-term memory: /memories/ is a folder on disk, kept between runs; every other file lives
    // for one run. The glossary is loaded into the agent's instructions at the start of each run.
    backend: new CompositeBackend(new StateBackend(), {
      '/memories/': new FilesystemBackend({ rootDir: resolve(run.memoryFolder), virtualMode: true })
    }),
    memory: ['/memories/glossary.md'],
    // Nothing is written without the owner: approve it, or send it back.
    interruptOn: { save_translation: { allowedDecisions: ['approve', 'reject'] } },
    checkpointer: new MemorySaver()
  });
}

/** The OpenRouter model and settings named in .env. ChatOpenRouter reads OPENROUTER_API_KEY itself. */
export function modelFromEnv() {
  return new ChatOpenRouter({
    model: process.env.OPENROUTER_MODEL || '~anthropic/claude-sonnet-latest',
    temperature: Number(process.env.OPENROUTER_TEMPERATURE || 0.2),
    maxTokens: Number(process.env.OPENROUTER_MAX_TOKENS || 16000)
  });
}

import { readFile } from 'node:fs/promises';
import { join } from 'node:path';
import type { SiteClient } from '@fetchski/attract-cms-sdk';
import { interrupt } from '@langchain/langgraph';
import { tool } from 'langchain';
import { z } from 'zod';
import { getDraft, problemsWith, type Wording, wordingOf, writeDraft } from './content.ts';
import type { Job, Language } from './language.ts';

/** What the tools share with the code around the agent. */
export interface Run {
  site: SiteClient;
  language: Language;
  /** The documents copied into the language, waiting for their words, by id. */
  jobs: Map<string, Job>;
  /** Each document's wording as read_document gave it, by id: what a translation is checked against. */
  originals: Map<string, Wording[]>;
  /** Every document a translation was saved for, by id. */
  saved: Map<string, Job>;
  /** The folder on disk that keeps the agent's long-term memory (the glossary). */
  memoryFolder: string;
  /** Shows the agent's progress. */
  log: (line: string) => void;
}

const texts = z
  .array(z.object({ key: z.string(), text: z.string() }))
  .describe('Every key read_document gave, each with its translated text');

/**
 * The names the glossary keeps as written: each "- " line under its "## Keep as written" heading.
 * The code checks every translation against them, whatever the model says.
 */
export async function keptNames(memoryFolder: string): Promise<string[]> {
  const glossary = await readFile(join(memoryFolder, 'glossary.md'), 'utf8').catch(() => '');
  const section = /^## Keep as written\s*$([\s\S]*?)(?=^## |(?![\s\S]))/m.exec(glossary)?.[1] ?? '';
  return section
    .split('\n')
    .map((line) => /^\s*[-*]\s+(.+?)\s*$/.exec(line)?.[1] ?? '')
    .filter(Boolean);
}

/** The SDK's calls, as the agent's tools. Every write is a draft in the run's language. */
export function siteTools(run: Run) {
  // A wrong id goes back to the agent, to correct; it never stops the run.
  const unknown = (id: string) =>
    `"${id}" isn't one of the documents list_documents gave. Use an id exactly as it gave it.`;

  const listDocuments = tool(
    async () => {
      run.log(`${run.jobs.size} document${run.jobs.size === 1 ? '' : 's'} to translate.`);
      return JSON.stringify(
        [...run.jobs.entries()].map(([id, { kind, title }]) => ({ id, kind, title }))
      );
    },
    {
      name: 'list_documents',
      description:
        'Lists every document to translate: id, kind (page, post, faq, settings: the header, footer and cookie banner, or shown: something the pages show) and title. Call it once.',
      schema: z.object({})
    }
  );

  const readDocument = tool(
    async ({ id }) => {
      const job = run.jobs.get(id);
      if (!job) {
        return unknown(id);
      }
      const copy = await getDraft(run.site, job, run.language.code);
      const wording = copy ? wordingOf(copy, job.fields) : [];
      run.originals.set(id, wording);
      run.log(`Read "${job.title ?? id}".`);
      return JSON.stringify(wording);
    },
    {
      name: 'read_document',
      description:
        "Reads one document's texts, each under a key. Rich text is HTML. Translate every one.",
      schema: z.object({ id: z.string().describe('The id from list_documents') })
    }
  );

  const check = async (id: string, translated: readonly Wording[]) => {
    const original = run.originals.get(id);
    if (!original) {
      return ['Read the document with read_document first.'];
    }
    return problemsWith(original, translated, await keptNames(run.memoryFolder));
  };

  const checkTranslation = tool(
    async (args) => {
      const problems = await check(args.id, args.texts);
      return problems.length
        ? `Fix these, then check again:\n${problems.join('\n')}`
        : 'Passes every check.';
    },
    {
      name: 'check_translation',
      description:
        'Checks a whole translation without saving it: every key there, every HTML tag and link kept, every glossary name kept.',
      schema: z.object({ id: z.string(), texts })
    }
  );

  const saveTranslation = tool(
    async (args) => {
      const job = run.jobs.get(args.id);
      if (!job) {
        return unknown(args.id);
      }
      const problems = await check(args.id, args.texts);
      if (problems.length) {
        return `Not saved. Fix these, then save again:\n${problems.join('\n')}`;
      }
      await writeDraft(run.site, job, run.language.code, [...args.texts, ...(job.links ?? [])]);
      run.saved.set(args.id, job);
      return `Saved as the ${run.language.name} draft, sent for review. Nothing is live until a person publishes it.`;
    },
    {
      name: 'save_translation',
      description:
        'Saves a translation that passed check_translation as the draft in the language, once the owner approves it.',
      schema: z.object({
        id: z.string(),
        title: z.string().describe("The document's title, shown to the owner"),
        texts
      })
    }
  );

  const askOwner = tool(
    // Pauses the run until the owner answers; their answer is the tool's result.
    async (args) => String(interrupt({ question: args.question })),
    {
      name: 'ask_owner',
      description: 'Asks the website owner one question, and waits for their answer.',
      schema: z.object({ question: z.string() })
    }
  );

  return { listDocuments, readDocument, checkTranslation, saveTranslation, askOwner };
}

import { NotFound, type SiteClient } from '@fetchski/attract-cms-sdk';

/** A document as the site's REST routes send it, under the API's own names. */
export interface StoredDoc {
  _id: string;
  aposDocId: string;
  type: string;
  slug?: string;
  title?: string;
  _url?: string;
  [field: string]: unknown;
}

/** Which document: enough to reach it in any language. */
export interface DocRef {
  aposDocId: string;
  type: string;
  slug?: string;
  title?: string;
}

/** One piece of a document's wording, under the key the site's PATCH writes it back by. */
export interface Wording {
  key: string;
  text: string;
}

// Copied into the language so pages keep them, never translated.
const MEDIA = new Set(['@apostrophecms/image', '@apostrophecms/file', 'video']);

// Keys that hold codes, addresses or settings, never words a visitor reads.
const NOT_WORDING_KEYS = new Set([
  'type',
  'metaType',
  'slug',
  'path',
  'aposDocId',
  'aposLocale',
  'aposMode',
  'visibility',
  'titleSortified',
  'highSearchText',
  'lowSearchText',
  'searchSummary',
  'parkedId',
  'source',
  'scopedArrayName',
  'submitted' // who sent it for review, and when
]);
const NOT_WORDING_SUFFIX =
  /(By|Url|Href|Color|ColorLight|Custom|Icon|IconFamily|Css|Slug|Email|Phone|Anchor|Code)$/;
const NOT_WORDS = [
  /^(https?:|mailto:|tel:|\/|#|--)/, // addresses, colors, design tokens
  /^[a-z0-9]+([-_.][a-z0-9]+)*$/, // choices, icon names, slugs
  /^\S*[a-z][A-Z]\S*$/, // one camelCase token: a choice (belowHeadline)
  /^[^\s<>]*[@/_{}=][^\s<>]*$/, // one token holding code characters (@apostrophecms/any-page-type)
  /^\d{4}-\d\d-\d\d/, // dates
  /^[^\p{L}]*$/u // no letters at all
];

/** A page lives under the page routes; every other type under its own name. */
export const isPage = (doc: DocRef) => typeof doc.slug === 'string' && doc.slug.startsWith('/');
export const isMedia = (doc: DocRef) => MEDIA.has(doc.type);

const route = (doc: DocRef) =>
  `/${isPage(doc) ? '@apostrophecms/page' : doc.type}/${doc.aposDocId}`;

/**
 * Every draft of a type in `locale`, as stored (so with the site's `localizedAt`, which the SDK's
 * own models leave out), a page at a time. Pages, posts and FAQs relate to documents of any type
 * (images, authors, categories, forms), so this module reaches each through the site's REST
 * routes with `request()`, the SDK's call for a route by name.
 */
export async function listDrafts(
  site: SiteClient,
  type: string,
  locale: string,
  query: Record<string, string | number> = {}
): Promise<StoredDoc[]> {
  const all: StoredDoc[] = [];
  for (let page = 1, pages = 1; page <= pages; page++) {
    const answer = await site.request<{ results: StoredDoc[]; pages?: number }>('GET', `/${type}`, {
      query: { ...query, aposLocale: locale, aposMode: 'draft', perPage: 100, page }
    });
    all.push(...(answer?.results ?? []));
    pages = answer?.pages ?? 1;
  }
  return all;
}

/** The draft in `locale`, with its related documents; null when the language has no copy yet. */
export async function getDraft(site: SiteClient, doc: DocRef, locale: string) {
  try {
    return await site.request<StoredDoc>('GET', route(doc), {
      query: { aposLocale: locale, aposMode: 'draft' }
    });
  } catch (error) {
    if (error instanceof NotFound) {
      return null;
    }
    throw error;
  }
}

/** Copies the draft in `from` into `to`, as a draft there; `update` overwrites a copy already there. */
export async function copyDraft(
  site: SiteClient,
  doc: DocRef,
  from: string,
  to: string,
  update: boolean
) {
  await site.request('POST', `${route(doc)}:${from}:draft/localize`, {
    body: { toLocale: to, update }
  });
}

/**
 * Writes the translated wording to the draft in `locale`, and puts it in the CMS's review queue
 * (Submitted Drafts). The site's PATCH takes `@<widget id>.<field>` keys, so only the words change.
 */
export async function writeDraft(
  site: SiteClient,
  doc: DocRef,
  locale: string,
  wording: readonly Wording[]
) {
  await site.request('PATCH', route(doc), {
    query: { aposLocale: locale, aposMode: 'draft' },
    body: Object.fromEntries(wording.map(({ key, text }) => [key, text]))
  });
  await site.request('POST', `${route(doc)}:${locale}:draft/submit`);
}

/**
 * The words a visitor reads in `doc`: titles, headings, text, rich text, button labels, SEO text.
 * Codes, colors, addresses and settings stay out, and so do related documents (they are
 * translated as documents of their own). `fields` limits it to those fields (the site settings'
 * own words: the rest of its fields are shared by every language).
 */
export function wordingOf(doc: StoredDoc, fields?: readonly string[]): Wording[] {
  const found: Wording[] = [];

  const visit = (value: unknown, owner: string | null, path: string) => {
    if (typeof value === 'string') {
      const text = value.trim();
      if (text && !NOT_WORDS.some((pattern) => pattern.test(text))) {
        found.push({ key: owner ? `@${owner}.${path}` : path, text: value });
      }
      return;
    }
    if (Array.isArray(value)) {
      value.forEach((item, index) => {
        // A widget or array row has an id: its fields are keyed by it.
        if (item && typeof item === 'object' && typeof item._id === 'string') {
          visit(item, item._id, '');
        } else {
          visit(item, owner, path ? `${path}.${index}` : String(index));
        }
      });
      return;
    }
    if (!value || typeof value !== 'object' || (value as StoredDoc).metaType === 'doc') {
      return;
    }
    for (const [key, child] of Object.entries(value)) {
      if (isWordingKey(key)) {
        visit(unwrap(child), owner, path ? `${path}.${key}` : key);
      }
    }
  };

  for (const [key, value] of Object.entries(doc)) {
    if (isWordingKey(key) && (!fields || fields.includes(key))) {
      visit(unwrap(value), null, key);
    }
  }
  return found;
}

/** The path of a document's `_url` (`/about-us`), without the site's address. */
export const pathOf = (url: unknown) =>
  typeof url === 'string' ? (URL.parse(url, 'https://site.invalid')?.pathname ?? null) : null;

/**
 * The links in `fields` of the site settings (a `url` in a list row, `headerCtaUrl`…) that point
 * at a page the language has, each written to point at that page in the language instead, under
 * the key the site's PATCH writes it back by. `pointTo` maps a page's path to its path in the
 * language. A link's `?query` and `#anchor` are kept; a link to another site is left alone.
 */
export function linksTo(
  doc: StoredDoc,
  fields: readonly string[],
  pointTo: ReadonlyMap<string, string>
): Wording[] {
  const found: Wording[] = [];
  const visit = (value: unknown, owner: string | null, key: string) => {
    if (Array.isArray(value)) {
      for (const row of value) {
        if (row && typeof row === 'object') {
          for (const [field, child] of Object.entries(row)) {
            visit(child, typeof row._id === 'string' ? row._id : owner, field);
          }
        }
      }
      return;
    }
    // A link on this site starts with one slash; another site's is left alone.
    if (typeof value !== 'string' || !/(^url|Url)$/.test(key) || !/^\/(?!\/)/.test(value)) {
      return;
    }
    const [, path = '', rest = ''] = /^([^?#]*)(.*)$/.exec(value) ?? [];
    const to = pointTo.get(path);
    if (to && `${to}${rest}` !== value) {
      found.push({ key: owner ? `@${owner}.${key}` : key, text: `${to}${rest}` });
    }
  };
  for (const field of fields) {
    visit(doc[field], null, field);
  }
  return found;
}

const isWordingKey = (key: string) =>
  !key.startsWith('_') && !NOT_WORDING_KEYS.has(key) && !NOT_WORDING_SUFFIX.test(key);

/** An area holds its widgets under `items`. */
const unwrap = (value: unknown) => {
  const area = value as { metaType?: string; items?: unknown[] } | null;
  return area?.metaType === 'area' ? area.items : value;
};

/**
 * The documents `doc` shows (images, authors, categories, forms…), once each. Pages are left
 * out: every page is in the run already.
 */
export function relatedTo(doc: StoredDoc): DocRef[] {
  const found = new Map<string, DocRef>();

  const visit = (value: unknown, root: boolean) => {
    if (Array.isArray(value)) {
      value.forEach((item) => visit(item, false));
      return;
    }
    if (!value || typeof value !== 'object') {
      return;
    }
    const item = value as StoredDoc;
    // A related document comes projected: its `_id` (`<aposDocId>:<locale>:<mode>`), not its
    // aposDocId. One with no locale in its `_id` (a media folder) is the same in every language.
    if (!root && item.metaType === 'doc' && typeof item._id === 'string') {
      const [aposDocId = '', locale] = item._id.split(':');
      if (locale && !isPage(item)) {
        found.set(aposDocId, { aposDocId, type: item.type, slug: item.slug, title: item.title });
      }
      return;
    }
    for (const [key, child] of Object.entries(item)) {
      // A page's tree and its published twin are pages, not what it shows.
      if (!['_ancestors', '_children', '_parent', '_publishedDoc'].includes(key)) {
        visit(child, false);
      }
    }
  };

  visit(doc, true);
  return [...found.values()];
}

const TAG = /<\/?([a-z][a-z0-9]*)\b[^>]*>/gi;
const LINK = /\b(?:href|src)\s*=\s*"([^"]*)"/gi;

const tagsOf = (text: string) =>
  [...text.matchAll(TAG)]
    .map((match) => (match[0].startsWith('</') ? `/${match[1]}` : match[1]))
    .join(' ');
const linksOf = (text: string) => [...text.matchAll(LINK)].map((match) => match[1]).join(' ');
/** The {year}-style words the site fills in, in any order (a language may move them). */
const placeholdersOf = (text: string) => (text.match(/\{[a-z]+\}/g) ?? []).sort().join(' ');
const plain = (text: string) => text.replace(TAG, ' ').replace(/\s+/g, ' ').trim();

/**
 * A sentence sent back as it was: not translated. A sentence: four words or more, glossary names
 * aside, two of them lowercase (a name or an address alone never counts).
 */
const leftAsIs = (text: string, translation: string, keep: readonly string[]) => {
  if (plain(text) !== plain(translation)) return false;
  const rest = keep.reduce((left, name) => left.replaceAll(name, ' '), plain(text));
  const words = rest.match(/\p{L}+/gu) ?? [];
  return words.length >= 4 && words.filter((word) => /^\p{Ll}/u.test(word)).length >= 2;
};

/**
 * Why a translation can't be saved: a key missing or invented, an HTML tag, a link or a
 * {placeholder} changed, a glossary name translated, a sentence left untranslated. Empty when it
 * can be.
 */
export function problemsWith(
  original: readonly Wording[],
  translated: readonly Wording[],
  keep: readonly string[]
): string[] {
  const problems: string[] = [];
  const given = new Map(translated.map(({ key, text }) => [key, text]));

  for (const { key, text } of original) {
    const translation = given.get(key);
    if (translation === undefined || !translation.trim()) {
      problems.push(`${key}: missing. Translate every key you were given.`);
      continue;
    }
    if (tagsOf(translation) !== tagsOf(text)) {
      problems.push(
        `${key}: the HTML tags changed. Keep exactly these, in order: ${tagsOf(text) || 'none'}.`
      );
    }
    if (linksOf(translation) !== linksOf(text)) {
      problems.push(`${key}: a link changed. Keep every href and src exactly as it was.`);
    }
    if (placeholdersOf(translation) !== placeholdersOf(text)) {
      const kept = placeholdersOf(text) || 'none';
      problems.push(`${key}: keep these exactly as written, the site fills them in: ${kept}.`);
    }
    for (const name of keep) {
      if (text.includes(name) && !translation.includes(name)) {
        problems.push(`${key}: "${name}" is in the glossary. Keep it exactly as written.`);
      }
    }
    if (leftAsIs(text, translation, keep)) {
      problems.push(`${key}: still in the original language. Translate it.`);
    }
  }
  const known = new Set(original.map(({ key }) => key));
  for (const key of given.keys()) {
    if (!known.has(key)) {
      problems.push(`${key}: not one of the keys you were given. Leave it out.`);
    }
  }
  return problems;
}

import type { SiteClient } from '@fetchski/attract-cms-sdk';
import {
  copyDraft,
  type DocRef,
  getDraft,
  isMedia,
  linksTo,
  listDrafts,
  pathOf,
  relatedTo,
  type StoredDoc,
  type Wording
} from './content.ts';

/**
 * What a document is to the run: a page, a blog post, an FAQ, the site settings (the header,
 * footer and cookie banner), or something they show.
 */
export type Kind = 'page' | 'post' | 'faq' | 'settings' | 'shown';

/** What each kind is called on screen and in the report. */
export const KIND_NAMES: Record<Kind, string> = {
  page: 'page',
  post: 'blog post',
  faq: 'FAQ',
  settings: 'site settings',
  shown: 'item'
};

/** A document to translate: copied into the language already, waiting for its words. */
export interface Job extends DocRef {
  kind: Kind;
  /** The only fields with words to translate (the site settings': the rest are shared). */
  fields?: string[];
  /** Links the code points at the language's pages, saved with the translation. */
  links?: Wording[];
}

/** The language a run translates into. */
export interface Language {
  code: string;
  name: string;
  /** The site's main language: the one without an address prefix. */
  main: string;
}

/**
 * Adds the language to the site, not public, when the site lacks it. Answers the language with
 * the site's main one, and whether it was added. Checks the site took it.
 */
export async function ensureLanguage(
  site: SiteClient,
  code: string,
  name: string
): Promise<{ language: Language; added: boolean }> {
  const [list] = (await site.languages().listSiteLanguages()).results;
  if (!list) {
    throw new Error(
      'The site has no language list: open Languages in the CMS once, then run this again.'
    );
  }
  const languages = list.languages ?? [];
  const added = !languages.some((language) => language.code === code);
  if (added) {
    await site.languages().updateSiteLanguages({
      id: list.id,
      languages: [...languages, { name, code, public: false }]
    });
  }

  const locales = await site.internationalization().getLocales();
  const main = Object.keys(locales).find((locale) => !locales[locale]?.prefix);
  if (!main || !locales[code] || code === main) {
    throw new Error(
      `"${code}" isn't one of the site's further languages: the site has ${Object.keys(locales).join(', ')}.`
    );
  }
  return { language: { code, name, main }, added };
}

/** Makes the language public: visitors see each page published in it. */
export async function makePublic(site: SiteClient, code: string) {
  const [list] = (await site.languages().listSiteLanguages()).results;
  if (list) {
    await site.languages().updateSiteLanguages({
      id: list.id,
      languages: (list.languages ?? []).map((language) =>
        language.code === code ? { ...language, public: true } : language
      )
    });
  }
}

/**
 * A copy is up to date when it was copied from the current original and written since: its
 * translation is saved. ISO dates compare as strings.
 */
export function isUpToDate(original: StoredDoc | null, copy: StoredDoc | null) {
  const copiedAt = typeof copy?.localizedAt === 'string' ? copy.localizedAt : null;
  return Boolean(
    original &&
    copiedAt &&
    copiedAt >= String(original.updatedAt) &&
    String(copy?.updatedAt) > copiedAt
  );
}

const KINDS = [
  { kind: 'page', type: '@apostrophecms/page', query: { all: 1, flat: 1 } },
  { kind: 'post', type: '@apostrophecms/blog', query: {} },
  { kind: 'faq', type: 'faq-item', query: {} }
] as const;

// Site settings and Site design: one document per language.
const SETTINGS = '@apostrophecms/global';

/**
 * Every page (parents first: a page is copied only under its parent), blog post and FAQ whose
 * translation isn't up to date, and the site settings when theirs isn't, copied into the language
 * over any older copy, after everything they show. Answers those, with what any of them shows that
 * needs words of its own (by the same rule, so a copy a stopped run left is translated by the
 * next), and how many were up to date. The settings' links to pages are pointed at the language's
 * pages. A key missing a scope stops here, before any copy is overwritten.
 */
export async function prepare(
  site: SiteClient,
  { code, main }: Language,
  log: (line: string) => void
): Promise<{ jobs: Job[]; upToDate: number }> {
  const listed: DocRef[] = [];
  const jobs: Job[] = [];
  // Each page's and post's path, by id, to point the settings' links at their copies.
  const paths = new Map<string, string>();
  let upToDate = 0;

  for (const { kind, type, query } of KINDS) {
    const copies = new Map(
      (await listDrafts(site, type, code, query)).map((copy) => [copy.aposDocId, copy])
    );
    const originals = (await listDrafts(site, type, main, query)).toSorted(
      (a, b) =>
        Number(a.level ?? 0) - Number(b.level ?? 0) || Number(a.rank ?? 0) - Number(b.rank ?? 0)
    );
    for (const original of originals) {
      listed.push(original);
      const path = pathOf(original._url);
      if (path && kind !== 'faq') {
        paths.set(original.aposDocId, path);
      }
      let copy = copies.get(original.aposDocId);
      // The page tree's list sends a child page's main fields only: read it whole.
      if (copy && !('localizedAt' in copy)) {
        copy = (await getDraft(site, copy, code)) ?? undefined;
      }
      if (isUpToDate(original, copy ?? null)) {
        upToDate++;
      } else {
        const { aposDocId, type: docType, slug, title } = original;
        jobs.push({ aposDocId, type: docType, slug, title, kind });
      }
    }
  }

  // The header, footer and cookie banner: the settings' fields the site says each language
  // translates.
  const [settings] = await listDrafts(site, SETTINGS, main);
  if (settings) {
    listed.push(settings);
    if (isUpToDate(settings, await getDraft(site, settings, code))) {
      upToDate++;
    } else {
      const { aposDocId, type, slug } = settings;
      const { translated } = await site.languages().getGlobalTranslatedFields();
      jobs.push({
        aposDocId,
        type,
        slug,
        title: 'Header, footer and cookie banner',
        kind: 'settings',
        fields: [...translated]
      });
    }
  }

  // What they show goes first: a write keeps a related document only once it exists in the language.
  const shown: Job[] = [];
  const seen = new Set<string>();
  const copyShown = async (doc: DocRef) => {
    const stored = await getDraft(site, doc, main);
    for (const related of stored ? relatedTo(stored) : []) {
      if (seen.has(related.aposDocId)) {
        continue;
      }
      seen.add(related.aposDocId);
      const copy = await getDraft(site, related, code);
      // A copy is all an image or a file needs; anything else needs its words, unless up to date.
      if (copy && (isMedia(related) || isUpToDate(await getDraft(site, related, main), copy))) {
        continue;
      }
      await copyDraft(site, related, main, code, copy !== null);
      if (!isMedia(related)) {
        shown.push({ ...related, kind: 'shown' });
      }
      await copyShown(related);
    }
  };
  for (const doc of listed) {
    await copyShown(doc);
  }
  for (const job of jobs) {
    await copyDraft(site, job, main, code, true);
  }
  const job = jobs.find(({ kind }) => kind === 'settings');
  if (settings && job?.fields) {
    job.links = linksTo(settings, job.fields, await pathsIn(site, code, paths));
  }
  log(
    `Copied ${jobs.length + shown.length} document${jobs.length + shown.length === 1 ? '' : 's'} into ${code}.`
  );
  return { jobs: [...jobs, ...shown], upToDate };
}

/** Each page's and post's path → its path in the language (`/about-us` → `/fr/about-us`). */
async function pathsIn(site: SiteClient, code: string, paths: ReadonlyMap<string, string>) {
  const pointTo = new Map<string, string>();
  for (const { type, query } of KINDS.filter(({ kind }) => kind !== 'faq')) {
    for (const copy of await listDrafts(site, type, code, query)) {
      const from = paths.get(copy.aposDocId);
      const to = pathOf(copy._url);
      if (from && to) {
        pointTo.set(from, to);
      }
    }
  }
  return pointTo;
}

/** A private link to a page's or post's draft, for anyone you send it to: no sign-in. */
export async function previewLink(
  site: SiteClient,
  job: Job,
  code: string
): Promise<string | null> {
  if (job.kind !== 'page' && job.kind !== 'post') {
    return null;
  }
  const id = `${job.aposDocId}:${code}:draft`;
  const copy = await getDraft(site, job, code);
  const shared =
    job.kind === 'page'
      ? await site.pages().sharePage({ id, share: true })
      : await site.blogPosts().shareBlog({ id, share: true });
  if (typeof copy?._url !== 'string' || !shared.aposShareKey) {
    return null;
  }
  return `${copy._url}?${new URLSearchParams({ aposShareKey: shared.aposShareKey, aposShareId: id })}`;
}

import { randomUUID } from 'node:crypto';
import { Command } from '@langchain/langgraph';
import type { buildAgent } from './agent.ts';
import type { Wording } from './content.ts';
import type { Run } from './tools.ts';
import type { Kind } from './language.ts';
import type { Usage } from './usage.ts';

/** A translation the agent wants to save: the document, and each text before and after. */
export interface Draft {
  kind: Kind;
  title: string;
  texts: { key: string; original: string; translation: string }[];
}

/** Your call on a translation: save it, send it back with notes, or skip it. */
export type Choice =
  { type: 'save' } | { type: 'revise'; feedback: string } | { type: 'skip'; reason: string };

/** The person at the keyboard. src/terminal.ts is the real one; the tests script one. */
export interface Owner {
  /** The agent is working (a spinner, in the terminal). */
  busy?(): void;
  /** Answers the agent's question. */
  answer(question: string): Promise<string>;
  /** Decides on a translation the agent wants to save. */
  decide(draft: Draft): Promise<Choice>;
}

interface ActionRequest {
  name: string;
  args: { id?: string; title?: string; texts?: Wording[] };
}

type Decision = { type: 'approve' } | { type: 'reject'; message: string };

/** The most steps (model calls and tool calls) one run may take: a ceiling on AI cost. */
const MAX_STEPS = Number(process.env.AGENT_MAX_STEPS ?? 1000);

// How many times the code sends the agent back when it stops with documents neither saved nor
// skipped: the code counts what is done, never the agent's word for it.
const FOLLOW_UPS = 2;

/**
 * Runs the agent to the end. Each time it pauses, the owner answers its question, or decides on
 * the translations it wants to save, each text beside the one it read.
 * Resolves the agent's closing summary.
 */
export async function runWithOwner(
  agent: ReturnType<typeof buildAgent>,
  task: string,
  owner: Owner,
  run: Pick<Run, 'jobs' | 'originals' | 'saved'>,
  usage?: Usage
): Promise<string> {
  // `usage` counts every model call's tokens and cost, the subagents' too.
  const config = {
    configurable: { thread_id: randomUUID() },
    recursionLimit: MAX_STEPS,
    callbacks: usage ? [usage] : []
  };
  let input: Parameters<typeof agent.invoke>[0] = { messages: [{ role: 'user', content: task }] };
  const skipped = new Set<string>();
  let followUps = 0;

  for (;;) {
    owner.busy?.();
    const result = await agent.invoke(input, config);
    // Several translators can pause at once: each pause gets its own answer, by its id.
    const pending = result.__interrupt__ ?? [];
    if (pending.length === 0) {
      const left = [...run.jobs].filter(([id]) => !run.saved.has(id) && !skipped.has(id));
      if (left.length > 0 && followUps < FOLLOW_UPS) {
        followUps++;
        const list = left.map(([id, job]) => `- ${id}: ${job.kind} "${job.title ?? id}"`);
        input = {
          messages: [
            {
              role: 'user',
              content: `Not finished: ${left.length} still need translating. Hand each one to the translator now, then finish with your summary.\n${list.join('\n')}`
            }
          ]
        };
        continue;
      }
      const last = result.messages.at(-1);
      return typeof last?.content === 'string' ? last.content : JSON.stringify(last?.content ?? '');
    }

    const answers: Record<string, unknown> = {};
    for (const { id, value } of pending) {
      const paused = value as { question?: string; actionRequests?: ActionRequest[] };
      if (paused.question) {
        answers[String(id)] = await owner.answer(paused.question);
        continue;
      }
      const decisions: Decision[] = [];
      for (const action of paused.actionRequests ?? []) {
        decisions.push(await review(action, owner, run, skipped));
      }
      answers[String(id)] = { decisions };
    }
    input = new Command({ resume: answers });
  }
}

/** Your choice, in the form the agent's approval step takes: approve or reject. */
async function review(
  action: ActionRequest,
  owner: Owner,
  run: Pick<Run, 'jobs' | 'originals'>,
  skipped: Set<string>
): Promise<Decision> {
  const id = action.args.id ?? '';
  const read = new Map((run.originals.get(id) ?? []).map(({ key, text }) => [key, text]));
  const choice = await owner.decide({
    kind: run.jobs.get(id)?.kind ?? 'page',
    title: action.args.title ?? '',
    texts: (action.args.texts ?? []).map(({ key, text }) => ({
      key,
      original: read.get(key) ?? '',
      translation: text
    }))
  });

  switch (choice.type) {
    case 'save':
      return { type: 'approve' };
    case 'revise':
      return {
        type: 'reject',
        message:
          `Not saved yet: the owner wants changes first. Their words: "${choice.feedback}". ` +
          'Revise the translation to match, check it, then call save_translation again with every key.'
      };
    case 'skip':
      skipped.add(id);
      return {
        type: 'reject',
        message: `The owner skipped this document${choice.reason ? `: ${choice.reason}` : ''}. Do not try it again.`
      };
  }
}

import { stdout } from 'node:process';
import { stripVTControlCharacters, styleText } from 'node:util';
import * as prompt from '@clack/prompts';
import { Marked, type Tokens } from 'marked';
import stringWidth from 'string-width';
import yoctoSpinner from 'yocto-spinner';
import { KIND_NAMES, type Language } from './language.ts';
import type { Choice, Draft, Owner } from './review.ts';
import type { Usage } from './usage.ts';

/** You pressed Ctrl+C or Esc at a prompt: the run stops, and the drafts saved so far stay drafts. */
export class Stopped extends Error {}

/**
 * The person at the keyboard, in the terminal: a spinner while the agent works, with what it is
 * doing and what the run has used so far; its words as formatted text.
 */
export function terminalOwner(
  language: Pick<Language, 'code' | 'main'>,
  usage?: Usage
): Owner & {
  step: (line: string) => void;
  report: (markdown: string) => void;
  failed: (error: unknown) => void;
} {
  // yocto-spinner's own dots; each frame redraws in one synchronized write, so it never flickers.
  const spin = yoctoSpinner({ stream: stdout });
  // Like yocto-spinner, only a terminal (not TERM=dumb, outside CI) gets live updates; anywhere
  // else each one would print a new line, so output sent to a file gets the start and the steps.
  const live = Boolean(stdout.isTTY) && process.env.TERM !== 'dumb' && !('CI' in process.env);
  let spinning = false;
  let started = 0;
  let clock: NodeJS.Timeout | undefined;
  let steps: string[] = [];
  let doing = 'The agent is working';

  // What the agent is doing, then the tokens, cost and time so far: one line, never wrapped.
  // On a narrow terminal the tokens and cost go first, then the time, before the task is cut.
  const line = () => {
    const seconds = Math.floor((Date.now() - started) / 1000);
    const time = seconds >= 60 ? `${Math.floor(seconds / 60)}m ${seconds % 60}s` : `${seconds}s`;
    const spent = usage && `${tokens(usage)} · $${usage.cost.toFixed(4)}`;
    const width = (stdout.columns || 80) - 3;
    const tail =
      [[spent, time], [time]]
        .map((parts) => parts.filter(Boolean).join(' · '))
        .find((text) => width - stringWidth(text) - 2 >= Math.min(stringWidth(doing), 20)) ?? '';
    const room = tail ? width - stringWidth(tail) - 2 : width;
    return tail ? `${fit(doing, room)}  ${styleText('dim', tail)}` : fit(doing, room);
  };
  const withUsage = (text: string) =>
    usage ? `${text}  ${styleText('dim', `${tokens(usage)} · $${usage.cost.toFixed(4)}`)}` : text;
  const refresh = () => {
    if (spinning && live) {
      spin.text = line();
    }
  };
  if (usage) {
    usage.onCall = refresh;
  }

  // The spinner sits on the run's guide line, like the rest of the terminal.
  const startSpinner = () => {
    started = Date.now();
    spinning = true;
    stdout.write(`${styleText('gray', prompt.S_BAR)}\n`);
    spin.start(line());
    if (live) {
      clock = setInterval(refresh, 1000);
    }
  };
  const stopSpinner = (finish: () => void) => {
    clearInterval(clock);
    spin.stop();
    spinning = false;
    finish();
  };

  // What the agent did since you last heard from it, as one dim line.
  const settle = () => {
    if (spinning) {
      const done = withUsage(styleText('dim', steps.join(' · ') || 'Done'));
      stopSpinner(() => prompt.log.step(done, { spacing: 0 }));
    }
    steps = [];
  };

  return {
    busy() {
      if (!spinning) {
        doing = 'The agent is working';
        startSpinner();
      }
    },
    step(text) {
      doing = text.replace(/\.$/, '');
      steps.push(doing);
      refresh();
    },
    async answer(question) {
      settle();
      say(markdown(question), 'cyan');
      return ask(() =>
        prompt.text({ message: 'Your answer', placeholder: 'Type, then press Enter' })
      );
    },
    async decide(draft) {
      settle();
      say(
        [
          styleText(
            'bold',
            `Save the ${KIND_NAMES[draft.kind]} "${draft.title}" in ${language.code}?`
          ),
          ...draft.texts.map((text, index) => showText(text, index, language))
        ].join('\n\n'),
        'cyan'
      );
      return decide();
    },
    /** Shows the run's report. */
    report(text) {
      settle();
      say(markdown(text), 'green');
    },
    /** The run stopped on an error: the spinner says so, and the error follows. */
    failed(error) {
      if (spinning) {
        stopSpinner(() => prompt.log.error('The run stopped', { spacing: 0 }));
      }
      prompt.log.error(error instanceof Error ? error.message : String(error));
    }
  };
}

/** "183,204 tokens": read and written together. */
function tokens(usage: Usage): string {
  return `${new Intl.NumberFormat('en-US').format(usage.inputTokens + usage.outputTokens)} tokens`;
}

/**
 * Text cut to a number of terminal cells, never through a character (an emoji counts as two).
 * yocto-spinner counts a line in UTF-16 code units, which can outnumber its cells (a family emoji
 * is 8), so the cut keeps both within the limit.
 */
function fit(text: string, cells: number): string {
  const size = (part: string) => Math.max(stringWidth(part), part.length);
  if (size(text) <= cells) {
    return text;
  }
  let cut = '';
  for (const { segment } of new Intl.Segmenter().segment(text)) {
    if (size(`${cut}${segment}…`) > cells) {
      break;
    }
    cut += segment;
  }
  return `${cut}…`;
}

/** Prints a block beside the run's guide line, each line wrapped to the terminal's width. */
function say(text: string, color: 'cyan' | 'green') {
  prompt.log.message(wrap(text, (stdout.columns || 80) - 4), { symbol: styleText(color, '◆') });
}

// The terminal's color and style codes (ESC [ … m), as styleText writes them.
const STYLE_CODES = new RegExp(`${String.fromCharCode(27)}\\[[\\d;]*m`, 'g');

/** Wraps each line at spaces, keeping its indent on the lines it wraps onto. Colors don't count. */
export function wrap(text: string, width: number): string {
  const visible = (line: string) => stripVTControlCharacters(line).length;
  return text
    .split('\n')
    .map((line) => {
      // A wrapped line sits under the text, past any list number or bullet.
      const lead = /^( *)((\d+\.|•) )?/.exec(stripVTControlCharacters(line))?.[0] ?? '';
      const indent = ' '.repeat(lead.length);
      const words = line.trimStart().split(' ');
      const lines: string[] = [];
      let current = line.slice(0, line.length - line.trimStart().length) + (words.shift() ?? '');
      for (const word of words) {
        if (visible(`${current} ${word}`) > width && current.trim()) {
          // The colors in force so far carry onto the next line: the guide line resets them.
          const colors = (lines.join('') + current).match(STYLE_CODES)?.join('') ?? '';
          lines.push(`${current}\x1b[0m`);
          current = indent + colors + word;
        } else {
          current = `${current} ${word}`;
        }
      }
      lines.push(current);
      return lines.join('\n');
    })
    .join('\n');
}

/** Starts and ends the run's frame in the terminal. */
export const intro = (title: string) => prompt.intro(styleText(['bgCyan', 'black'], ` ${title} `));
export const outro = (line: string) => prompt.outro(line);
export const note = (line: string) => prompt.log.info(line);

async function decide(): Promise<Choice> {
  const choice = await ask(() =>
    prompt.select({
      message: 'Your call',
      options: [
        { value: 'save', label: 'Save it' },
        { value: 'revise', label: 'Tell the agent what to change', hint: 'it tries again' },
        { value: 'skip', label: 'Skip this one', hint: 'nothing is saved' }
      ]
    })
  );
  if (choice === 'revise') {
    return {
      type: 'revise',
      feedback: await ask(() =>
        prompt.text({
          message: 'What should change?',
          validate: (value) => (value?.trim() ? undefined : 'Say what should change.')
        })
      )
    };
  }
  if (choice === 'skip') {
    return {
      type: 'skip',
      reason: await ask(() =>
        prompt.text({ message: 'Why? (optional, the agent reads it)', defaultValue: '' })
      )
    };
  }
  return { type: 'save' };
}

/** Waits for one prompt; Ctrl+C or Esc stops the run. */
async function ask<T>(show: () => Promise<T | symbol>): Promise<Exclude<T, symbol>> {
  const value = await show();
  if (prompt.isCancel(value)) {
    throw new Stopped('You stopped the run.');
  }
  return value as Exclude<T, symbol>;
}

/**
 * One text, numbered: what it said, under the main language's code, and its translation in green,
 * under the new one's. Still readable with the colors gone, pasted elsewhere.
 */
export function showText(
  text: Draft['texts'][number],
  index: number,
  { main, code }: Pick<Language, 'code' | 'main'>
): string {
  return [
    `${styleText(['bold', 'cyan'], `${index + 1}.`)} ${styleText('dim', text.key)}`,
    `   ${styleText('dim', main.padEnd(code.length))} ${dimTags(text.original)}`,
    `   ${code.padEnd(main.length)} ${styleText('green', text.translation)}`
  ].join('\n');
}

/** Rich text's HTML tags, dimmed, so the words stand out. */
const dimTags = (text: string) => text.replace(/<[^>]+>/g, (tag) => styleText('dim', tag));

// The agent writes Markdown. This renders it for the terminal, with a blank line after every
// paragraph and every list item, so a long answer reads as separate blocks, not a wall of text.
const terminal = new Marked({
  renderer: {
    heading({ tokens }) {
      return `${styleText(['bold', 'cyan'], this.parser.parseInline(tokens))}\n\n`;
    },
    paragraph({ tokens }) {
      return `${this.parser.parseInline(tokens)}\n\n`;
    },
    list(list: Tokens.List) {
      const start = typeof list.start === 'number' ? list.start : 1;
      return list.items
        .map((item, index) => {
          const bullet = list.ordered ? `${start + index}.` : '•';
          const body = this.parser.parse(item.tokens).trim();
          // Lines under the first are indented to sit under its text.
          const pad = ' '.repeat(bullet.length + 1);
          return `${styleText('cyan', bullet)} ${body.replace(/\n/g, `\n${pad}`)}\n\n`;
        })
        .join('');
    },
    text(token) {
      return 'tokens' in token && token.tokens ? this.parser.parseInline(token.tokens) : token.text;
    },
    strong({ tokens }) {
      return styleText('bold', this.parser.parseInline(tokens));
    },
    em({ tokens }) {
      return styleText('italic', this.parser.parseInline(tokens));
    },
    codespan({ text }) {
      return styleText('cyan', text);
    },
    code({ text }) {
      return `${styleText('cyan', text)}\n\n`;
    },
    blockquote({ tokens }) {
      return `${styleText('dim', `│ ${this.parser.parse(tokens).trim().replace(/\n/g, '\n│ ')}`)}\n\n`;
    },
    link({ href, tokens }) {
      const text = this.parser.parseInline(tokens);
      return text === href
        ? styleText('underline', href)
        : `${text} ${styleText('dim', `(${href})`)}`;
    },
    hr() {
      return `${styleText('dim', '─'.repeat(Math.min(stdout.columns || 60, 60)))}\n\n`;
    },
    br() {
      return '\n';
    },
    html({ text }) {
      return text;
    }
  }
});

/** The agent's Markdown, formatted for the terminal. */
export function markdown(text: string): string {
  return String(terminal.parse(text, { async: false })).trim();
}