Translate your whole site into a new language

npm create adonisjs@latest site-translate -- --kit=api
cd site-translate

@fetchski:registry=https://acme.example/api/v1/sdk/npm/
//acme.example/api/v1/sdk/npm/:_authToken=fsk_acme_…

.npmrc

node ace configure @fetchski/attract-cms-adonis

# OpenRouter: your key, and the model the agent runs on
OPENROUTER_API_KEY=sk-or-v1-…
OPENROUTER_MODEL=~anthropic/claude-sonnet-latest

/*
|--------------------------------------------------------------------------
| Environment variables service
|--------------------------------------------------------------------------
|
| The `Env.create` method creates an instance of the Env service. The
| service validates the environment variables and also cast values
| to JavaScript data types.
|
*/

import { Env } from '@adonisjs/core/env'

export default await Env.create(new URL('../', import.meta.url), {
  // Node
  NODE_ENV: Env.schema.enum(['development', 'production', 'test'] as const),
  PORT: Env.schema.number(),
  HOST: Env.schema.string({ format: 'host' }),
  LOG_LEVEL: Env.schema.string(),

  // App
  APP_KEY: Env.schema.secret(),
  APP_URL: Env.schema.string({ format: 'url', tld: false }),

  // Session
  SESSION_DRIVER: Env.schema.enum(['cookie', 'memory', 'database'] as const),

  /*
  |----------------------------------------------------------
  | Variables for configuring @fetchski/attract-cms-adonis
  |----------------------------------------------------------
  */
  FETCHSKI_ATTRACT_SITE: Env.schema.string(),
  FETCHSKI_ATTRACT_URL: Env.schema.string({ format: 'url', tld: false }),
  FETCHSKI_ATTRACT_API_KEY: Env.schema.secret(),
  FETCHSKI_ATTRACT_SIGNING_SECRET: Env.schema.secret.optional(),

  // OpenRouter
  OPENROUTER_API_KEY: Env.schema.secret(),
  OPENROUTER_MODEL: Env.schema.string(),
})

SESSION_DRIVER=memory

# Tests fake the site and the model: placeholders only
FETCHSKI_ATTRACT_SITE=acme
FETCHSKI_ATTRACT_URL=https://acme.example
FETCHSKI_ATTRACT_API_KEY=fsk_acme_0a1b2c3d_not-a-real-secret
OPENROUTER_API_KEY=sk-or-v1-not-a-real-key
OPENROUTER_MODEL=test/fake-model

import env from '#start/env'

/**
 * The model the translator runs on, and the limits every run stays inside.
 */
const openrouterConfig = {
  apiKey: env.get('OPENROUTER_API_KEY'),
  model: env.get('OPENROUTER_MODEL'),
  maxSteps: 6, // tool rounds per document, at most
  attempts: 3, // tries per document when the model doesn't answer (a timeout, an outage, a rate limit)
  // How long one model call may take: room for your longest page on a slow model, so it only ends a call that hangs.
  timeoutSeconds: 600,
  maxCostPerDocumentUsd: 0.5, // one document stops once it has spent this much
  maxCostPerRunUsd: 10, // the run starts no new document past this
}

export default openrouterConfig

node ace make:provider openrouter --register

import { OpenRouter } from '@openrouter/agent'
import type { ApplicationService } from '@adonisjs/core/types'
import type openrouterConfig from '#config/openrouter'

export default class OpenrouterProvider {
  constructor(protected app: ApplicationService) {}

  /**
   * One OpenRouter client for the whole app. Tests swap it for a fake model.
   */
  register() {
    this.app.container.singleton(OpenRouter, () => {
      const config = this.app.config.get<typeof openrouterConfig>('openrouter')

      return new OpenRouter({ apiKey: config.apiKey.release() })
    })
  }
}

# Names the translator never translates, one per line: your business name,
# your services' and products' names, place names you keep as they are.
Acme
Acme Outfitters
Trailhead Tune-Up

import { NotFound, type SiteClient } from '@fetchski/attract-cms-sdk'

/** A document as the site's REST routes send it, under the API's own names. */
export interface StoredDoc {
  _id: string
  aposDocId: string
  type: string
  slug?: string
  title?: string
  _url?: string
  [field: string]: unknown
}

/** Which document: enough to reach it in any language. */
export interface DocRef {
  aposDocId: string
  type: string
  slug?: string
  title?: string
}

/** One piece of a document's wording, under the key core's PATCH writes it back by. */
export interface Wording {
  key: string
  text: string
}

// Copied into the language so pages keep them, never translated.
const MEDIA = new Set(['@apostrophecms/image', '@apostrophecms/file', 'video'])

// Keys that hold codes, addresses or settings, never words a visitor reads.
const NOT_WORDING_KEYS = new Set([
  'type',
  'metaType',
  'slug',
  'path',
  'aposDocId',
  'aposLocale',
  'aposMode',
  'visibility',
  'titleSortified',
  'highSearchText',
  'lowSearchText',
  'searchSummary',
  'parkedId',
  'source',
  'scopedArrayName',
  'submitted', // who sent it for review, and when
])
const NOT_WORDING_SUFFIX =
  /(By|Url|Href|Color|ColorLight|Custom|Icon|IconFamily|Css|Slug|Email|Phone|Anchor|Code)$/
const NOT_WORDS = [
  /^(https?:|mailto:|tel:|\/|#|--)/, // addresses, colors, design tokens
  /^[a-z0-9]+([-_.][a-z0-9]+)*$/, // choices, icon names, slugs
  /^\S*[a-z][A-Z]\S*$/, // one camelCase token: a choice (belowHeadline)
  /^[^\s<>]*[@/_{}=][^\s<>]*$/, // one token holding code characters (@apostrophecms/any-page-type)
  /^\d{4}-\d\d-\d\d/, // dates
  /^[^\p{L}]*$/u, // no letters at all
]

/** A page lives under the page routes; every other type under its own name. */
export const isPage = (doc: DocRef) => typeof doc.slug === 'string' && doc.slug.startsWith('/')
export const isMedia = (doc: DocRef) => MEDIA.has(doc.type)

const route = (doc: DocRef) => `/${isPage(doc) ? '@apostrophecms/page' : doc.type}/${doc.aposDocId}`

/**
 * Any document on the site, by its type: read in a language, copied into one,
 * its wording written back. Pages, posts and FAQs relate to documents of any
 * type (images, authors, categories, forms), so this reaches each through the
 * site's REST routes with `request()`, the SDK's call for a route by name.
 */
export class SiteDocuments {
  constructor(protected site: SiteClient) {}

  /**
   * Every draft of a type in `locale`, as stored (so with core's
   * `localizedAt`, which the SDK's own models leave out), a page at a time.
   */
  async list(type: string, locale: string, query: Record<string, string | number> = {}) {
    const all: StoredDoc[] = []
    for (let page = 1, pages = 1; page <= pages; page++) {
      const answer = await this.site.request<{ results: StoredDoc[]; pages?: number }>(
        'GET',
        `/${type}`,
        { query: { ...query, aposLocale: locale, aposMode: 'draft', perPage: 100, page } }
      )
      all.push(...(answer?.results ?? []))
      pages = answer?.pages ?? 1
    }
    return all
  }

  /** The draft in `locale`, with its related documents; null when the language has no copy yet. */
  async get(doc: DocRef, locale: string) {
    try {
      return await this.site.request<StoredDoc>('GET', route(doc), {
        query: { aposLocale: locale, aposMode: 'draft' },
      })
    } catch (error) {
      if (error instanceof NotFound) {
        return null
      }
      throw error
    }
  }

  /** Copies the draft in `from` into `to`, as a draft there; `update` overwrites a copy already there. */
  async copy(doc: DocRef, from: string, to: string, update: boolean) {
    await this.site.request('POST', `${route(doc)}:${from}:draft/localize`, {
      body: { toLocale: to, update },
    })
  }

  /**
   * Writes the translated wording to the draft in `locale`. Core's PATCH takes
   * `@<widget id>.<field>` keys, so only the words change, never a setting.
   */
  async write(doc: DocRef, locale: string, wording: readonly Wording[]) {
    await this.site.request('PATCH', route(doc), {
      query: { aposLocale: locale, aposMode: 'draft' },
      body: Object.fromEntries(wording.map(({ key, text }) => [key, text])),
    })
  }

  /** Puts the draft in `locale` in the CMS's review queue (Submitted Drafts). */
  async submit(doc: DocRef, locale: string) {
    await this.site.request('POST', `${route(doc)}:${locale}:draft/submit`)
  }
}

/**
 * A copy is up to date when it was copied from the current original and
 * written since: its translation is saved. ISO dates compare as strings.
 */
export function isUpToDate(original: StoredDoc | null, copy: StoredDoc | null) {
  const copiedAt = typeof copy?.localizedAt === 'string' ? copy.localizedAt : null
  return Boolean(
    original &&
    copiedAt &&
    copiedAt >= String(original.updatedAt) &&
    String(copy?.updatedAt) > copiedAt
  )
}

/**
 * The words a visitor reads in `doc`: titles, headings, text, rich text,
 * button labels, SEO text. Codes, colors, addresses and settings stay out,
 * and so do related documents (they are translated as documents of their own).
 * `fields` limits it to those fields (the site settings' own words: the rest
 * of their fields are shared by every language).
 */
export function wordingOf(doc: StoredDoc, fields?: readonly string[]): Wording[] {
  const found: Wording[] = []

  const visit = (value: unknown, owner: string | null, path: string) => {
    if (typeof value === 'string') {
      const text = value.trim()
      if (text && !NOT_WORDS.some((pattern) => pattern.test(text))) {
        found.push({ key: owner ? `@${owner}.${path}` : path, text: value })
      }
      return
    }
    if (Array.isArray(value)) {
      value.forEach((item, index) => {
        // A widget or array row has an id: its fields are keyed by it.
        if (item && typeof item === 'object' && typeof item._id === 'string') {
          visit(item, item._id, '')
        } else {
          visit(item, owner, path ? `${path}.${index}` : String(index))
        }
      })
      return
    }
    if (!value || typeof value !== 'object' || (value as StoredDoc).metaType === 'doc') {
      return
    }
    for (const [key, child] of Object.entries(value)) {
      if (key.startsWith('_') || NOT_WORDING_KEYS.has(key) || NOT_WORDING_SUFFIX.test(key)) {
        continue
      }
      const area = child as { metaType?: string; items?: unknown[] } | null
      visit(area?.metaType === 'area' ? area.items : child, owner, path ? `${path}.${key}` : key)
    }
  }

  for (const [key, value] of Object.entries(doc)) {
    if (
      !key.startsWith('_') &&
      !NOT_WORDING_KEYS.has(key) &&
      !NOT_WORDING_SUFFIX.test(key) &&
      (!fields || fields.includes(key))
    ) {
      const area = value as { metaType?: string; items?: unknown[] } | null
      visit(area?.metaType === 'area' ? area.items : value, null, key)
    }
  }
  return found
}

/** The path of a document's `_url` (`/about-us`), without the site's address. */
export const pathOf = (url: unknown) =>
  typeof url === 'string' ? (URL.parse(url, 'https://site.invalid')?.pathname ?? null) : null

/**
 * The links in `fields` of the site settings (a `url` in a list row,
 * `headerCtaUrl`…) that point at a page the language has, each written to
 * point at that page in the language instead, under the key core's PATCH
 * writes it back by. `pointTo` maps a page's path to its path in the
 * language. A link's `?query` and `#anchor` are kept; a link to another site
 * is left alone.
 */
export function linksTo(
  doc: StoredDoc,
  fields: readonly string[],
  pointTo: ReadonlyMap<string, string>
): Wording[] {
  const found: Wording[] = []
  const visit = (value: unknown, owner: string | null, key: string) => {
    if (Array.isArray(value)) {
      for (const row of value) {
        if (row && typeof row === 'object') {
          for (const [field, child] of Object.entries(row)) {
            visit(child, typeof row._id === 'string' ? row._id : owner, field)
          }
        }
      }
      return
    }
    // A link on this site starts with one slash; another site's is left alone.
    if (typeof value !== 'string' || !/(^url|Url)$/.test(key) || !/^\/(?!\/)/.test(value)) {
      return
    }
    const [, path = '', rest = ''] = /^([^?#]*)(.*)$/.exec(value) ?? []
    const to = pointTo.get(path)
    if (to && `${to}${rest}` !== value) {
      found.push({ key: owner ? `@${owner}.${key}` : key, text: `${to}${rest}` })
    }
  }
  for (const field of fields) {
    visit(doc[field], null, field)
  }
  return found
}

/**
 * The documents `doc` shows (images, authors, categories, forms…), once each.
 * Pages are left out: every page is in the run already.
 */
export function relatedTo(doc: StoredDoc): DocRef[] {
  const found = new Map<string, DocRef>()

  const visit = (value: unknown, root: boolean) => {
    if (Array.isArray(value)) {
      value.forEach((item) => visit(item, false))
      return
    }
    if (!value || typeof value !== 'object') {
      return
    }
    const item = value as StoredDoc
    // A related document comes projected: its `_id` (`<aposDocId>:<locale>:<mode>`),
    // not its aposDocId. One with no locale in its `_id` (a media folder) is
    // the same in every language: nothing to copy.
    if (!root && item.metaType === 'doc' && typeof item._id === 'string') {
      const [aposDocId, locale] = item._id.split(':')
      if (locale && !isPage(item)) {
        found.set(aposDocId, {
          aposDocId,
          type: item.type,
          slug: item.slug,
          title: item.title,
        })
      }
      return
    }
    for (const [key, child] of Object.entries(item)) {
      // A page's tree and its published twin are pages, not what it shows.
      if (!['_ancestors', '_children', '_parent', '_publishedDoc'].includes(key)) {
        visit(child, false)
      }
    }
  }

  visit(doc, true)
  return [...found.values()]
}

import { tool } from '@openrouter/agent'
import { z } from 'zod'
import type { Wording } from '#services/site_documents'

/** What the run saved, for the code around the agent (never the model's word for it). */
export interface TranslationRun {
  saved: boolean
  /** What the last check found, for the report when nothing was saved. */
  problems: string[]
}

const TAG = /<\/?([a-z][a-z0-9]*)\b[^>]*>/gi
const LINK = /\b(?:href|src)\s*=\s*"([^"]*)"/gi

const tagsOf = (text: string) =>
  [...text.matchAll(TAG)]
    .map((match) => (match[0].startsWith('</') ? `/${match[1]}` : match[1]))
    .join(' ')
const linksOf = (text: string) => [...text.matchAll(LINK)].map((match) => match[1]).join(' ')
/** The {year}-style words the site fills in, in any order (a language may move them). */
const placeholdersOf = (text: string) => (text.match(/\{[a-z]+\}/g) ?? []).sort().join(' ')
const plain = (text: string) => text.replace(TAG, ' ').replace(/\s+/g, ' ').trim()

/**
 * A sentence sent back as it was: not translated. A sentence: four words or more, glossary names
 * aside, two of them lowercase (a name or an address alone never counts).
 */
const leftAsIs = (text: string, translation: string, glossary: readonly string[]) => {
  if (plain(text) !== plain(translation)) return false
  const rest = glossary.reduce((left, name) => left.replaceAll(name, ' '), plain(text))
  const words = rest.match(/\p{L}+/gu) ?? []
  return words.length >= 4 && words.filter((word) => /^\p{Ll}/u.test(word)).length >= 2
}

/**
 * Why a translation can't be saved: a key missing or invented, an HTML tag,
 * a link or a {placeholder} changed, a glossary name translated, a sentence
 * left untranslated. Empty when it can be.
 */
export function problemsWith(
  original: readonly Wording[],
  translated: readonly Wording[],
  glossary: readonly string[]
): string[] {
  const problems: string[] = []
  const given = new Map(translated.map(({ key, text }) => [key, text]))

  for (const { key, text } of original) {
    const translation = given.get(key)
    if (translation === undefined || !translation.trim()) {
      problems.push(`${key}: missing. Translate every key you were given.`)
      continue
    }
    if (tagsOf(translation) !== tagsOf(text)) {
      problems.push(
        `${key}: the HTML tags changed. Keep exactly these, in order: ${tagsOf(text) || 'none'}.`
      )
    }
    if (linksOf(translation) !== linksOf(text)) {
      problems.push(`${key}: a link changed. Keep every href and src exactly as it was.`)
    }
    if (placeholdersOf(translation) !== placeholdersOf(text)) {
      const kept = placeholdersOf(text) || 'none'
      problems.push(`${key}: keep these exactly as written, the site fills them in: ${kept}.`)
    }
    for (const name of glossary) {
      if (text.includes(name) && !translation.includes(name)) {
        problems.push(`${key}: "${name}" is in the glossary. Keep it exactly as written.`)
      }
    }
    if (leftAsIs(text, translation, glossary)) {
      problems.push(`${key}: still in the original language. Translate it.`)
    }
  }
  const known = new Set(original.map(({ key }) => key))
  for (const key of given.keys()) {
    if (!known.has(key)) {
      problems.push(`${key}: not one of the keys you were given. Leave it out.`)
    }
  }
  return problems
}

/**
 * The agent's one tool: check a whole translation and, once it passes, save
 * it to the language's draft with `save`. Nothing is saved until it passes.
 */
export function translationTools(
  original: readonly Wording[],
  glossary: readonly string[],
  save: (translated: readonly Wording[]) => Promise<void>,
  run: TranslationRun
) {
  const saveTranslation = tool({
    name: 'save_translation',
    description:
      'Check the whole translation and save it as the draft in the new language. It names every problem to fix; call it again with all the texts once they are fixed.',
    inputSchema: z.object({
      texts: z
        .array(z.object({ key: z.string(), text: z.string() }))
        .describe('Every key you were given, each with its translated text'),
    }),
    execute: async ({ texts }) => {
      if (run.saved) {
        return { error: 'Already saved. Stop here.' }
      }
      const problems = problemsWith(original, texts, glossary)
      run.problems = problems
      if (problems.length) {
        return { saved: false, problems }
      }
      await save(texts)
      run.saved = true
      return { saved: true }
    },
  })

  return [saveTranslation] as const
}

import { readFile } from 'node:fs/promises'
import { inject } from '@adonisjs/core'
import app from '@adonisjs/core/services/app'
import { maxCost, OpenRouter, stepCountIs } from '@openrouter/agent'
import { SiteClient } from '@fetchski/attract-cms-sdk'
import type openrouterConfig from '#config/openrouter'
import {
  type DocRef,
  isMedia,
  isUpToDate,
  linksTo,
  pathOf,
  relatedTo,
  SiteDocuments,
  type StoredDoc,
  type Wording,
  wordingOf,
} from '#services/site_documents'
import { translationTools, type TranslationRun } from '#services/translation_tools'

const INSTRUCTIONS = `You translate a small business's website, one document at a time, for its own visitors.

- You get the language, a glossary and the document's texts, each under a key.
  Translate every text into the language and call save_translation once with
  every key.
- Write as a native speaker would for this business: natural, warm, plain.
  Keep the meaning, the facts and the tone. Never add or drop information.
- Keep every glossary name exactly as written, in any text.
- Keep every word in braces, such as {year} or {business}, exactly as
  written: the site fills it in.
- Rich text is HTML: translate the words between the tags, and keep every tag,
  attribute and link exactly as it is.
- Leave a text as it is when it isn't words for a visitor (a code, a hashtag,
  a product name) or is already in the language.
- If save_translation names problems, fix them and call it again with every key.`

/** One document's outcome, for the report. */
export interface Translated {
  title: string
  kind: 'page' | 'blog post' | 'FAQ' | 'site settings' | 'shown on them'
  /** The draft on the site, for anyone with the link, without signing in. */
  previewUrl: string | null
  /** Why it was skipped or failed; null once saved. */
  problem: string | null
}

export interface TranslationReport {
  added: boolean
  documents: Translated[]
  upToDate: number
  inputTokens: number
  outputTokens: number
  costUsd: number
  published: boolean
}

interface Job extends DocRef {
  kind: Translated['kind']
  share?: (id: string) => Promise<{ aposShareKey?: string | null }>
  /** The only fields with words to translate (the site settings': the rest are shared). */
  fields?: string[]
  /** Links pointed at the language's pages, saved with the translation. */
  links?: Wording[]
}

// Site settings and Site design: one document per language.
const SETTINGS = '@apostrophecms/global'

@inject()
export class SiteTranslator {
  protected docs: SiteDocuments

  constructor(
    protected site: SiteClient,
    protected openrouter: OpenRouter
  ) {
    this.docs = new SiteDocuments(site)
  }

  /**
   * Translates the site's pages, blog posts and FAQs (and what they show), and
   * its header, footer and cookie banner, into `code`, adding the language
   * first if the site lacks it. Every translation is a draft; `publish` makes
   * the language public, nothing else.
   */
  async translate(code: string, name: string, publish = false): Promise<TranslationReport> {
    const added = await this.ensureLanguage(code, name, publish)
    const main = await this.mainLocale(code)
    const glossary = await this.glossary()
    const report: TranslationReport = {
      added,
      documents: [],
      upToDate: 0,
      inputTokens: 0,
      outputTokens: 0,
      costUsd: 0,
      published: publish,
    }

    // 1. What needs translating: never copied, or changed since its copy.
    const { all, paths } = await this.listAll(main, code)
    const jobs: Job[] = []
    for (const listed of all) {
      if (listed.upToDate) {
        report.upToDate++
      } else {
        jobs.push(listed.job)
      }
    }

    // 2. Copy what any of them shows that needs its words (by the same rule, so
    //    a copy a stopped run left is translated by the next), then each one: a
    //    write keeps a related document only once it exists in the language.
    //    A key missing a scope stops here, before any copy is overwritten.
    const shown: Job[] = []
    const seen = new Set<string>()
    const copyShown = async (doc: DocRef) => {
      const stored = await this.docs.get(doc, main)
      for (const related of stored ? relatedTo(stored) : []) {
        if (seen.has(related.aposDocId)) {
          continue
        }
        seen.add(related.aposDocId)
        const copy = await this.docs.get(related, code)
        // A copy is all an image or a file needs; anything else needs its words, unless up to date.
        if (copy && (isMedia(related) || isUpToDate(await this.docs.get(related, main), copy))) {
          continue
        }
        await this.docs.copy(related, main, code, copy !== null)
        if (!isMedia(related)) {
          shown.push({ ...related, kind: 'shown on them' })
        }
        await copyShown(related)
      }
    }
    for (const { job } of all) {
      await copyShown(job)
    }
    for (const job of jobs) {
      await this.docs.copy(job, main, code, true)
    }
    // The settings' links to pages point at the language's pages.
    const settings = all.find(({ job }) => job.kind === 'site settings')
    if (settings?.job.fields && !settings.upToDate) {
      settings.job.links = linksTo(
        settings.original,
        settings.job.fields,
        await this.pathsIn(code, paths)
      )
    }

    // 3. Translate each copy and send it for review, within the run's budget.
    const config = app.config.get<typeof openrouterConfig>('openrouter')
    for (const job of [...jobs, ...shown]) {
      if (report.costUsd >= config.maxCostPerRunUsd) {
        report.documents.push(
          this.outcome(
            job,
            null,
            `Not translated: the run reached $${config.maxCostPerRunUsd}. Run it again to go on.`
          )
        )
        continue
      }
      report.documents.push(await this.translateOne(job, code, name, glossary, report))
    }
    return report
  }

  /** Adds the language, not public, when the site lacks it; `publish` makes it public. */
  protected async ensureLanguage(code: string, name: string, publish: boolean) {
    const lists = await this.site.languages().listSiteLanguages()
    const [list] = lists.results
    if (!list) {
      throw new Error(
        'The site has no language list: open Languages in the CMS once, then run this again.'
      )
    }
    const languages = list.languages ?? []
    const known = languages.find((language) => language.code === code)
    if (known && (!publish || known.public)) {
      return false
    }
    await this.site.languages().updateSiteLanguages({
      id: list.id,
      languages: known
        ? languages.map((language) =>
            language.code === code ? { ...language, public: true } : language
          )
        : [...languages, { name, code, public: publish }],
    })
    return !known
  }

  /** The site's main language: the one without an address prefix. Checks `code` is one of its languages now. */
  protected async mainLocale(code: string) {
    const locales = await this.site.internationalization().getLocales()
    const main = Object.keys(locales).find((locale) => !locales[locale]?.prefix)
    if (!main || !locales[code] || code === main) {
      throw new Error(
        `"${code}" isn't one of the site's further languages: the site has ${Object.keys(locales).join(', ')}.`
      )
    }
    return main
  }

  /** The names never translated, one per line in storage/glossary.txt. */
  protected async glossary() {
    const text = await readFile(app.makePath('storage/glossary.txt'), 'utf8').catch(() => '')
    return text
      .split('\n')
      .map((line) => line.trim())
      .filter((line) => line && !line.startsWith('#'))
  }

  /**
   * Every page (parents first: a page is copied only under its parent),
   * blog post and FAQ, and the site settings, each with whether its
   * translation is up to date: copied from the current original, and written
   * since the copy. With each page's and post's path, by id.
   */
  protected async listAll(main: string, code: string) {
    const kinds = [
      {
        kind: 'page' as const,
        type: '@apostrophecms/page',
        query: { all: 1, flat: 1 },
        share: (id: string) => this.site.pages().sharePage({ id, share: true }),
      },
      {
        kind: 'blog post' as const,
        type: '@apostrophecms/blog',
        share: (id: string) => this.site.blogPosts().shareBlog({ id, share: true }),
      },
      { kind: 'FAQ' as const, type: 'faq-item' },
    ]
    const out: { job: Job; original: StoredDoc; upToDate: boolean }[] = []
    const paths = new Map<string, string>()

    for (const { kind, type, query, share } of kinds) {
      const copyList = await this.docs.list(type, code, query)
      const copies = new Map(copyList.map((copy) => [copy.aposDocId, copy]))
      const originalList = await this.docs.list(type, main, query)
      const originals = originalList.toSorted(
        (a, b) =>
          Number(a.level ?? 0) - Number(b.level ?? 0) || Number(a.rank ?? 0) - Number(b.rank ?? 0)
      )
      for (const original of originals) {
        const path = pathOf(original._url)
        if (path && kind !== 'FAQ') {
          paths.set(original.aposDocId, path)
        }
        let copy = copies.get(original.aposDocId)
        // The page tree's list sends a child page's main fields only: read it whole.
        if (copy && !('localizedAt' in copy)) {
          copy = (await this.docs.get(copy, code)) ?? undefined
        }
        out.push({
          job: {
            aposDocId: original.aposDocId,
            type: original.type,
            slug: original.slug,
            title: original.title,
            kind,
            share,
          },
          original,
          upToDate: isUpToDate(original, copy ?? null),
        })
      }
    }

    // The header, footer and cookie banner: the settings' fields the site says
    // each language translates.
    const [settings] = await this.docs.list(SETTINGS, main)
    if (settings) {
      const upToDate = isUpToDate(settings, await this.docs.get(settings, code))
      const { translated } = upToDate
        ? { translated: [] }
        : await this.site.languages().getGlobalTranslatedFields()
      out.push({
        job: {
          aposDocId: settings.aposDocId,
          type: settings.type,
          slug: settings.slug,
          title: 'Header, footer and cookie banner',
          kind: 'site settings',
          fields: [...translated],
        },
        original: settings,
        upToDate,
      })
    }
    return { all: out, paths }
  }

  /** Each page's and post's path → its path in the language (`/about-us` → `/fr/about-us`). */
  protected async pathsIn(code: string, paths: ReadonlyMap<string, string>) {
    const pointTo = new Map<string, string>()
    for (const [type, query] of [
      ['@apostrophecms/page', { all: 1, flat: 1 }],
      ['@apostrophecms/blog', {}],
    ] as const) {
      for (const copy of await this.docs.list(type, code, query)) {
        const from = paths.get(copy.aposDocId)
        const to = pathOf(copy._url)
        if (from && to) {
          pointTo.set(from, to)
        }
      }
    }
    return pointTo
  }

  /** Has the agent translate one copy, saves it, and sends it for review. */
  protected async translateOne(
    job: Job,
    code: string,
    name: string,
    glossary: string[],
    report: TranslationReport
  ) {
    const config = app.config.get<typeof openrouterConfig>('openrouter')
    const copy = await this.docs.get(job, code)
    const texts = copy ? wordingOf(copy, job.fields) : []
    if (!copy || !texts.length) {
      return this.outcome(job, null, copy ? 'Nothing to translate.' : `No copy in ${code}.`)
    }

    const run: TranslationRun = { saved: false, problems: [] }
    let failure: string | null = null
    for (let attempt = 1; attempt <= config.attempts && !run.saved; attempt++) {
      const result = this.openrouter.callModel(
        {
          model: config.model,
          instructions: INSTRUCTIONS,
          input: JSON.stringify({ language: `${name} (${code})`, glossary, texts }),
          tools: translationTools(
            texts,
            glossary,
            (translated) => this.docs.write(job, code, [...translated, ...(job.links ?? [])]),
            run
          ),
          stopWhen: [
            stepCountIs(config.maxSteps),
            maxCost(config.maxCostPerDocumentUsd),
            () => run.saved,
          ],
        },
        { timeoutMs: config.timeoutSeconds * 1000 }
      )
      try {
        await result.getText() // throws when the model never answered
        failure = null
        break
      } catch (error) {
        // The model didn't answer: a timeout, an outage, a rate limit. Try again.
        failure = (error as Error).message.replace(/\.$/, '')
      } finally {
        // Every attempt counts toward the run's budget, a failed one too.
        const usage = await result.getUsage()
        report.inputTokens += usage.inputTokens
        report.outputTokens += usage.outputTokens
        report.costUsd += usage.cost ?? 0
      }
    }

    if (!run.saved) {
      const problem = failure
        ? `Not translated: the model didn't answer (${failure}). Run it again to go on.`
        : `Not saved: ${run.problems.slice(0, 3).join(' ') || 'the model stopped before saving.'}`
      return this.outcome(job, null, problem)
    }
    await this.docs.submit(job, code)
    const shared = job.share ? await job.share(`${job.aposDocId}:${code}:draft`) : null
    const url = typeof copy._url === 'string' ? copy._url : null
    const previewUrl =
      url && shared?.aposShareKey
        ? `${url}?${new URLSearchParams({ aposShareKey: shared.aposShareKey, aposShareId: `${job.aposDocId}:${code}:draft` })}`
        : null
    return this.outcome(job, previewUrl, null)
  }

  protected outcome(job: Job, previewUrl: string | null, problem: string | null): Translated {
    return { title: job.title ?? job.slug ?? job.aposDocId, kind: job.kind, previewUrl, problem }
  }
}