Find where your website contradicts itself, and fix it

import type { SiteClient } from '@fetchski/attract-cms-sdk';
import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
import { MemorySaver } from '@langchain/langgraph';
import { ChatOpenRouter } from '@langchain/openrouter';
import { createDeepAgent } from 'deepagents';
import { todoListMiddleware } from 'langchain';
import { type SavedDraft, siteTools } from './tools.ts';

const FIND = `You check a small business website for places where it contradicts itself.

A contradiction is two places on the site stating one fact differently: a price, opening hours, a phone
number or address, a year founded, a team size, a guarantee, a delivery time, a policy, a name.
Different wording of the same fact is not a contradiction.

Work in this order:
1. Plan your work with write_todos.
2. Call list_site_content once.
3. Hand the documents to the fact-collector subagent with the task tool, about eight at a time. Give it
   the kind, id and title of each, and the file to write: /facts/batch-1.md, /facts/batch-2.md and so on.
4. Read the /facts files. Write /contradictions.md: each contradiction, every claim with its kind, id
   and path. If there are none, say so and stop.`;

const FIX = `
5. For each contradiction, call ask_owner once: quote each version and where it appears, and ask which is
   true (the owner may also type the correct fact).
6. For each document that needs a fix, call save_draft_changes once with every change it needs. Change
   only the words that are wrong, and keep the HTML around them. The owner approves each document.
   If the owner sends one back with notes, change it to match and call save_draft_changes again. If
   they skip one, do not try it again.
7. Finish with a short summary: what you found, and which documents now have a draft waiting.

Each ask_owner call asks ONE question. Write it in Markdown, with a blank line between paragraphs and
between list items.`;

const REPORT_ONLY = `
5. Nobody is at the keyboard for this run, and you can't change the site. Finish with a report for the
   owner: each contradiction as a short heading, then every version quoted, with the title of the
   document it appears in.`;

const FACT_COLLECTOR = `You collect facts from website documents. For each document you are given, call
read_document, then write every concrete, checkable claim (prices, hours, numbers, years, names, contact
details, guarantees, policies, delivery times) to the file you are told, one line each:
- <claim in your own words> | <kind> <id> <path> | "<the exact words>"
Skip opinions and general marketing language. Answer with only the file's path when you are done.`;

export interface AgentOptions {
  /** Nobody is at the keyboard: the agent only reports, with no tool to ask or to write. */
  reportOnly?: boolean;
}

/** The deep agent: our SDK's operations as its tools, a subagent per batch of documents, and the owner's approval before any draft is written. */
export function buildAgent(
  site: SiteClient,
  model: BaseChatModel,
  saved: Map<string, SavedDraft>,
  log: (line: string) => void,
  { reportOnly = false }: AgentOptions = {}
) {
  const tools = siteTools(site, saved, log);
  return createDeepAgent({
    model,
    systemPrompt: FIND + (reportOnly ? REPORT_ONLY : FIX),
    tools: reportOnly
      ? [tools.listSiteContent, tools.readDocument]
      : [tools.listSiteContent, tools.readDocument, tools.askOwner, tools.saveDraftChanges],
    middleware: [todoListMiddleware()],
    subagents: [
      {
        name: 'fact-collector',
        description: 'Reads a batch of documents and writes their concrete facts to a /facts file.',
        systemPrompt: FACT_COLLECTOR,
        tools: [tools.readDocument]
      }
    ],
    // Nothing is written without the owner: approve it, edit it, or reject it.
    interruptOn: { save_draft_changes: { allowedDecisions: ['approve', 'edit', 'reject'] } },
    checkpointer: new MemorySaver()
  });
}

/** The OpenRouter model and settings named in .env. ChatOpenRouter reads OPENROUTER_API_KEY itself. */
export function modelFromEnv() {
  return new ChatOpenRouter({
    model: process.env.OPENROUTER_MODEL || '~google/gemini-flash-latest',
    temperature: Number(process.env.OPENROUTER_TEMPERATURE || 0),
    maxTokens: Number(process.env.OPENROUTER_MAX_TOKENS || 16000)
  });
}

import type { SiteClient } from '@fetchski/attract-cms-sdk';
import { interrupt } from '@langchain/langgraph';
import { tool } from 'langchain';
import { z } from 'zod';
import { type Kind, listDocuments, readWording, saveDraftChanges } from './content.ts';

const kind = z.enum(['page', 'post', 'faq']).describe('page, post (a blog post) or faq');

/** A document the agent saved a draft of. */
export interface SavedDraft {
  kind: Kind;
  title: string;
}

/**
 * Our SDK's operations, as the agent's tools. `saved` collects every document a draft was written
 * for, by id; `log` shows the agent's progress.
 */
export function siteTools(
  site: SiteClient,
  saved: Map<string, SavedDraft>,
  log: (line: string) => void
) {
  const listSiteContent = tool(
    async () => {
      const documents = await listDocuments(site);
      log(`Found ${documents.length} document${documents.length === 1 ? '' : 's'}.`);
      return JSON.stringify(documents);
    },
    {
      name: 'list_site_content',
      description:
        'Lists every page, blog post and FAQ on the website: kind, id and title. Call it once.',
      schema: z.object({})
    }
  );

  const readDocument = tool(
    async (args) => {
      const document = await readWording(site, args.kind, args.id);
      log(`Read the ${args.kind} "${document.title}".`);
      return JSON.stringify(document.wording);
    },
    {
      name: 'read_document',
      description:
        'Reads one document: every piece of its wording, each with the path that finds it again. Rich text is HTML.',
      schema: z.object({ kind, id: z.string().describe('The id from list_site_content') })
    }
  );

  const saveDraftChangesTool = tool(
    async (args) => {
      const answer = await saveDraftChanges(site, args.kind, args.id, args.changes);
      if (answer.startsWith('Saved')) {
        saved.set(args.id, { kind: args.kind, title: args.title });
      }
      return answer;
    },
    {
      name: 'save_draft_changes',
      description:
        'Saves edits to ONE document as a draft, once the owner approves them. Keep every HTML tag and link ' +
        'exactly; change only words. `before` is the exact text read_document gave for that path.',
      schema: z.object({
        kind,
        id: z.string(),
        title: z.string().describe("The document's title, shown to the owner"),
        changes: z
          .array(z.object({ path: z.string(), before: z.string(), after: z.string() }))
          .min(1)
          .describe('Every edit to this document, each at its path')
      })
    }
  );

  const askOwner = tool(
    // Pauses the run until the owner answers; their answer is the tool's result.
    async (args) => String(interrupt({ question: args.question })),
    {
      name: 'ask_owner',
      description: 'Asks the website owner one question, and waits for their answer.',
      schema: z.object({ question: z.string() })
    }
  );

  return { listSiteContent, readDocument, saveDraftChanges: saveDraftChangesTool, askOwner };
}

import type { SiteClient } from '@fetchski/attract-cms-sdk';

/** The three kinds of document the agent reads and edits. */
export type Kind = 'page' | 'post' | 'faq';

export interface DocumentRow {
  kind: Kind;
  id: string; // the draft's id, `…:en:draft`
  title: string;
}

/** One piece of wording in a document: where it sits, and what it says (rich text is HTML). */
export interface Wording {
  path: string;
  text: string;
}

/** One edit: the wording at `path` goes from `before` to `after`. */
export interface Change {
  path: string;
  before: string;
  after: string;
}

/** The most documents one run reads: a ceiling on time and AI cost. */
export const MAX_DOCUMENTS = Number(process.env.MAX_DOCUMENTS ?? 80);

/** The fields that hold a document's wording, per kind. Nothing else is ever written. */
export const EDITABLE: Record<Kind, readonly string[]> = {
  page: ['title', 'seoDescription', 'sections'],
  post: ['title', 'excerpt', 'seoDescription', 'main'],
  faq: ['title', 'answer']
};

// Keys inside a widget that never hold wording: ids, types, styling, links.
const NOT_WORDING = /^(_|metaType$|type$|id$)|css|color|icon|font|url|href|slug|image|video/i;

/** Every page, blog post and FAQ, as drafts: the latest wording, published or not. */
export async function listDocuments(site: SiteClient): Promise<DocumentRow[]> {
  const draft = { aposMode: 'draft' } as const;
  const [pages, posts, faqs] = await Promise.all([
    site.pages().listPagesPick(['title', 'type'], draft),
    site.blogPosts().listBlogsPick(['title'], draft),
    site.faqItems().listFaqItemsPick(['title'], draft)
  ]);
  const rows: DocumentRow[] = [
    ...pages
      .filter((page) => page.type !== '@apostrophecms/archive-page')
      .map((page) => ({ kind: 'page' as const, id: page.id, title: String(page.title) })),
    ...posts.map((post) => ({ kind: 'post' as const, id: post.id, title: String(post.title) })),
    ...faqs.map((faq) => ({ kind: 'faq' as const, id: faq.id, title: String(faq.title) }))
  ];
  return rows.slice(0, MAX_DOCUMENTS);
}

/** A document's draft, as a plain object. */
async function readDraft(
  site: SiteClient,
  kind: Kind,
  id: string
): Promise<Record<string, unknown>> {
  const draftId = id.replace(/:published$/, ':draft');
  switch (kind) {
    case 'page':
      return { ...(await site.pages().getPage({ id: draftId })) };
    case 'post':
      return { ...(await site.blogPosts().getBlog({ id: draftId })) };
    case 'faq':
      return { ...(await site.faqItems().getFaqItem({ id: draftId })) };
  }
}

/** A document's title, and every piece of wording in its draft with the path that finds it again. */
export async function readWording(
  site: SiteClient,
  kind: Kind,
  id: string
): Promise<{ title: string; wording: Wording[] }> {
  const doc = await readDraft(site, kind, id);
  const found: Wording[] = [];
  const visit = (value: unknown, path: string, topLevel: boolean) => {
    if (typeof value === 'string') {
      // A top-level field is wording even as one word; inside a widget, wording has a space in it.
      if (value.trim() && (topLevel || /\s/.test(value.trim())) && !/^https?:\/\//.test(value)) {
        found.push({ path, text: value });
      }
    } else if (Array.isArray(value)) {
      for (const [index, item] of value.entries()) {
        visit(item, `${path}.${index}`, false);
      }
    } else if (value && typeof value === 'object') {
      for (const [key, item] of Object.entries(value)) {
        if (!NOT_WORDING.test(key)) {
          visit(item, `${path}.${key}`, false);
        }
      }
    }
  };
  for (const field of EDITABLE[kind]) {
    visit(doc[field], field, true);
  }
  return { title: String(doc.title), wording: found };
}

/**
 * Writes the changes to the document's DRAFT. Each change must still match what the agent read
 * (`before`), so an edit someone made meanwhile is never overwritten. Answers in words, for the agent.
 */
export async function saveDraftChanges(
  site: SiteClient,
  kind: Kind,
  id: string,
  changes: readonly Change[]
): Promise<string> {
  const draftId = id.replace(/:published$/, ':draft');
  const doc = await readDraft(site, kind, draftId);
  const fields = new Map<string, unknown>();
  for (const change of changes) {
    const [field = '', ...rest] = change.path.split('.');
    if (!EDITABLE[kind].includes(field)) {
      return `Not saved: "${change.path}" is not wording this tool may change (allowed: ${EDITABLE[kind].join(', ')}).`;
    }
    if (!fields.has(field)) {
      fields.set(field, structuredClone(doc[field]));
    }
    if (rest.length === 0) {
      if (fields.get(field) !== change.before) {
        return `Not saved: "${change.path}" changed since you read it. Read the document again.`;
      }
      fields.set(field, change.after);
      continue;
    }
    let parent = fields.get(field) as Record<string, unknown> | undefined;
    for (const key of rest.slice(0, -1)) {
      parent = parent?.[key] as Record<string, unknown> | undefined;
    }
    const last = rest[rest.length - 1] ?? '';
    if (!parent || parent[last] !== change.before) {
      return `Not saved: "${change.path}" changed since you read it. Read the document again.`;
    }
    parent[last] = change.after;
  }

  const text = (name: string) => fields.get(name) as string | undefined;
  const area = (name: string) => fields.get(name) as Record<string, unknown> | undefined;
  switch (kind) {
    case 'page':
      await site.pages().updatePage({
        id: draftId,
        title: text('title'),
        seoDescription: text('seoDescription'),
        sections: area('sections')
      });
      break;
    case 'post':
      await site.blogPosts().updateBlog({
        id: draftId,
        title: text('title'),
        excerpt: text('excerpt'),
        seoDescription: text('seoDescription'),
        main: area('main')
      });
      break;
    case 'faq':
      await site
        .faqItems()
        .updateFaqItem({ id: draftId, title: text('title'), answer: area('answer') });
      break;
  }
  return `Saved ${changes.length} change(s) to the draft of ${draftId}. Nothing is live until a person publishes it.`;
}

/** A private link to a page's or post's draft, for anyone you send it to: no sign-in. FAQs have no page of their own. */
export async function previewLink(
  site: SiteClient,
  kind: Kind,
  id: string
): Promise<string | null> {
  if (kind === 'faq') {
    return null;
  }
  const doc = await readDraft(site, kind, id);
  const shared =
    kind === 'page'
      ? await site.pages().sharePage({ id, share: true })
      : await site.blogPosts().shareBlog({ id, share: true });
  return `${String(doc.url)}?${new URLSearchParams({ aposShareKey: shared.aposShareKey ?? '', aposShareId: id })}`;
}

import { randomUUID } from 'node:crypto';
import { Command } from '@langchain/langgraph';
import type { buildAgent } from './agent.ts';
import type { Change } from './content.ts';
import type { Usage } from './usage.ts';

/** A draft the agent wants to save: the document, and every change to it. */
export interface Draft {
  kind: string;
  title: string;
  changes: Change[];
}

/** Your call on a draft: save it all, keep some changes, send it back with notes, or skip it. */
export type Choice =
  | { type: 'all' }
  | { type: 'some'; keep: number[] }
  | { type: 'revise'; feedback: string }
  | { type: 'skip'; reason: string };

/** The person at the keyboard. src/terminal.ts is the real one; the tests script one. */
export interface Owner {
  /** The agent is working (a spinner, in the terminal). */
  busy?(): void;
  /** Answers the agent's question. */
  answer(question: string): Promise<string>;
  /** Decides on a draft the agent wants to save. */
  decide(draft: Draft): Promise<Choice>;
}

/** Nobody at the keyboard: a report-only run, whose agent has no tool to ask or to save. */
export const nobody: Owner = {
  answer: async () => {
    throw new Error('A report-only run never asks.');
  },
  decide: async () => {
    throw new Error('A report-only run never saves.');
  }
};

interface ActionRequest {
  name: string;
  args: { kind?: string; id?: string; title?: string; changes?: Change[] };
}

type Decision =
  | { type: 'approve' }
  | { type: 'reject'; message: string }
  | { type: 'edit'; editedAction: { name: string; args: ActionRequest['args'] } };

/** The most steps (model calls and tool calls) one run may take: a ceiling on AI cost. */
const MAX_STEPS = Number(process.env.AGENT_MAX_STEPS ?? 300);

/**
 * Runs the agent to the end. Each time it pauses, the owner answers its question, or decides on the
 * drafts it wants to write. Resolves the agent's closing summary.
 */
export async function runWithOwner(
  agent: ReturnType<typeof buildAgent>,
  task: string,
  owner: Owner,
  usage?: Usage
): Promise<string> {
  // `usage` counts every model call's tokens and cost, the subagents' too.
  const config = {
    configurable: { thread_id: randomUUID() },
    recursionLimit: MAX_STEPS,
    callbacks: usage ? [usage] : []
  };
  let input: Parameters<typeof agent.invoke>[0] = { messages: [{ role: 'user', content: task }] };

  for (;;) {
    owner.busy?.();
    const result = await agent.invoke(input, config);
    // Several rewriters can pause at once: each pause gets its own answer, by its id.
    const pending = result.__interrupt__ ?? [];
    if (pending.length === 0) {
      const last = result.messages.at(-1);
      return typeof last?.content === 'string' ? last.content : JSON.stringify(last?.content ?? '');
    }

    const answers: Record<string, unknown> = {};
    for (const { id, value } of pending) {
      const paused = value as { question?: string; actionRequests?: ActionRequest[] };
      if (paused.question) {
        answers[String(id)] = await owner.answer(paused.question);
        continue;
      }
      const decisions: Decision[] = [];
      for (const action of paused.actionRequests ?? []) {
        decisions.push(await review(action, owner));
      }
      answers[String(id)] = { decisions };
    }
    input = new Command({ resume: answers });
  }
}

/** Your choice, in the form the agent's approval step takes: approve, edit (fewer changes) or reject. */
async function review(action: ActionRequest, owner: Owner): Promise<Decision> {
  const changes = action.args.changes ?? [];
  const choice = await owner.decide({
    kind: action.args.kind ?? 'page',
    title: action.args.title ?? '',
    changes
  });

  switch (choice.type) {
    case 'all':
      return { type: 'approve' };
    case 'some': {
      const kept = changes.filter((_, index) => choice.keep.includes(index));
      if (kept.length === 0) {
        return {
          type: 'reject',
          message: 'The owner kept none of these changes. Do not try them again.'
        };
      }
      return {
        type: 'edit',
        editedAction: { name: action.name, args: { ...action.args, changes: kept } }
      };
    }
    case 'revise':
      return {
        type: 'reject',
        message:
          `Not saved yet: the owner wants changes first. Their words: "${choice.feedback}". ` +
          'Revise your changes to match, then call the same tool again with every change.'
      };
    case 'skip':
      return {
        type: 'reject',
        message: `The owner skipped this draft${choice.reason ? `: ${choice.reason}` : ''}. Do not try it again.`
      };
  }
}

import { stdout } from 'node:process';
import { stripVTControlCharacters, styleText } from 'node:util';
import * as prompt from '@clack/prompts';
import { diffWordsWithSpace } from 'diff';
import { Marked, type Tokens } from 'marked';
import stringWidth from 'string-width';
import yoctoSpinner from 'yocto-spinner';
import type { Change } from './content.ts';
import type { Choice, Draft, Owner } from './review.ts';
import type { Usage } from './usage.ts';

/** You pressed Ctrl+C or Esc at a prompt: the run stops, and the drafts saved so far stay drafts. */
export class Stopped extends Error {}

/**
 * The person at the keyboard, in the terminal: a spinner while the agent works, with what it is
 * doing and what the run has used so far; its words as formatted text.
 */
export function terminalOwner(usage?: Usage): Owner & {
  step: (line: string) => void;
  report: (markdown: string) => void;
  failed: (error: unknown) => void;
} {
  // yocto-spinner's own dots; each frame redraws in one synchronized write, so it never flickers.
  const spin = yoctoSpinner({ stream: stdout });
  // Like yocto-spinner, only a terminal (not TERM=dumb, outside CI) gets live updates; anywhere
  // else each one would print a new line, so output sent to a file gets the start and the steps.
  const live = Boolean(stdout.isTTY) && process.env.TERM !== 'dumb' && !('CI' in process.env);
  let spinning = false;
  let started = 0;
  let clock: NodeJS.Timeout | undefined;
  let steps: string[] = [];
  let doing = 'The agent is working';

  // What the agent is doing, then the tokens, cost and time so far: one line, never wrapped.
  // On a narrow terminal the tokens and cost go first, then the time, before the task is cut.
  const line = () => {
    const seconds = Math.floor((Date.now() - started) / 1000);
    const time = seconds >= 60 ? `${Math.floor(seconds / 60)}m ${seconds % 60}s` : `${seconds}s`;
    const spent = usage && `${tokens(usage)} · $${usage.cost.toFixed(4)}`;
    const width = (stdout.columns || 80) - 3;
    const tail =
      [[spent, time], [time]]
        .map((parts) => parts.filter(Boolean).join(' · '))
        .find((text) => width - stringWidth(text) - 2 >= Math.min(stringWidth(doing), 20)) ?? '';
    const room = tail ? width - stringWidth(tail) - 2 : width;
    return tail ? `${fit(doing, room)}  ${styleText('dim', tail)}` : fit(doing, room);
  };
  const withUsage = (text: string) =>
    usage ? `${text}  ${styleText('dim', `${tokens(usage)} · $${usage.cost.toFixed(4)}`)}` : text;
  const refresh = () => {
    if (spinning && live) {
      spin.text = line();
    }
  };
  if (usage) {
    usage.onCall = refresh;
  }

  // The spinner sits on the run's guide line, like the rest of the terminal.
  const startSpinner = () => {
    started = Date.now();
    spinning = true;
    stdout.write(`${styleText('gray', prompt.S_BAR)}\n`);
    spin.start(line());
    if (live) {
      clock = setInterval(refresh, 1000);
    }
  };
  const stopSpinner = (finish: () => void) => {
    clearInterval(clock);
    spin.stop();
    spinning = false;
    finish();
  };

  // What the agent did since you last heard from it, as one dim line.
  const settle = () => {
    if (spinning) {
      const done = withUsage(styleText('dim', steps.join(' · ') || 'Done'));
      stopSpinner(() => prompt.log.step(done, { spacing: 0 }));
    }
    steps = [];
  };

  return {
    busy() {
      if (!spinning) {
        doing = 'The agent is working';
        startSpinner();
      }
    },
    step(text) {
      doing = text.replace(/\.$/, '');
      steps.push(doing);
      refresh();
    },
    async answer(question) {
      settle();
      say(markdown(question), 'cyan');
      return ask(() =>
        prompt.text({ message: 'Your answer', placeholder: 'Type, then press Enter' })
      );
    },
    async decide(draft) {
      settle();
      say(
        [
          styleText('bold', `Save a draft of the ${draft.kind} "${draft.title}"?`),
          ...draft.changes.map((change, index) => showChange(change, index))
        ].join('\n\n'),
        'cyan'
      );
      return decide(draft);
    },
    /** Shows the run's report. */
    report(text) {
      settle();
      say(markdown(text), 'green');
    },
    /** The run stopped on an error: the spinner says so, and the error follows. */
    failed(error) {
      if (spinning) {
        stopSpinner(() => prompt.log.error('The run stopped', { spacing: 0 }));
      }
      prompt.log.error(error instanceof Error ? error.message : String(error));
    }
  };
}

/** "183,204 tokens": read and written together. */
function tokens(usage: Usage): string {
  return `${new Intl.NumberFormat('en-US').format(usage.inputTokens + usage.outputTokens)} tokens`;
}

/**
 * Text cut to a number of terminal cells, never through a character (an emoji counts as two).
 * yocto-spinner counts a line in UTF-16 code units, which can outnumber its cells (a family emoji
 * is 8), so the cut keeps both within the limit.
 */
function fit(text: string, cells: number): string {
  const size = (part: string) => Math.max(stringWidth(part), part.length);
  if (size(text) <= cells) {
    return text;
  }
  let cut = '';
  for (const { segment } of new Intl.Segmenter().segment(text)) {
    if (size(`${cut}${segment}…`) > cells) {
      break;
    }
    cut += segment;
  }
  return `${cut}…`;
}

/** Prints a block beside the run's guide line, each line wrapped to the terminal's width. */
function say(text: string, color: 'cyan' | 'green') {
  prompt.log.message(wrap(text, (stdout.columns || 80) - 4), { symbol: styleText(color, '◆') });
}

// The terminal's color and style codes (ESC [ … m), as styleText writes them.
const STYLE_CODES = new RegExp(`${String.fromCharCode(27)}\\[[\\d;]*m`, 'g');

/** Wraps each line at spaces, keeping its indent on the lines it wraps onto. Colors don't count. */
export function wrap(text: string, width: number): string {
  const visible = (line: string) => stripVTControlCharacters(line).length;
  return text
    .split('\n')
    .map((line) => {
      // A wrapped line sits under the text, past any list number or bullet.
      const lead =
        /^( *)((\d+\.|•|before|after ) )?/.exec(stripVTControlCharacters(line))?.[0] ?? '';
      const indent = ' '.repeat(lead.length);
      const words = line.trimStart().split(' ');
      const lines: string[] = [];
      let current = line.slice(0, line.length - line.trimStart().length) + (words.shift() ?? '');
      for (const word of words) {
        if (visible(`${current} ${word}`) > width && current.trim()) {
          // The colors in force so far carry onto the next line: the guide line resets them.
          const colors = (lines.join('') + current).match(STYLE_CODES)?.join('') ?? '';
          lines.push(`${current}\x1b[0m`);
          current = indent + colors + word;
        } else {
          current = `${current} ${word}`;
        }
      }
      lines.push(current);
      return lines.join('\n');
    })
    .join('\n');
}

/** Starts and ends the run's frame in the terminal. */
export const intro = (title: string) => prompt.intro(styleText(['bgCyan', 'black'], ` ${title} `));
export const outro = (line: string) => prompt.outro(line);
export const note = (line: string) => prompt.log.info(line);

async function decide(draft: Draft): Promise<Choice> {
  const many = draft.changes.length > 1;
  const choice = await ask(() =>
    prompt.select({
      message: 'Your call',
      options: [
        { value: 'all', label: many ? `Save all ${draft.changes.length} changes` : 'Save it' },
        ...(many ? [{ value: 'some', label: 'Pick which changes to keep' }] : []),
        { value: 'revise', label: 'Tell the agent what to change', hint: 'it tries again' },
        { value: 'skip', label: 'Skip this one', hint: 'nothing is saved' }
      ]
    })
  );
  if (choice === 'some') {
    const keep = await ask(() =>
      prompt.multiselect({
        message: 'Keep these (Space to toggle, Enter to save)',
        options: draft.changes.map((change, index) => ({
          value: index,
          label: `${index + 1}. ${plain(change.after)}`
        })),
        initialValues: draft.changes.map((_, index) => index),
        required: false
      })
    );
    return { type: 'some', keep };
  }
  if (choice === 'revise') {
    return {
      type: 'revise',
      feedback: await ask(() =>
        prompt.text({
          message: 'What should change?',
          validate: (value) => (value?.trim() ? undefined : 'Say what should change.')
        })
      )
    };
  }
  if (choice === 'skip') {
    return {
      type: 'skip',
      reason: await ask(() =>
        prompt.text({ message: 'Why? (optional, the agent reads it)', defaultValue: '' })
      )
    };
  }
  return { type: 'all' };
}

/** Waits for one prompt; Ctrl+C or Esc stops the run. */
async function ask<T>(show: () => Promise<T | symbol>): Promise<Exclude<T, symbol>> {
  const value = await show();
  if (prompt.isCancel(value)) {
    throw new Stopped('You stopped the run.');
  }
  return value as Exclude<T, symbol>;
}

/**
 * One change, numbered: the text before and after, whole, with the words that change marked (red
 * and struck through before, green after). Still readable with the colors gone, pasted elsewhere.
 */
export function showChange(change: Change, index: number): string {
  const parts = diffWordsWithSpace(change.before, change.after);
  const kept = parts.filter((part) => !part.added && !part.removed);
  const label = `${styleText(['bold', 'cyan'], `${index + 1}.`)} ${styleText('dim', change.path)}`;
  // Mostly rewritten: the old text in red and the new in green, with no marks to read through.
  if (
    kept
      .map((part) => part.value)
      .join('')
      .trim().length <
    change.after.length / 2
  ) {
    return [
      label,
      `   ${styleText('dim', 'before')} ${styleText('red', change.before)}`,
      `   ${styleText('dim', 'after ')} ${styleText('green', change.after)}`
    ].join('\n');
  }
  const side = (drop: 'added' | 'removed', mark: Parameters<typeof styleText>[0]) =>
    parts
      .filter((part) => !part[drop])
      .map((part) =>
        part.added || part.removed ? styleText(mark, part.value) : dimTags(part.value)
      )
      .join('');
  return [
    label,
    `   ${styleText('dim', 'before')} ${side('added', ['red', 'strikethrough'])}`,
    `   ${styleText('dim', 'after ')} ${side('removed', ['green', 'bold'])}`
  ].join('\n');
}

/** Rich text's HTML tags, dimmed, so the words stand out. */
const dimTags = (text: string) => text.replace(/<[^>]+>/g, (tag) => styleText('dim', tag));

/** Text with its HTML tags taken out, on one line, for a short label. */
function plain(text: string, max = 70): string {
  const flat = text
    .replace(/<[^>]+>/g, ' ')
    .replace(/\s+/g, ' ')
    .trim();
  return flat.length > max ? `${flat.slice(0, max - 1)}…` : flat;
}

// The agent writes Markdown. This renders it for the terminal, with a blank line after every
// paragraph and every list item, so a long answer reads as separate blocks, not a wall of text.
const terminal = new Marked({
  renderer: {
    heading({ tokens }) {
      return `${styleText(['bold', 'cyan'], this.parser.parseInline(tokens))}\n\n`;
    },
    paragraph({ tokens }) {
      return `${this.parser.parseInline(tokens)}\n\n`;
    },
    list(list: Tokens.List) {
      const start = typeof list.start === 'number' ? list.start : 1;
      return list.items
        .map((item, index) => {
          const bullet = list.ordered ? `${start + index}.` : '•';
          const body = this.parser.parse(item.tokens).trim();
          // Lines under the first are indented to sit under its text.
          const pad = ' '.repeat(bullet.length + 1);
          return `${styleText('cyan', bullet)} ${body.replace(/\n/g, `\n${pad}`)}\n\n`;
        })
        .join('');
    },
    text(token) {
      return 'tokens' in token && token.tokens ? this.parser.parseInline(token.tokens) : token.text;
    },
    strong({ tokens }) {
      return styleText('bold', this.parser.parseInline(tokens));
    },
    em({ tokens }) {
      return styleText('italic', this.parser.parseInline(tokens));
    },
    codespan({ text }) {
      return styleText('cyan', text);
    },
    code({ text }) {
      return `${styleText('cyan', text)}\n\n`;
    },
    blockquote({ tokens }) {
      return `${styleText('dim', `│ ${this.parser.parse(tokens).trim().replace(/\n/g, '\n│ ')}`)}\n\n`;
    },
    link({ href, tokens }) {
      const text = this.parser.parseInline(tokens);
      return text === href
        ? styleText('underline', href)
        : `${text} ${styleText('dim', `(${href})`)}`;
    },
    hr() {
      return `${styleText('dim', '─'.repeat(Math.min(stdout.columns || 60, 60)))}\n\n`;
    },
    br() {
      return '\n';
    },
    html({ text }) {
      return text;
    }
  }
});

/** The agent's Markdown, formatted for the terminal. */
export function markdown(text: string): string {
  return String(terminal.parse(text, { async: false })).trim();
}

import { BaseCallbackHandler } from '@langchain/core/callbacks/base';
import { AIMessage } from '@langchain/core/messages';
import type { ChatGeneration, LLMResult } from '@langchain/core/outputs';

/**
 * Counts every model call in a run, the subagents' too: tokens read and written, and what
 * OpenRouter charged. Pass it in the run's `callbacks`; LangChain calls handleLLMEnd after each call.
 */
export class Usage extends BaseCallbackHandler {
  name = 'usage';
  calls = 0;
  inputTokens = 0;
  outputTokens = 0;
  /** In US dollars, from OpenRouter's own answer (`usage.cost`), never estimated from a price list. */
  cost = 0;
  /** Calls whose answer carried no cost (a streamed answer, or a provider that leaves it out). */
  uncosted = 0;
  /** Called after each model call, e.g. to refresh a running total on screen. */
  onCall?: () => void;

  override handleLLMEnd(output: LLMResult) {
    this.calls += 1;
    const message = (output.generations[0]?.[0] as ChatGeneration | undefined)?.message;
    const usage = AIMessage.isInstance(message) ? message.usage_metadata : undefined;
    this.inputTokens += usage?.input_tokens ?? 0;
    this.outputTokens += usage?.output_tokens ?? 0;
    const cost = output.llmOutput?.tokenUsage?.cost;
    if (typeof cost === 'number') {
      this.cost += cost;
    } else {
      this.uncosted += 1;
    }
    this.onCall?.();
  }

  /** One line: "42 model calls · 183,204 tokens in · 9,876 out · $0.0412". */
  line(): string {
    const count = new Intl.NumberFormat('en-US');
    const parts = [
      `${count.format(this.calls)} model call${this.calls === 1 ? '' : 's'}`,
      `${count.format(this.inputTokens)} tokens in`,
      `${count.format(this.outputTokens)} out`,
      `$${this.cost.toFixed(4)}`
    ];
    const line = parts.join(' · ');
    return this.uncosted > 0
      ? `${line} (${this.uncosted} call${this.uncosted === 1 ? '' : 's'} without a cost)`
      : line;
  }

  /** The run's usage, as a Markdown section for the report and its email. */
  markdown(): string {
    return `## Usage\n\n${this.line()}. The cost is what OpenRouter charged for this run.`;
  }
}

import type { SiteClient } from '@fetchski/attract-cms-sdk';
import { type Kind, previewLink } from './content.ts';
import type { SavedDraft } from './tools.ts';
import type { Usage } from './usage.ts';

/** A draft the run saved, and the private link that previews it (FAQs have no page of their own). */
export interface WaitingDraft {
  kind: Kind;
  title: string;
  link: string | null;
}

/** Every draft the run saved, each with its preview link. */
export async function draftsWaiting(
  site: SiteClient,
  saved: Map<string, SavedDraft>
): Promise<WaitingDraft[]> {
  const drafts: WaitingDraft[] = [];
  for (const [id, draft] of saved) {
    drafts.push({ ...draft, link: await previewLink(site, draft.kind, id) });
  }
  return drafts;
}

/**
 * The run's report, in Markdown: a heading, the agent's summary, the drafts waiting for you, then
 * what the run used (tokens and cost).
 */
export function reportMarkdown(
  summary: string,
  drafts: readonly WaitingDraft[],
  usage?: Usage,
  date = new Date()
) {
  // The site's host for the heading; a missing or mistyped URL still gets a report that says why.
  const site = URL.parse(process.env.FETCHSKI_ATTRACT_URL ?? '')?.host || 'your site';
  const lines = [
    `# Contradictions on ${site}, ${date.toLocaleDateString('en-CA')}`,
    '',
    summary.trim()
  ];
  if (drafts.length > 0) {
    lines.push('', '## Drafts waiting for you', '');
    lines.push('Publish each one in the CMS when it reads right.', '');
    for (const draft of drafts) {
      const link = draft.link ? `, preview: ${draft.link}` : '';
      lines.push(`- ${draft.kind} "${draft.title}"${link}`);
    }
  }
  if (usage) {
    lines.push('', usage.markdown());
  }
  return `${lines.join('\n')}\n`;
}

import { createSiteClient } from '@fetchski/attract-cms-sdk';
import { buildAgent, modelFromEnv } from './agent.ts';
import { draftsWaiting, reportMarkdown } from './report.ts';
import { runWithOwner } from './review.ts';
import { intro, outro, Stopped, terminalOwner } from './terminal.ts';
import type { SavedDraft } from './tools.ts';
import { Usage } from './usage.ts';

// FETCHSKI_ATTRACT_URL, FETCHSKI_ATTRACT_API_KEY and FETCHSKI_ATTRACT_SIGNING_SECRET, from .env
const site = createSiteClient();

const saved = new Map<string, SavedDraft>();
const usage = new Usage();
const owner = terminalOwner(usage);
const agent = buildAgent(site, modelFromEnv(), saved, owner.step);

intro('Find contradictions');
let summary: string;
try {
  summary = await runWithOwner(
    agent,
    'Check this whole website for places where it contradicts itself, and fix them with me.',
    owner,
    usage
  );
} catch (error) {
  if (!(error instanceof Stopped)) {
    owner.failed(error);
    throw error;
  }
  // Ctrl+C at a prompt: the drafts saved so far stay drafts, and the report lists them.
  summary = 'You stopped the run before it finished.';
}

const report = reportMarkdown(summary, await draftsWaiting(site, saved), usage);
owner.report(report);
outro(`Done: ${saved.size} draft${saved.size === 1 ? '' : 's'} saved · ${usage.line()}`);

import { deepStrictEqual, ok, strictEqual } from 'node:assert';
import { test } from 'node:test';
import { factories, fakeSite } from '@fetchski/attract-cms-sdk/testing';
import { AIMessage } from '@langchain/core/messages';
import type { LLMResult } from '@langchain/core/outputs';
import { fakeModel } from 'langchain';
import { buildAgent } from '../src/agent.ts';
import { reportMarkdown } from '../src/report.ts';
import { type Choice, nobody, type Owner, runWithOwner } from '../src/review.ts';
import type { SavedDraft } from '../src/tools.ts';
import { Usage } from '../src/usage.ts';

const PAGE_ID = 'page-about:en:draft';
const FAQ_ID = 'faq-hours:en:draft';

function richText(html: string) {
  return {
    metaType: 'area',
    items: [{ _id: 'w1', metaType: 'widget', type: '@apostrophecms/rich-text', content: html }]
  };
}

// A site whose About page and FAQ disagree about Saturdays.
function siteThatContradictsItself() {
  return fakeSite({
    'pages.listPages': factories.ListPagesResponse({
      results: [{ id: PAGE_ID, title: 'About us', type: 'default-page' }]
    }),
    'blogPosts.listBlogs': factories.BlogList({ results: [], pages: 1, currentPage: 1 }),
    'faqItems.listFaqItems': factories.FaqItemList({
      results: [{ id: FAQ_ID, title: 'When are you open?' }],
      pages: 1,
      currentPage: 1
    }),
    'pages.getPage': factories.Page({
      id: PAGE_ID,
      title: 'About us',
      sections: richText('<p>We are open Monday to Friday, 9 to 5.</p>')
    }),
    'faqItems.getFaqItem': factories.FaqItem({
      id: FAQ_ID,
      title: 'When are you open?',
      answer: richText('<p>Monday to Saturday, 9 to 5.</p>')
    })
  });
}

// The fix the agent proposes: the FAQ's answer, from Saturday to Friday.
const fix = (after: string) => ({
  name: 'save_draft_changes',
  args: {
    kind: 'faq',
    id: FAQ_ID,
    title: 'When are you open?',
    changes: [
      { path: 'answer.items.0.content', before: '<p>Monday to Saturday, 9 to 5.</p>', after }
    ]
  }
});

// The model's moves, in order: what a real model would decide, scripted. With a `revision`, the
// owner sends the first fix back, and the agent saves that instead.
function scriptedModel(revision?: string) {
  let model = fakeModel()
    .respondWithTools([{ name: 'list_site_content', args: {} }])
    .respondWithTools([
      { name: 'read_document', args: { kind: 'page', id: PAGE_ID } },
      { name: 'read_document', args: { kind: 'faq', id: FAQ_ID } }
    ])
    .respondWithTools([
      {
        name: 'ask_owner',
        args: {
          question:
            'About us says Monday to Friday; the FAQ says Monday to Saturday. Which is true?'
        }
      }
    ])
    .respondWithTools([fix('<p>Monday to Friday, 9 to 5.</p>')]);
  if (revision) {
    model = model.respondWithTools([fix(revision)]);
  }
  return model.respond(
    new AIMessage(
      'Fixed one contradiction: the FAQ now matches the About page. Its draft is waiting.'
    )
  );
}

// The person at the keyboard, scripted: their answers, then their call on each draft.
function scriptedOwner(answers: string[], choices: Choice[]): Owner & { questions: string[] } {
  const questions: string[] = [];
  return {
    questions,
    answer: async (question) => {
      questions.push(question);
      return answers.shift() ?? '';
    },
    decide: async () => choices.shift() ?? { type: 'skip', reason: '' }
  };
}

test('asks the owner, then saves the approved fix as a draft', async () => {
  const site = siteThatContradictsItself();
  const saved = new Map<string, SavedDraft>();
  const owner = scriptedOwner(['Monday to Friday'], [{ type: 'all' }]);

  const summary = await runWithOwner(
    buildAgent(site, scriptedModel(), saved, () => {}),
    'Check this whole website for contradictions.',
    owner
  );

  strictEqual(
    summary,
    'Fixed one contradiction: the FAQ now matches the About page. Its draft is waiting.'
  );
  site.assertCalled('faqItems.updateFaqItem', (args) => {
    deepStrictEqual(args.answer, richText('<p>Monday to Friday, 9 to 5.</p>'));
    return args.id === FAQ_ID;
  });
  site.assertNotCalled('pages.updatePage');
  deepStrictEqual([...saved], [[FAQ_ID, { kind: 'faq', title: 'When are you open?' }]]);
  deepStrictEqual(owner.questions, [
    'About us says Monday to Friday; the FAQ says Monday to Saturday. Which is true?'
  ]);
});

test('the owner sends a fix back with notes, and the agent saves its revision', async () => {
  const site = siteThatContradictsItself();
  const owner = scriptedOwner(
    ['Monday to Friday'],
    [{ type: 'revise', feedback: 'Say weekdays' }, { type: 'all' }]
  );

  await runWithOwner(
    buildAgent(site, scriptedModel('<p>Weekdays, 9 to 5.</p>'), new Map(), () => {}),
    'Check this whole website for contradictions.',
    owner
  );

  site.assertCalledTimes('faqItems.updateFaqItem', 1);
  site.assertCalled('faqItems.updateFaqItem', (args) => {
    deepStrictEqual(args.answer, richText('<p>Weekdays, 9 to 5.</p>'));
    return true;
  });
});

test('a skipped fix is never written', async () => {
  const site = siteThatContradictsItself();
  const owner = scriptedOwner(
    ['Monday to Friday'],
    [{ type: 'skip', reason: 'We are changing our hours next month anyway' }]
  );

  await runWithOwner(
    buildAgent(site, scriptedModel(), new Map(), () => {}),
    'Check this whole website for contradictions.',
    owner
  );

  site.assertNotCalled('faqItems.updateFaqItem');
});

test('a report-only run reads and reports, and never asks or writes', async () => {
  const site = siteThatContradictsItself();
  const model = fakeModel()
    .respondWithTools([{ name: 'list_site_content', args: {} }])
    .respondWithTools([
      { name: 'read_document', args: { kind: 'page', id: PAGE_ID } },
      { name: 'read_document', args: { kind: 'faq', id: FAQ_ID } }
    ])
    .respond(
      new AIMessage(
        '## Opening days\n- About us: "Monday to Friday"\n- When are you open?: "Monday to Saturday"'
      )
    );

  const summary = await runWithOwner(
    buildAgent(site, model, new Map(), () => {}, { reportOnly: true }),
    'Check this whole website for contradictions, and report them to me.',
    nobody
  );

  strictEqual(summary.startsWith('## Opening days'), true);
  site.assertNotCalled('faqItems.updateFaqItem');
  site.assertNotCalled('pages.updatePage');
});

test('the report lists each draft with its preview link', () => {
  process.env.FETCHSKI_ATTRACT_URL = 'https://acme.example';
  const report = reportMarkdown(
    'Fixed one contradiction.',
    [
      { kind: 'faq', title: 'When are you open?', link: null },
      { kind: 'page', title: 'Pricing', link: 'https://acme.example/pricing?aposShareKey=k8Jq2w' }
    ],
    undefined,
    new Date(2026, 9, 28)
  );

  strictEqual(
    report,
    [
      '# Contradictions on acme.example, 2026-10-28',
      '',
      'Fixed one contradiction.',
      '',
      '## Drafts waiting for you',
      '',
      'Publish each one in the CMS when it reads right.',
      '',
      '- faq "When are you open?"',
      '- page "Pricing", preview: https://acme.example/pricing?aposShareKey=k8Jq2w',
      ''
    ].join('\n')
  );
});

test('usage adds up tokens and the cost OpenRouter reports for each call', () => {
  const usage = new Usage();
  const call = (input: number, output: number, cost?: number) =>
    usage.handleLLMEnd({
      generations: [
        [
          {
            text: '',
            message: new AIMessage({
              content: '',
              usage_metadata: {
                input_tokens: input,
                output_tokens: output,
                total_tokens: input + output
              }
            })
          }
        ]
      ],
      // OpenRouter's own `usage` object, as ChatOpenRouter passes it on.
      llmOutput: { tokenUsage: cost === undefined ? {} : { cost } }
    } as unknown as LLMResult);
  call(1000, 200, 0.0012);
  call(3000, 50, 0.0021);
  call(10, 5);

  strictEqual(
    usage.line(),
    '3 model calls · 4,010 tokens in · 255 out · $0.0033 (1 call without a cost)'
  );
  ok(usage.markdown().startsWith('## Usage\n\n3 model calls'));
});