From ec0c81b4b3c2f91802fc09698a7af03f7d9f4ab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Mon, 20 Apr 2026 12:28:23 -0400 Subject: [PATCH] Harden record-fixture CLI to emit real LLM trace JSONL (B5 step 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the stub recorder with a real-provider CLI. The command now: 1. Reads system prompt + user message from files on disk (empty files are rejected — a fixture with no prompt captures no behaviour). 2. Resolves the provider spec from `--provider anthropic|openai|google` and pulls the API key from the matching environment variable. Missing env → hard error with a copy-pasteable export hint; no fallback, no mock. 3. Calls `streamSimple` from @mariozechner/pi-ai with the user-picked model. On provider error (stopReason=error|aborted) the CLI exits non-zero and writes no JSONL. Users re-run if they think the failure was transient. 4. Serialises one user turn + one assistant turn + an empty assertions block into the JSONL shape documented in tests/eval/README.md. Token ledger comes straight from the assistant message's usage field (renamed to snake_case for provider-neutral on-disk format). Scope is deliberately single-turn: tool use / multi-turn agent loops are B5.1's job. Importing the AgentCompany worker's system prompt builder would require the Electron runtime; the CLI stays node-only so it can run standalone. `--help` works and prints the argument matrix. All failure modes exit non-zero with a specific, actionable message. Also document the new recording workflow in tests/eval/README.md — four-step procedure, example scenario files, assertion templates, and explicit list of what the recorder still does not do. --- tests/eval/README.md | 134 +++++++++++++++-- tests/eval/record-fixture.ts | 274 +++++++++++++++++++++++++++++------ 2 files changed, 353 insertions(+), 55 deletions(-) diff --git a/tests/eval/README.md b/tests/eval/README.md index 2977ad21..4af2c133 100644 --- a/tests/eval/README.md +++ b/tests/eval/README.md @@ -25,14 +25,125 @@ yet; that tier is proposed in ```bash # Assertion-only eval (no fixtures needed; passes on empty fixture dir) pnpm test:eval - -# Record a stub fixture header (full recorder lands in B5.1) -pnpm eval:record -- \ - --fixture-id ceo-team-building-empty \ - --scenario "Empty company, CEO bootstrap" \ - --out tests/eval/fixtures/ceo-team-building-empty.jsonl ``` +## Recording your first fixture + +The recorder drives the **real** `@mariozechner/pi-ai` client against a +live provider. No mocks, no fallbacks. A fixture that exists was +produced by a real API call, end of story. + +### Step 1 — export an API key + +Pick the provider you want to record against and export the matching +variable: + +```bash +export ANTHROPIC_API_KEY=sk-ant-… +# or +export OPENAI_API_KEY=sk-… +# or +export GOOGLE_API_KEY=… +``` + +The recorder **fails hard** if the variable is absent. This is +deliberate: silently skipping the call would produce a fixture that +documents the wrong thing. + +### Step 2 — write the system prompt and user message to disk + +The recorder reads both from files so you can version your scenarios +without pasting long prompts into shell args. Put them anywhere; the +conventional location is under `tests/eval/scenarios//`: + +```bash +mkdir -p tests/eval/scenarios/ceo-bootstrap-empty +cat > tests/eval/scenarios/ceo-bootstrap-empty/system.txt <<'EOF' +You are the CEO of AgentCompany. Your team is empty. Call the `hire_agent` +tool to file the first three hires for the engineering, design, and +research departments. After all three, call `approve_ceo_strategy` once. +EOF + +cat > tests/eval/scenarios/ceo-bootstrap-empty/user.txt <<'EOF' +Build me a team. +EOF +``` + +Empty files are rejected. A fixture with no real prompt captures no +real behaviour. + +### Step 3 — record + +```bash +pnpm eval:record \ + --scenario "Empty company, CEO bootstrap" \ + --fixture-id ceo-bootstrap-empty \ + --provider anthropic \ + --model claude-sonnet-4-5-20250929 \ + --system-prompt-file tests/eval/scenarios/ceo-bootstrap-empty/system.txt \ + --user-message-file tests/eval/scenarios/ceo-bootstrap-empty/user.txt \ + --output tests/eval/fixtures/ceo-bootstrap-empty.jsonl +``` + +On success you'll see the usage ledger printed to stdout and the JSONL +written at `--output`. The assertions block is written empty; the next +step is to fill it in. + +### Step 4 — add assertions + +Open the fixture you just wrote and replace the empty assertions block: + +```diff +- {"type":"assertions","assertions":[]} ++ {"type":"assertions","assertions":[ ++ {"kind":"tool_called","name":"hire_agent","min_count":3,"max_count":5}, ++ {"kind":"message_contains_substring","role":"assistant","substring":"engineering"}, ++ {"kind":"trace_max_turns","max":40} ++ ]} +``` + +Common assertion shapes: + +```json +{"kind":"tool_called","name":"hire_agent","min_count":3} +{"kind":"no_tool_called","name":"delete_everything"} +{"kind":"message_contains_substring","role":"assistant","substring":"approve_ceo_strategy"} +{"kind":"tool_call_arg_matches","name":"hire_agent","arg_path":"department","matches_regex":"^(engineering|design|research)$"} +``` + +See `types.ts` for the exhaustive schema. + +### Step 5 — verify + +```bash +pnpm test:eval --run +``` + +A green run means: given the recorded assistant turn, every assertion +you wrote passes. A red run means either the assertion is wrong or the +assistant did not produce the behaviour you expected — inspect the +fixture and decide which. + +### Re-recording + +Fixtures drift when the system prompt changes (the `systemPromptHash` +header surfaces that). To re-record, delete the old fixture and run the +recorder again. Do **not** hand-edit the `turn` records; that defeats +the point of recording against a real provider. + +### What the recorder does NOT do + +- It does not loop tool calls back to the provider. Today's recorder + captures one turn: user message in, assistant reply out. Multi-turn + with tool execution is B5.1's scope. +- It does not retry on provider errors. A 529 or a 5xx surfaces as a + non-zero exit and nothing is written. Re-run manually if you think + the failure was transient. +- It does not reach into AgentCompany's `src/main/` to import the + worker's system prompt builder or tool catalogue. You bring your own + prompt and message text on the CLI. Importing the worker would + require the Electron runtime; the CLI is node-only on purpose. + ## Fixture JSONL format Every fixture begins with a `meta` header, contains one `turn` per line in @@ -78,11 +189,14 @@ belongs in the LLM-as-judge tier. See ## What this scaffold does NOT do tonight -- **No synthesized fixtures.** `fixtures/` ships empty. The user records - real traces by running the recorder against a live provider. +- **No synthesized fixtures.** `fixtures/` ships empty. Fixtures arrive + by running the recorder against a live provider. +- **No multi-turn tool-use recording.** The recorder captures one + user→assistant turn. Multi-turn runs with real tool execution are + B5.1's scope. - **No live-replay.** `replay-client.ts` is ready to swap into the agent - loop; wiring it into a real `pi-agent-core` Agent instance is the B5.1 - task. Tonight the runner asserts against recorded turns directly. + loop; wiring it into a real `pi-agent-core` Agent instance is also + B5.1. Tonight the runner asserts against recorded turns directly. - **No LLM-as-judge.** Assertions are structural only. Semantic judgment tier is proposed separately. - **No logging / telemetry.** Results are returned from `runEval` as diff --git a/tests/eval/record-fixture.ts b/tests/eval/record-fixture.ts index 0bbd5e9d..da559121 100644 --- a/tests/eval/record-fixture.ts +++ b/tests/eval/record-fixture.ts @@ -1,46 +1,76 @@ /** - * Fixture recorder CLI for B5 eval harness. + * Fixture recorder CLI for the B5 eval harness. * - * Entrypoint: `pnpm eval:record -- --out tests/eval/fixtures/.jsonl` + * Entrypoint: + * pnpm eval:record \ + * --scenario \ + * --fixture-id \ + * --provider \ + * --model \ + * --system-prompt-file \ + * --user-message-file \ + * --output tests/eval/fixtures/.jsonl * - * This CLI is a *scaffold*: it accepts a scenario config path and an output - * path, validates the config shape, and writes a stub fixture containing - * only the meta header plus an empty assertions block. The real recording - * loop — which drives `pi-agent-core` against a real provider with the - * AgentCompany system prompt and tool catalogue — lives in a follow-up - * commit. Tonight's scope is the harness skeleton; the user populates - * `tests/eval/fixtures/` by running the full recorder once it is wired. + * What this CLI does: * - * Why a stub and not a full recording loop tonight: - * - Wiring the full loop requires importing the AgentCompany worker's - * system prompt builder and tool registry. Those modules live in - * `src/main/` and depend on the Electron runtime; importing them from a - * Node CLI would require either a recorder-only entrypoint or a vite - * build step. Both are bigger than tonight's surgical scope. - * - Keeping the CLI as a stub today forces a deliberate follow-up rather - * than silently recording fixtures from the wrong context — a Karpathy - * "let it crash" moment. + * 1. Reads the system prompt and the user message from the paths given on + * the CLI. + * 2. Calls the real `@mariozechner/pi-ai` `streamSimple` against the + * provider+model the user picked. The API key is taken from the + * corresponding environment variable (`ANTHROPIC_API_KEY`, + * `OPENAI_API_KEY`, `GOOGLE_API_KEY`). Missing key → hard error; no + * fallback, no retry. + * 3. Serialises the single turn (user message + assistant reply + token + * ledger) into the JSONL shape documented in tests/eval/README.md. * - * Usage: - * tsx tests/eval/record-fixture.ts \ - * --fixture-id ceo-team-building-empty \ - * --scenario "Empty company, CEO bootstrap" \ - * --out tests/eval/fixtures/ceo-team-building-empty.jsonl + * What this CLI does NOT do: + * + * - Multi-turn agent loops. Tool use / tool results are not yet wired in; + * the B5.1 live-replay tier is where that work belongs. Tonight's scope + * is a single-turn record. + * - Fixture synthesis. If the provider call fails for any reason, the + * error is rethrown and no JSONL is written. `Let it crash`. + * - Retrying, resuming, or merging with an existing fixture. + * - Seeding assertions. The assertions block is written empty; the user + * fills it in by hand using the kinds documented in the README. + * + * All failure modes surface as non-zero exits with a specific message — + * the user should read the stderr line and fix the cause (missing env, + * missing file, provider error), not the harness. */ import { createHash } from "node:crypto"; -import { writeFile } from "node:fs/promises"; -import { resolve } from "node:path"; +import { readFile, writeFile, mkdir } from "node:fs/promises"; +import { resolve, dirname } from "node:path"; import { createRequire } from "node:module"; -import type { FixtureMeta } from "./types.js"; +import { + getModel, + streamSimple, + type AssistantMessage, + type Context, + type Provider, + type UserMessage, +} from "@mariozechner/pi-ai"; +import type { + AssertionsBlock, + AssistantTurn, + FixtureMeta, + TokenLedger, + UserTurn, +} from "./types.js"; interface CliArgs { fixtureId: string; scenario: string; + provider: string; + modelId: string; + systemPromptFile: string; + userMessageFile: string; out: string; - systemPromptPreview: string; } +function readArg(args: string[], flag: string, required: true): string; +function readArg(args: string[], flag: string, required: false): string | undefined; function readArg(args: string[], flag: string, required: boolean): string | undefined { const idx = args.indexOf(flag); if (idx === -1) { @@ -58,27 +88,87 @@ function readArg(args: string[], flag: string, required: boolean): string | unde function parseArgs(argv: string[]): CliArgs { const args = argv.slice(2); - const fixtureId = readArg(args, "--fixture-id", true)!; - const scenario = readArg(args, "--scenario", true)!; - const out = readArg(args, "--out", true)!; - const systemPromptPreview = readArg(args, "--system-prompt-preview", false) ?? ""; - return { fixtureId, scenario, out, systemPromptPreview }; + if (args.includes("--help") || args.includes("-h")) { + process.stdout.write(usage()); + process.exit(0); + } + return { + fixtureId: readArg(args, "--fixture-id", true), + scenario: readArg(args, "--scenario", true), + provider: readArg(args, "--provider", true), + modelId: readArg(args, "--model", true), + systemPromptFile: readArg(args, "--system-prompt-file", true), + userMessageFile: readArg(args, "--user-message-file", true), + out: readArg(args, "--output", true), + }; +} + +function usage(): string { + return [ + "pnpm eval:record — record a single-turn LLM fixture against a real provider", + "", + "Required flags:", + " --scenario Human description of what the scenario exercises", + " --fixture-id Stable identifier; used as filename stem", + " --provider anthropic | openai | google", + " --model Provider-specific model id, e.g. claude-sonnet-4-5-20250929", + " --system-prompt-file File containing the system prompt", + " --user-message-file File containing the first user message", + " --output Destination JSONL path, e.g. tests/eval/fixtures/ceo-bootstrap-empty.jsonl", + "", + "Environment:", + " ANTHROPIC_API_KEY | OPENAI_API_KEY | GOOGLE_API_KEY — required for the chosen provider", + "", + "See tests/eval/README.md for an end-to-end walkthrough.", + "", + ].join("\n"); } function resolvePiAgentCoreVersion(): string { - const require = createRequire(import.meta.url); - // Narrow shape of the pi-agent-core package.json read at runtime. - const pkg = require("@mariozechner/pi-agent-core/package.json") as { version?: unknown }; + const req = createRequire(import.meta.url); + const pkg = req("@mariozechner/pi-agent-core/package.json") as { version?: unknown }; if (typeof pkg.version !== "string") { - throw new Error( - "record-fixture: could not resolve @mariozechner/pi-agent-core version", - ); + throw new Error("record-fixture: could not resolve @mariozechner/pi-agent-core version"); } return pkg.version; } -function buildMeta(args: CliArgs): FixtureMeta { - const hash = createHash("sha256").update(args.systemPromptPreview).digest("hex"); +interface ProviderSpec { + id: Provider; + envVar: string; +} + +const PROVIDER_TABLE: Record = { + anthropic: { id: "anthropic", envVar: "ANTHROPIC_API_KEY" }, + openai: { id: "openai", envVar: "OPENAI_API_KEY" }, + google: { id: "google", envVar: "GOOGLE_API_KEY" }, +}; + +function resolveProvider(rawProvider: string): ProviderSpec { + const spec = PROVIDER_TABLE[rawProvider]; + if (!spec) { + throw new Error( + `record-fixture: unsupported provider "${rawProvider}". ` + + `Supported: ${Object.keys(PROVIDER_TABLE).join(", ")}.`, + ); + } + return spec; +} + +function requireApiKey(envVar: string): string { + const key = process.env[envVar]; + if (!key || key.length === 0) { + throw new Error( + `record-fixture: missing ${envVar} in environment. Export it before re-running:\n` + + ` export ${envVar}=...\n` + + ` pnpm eval:record ...`, + ); + } + return key; +} + +function buildMeta(args: CliArgs, systemPrompt: string): FixtureMeta { + const hash = createHash("sha256").update(systemPrompt).digest("hex"); return { type: "meta", fixtureId: args.fixtureId, @@ -89,18 +179,112 @@ function buildMeta(args: CliArgs): FixtureMeta { }; } +function buildUserTurn(text: string): UserTurn { + return { + type: "turn", + turnIndex: 0, + role: "user", + content: [{ type: "text", text }], + }; +} + +function buildAssistantTurn(final: AssistantMessage): AssistantTurn { + const tokenLedger: TokenLedger = { + input: final.usage.input, + cache_read: final.usage.cacheRead, + cache_creation: final.usage.cacheWrite, + output: final.usage.output, + }; + return { + type: "turn", + turnIndex: 1, + role: "assistant", + content: final.content, + tokenLedger, + }; +} + +function buildEmptyAssertions(): AssertionsBlock { + return { type: "assertions", assertions: [] }; +} + async function main(): Promise { const args = parseArgs(process.argv); - const meta = buildMeta(args); - const lines: string[] = [ + const providerSpec = resolveProvider(args.provider); + const apiKey = requireApiKey(providerSpec.envVar); + + const [systemPrompt, userMessage] = await Promise.all([ + readFile(resolve(args.systemPromptFile), "utf8"), + readFile(resolve(args.userMessageFile), "utf8"), + ]); + + if (systemPrompt.trim().length === 0) { + throw new Error( + `record-fixture: ${args.systemPromptFile} is empty. A fixture must carry a real system prompt.`, + ); + } + if (userMessage.trim().length === 0) { + throw new Error( + `record-fixture: ${args.userMessageFile} is empty. A fixture must carry a real user message.`, + ); + } + + const model = getModel(providerSpec.id as never, args.modelId as never); + if (!model) { + throw new Error( + `record-fixture: unknown model "${args.modelId}" for provider "${providerSpec.id}". ` + + `Consult @mariozechner/pi-ai's models registry for valid ids.`, + ); + } + + const userMessagePayload: UserMessage = { + role: "user", + content: userMessage, + timestamp: Date.now(), + }; + + const context: Context = { + systemPrompt, + messages: [userMessagePayload], + }; + + process.stderr.write( + `record-fixture: calling ${providerSpec.id}/${args.modelId} (single turn, no tool use)…\n`, + ); + + const stream = streamSimple(model, context, { apiKey }); + const final = await stream.result(); + + if (final.stopReason === "error" || final.stopReason === "aborted") { + const detail = final.errorMessage ?? `provider returned stopReason=${final.stopReason}`; + throw new Error( + `record-fixture: provider call failed — ${detail}. No fixture written; fix the cause and re-run.`, + ); + } + + const meta = buildMeta(args, systemPrompt); + const userTurn = buildUserTurn(userMessage); + const assistantTurn = buildAssistantTurn(final); + const assertions = buildEmptyAssertions(); + + const lines = [ JSON.stringify(meta), - JSON.stringify({ type: "assertions", assertions: [] }), + JSON.stringify(userTurn), + JSON.stringify(assistantTurn), + JSON.stringify(assertions), ]; const outPath = resolve(args.out); + await mkdir(dirname(outPath), { recursive: true }); await writeFile(outPath, `${lines.join("\n")}\n`, "utf8"); + + const usageSummary = `in=${final.usage.input} cacheRead=${final.usage.cacheRead} cacheWrite=${final.usage.cacheWrite} out=${final.usage.output}`; process.stdout.write( - `record-fixture: wrote stub fixture ${args.fixtureId} to ${outPath}\n` + - `record-fixture: NEXT STEP — replace the stub with recorded turns by running the full recorder (see tests/eval/README.md).\n`, + [ + `record-fixture: wrote fixture ${args.fixtureId} to ${outPath}`, + `record-fixture: usage ${usageSummary} stopReason=${final.stopReason}`, + `record-fixture: assertions block is empty — edit ${outPath} and add one or more assertion entries`, + `record-fixture: then verify with pnpm test:eval --run`, + ].join("\n") + "\n", ); }