/** * Fixture recorder CLI for B5 eval harness. * * Entrypoint: `pnpm eval:record -- --out tests/eval/fixtures/.jsonl` * * This CLI is a *scaffold*: it accepts a scenario config path and an output * path, validates the config shape, and writes a stub fixture containing * only the meta header plus an empty assertions block. The real recording * loop — which drives `pi-agent-core` against a real provider with the * AgentCompany system prompt and tool catalogue — lives in a follow-up * commit. Tonight's scope is the harness skeleton; the user populates * `tests/eval/fixtures/` by running the full recorder once it is wired. * * Why a stub and not a full recording loop tonight: * - Wiring the full loop requires importing the AgentCompany worker's * system prompt builder and tool registry. Those modules live in * `src/main/` and depend on the Electron runtime; importing them from a * Node CLI would require either a recorder-only entrypoint or a vite * build step. Both are bigger than tonight's surgical scope. * - Keeping the CLI as a stub today forces a deliberate follow-up rather * than silently recording fixtures from the wrong context — a Karpathy * "let it crash" moment. * * Usage: * tsx tests/eval/record-fixture.ts \ * --fixture-id ceo-team-building-empty \ * --scenario "Empty company, CEO bootstrap" \ * --out tests/eval/fixtures/ceo-team-building-empty.jsonl */ import { createHash } from "node:crypto"; import { writeFile } from "node:fs/promises"; import { resolve } from "node:path"; import { createRequire } from "node:module"; import type { FixtureMeta } from "./types.js"; interface CliArgs { fixtureId: string; scenario: string; out: string; systemPromptPreview: string; } function readArg(args: string[], flag: string, required: boolean): string | undefined { const idx = args.indexOf(flag); if (idx === -1) { if (required) { throw new Error(`record-fixture: missing required flag ${flag}`); } return undefined; } const value = args[idx + 1]; if (typeof value !== "string" || value.startsWith("--")) { throw new Error(`record-fixture: flag ${flag} requires a value`); } return value; } function parseArgs(argv: string[]): CliArgs { const args = argv.slice(2); const fixtureId = readArg(args, "--fixture-id", true)!; const scenario = readArg(args, "--scenario", true)!; const out = readArg(args, "--out", true)!; const systemPromptPreview = readArg(args, "--system-prompt-preview", false) ?? ""; return { fixtureId, scenario, out, systemPromptPreview }; } function resolvePiAgentCoreVersion(): string { const require = createRequire(import.meta.url); // Narrow shape of the pi-agent-core package.json read at runtime. const pkg = require("@mariozechner/pi-agent-core/package.json") as { version?: unknown }; if (typeof pkg.version !== "string") { throw new Error( "record-fixture: could not resolve @mariozechner/pi-agent-core version", ); } return pkg.version; } function buildMeta(args: CliArgs): FixtureMeta { const hash = createHash("sha256").update(args.systemPromptPreview).digest("hex"); return { type: "meta", fixtureId: args.fixtureId, piAgentCoreVersion: resolvePiAgentCoreVersion(), recordedAt: new Date().toISOString(), scenario: args.scenario, systemPromptHash: `sha256:${hash}`, }; } async function main(): Promise { const args = parseArgs(process.argv); const meta = buildMeta(args); const lines: string[] = [ JSON.stringify(meta), JSON.stringify({ type: "assertions", assertions: [] }), ]; const outPath = resolve(args.out); await writeFile(outPath, `${lines.join("\n")}\n`, "utf8"); process.stdout.write( `record-fixture: wrote stub fixture ${args.fixtureId} to ${outPath}\n` + `record-fixture: NEXT STEP — replace the stub with recorded turns by running the full recorder (see tests/eval/README.md).\n`, ); } main().catch((error) => { const detail = error instanceof Error ? error.message : String(error); process.stderr.write(`record-fixture: ${detail}\n`); process.exit(1); });