From e7f879bed0da38981311cf60c3413f8f8319a236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Mon, 20 Apr 2026 11:39:53 -0400 Subject: [PATCH] =?UTF-8?q?Build=20minimal=20LLM=20eval=20harness=20scaffo?= =?UTF-8?q?ld=20(B5)=20=E2=80=94=20fixtures=20recorded=20separately?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Land the B5 harness skeleton: fixture loader (strict JSONL contract with meta header + ordered turn records + assertions tail), replay-client that swaps in for pi-ai's streamSimple by replaying recorded assistant turns, structural assertion judge (tool_called / no_tool_called / message_contains_substring / tool_call_arg_matches / trace_max_turns), an eval-runner that sums the token ledger and produces an EvalResult, a vitest driver (tests/eval/harness.test.ts) that discovers *.jsonl fixtures, and a record-fixture CLI that writes fixture meta headers. Hard contracts: - No fallback: replay-client and fixture-loader throw on exhaustion / malformed input rather than synthesizing defaults. - No mock: types reuse @mariozechner/pi-ai AgentMessage / ToolCall verbatim; shadow types are forbidden. - No fake fixture data: tests/eval/fixtures/ ships only README + .gitkeep. Real traces are recorded by the user against live providers. package.json adds test:eval (vitest --passWithNoTests so an empty fixture dir is green) and eval:record (CLI entrypoint). electron-builder already excludes tests/ via its files: glob — fixtures never enter the shipped .asar. --- .gitignore | 1 + package.json | 2 + tests/eval/README.md | 100 ++++++++++++++++++ tests/eval/assertion-judge.ts | 188 ++++++++++++++++++++++++++++++++++ tests/eval/eval-runner.ts | 55 ++++++++++ tests/eval/fixture-loader.ts | 109 ++++++++++++++++++++ tests/eval/fixtures/.gitkeep | 0 tests/eval/fixtures/README.md | 44 ++++++++ tests/eval/harness.test.ts | 49 +++++++++ tests/eval/record-fixture.ts | 111 ++++++++++++++++++++ tests/eval/replay-client.ts | 110 ++++++++++++++++++++ tests/eval/types.ts | 162 +++++++++++++++++++++++++++++ 12 files changed, 931 insertions(+) create mode 100644 tests/eval/README.md create mode 100644 tests/eval/assertion-judge.ts create mode 100644 tests/eval/eval-runner.ts create mode 100644 tests/eval/fixture-loader.ts create mode 100644 tests/eval/fixtures/.gitkeep create mode 100644 tests/eval/fixtures/README.md create mode 100644 tests/eval/harness.test.ts create mode 100644 tests/eval/record-fixture.ts create mode 100644 tests/eval/replay-client.ts create mode 100644 tests/eval/types.ts diff --git a/.gitignore b/.gitignore index 37e81038..d6267cd1 100644 --- a/.gitignore +++ b/.gitignore @@ -64,6 +64,7 @@ oldreference/ # the parent directory; losing SKILL.md to the blanket *.md ignore meant a # fresh checkout silently dropped every skill from the catalogue. !/builtin-skills/**/*.md +!/tests/eval/**/*.md *.zip *.blockmap *.exe diff --git a/package.json b/package.json index a148fc41..e35d22bd 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,8 @@ "typecheck": "tsc --noEmit", "test:unit": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/unit", "test:integration": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/integration", + "test:eval": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/eval --passWithNoTests", + "eval:record": "node --disable-warning=ExperimentalWarning --experimental-strip-types tests/eval/record-fixture.ts", "test:e2e": "node --no-warnings ./node_modules/@playwright/test/cli.js test -c tests/e2e/playwright.config.ts", "test:run": "pnpm test:unit && pnpm test:integration && pnpm test:e2e", "verify": "pnpm typecheck && pnpm lint && pnpm test:unit && pnpm test:integration && pnpm test:e2e", diff --git a/tests/eval/README.md b/tests/eval/README.md new file mode 100644 index 00000000..2977ad21 --- /dev/null +++ b/tests/eval/README.md @@ -0,0 +1,100 @@ +# B5 eval harness (minimal scaffold) + +This directory holds the minimal LLM behavior eval tier for AgentCompany. +It replays recorded LLM traces (fixtures) and checks structural assertions +against them — tool names, call counts, argument shapes, message +substrings, and turn caps. It does *not* perform semantic quality judgment +yet; that tier is proposed in +`docs/proposals/B5.1-llm-as-judge-followup.md`. + +## File layout + +| File | Role | +|---|---| +| `types.ts` | Wire types — fixtures, assertions, results. Reuses `@mariozechner/pi-ai` primitives. | +| `fixture-loader.ts` | JSONL → `FixtureFile`. Rejects malformed or missing meta/assertions blocks. | +| `replay-client.ts` | Swaps in for `pi-ai`'s `streamSimple`; replays recorded assistant turns. Used by the live-replay tier in B5.1. | +| `assertion-judge.ts` | Structural assertions over recorded trace. No semantic judgment. | +| `eval-runner.ts` | Parse fixture → judge assertions → produce `EvalResult`. | +| `record-fixture.ts` | CLI for writing fixture meta headers. Full record loop lands in B5.1. | +| `harness.test.ts` | vitest driver that walks `fixtures/` and asserts every fixture passes. | +| `fixtures/` | Recorded JSONL traces. Git-tracked. Not bundled into production (`electron-builder.yml` only ships `out/**`). | + +## Running + +```bash +# Assertion-only eval (no fixtures needed; passes on empty fixture dir) +pnpm test:eval + +# Record a stub fixture header (full recorder lands in B5.1) +pnpm eval:record -- \ + --fixture-id ceo-team-building-empty \ + --scenario "Empty company, CEO bootstrap" \ + --out tests/eval/fixtures/ceo-team-building-empty.jsonl +``` + +## Fixture JSONL format + +Every fixture begins with a `meta` header, contains one `turn` per line in +ascending `turnIndex` order, and ends with an `assertions` block. + +```jsonl +{"type":"meta","fixtureId":"ceo-team-building-empty","piAgentCoreVersion":"0.67.68","recordedAt":"2026-04-20T21:14:00.000Z","scenario":"Empty company, CEO bootstrap","systemPromptHash":"sha256:…"} +{"type":"turn","turnIndex":0,"role":"user","content":[{"type":"text","text":"build me a team"}]} +{"type":"turn","turnIndex":1,"role":"assistant","content":[{"type":"text","text":"Starting team build."},{"type":"toolCall","id":"call_1","name":"hire_agent","arguments":{"role":"engineer"}}],"tokenLedger":{"input":12345,"cache_read":11000,"cache_creation":345,"output":234}} +{"type":"turn","turnIndex":2,"role":"toolResult","toolUseId":"call_1","content":[{"type":"text","text":"hired agent-42"}]} +{"type":"assertions","assertions":[{"kind":"tool_called","name":"hire_agent","min_count":3,"max_count":5},{"kind":"trace_max_turns","max":40}]} +``` + +Field contracts: + +- `meta.fixtureId` — stable kebab-case identifier used as the scenario name. +- `meta.piAgentCoreVersion` — the `@mariozechner/pi-agent-core` version + used when the fixture was recorded. Drift against the installed version + is allowed but future tooling will surface a warning. +- `meta.systemPromptHash` — sha256 of the system prompt text. Changes in + this hash are the canonical signal that a fixture is stale. +- `turn.turnIndex` — strictly increasing from 0. Gaps or duplicates are + hard errors. +- `turn.role = "assistant"` requires a `tokenLedger` in snake_case + (`input`, `cache_read`, `cache_creation`, `output`). This maps 1:1 to + pi-ai's `Usage.{input, cacheRead, cacheWrite, output}` — renamed to + provider-neutral snake_case on disk. +- `assertions.assertions` — see `types.ts` for the complete schema. + +## Assertion kinds + +| Kind | Semantics | +|---|---| +| `tool_called` | Matches tool calls by `name`. Supports `min_count` (default 1), `max_count`, and `order` (0-based ordinal across all tool calls in the trace). | +| `no_tool_called` | Fails if any tool call with the given name appears. | +| `message_contains_substring` | Any `role`-matching message whose text content contains `substring` passes. | +| `tool_call_arg_matches` | At least one call to `name` has the dotted `arg_path` resolving to a string matching `matches_regex`. | +| `trace_max_turns` | Trace must have at most `max` turns. | + +Anything that requires semantic judgment (tone, strategy, plan quality) +belongs in the LLM-as-judge tier. See +`docs/proposals/B5.1-llm-as-judge-followup.md`. + +## What this scaffold does NOT do tonight + +- **No synthesized fixtures.** `fixtures/` ships empty. The user records + real traces by running the recorder against a live provider. +- **No live-replay.** `replay-client.ts` is ready to swap into the agent + loop; wiring it into a real `pi-agent-core` Agent instance is the B5.1 + task. Tonight the runner asserts against recorded turns directly. +- **No LLM-as-judge.** Assertions are structural only. Semantic judgment + tier is proposed separately. +- **No logging / telemetry.** Results are returned from `runEval` as + plain values; the vitest reporter is the only output surface. + +Each omission is deliberate and documented. Nothing here silently degrades +to a placeholder — missing fixture = hard error, malformed fixture = hard +error, exhausted fixture = hard error. + +## Production bundle posture + +`electron-builder.yml` ships only `out/**` and `package.json`. `tests/` — +including `tests/eval/fixtures/` — is never packaged into the shipped +`.asar`. Fixtures are dev-time / CI-time artifacts; they do not leak into +end-user installs. diff --git a/tests/eval/assertion-judge.ts b/tests/eval/assertion-judge.ts new file mode 100644 index 00000000..3f90d4cc --- /dev/null +++ b/tests/eval/assertion-judge.ts @@ -0,0 +1,188 @@ +/** + * Assertion judge for B5 eval harness. + * + * Evaluates a list of structural `Assertion` records against a recorded + * fixture trace. This is the cheap, deterministic tier — it understands tool + * call names, counts, order, and substrings. Semantic judgment (tone, plan + * quality) is out of scope and handled by the LLM-as-judge tier described in + * B5.1. + * + * Design notes: + * - Pure function of (assertions, turns) → AssertionResult[]. No I/O. + * - Never throws. Every assertion either passes or fails with a `reason`. + * The runner aggregates these into a single `EvalResult`. + * - `tool_call_arg_matches.arg_path` supports dotted paths through nested + * objects and numeric indexes through arrays (`items.0.name`). If the path + * misses or resolves to a non-string, the assertion fails with a reason. + */ + +import type { ToolCall } from "@mariozechner/pi-ai"; +import type { + Assertion, + AssertionResult, + AssistantTurn, + TraceTurn, +} from "./types.js"; + +interface ToolCallOccurrence { + toolCall: ToolCall; + /** Ordinal index among all tool calls across the full trace. */ + ordinal: number; +} + +function collectToolCalls(turns: TraceTurn[]): ToolCallOccurrence[] { + const occurrences: ToolCallOccurrence[] = []; + for (const turn of turns) { + if (turn.role !== "assistant") continue; + const assistantTurn = turn as AssistantTurn; + for (const block of assistantTurn.content) { + if (block.type === "toolCall") { + occurrences.push({ + toolCall: block, + ordinal: occurrences.length, + }); + } + } + } + return occurrences; +} + +function resolveArgPath(root: unknown, path: string): unknown { + const parts = path.split("."); + let cursor: unknown = root; + for (const part of parts) { + if (cursor === null || cursor === undefined) return undefined; + if (typeof cursor !== "object") return undefined; + cursor = (cursor as Record)[part]; + } + return cursor; +} + +function evaluateOne( + assertion: Assertion, + turns: TraceTurn[], + toolCalls: ToolCallOccurrence[], +): AssertionResult { + switch (assertion.kind) { + case "tool_called": { + const matches = toolCalls.filter( + (occ) => occ.toolCall.name === assertion.name, + ); + const minCount = assertion.min_count ?? 1; + if (matches.length < minCount) { + return { + assertion, + passed: false, + reason: `expected at least ${minCount} call(s) to "${assertion.name}", saw ${matches.length}`, + }; + } + if (assertion.max_count !== undefined && matches.length > assertion.max_count) { + return { + assertion, + passed: false, + reason: `expected at most ${assertion.max_count} call(s) to "${assertion.name}", saw ${matches.length}`, + }; + } + if (assertion.order !== undefined) { + const match = matches.find((occ) => occ.ordinal === assertion.order); + if (!match) { + const seenOrdinals = matches.map((occ) => occ.ordinal).join(","); + return { + assertion, + passed: false, + reason: `expected "${assertion.name}" at ordinal ${assertion.order}, matching calls at [${seenOrdinals}]`, + }; + } + } + return { assertion, passed: true }; + } + case "no_tool_called": { + const matches = toolCalls.filter( + (occ) => occ.toolCall.name === assertion.name, + ); + if (matches.length > 0) { + return { + assertion, + passed: false, + reason: `expected no call to "${assertion.name}", saw ${matches.length}`, + }; + } + return { assertion, passed: true }; + } + case "message_contains_substring": { + for (const turn of turns) { + if (turn.role !== assertion.role) continue; + const content = (turn as AssistantTurn | { content: unknown[] }).content; + if (!Array.isArray(content)) continue; + for (const block of content) { + if ( + block && + typeof block === "object" && + (block as { type?: string }).type === "text" && + typeof (block as { text?: unknown }).text === "string" && + ((block as { text: string }).text).includes(assertion.substring) + ) { + return { assertion, passed: true }; + } + } + } + return { + assertion, + passed: false, + reason: `no ${assertion.role} message contained substring "${assertion.substring}"`, + }; + } + case "tool_call_arg_matches": { + const matches = toolCalls.filter( + (occ) => occ.toolCall.name === assertion.name, + ); + if (matches.length === 0) { + return { + assertion, + passed: false, + reason: `no tool call named "${assertion.name}" to match arg against`, + }; + } + let regex: RegExp; + try { + regex = new RegExp(assertion.matches_regex); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + return { + assertion, + passed: false, + reason: `invalid regex "${assertion.matches_regex}": ${detail}`, + }; + } + for (const occurrence of matches) { + const value = resolveArgPath(occurrence.toolCall.arguments, assertion.arg_path); + if (typeof value === "string" && regex.test(value)) { + return { assertion, passed: true }; + } + } + return { + assertion, + passed: false, + reason: `no "${assertion.name}" call had arg "${assertion.arg_path}" matching /${assertion.matches_regex}/`, + }; + } + case "trace_max_turns": { + if (turns.length > assertion.max) { + return { + assertion, + passed: false, + reason: `trace has ${turns.length} turns, max allowed is ${assertion.max}`, + }; + } + return { assertion, passed: true }; + } + } +} + +export function judgeAssertions( + assertions: Assertion[], + turns: TraceTurn[], +): AssertionResult[] { + const toolCalls = collectToolCalls(turns); + return assertions.map((assertion) => evaluateOne(assertion, turns, toolCalls)); +} diff --git a/tests/eval/eval-runner.ts b/tests/eval/eval-runner.ts new file mode 100644 index 00000000..922bcdd8 --- /dev/null +++ b/tests/eval/eval-runner.ts @@ -0,0 +1,55 @@ +/** + * Eval runner for B5 harness. + * + * Given a parsed `FixtureFile`, produces an `EvalResult` by: + * 1. Summing the token ledger across all recorded assistant turns. + * 2. Running the assertion judge against the recorded turns. + * 3. Measuring wall-clock duration of steps (1)–(2). + * + * Tonight's scope deliberately evaluates assertions against the RECORDED + * trace, not a live replay-driven agent loop. That keeps the harness + * useful as a snapshot-regression tool the moment the first fixture lands, + * without wiring in the full `pi-agent-core` Agent loop (which requires + * tools + hooks and is better done incrementally once we have fixtures). + * + * The replay-client + fixture-loader are written now so the live-replay + * tier can be added in B5.1 without reshaping types. + */ + +import { judgeAssertions } from "./assertion-judge.js"; +import type { EvalResult, FixtureFile, TokenLedger } from "./types.js"; + +const ZERO_LEDGER: TokenLedger = { + input: 0, + cache_read: 0, + cache_creation: 0, + output: 0, +}; + +function sumLedger(fixture: FixtureFile): TokenLedger { + return fixture.turns.reduce((acc, turn) => { + if (turn.role !== "assistant") return acc; + return { + input: acc.input + turn.tokenLedger.input, + cache_read: acc.cache_read + turn.tokenLedger.cache_read, + cache_creation: acc.cache_creation + turn.tokenLedger.cache_creation, + output: acc.output + turn.tokenLedger.output, + }; + }, ZERO_LEDGER); +} + +export function runEval(fixture: FixtureFile): EvalResult { + const start = Date.now(); + const results = judgeAssertions(fixture.assertions, fixture.turns); + const tokenLedger = sumLedger(fixture); + const durationMs = Date.now() - start; + const passed = results.every((result) => result.passed); + + return { + fixtureId: fixture.meta.fixtureId, + passed, + assertions: results, + tokenLedger, + durationMs, + }; +} diff --git a/tests/eval/fixture-loader.ts b/tests/eval/fixture-loader.ts new file mode 100644 index 00000000..bc07d384 --- /dev/null +++ b/tests/eval/fixture-loader.ts @@ -0,0 +1,109 @@ +/** + * Fixture loader for B5 eval harness. + * + * Reads a JSONL file and parses it into a strongly-typed `FixtureFile`. + * + * Non-negotiable invariants: + * - The first non-empty line MUST be a `meta` record. Otherwise the loader + * throws — no synthesized default metadata. + * - The last non-empty line MUST be an `assertions` record. Otherwise throw. + * - Turn `turnIndex` must be strictly increasing starting at 0. Gaps or + * duplicates throw. + * - Unknown `type` values throw. We do not silently ignore junk lines. + * + * The loader does no I/O beyond `readFile`. It does not tolerate partial or + * corrupt fixtures by "best-effort parsing" — Karpathy rule: bad data must + * crash loud so the user fixes the fixture, not the harness. + */ + +import { readFile } from "node:fs/promises"; +import type { + AssertionsBlock, + FixtureFile, + FixtureLine, + FixtureMeta, + TraceTurn, +} from "./types.js"; + +function parseLine(raw: string, lineNumber: number, source: string): FixtureLine { + try { + return JSON.parse(raw) as FixtureLine; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error( + `fixture ${source}: line ${lineNumber} is not valid JSON: ${detail}`, + ); + } +} + +function isMeta(value: FixtureLine): value is FixtureMeta { + return value.type === "meta"; +} + +function isAssertions(value: FixtureLine): value is AssertionsBlock { + return value.type === "assertions"; +} + +function isTurn(value: FixtureLine): value is TraceTurn { + return value.type === "turn"; +} + +export async function loadFixture(path: string): Promise { + const raw = await readFile(path, "utf8"); + const lines = raw + .split("\n") + .map((line, idx) => ({ line: line.trim(), lineNumber: idx + 1 })) + .filter((entry) => entry.line.length > 0); + + if (lines.length < 3) { + throw new Error( + `fixture ${path}: must contain at least a meta record, one turn, and an assertions record`, + ); + } + + const parsed: { record: FixtureLine; lineNumber: number }[] = lines.map( + ({ line, lineNumber }) => ({ + record: parseLine(line, lineNumber, path), + lineNumber, + }), + ); + + const first = parsed[0]; + if (!first || !isMeta(first.record)) { + throw new Error( + `fixture ${path}: first line must be a {"type":"meta"} record`, + ); + } + const last = parsed[parsed.length - 1]; + if (!last || !isAssertions(last.record)) { + throw new Error( + `fixture ${path}: last line must be a {"type":"assertions"} record`, + ); + } + + const turns: TraceTurn[] = []; + for (let i = 1; i < parsed.length - 1; i += 1) { + const entry = parsed[i]; + if (!entry) { + throw new Error(`fixture ${path}: internal index error`); + } + if (!isTurn(entry.record)) { + throw new Error( + `fixture ${path}: line ${entry.lineNumber} has unexpected type "${entry.record.type}" (only turn records allowed between meta and assertions)`, + ); + } + const expectedIndex = turns.length; + if (entry.record.turnIndex !== expectedIndex) { + throw new Error( + `fixture ${path}: line ${entry.lineNumber} has turnIndex=${entry.record.turnIndex}, expected ${expectedIndex}`, + ); + } + turns.push(entry.record); + } + + return { + meta: first.record, + turns, + assertions: last.record.assertions, + }; +} diff --git a/tests/eval/fixtures/.gitkeep b/tests/eval/fixtures/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/eval/fixtures/README.md b/tests/eval/fixtures/README.md new file mode 100644 index 00000000..787ef46d --- /dev/null +++ b/tests/eval/fixtures/README.md @@ -0,0 +1,44 @@ +# Eval fixtures + +Every file in this directory is a **recorded trace** of a real AgentCompany +turn against a real LLM provider. Fixtures are recorded-truth, not +synthesized mock data — the distinction is what keeps the B5 harness on the +"fixture, not fallback" side of the roundtable line. + +## File format + +Each fixture is a `.jsonl` file with one JSON object per line: + +1. First line: a `{ "type": "meta", ... }` record (see `FixtureMeta` in + `../types.ts`). +2. Middle lines: one `{ "type": "turn", "turnIndex": N, ... }` record per + conversation turn, in order. `turnIndex` is strictly increasing from `0`. +3. Last line: a `{ "type": "assertions", "assertions": [...] }` record with + the expected behavior contract. + +The loader rejects files that do not match this shape. No silent skipping. + +## Recording workflow + +```bash +pnpm eval:record -- \ + --fixture-id ceo-team-building-empty \ + --scenario "Empty company, CEO bootstrap" \ + --out tests/eval/fixtures/ceo-team-building-empty.jsonl \ + --system-prompt-preview "$(cat /tmp/current-system-prompt.txt)" +``` + +The current CLI writes a stub meta + empty assertions. The full recorder +(which drives `pi-agent-core` against a real provider using the +AgentCompany worker's system prompt and tool registry) lands in a +follow-up commit — see `../README.md` for the B5 roadmap. + +## Committing fixtures + +Fixtures belong in git. They are small, deterministic, and the whole point +of the eval tier is to diff behavior against recorded truth. Do NOT add +`tests/eval/fixtures/` to `.gitignore`. + +Fixtures are excluded from the electron-builder production bundle; see +`electron-builder.yml` (the `files` glob only picks `out/**`, so `tests/` +never enters the packaged `.asar`). diff --git a/tests/eval/harness.test.ts b/tests/eval/harness.test.ts new file mode 100644 index 00000000..7a9b13d0 --- /dev/null +++ b/tests/eval/harness.test.ts @@ -0,0 +1,49 @@ +/** + * B5 eval harness — driver spec. + * + * This spec discovers every `*.jsonl` fixture under `tests/eval/fixtures/` + * and runs it through the assertion judge. If there are zero fixtures, the + * spec file has zero tests — vitest is invoked with `--passWithNoTests` + * from the `test:eval` script so this is a green no-op until the user + * records real fixtures. + * + * The harness does NOT synthesize placeholder fixtures under any + * circumstance. An empty `fixtures/` directory is a real state — the tier + * is simply not yet exercising anything — and the green run reflects that + * honestly. + */ + +import { readdir } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { loadFixture } from "./fixture-loader.js"; +import { runEval } from "./eval-runner.js"; + +const FIXTURE_DIR = join(__dirname, "fixtures"); + +async function listFixtures(): Promise { + const entries = await readdir(FIXTURE_DIR, { withFileTypes: true }); + return entries + .filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")) + .map((entry) => join(FIXTURE_DIR, entry.name)) + .sort(); +} + +const fixturePaths = await listFixtures(); + +describe.each(fixturePaths)("eval fixture %s", (fixturePath) => { + it("passes all assertions", async () => { + const fixture = await loadFixture(fixturePath); + const result = runEval(fixture); + if (!result.passed) { + const failures = result.assertions + .filter((assertion) => !assertion.passed) + .map((assertion) => `- ${assertion.assertion.kind}: ${assertion.reason ?? ""}`) + .join("\n"); + throw new Error( + `fixture ${result.fixtureId} failed:\n${failures}\ntoken ledger: ${JSON.stringify(result.tokenLedger)}`, + ); + } + expect(result.passed).toBe(true); + }); +}); diff --git a/tests/eval/record-fixture.ts b/tests/eval/record-fixture.ts new file mode 100644 index 00000000..0bbd5e9d --- /dev/null +++ b/tests/eval/record-fixture.ts @@ -0,0 +1,111 @@ +/** + * Fixture recorder CLI for B5 eval harness. + * + * Entrypoint: `pnpm eval:record -- --out tests/eval/fixtures/.jsonl` + * + * This CLI is a *scaffold*: it accepts a scenario config path and an output + * path, validates the config shape, and writes a stub fixture containing + * only the meta header plus an empty assertions block. The real recording + * loop — which drives `pi-agent-core` against a real provider with the + * AgentCompany system prompt and tool catalogue — lives in a follow-up + * commit. Tonight's scope is the harness skeleton; the user populates + * `tests/eval/fixtures/` by running the full recorder once it is wired. + * + * Why a stub and not a full recording loop tonight: + * - Wiring the full loop requires importing the AgentCompany worker's + * system prompt builder and tool registry. Those modules live in + * `src/main/` and depend on the Electron runtime; importing them from a + * Node CLI would require either a recorder-only entrypoint or a vite + * build step. Both are bigger than tonight's surgical scope. + * - Keeping the CLI as a stub today forces a deliberate follow-up rather + * than silently recording fixtures from the wrong context — a Karpathy + * "let it crash" moment. + * + * Usage: + * tsx tests/eval/record-fixture.ts \ + * --fixture-id ceo-team-building-empty \ + * --scenario "Empty company, CEO bootstrap" \ + * --out tests/eval/fixtures/ceo-team-building-empty.jsonl + */ + +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { createRequire } from "node:module"; +import type { FixtureMeta } from "./types.js"; + +interface CliArgs { + fixtureId: string; + scenario: string; + out: string; + systemPromptPreview: string; +} + +function readArg(args: string[], flag: string, required: boolean): string | undefined { + const idx = args.indexOf(flag); + if (idx === -1) { + if (required) { + throw new Error(`record-fixture: missing required flag ${flag}`); + } + return undefined; + } + const value = args[idx + 1]; + if (typeof value !== "string" || value.startsWith("--")) { + throw new Error(`record-fixture: flag ${flag} requires a value`); + } + return value; +} + +function parseArgs(argv: string[]): CliArgs { + const args = argv.slice(2); + const fixtureId = readArg(args, "--fixture-id", true)!; + const scenario = readArg(args, "--scenario", true)!; + const out = readArg(args, "--out", true)!; + const systemPromptPreview = readArg(args, "--system-prompt-preview", false) ?? ""; + return { fixtureId, scenario, out, systemPromptPreview }; +} + +function resolvePiAgentCoreVersion(): string { + const require = createRequire(import.meta.url); + // Narrow shape of the pi-agent-core package.json read at runtime. + const pkg = require("@mariozechner/pi-agent-core/package.json") as { version?: unknown }; + if (typeof pkg.version !== "string") { + throw new Error( + "record-fixture: could not resolve @mariozechner/pi-agent-core version", + ); + } + return pkg.version; +} + +function buildMeta(args: CliArgs): FixtureMeta { + const hash = createHash("sha256").update(args.systemPromptPreview).digest("hex"); + return { + type: "meta", + fixtureId: args.fixtureId, + piAgentCoreVersion: resolvePiAgentCoreVersion(), + recordedAt: new Date().toISOString(), + scenario: args.scenario, + systemPromptHash: `sha256:${hash}`, + }; +} + +async function main(): Promise { + const args = parseArgs(process.argv); + const meta = buildMeta(args); + const lines: string[] = [ + JSON.stringify(meta), + JSON.stringify({ type: "assertions", assertions: [] }), + ]; + const outPath = resolve(args.out); + await writeFile(outPath, `${lines.join("\n")}\n`, "utf8"); + process.stdout.write( + `record-fixture: wrote stub fixture ${args.fixtureId} to ${outPath}\n` + + `record-fixture: NEXT STEP — replace the stub with recorded turns by running the full recorder (see tests/eval/README.md).\n`, + ); +} + +main().catch((error) => { + const detail = error instanceof Error ? error.message : String(error); + process.stderr.write(`record-fixture: ${detail}\n`); + process.exit(1); +}); diff --git a/tests/eval/replay-client.ts b/tests/eval/replay-client.ts new file mode 100644 index 00000000..aca37152 --- /dev/null +++ b/tests/eval/replay-client.ts @@ -0,0 +1,110 @@ +/** + * Replay client for B5 eval harness. + * + * Swaps in for `streamSimple` (the pi-ai LLM call) during eval runs. Instead + * of hitting a real provider, it replays the assistant turns from the fixture + * in order, emitting the `AssistantMessageEvent` protocol expected by + * `pi-agent-core`. + * + * Hard contract (Karpathy rule: no fallback): + * - If the agent loop asks for the Nth assistant turn and the fixture has + * fewer assistant turns, the replay client THROWS. It does not return a + * placeholder response, does not return an empty assistant message, and + * does not invent an `errorMessage`. Missing fixture = bug in fixture, and + * the runner must surface that loud. + * - The client ignores the incoming `context.messages` because the fixture + * is the source of truth. A future harness tier (B5.1 or later) can diff + * the incoming messages against the fixture's recorded prior messages to + * detect context drift — out of scope tonight. + */ + +import type { AssistantMessage } from "@mariozechner/pi-ai"; +import { + AssistantMessageEventStream, + createAssistantMessageEventStream, +} from "@mariozechner/pi-ai"; +import type { StreamFn } from "@mariozechner/pi-agent-core"; +import type { AssistantTurn, FixtureFile } from "./types.js"; + +function collectAssistantTurns(fixture: FixtureFile): AssistantTurn[] { + return fixture.turns.filter( + (turn): turn is AssistantTurn => turn.role === "assistant", + ); +} + +function buildAssistantMessage( + turn: AssistantTurn, + modelId: string, + provider: string, + api: string, +): AssistantMessage { + const ledger = turn.tokenLedger; + const totalTokens = + ledger.input + ledger.cache_read + ledger.cache_creation + ledger.output; + const hasToolCall = turn.content.some((block) => block.type === "toolCall"); + return { + role: "assistant", + content: turn.content, + api, + provider, + model: modelId, + usage: { + input: ledger.input, + output: ledger.output, + cacheRead: ledger.cache_read, + cacheWrite: ledger.cache_creation, + totalTokens, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: hasToolCall ? "toolUse" : "stop", + timestamp: Date.now(), + }; +} + +function pushRecordedMessage( + stream: AssistantMessageEventStream, + message: AssistantMessage, +): void { + stream.push({ type: "start", partial: message }); + const reason = (message.stopReason === "toolUse" || message.stopReason === "length" + ? message.stopReason + : "stop") as "stop" | "toolUse" | "length"; + stream.push({ type: "done", reason, message }); + stream.end(message); +} + +/** + * Create a replay stream function bound to a fixture. + * + * The returned function tracks its own cursor. Each invocation consumes the + * next assistant turn. If the caller exhausts the fixture, the next + * invocation throws. + */ +export function createReplayStreamFn(fixture: FixtureFile): StreamFn { + const assistantTurns = collectAssistantTurns(fixture); + let cursor = 0; + + return ((model, _context, _options) => { + if (cursor >= assistantTurns.length) { + throw new Error( + `replay-client: fixture "${fixture.meta.fixtureId}" exhausted after ${assistantTurns.length} assistant turns; agent requested turn ${cursor + 1}`, + ); + } + const turn = assistantTurns[cursor]; + cursor += 1; + if (!turn) { + throw new Error( + `replay-client: internal cursor error at position ${cursor - 1}`, + ); + } + const message = buildAssistantMessage( + turn, + model.id, + model.provider, + model.api, + ); + const stream = createAssistantMessageEventStream(); + pushRecordedMessage(stream, message); + return stream; + }) satisfies StreamFn; +} diff --git a/tests/eval/types.ts b/tests/eval/types.ts new file mode 100644 index 00000000..90b825ac --- /dev/null +++ b/tests/eval/types.ts @@ -0,0 +1,162 @@ +/** + * Type definitions for the B5 minimal eval harness. + * + * Design invariants: + * - Fixtures are recorded LLM traces (history real runs), not synthesized mocks. + * A fixture is the serialized truth of a turn that actually happened against + * a real provider; replaying it tells us whether the *harness* still reaches + * the same tool-call shape given the same prompt/context. + * - Every type here reuses `@mariozechner/pi-ai` primitives verbatim. Shadow + * types are forbidden; if pi-ai changes its `AssistantMessage` shape, this + * harness breaks loud — that is the intent. + * - No runtime field here is optional for "fallback reasons". If a field is + * optional, it encodes real optionality of the source data (e.g. a turn + * record that does not carry usage because it is a user turn). + */ + +import type { + AssistantMessage, + ToolResultMessage, + UserMessage, +} from "@mariozechner/pi-ai"; + +/** + * Top-level fixture file meta header. + * + * Every JSONL fixture begins with exactly one `meta` record. The runner will + * throw if the first line is not a meta record — no fallback default metadata + * is synthesized. + */ +export interface FixtureMeta { + type: "meta"; + /** Stable fixture identifier; used as the scenario name in eval output. */ + fixtureId: string; + /** pi-agent-core version used to record this fixture. */ + piAgentCoreVersion: string; + /** ISO-8601 timestamp when the fixture was recorded against a real provider. */ + recordedAt: string; + /** Short human description of what the scenario exercises. */ + scenario: string; + /** sha256 of the system prompt used at recording time. Used to detect drift. */ + systemPromptHash: string; +} + +/** + * Token ledger captured for assistant turns. + * + * Shape mirrors `AssistantMessage.usage` but uses snake_case fields that match + * the JSONL spec printed in `README.md`. The mapping is 1:1 with pi-ai's + * `Usage` primitives (`input`, `output`, `cacheRead`, `cacheWrite`) — renamed + * for the on-disk format to stay provider-neutral. + */ +export interface TokenLedger { + input: number; + cache_read: number; + cache_creation: number; + output: number; +} + +export interface UserTurn { + type: "turn"; + turnIndex: number; + role: "user"; + /** Verbatim user content blocks from the real recording. */ + content: UserMessage["content"]; +} + +export interface AssistantTurn { + type: "turn"; + turnIndex: number; + role: "assistant"; + /** Verbatim assistant content blocks (text / thinking / toolCall). */ + content: AssistantMessage["content"]; + /** Token ledger recorded for this assistant turn. Required on assistant turns. */ + tokenLedger: TokenLedger; +} + +export interface ToolResultTurn { + type: "turn"; + turnIndex: number; + role: "toolResult"; + toolUseId: string; + content: ToolResultMessage["content"]; +} + +export type TraceTurn = UserTurn | AssistantTurn | ToolResultTurn; + +/** + * Structural assertions that can be evaluated against a replayed trace. + * + * Deliberately minimal: assertions express tool-call shape, order, count, and + * textual substrings. Semantic quality judgments (tone, plan soundness) are + * out of scope for this tier and will be handled by the LLM-as-judge tier + * described in B5.1. + */ +export type Assertion = + | { + kind: "tool_called"; + name: string; + min_count?: number; + max_count?: number; + /** Ordinal position in the trace (0-based) among all tool calls. */ + order?: number; + } + | { kind: "no_tool_called"; name: string } + | { + kind: "message_contains_substring"; + role: "assistant" | "user"; + substring: string; + } + | { + kind: "tool_call_arg_matches"; + name: string; + /** Dotted JSON path into the tool call's arguments object, e.g. "agent.role". */ + arg_path: string; + /** ECMAScript regex source evaluated against the argument string. */ + matches_regex: string; + } + | { kind: "trace_max_turns"; max: number }; + +export interface AssertionsBlock { + type: "assertions"; + assertions: Assertion[]; +} + +/** + * A JSONL fixture file is parsed into exactly one of these records per line. + * The loader rejects files that do not begin with `meta` and end with + * `assertions`. + */ +export type FixtureLine = FixtureMeta | TraceTurn | AssertionsBlock; + +/** + * Parsed fixture file ready to be replayed by the runner. + */ +export interface FixtureFile { + meta: FixtureMeta; + turns: TraceTurn[]; + assertions: Assertion[]; +} + +/** + * Per-assertion outcome within a single eval run. + */ +export interface AssertionResult { + assertion: Assertion; + passed: boolean; + /** Populated when `passed === false`; concise human-readable explanation. */ + reason?: string; +} + +/** + * Result of evaluating one fixture against the current code path. + */ +export interface EvalResult { + fixtureId: string; + passed: boolean; + assertions: AssertionResult[]; + /** Aggregated token ledger summed across all assistant turns in the fixture. */ + tokenLedger: TokenLedger; + /** Wall-clock duration of the replay, in milliseconds. */ + durationMs: number; +}