Build minimal LLM eval harness scaffold (B5) — fixtures recorded separately

Land the B5 harness skeleton: fixture loader (strict JSONL contract with
meta header + ordered turn records + assertions tail), replay-client that
swaps in for pi-ai's streamSimple by replaying recorded assistant turns,
structural assertion judge (tool_called / no_tool_called /
message_contains_substring / tool_call_arg_matches / trace_max_turns), an
eval-runner that sums the token ledger and produces an EvalResult, a
vitest driver (tests/eval/harness.test.ts) that discovers *.jsonl
fixtures, and a record-fixture CLI that writes fixture meta headers.

Hard contracts:
- No fallback: replay-client and fixture-loader throw on exhaustion /
  malformed input rather than synthesizing defaults.
- No mock: types reuse @mariozechner/pi-ai AgentMessage / ToolCall
  verbatim; shadow types are forbidden.
- No fake fixture data: tests/eval/fixtures/ ships only README +
  .gitkeep. Real traces are recorded by the user against live providers.

package.json adds test:eval (vitest --passWithNoTests so an empty
fixture dir is green) and eval:record (CLI entrypoint). electron-builder
already excludes tests/ via its files: glob — fixtures never enter the
shipped .asar.

.gitignore

@@ -64,6 +64,7 @@ oldreference/
 # the parent directory; losing SKILL.md to the blanket *.md ignore meant a
 # fresh checkout silently dropped every skill from the catalogue.
 !/builtin-skills/**/*.md
+!/tests/eval/**/*.md
 *.zip
 *.blockmap
 *.exe

package.json

@@ -26,6 +26,8 @@
     "typecheck": "tsc --noEmit",
     "test:unit": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/unit",
     "test:integration": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/integration",
+    "test:eval": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/eval --passWithNoTests",
+    "eval:record": "node --disable-warning=ExperimentalWarning --experimental-strip-types tests/eval/record-fixture.ts",
     "test:e2e": "node --no-warnings ./node_modules/@playwright/test/cli.js test -c tests/e2e/playwright.config.ts",
     "test:run": "pnpm test:unit && pnpm test:integration && pnpm test:e2e",
     "verify": "pnpm typecheck && pnpm lint && pnpm test:unit && pnpm test:integration && pnpm test:e2e",

tests/eval/README.md

@@ -0,0 +1,100 @@
+# B5 eval harness (minimal scaffold)
+
+This directory holds the minimal LLM behavior eval tier for AgentCompany.
+It replays recorded LLM traces (fixtures) and checks structural assertions
+against them — tool names, call counts, argument shapes, message
+substrings, and turn caps. It does *not* perform semantic quality judgment
+yet; that tier is proposed in
+`docs/proposals/B5.1-llm-as-judge-followup.md`.
+
+## File layout
+
+| File | Role |
+|---|---|
+| `types.ts` | Wire types — fixtures, assertions, results. Reuses `@mariozechner/pi-ai` primitives. |
+| `fixture-loader.ts` | JSONL → `FixtureFile`. Rejects malformed or missing meta/assertions blocks. |
+| `replay-client.ts` | Swaps in for `pi-ai`'s `streamSimple`; replays recorded assistant turns. Used by the live-replay tier in B5.1. |
+| `assertion-judge.ts` | Structural assertions over recorded trace. No semantic judgment. |
+| `eval-runner.ts` | Parse fixture → judge assertions → produce `EvalResult`. |
+| `record-fixture.ts` | CLI for writing fixture meta headers. Full record loop lands in B5.1. |
+| `harness.test.ts` | vitest driver that walks `fixtures/` and asserts every fixture passes. |
+| `fixtures/` | Recorded JSONL traces. Git-tracked. Not bundled into production (`electron-builder.yml` only ships `out/**`). |
+
+## Running
+
+```bash
+# Assertion-only eval (no fixtures needed; passes on empty fixture dir)
+pnpm test:eval
+
+# Record a stub fixture header (full recorder lands in B5.1)
+pnpm eval:record -- \
+  --fixture-id ceo-team-building-empty \
+  --scenario "Empty company, CEO bootstrap" \
+  --out tests/eval/fixtures/ceo-team-building-empty.jsonl
+```
+
+## Fixture JSONL format
+
+Every fixture begins with a `meta` header, contains one `turn` per line in
+ascending `turnIndex` order, and ends with an `assertions` block.
+
+```jsonl
+{"type":"meta","fixtureId":"ceo-team-building-empty","piAgentCoreVersion":"0.67.68","recordedAt":"2026-04-20T21:14:00.000Z","scenario":"Empty company, CEO bootstrap","systemPromptHash":"sha256:…"}
+{"type":"turn","turnIndex":0,"role":"user","content":[{"type":"text","text":"build me a team"}]}
+{"type":"turn","turnIndex":1,"role":"assistant","content":[{"type":"text","text":"Starting team build."},{"type":"toolCall","id":"call_1","name":"hire_agent","arguments":{"role":"engineer"}}],"tokenLedger":{"input":12345,"cache_read":11000,"cache_creation":345,"output":234}}
+{"type":"turn","turnIndex":2,"role":"toolResult","toolUseId":"call_1","content":[{"type":"text","text":"hired agent-42"}]}
+{"type":"assertions","assertions":[{"kind":"tool_called","name":"hire_agent","min_count":3,"max_count":5},{"kind":"trace_max_turns","max":40}]}
+```
+
+Field contracts:
+
+- `meta.fixtureId` — stable kebab-case identifier used as the scenario name.
+- `meta.piAgentCoreVersion` — the `@mariozechner/pi-agent-core` version
+  used when the fixture was recorded. Drift against the installed version
+  is allowed but future tooling will surface a warning.
+- `meta.systemPromptHash` — sha256 of the system prompt text. Changes in
+  this hash are the canonical signal that a fixture is stale.
+- `turn.turnIndex` — strictly increasing from 0. Gaps or duplicates are
+  hard errors.
+- `turn.role = "assistant"` requires a `tokenLedger` in snake_case
+  (`input`, `cache_read`, `cache_creation`, `output`). This maps 1:1 to
+  pi-ai's `Usage.{input, cacheRead, cacheWrite, output}` — renamed to
+  provider-neutral snake_case on disk.
+- `assertions.assertions` — see `types.ts` for the complete schema.
+
+## Assertion kinds
+
+| Kind | Semantics |
+|---|---|
+| `tool_called` | Matches tool calls by `name`. Supports `min_count` (default 1), `max_count`, and `order` (0-based ordinal across all tool calls in the trace). |
+| `no_tool_called` | Fails if any tool call with the given name appears. |
+| `message_contains_substring` | Any `role`-matching message whose text content contains `substring` passes. |
+| `tool_call_arg_matches` | At least one call to `name` has the dotted `arg_path` resolving to a string matching `matches_regex`. |
+| `trace_max_turns` | Trace must have at most `max` turns. |
+
+Anything that requires semantic judgment (tone, strategy, plan quality)
+belongs in the LLM-as-judge tier. See
+`docs/proposals/B5.1-llm-as-judge-followup.md`.
+
+## What this scaffold does NOT do tonight
+
+- **No synthesized fixtures.** `fixtures/` ships empty. The user records
+  real traces by running the recorder against a live provider.
+- **No live-replay.** `replay-client.ts` is ready to swap into the agent
+  loop; wiring it into a real `pi-agent-core` Agent instance is the B5.1
+  task. Tonight the runner asserts against recorded turns directly.
+- **No LLM-as-judge.** Assertions are structural only. Semantic judgment
+  tier is proposed separately.
+- **No logging / telemetry.** Results are returned from `runEval` as
+  plain values; the vitest reporter is the only output surface.
+
+Each omission is deliberate and documented. Nothing here silently degrades
+to a placeholder — missing fixture = hard error, malformed fixture = hard
+error, exhausted fixture = hard error.
+
+## Production bundle posture
+
+`electron-builder.yml` ships only `out/**` and `package.json`. `tests/` —
+including `tests/eval/fixtures/` — is never packaged into the shipped
+`.asar`. Fixtures are dev-time / CI-time artifacts; they do not leak into
+end-user installs.

tests/eval/assertion-judge.ts

@@ -0,0 +1,188 @@
+/**
+ * Assertion judge for B5 eval harness.
+ *
+ * Evaluates a list of structural `Assertion` records against a recorded
+ * fixture trace. This is the cheap, deterministic tier — it understands tool
+ * call names, counts, order, and substrings. Semantic judgment (tone, plan
+ * quality) is out of scope and handled by the LLM-as-judge tier described in
+ * B5.1.
+ *
+ * Design notes:
+ * - Pure function of (assertions, turns) → AssertionResult[]. No I/O.
+ * - Never throws. Every assertion either passes or fails with a `reason`.
+ *   The runner aggregates these into a single `EvalResult`.
+ * - `tool_call_arg_matches.arg_path` supports dotted paths through nested
+ *   objects and numeric indexes through arrays (`items.0.name`). If the path
+ *   misses or resolves to a non-string, the assertion fails with a reason.
+ */
+
+import type { ToolCall } from "@mariozechner/pi-ai";
+import type {
+  Assertion,
+  AssertionResult,
+  AssistantTurn,
+  TraceTurn,
+} from "./types.js";
+
+interface ToolCallOccurrence {
+  toolCall: ToolCall;
+  /** Ordinal index among all tool calls across the full trace. */
+  ordinal: number;
+}
+
+function collectToolCalls(turns: TraceTurn[]): ToolCallOccurrence[] {
+  const occurrences: ToolCallOccurrence[] = [];
+  for (const turn of turns) {
+    if (turn.role !== "assistant") continue;
+    const assistantTurn = turn as AssistantTurn;
+    for (const block of assistantTurn.content) {
+      if (block.type === "toolCall") {
+        occurrences.push({
+          toolCall: block,
+          ordinal: occurrences.length,
+        });
+      }
+    }
+  }
+  return occurrences;
+}
+
+function resolveArgPath(root: unknown, path: string): unknown {
+  const parts = path.split(".");
+  let cursor: unknown = root;
+  for (const part of parts) {
+    if (cursor === null || cursor === undefined) return undefined;
+    if (typeof cursor !== "object") return undefined;
+    cursor = (cursor as Record<string, unknown>)[part];
+  }
+  return cursor;
+}
+
+function evaluateOne(
+  assertion: Assertion,
+  turns: TraceTurn[],
+  toolCalls: ToolCallOccurrence[],
+): AssertionResult {
+  switch (assertion.kind) {
+    case "tool_called": {
+      const matches = toolCalls.filter(
+        (occ) => occ.toolCall.name === assertion.name,
+      );
+      const minCount = assertion.min_count ?? 1;
+      if (matches.length < minCount) {
+        return {
+          assertion,
+          passed: false,
+          reason: `expected at least ${minCount} call(s) to "${assertion.name}", saw ${matches.length}`,
+        };
+      }
+      if (assertion.max_count !== undefined && matches.length > assertion.max_count) {
+        return {
+          assertion,
+          passed: false,
+          reason: `expected at most ${assertion.max_count} call(s) to "${assertion.name}", saw ${matches.length}`,
+        };
+      }
+      if (assertion.order !== undefined) {
+        const match = matches.find((occ) => occ.ordinal === assertion.order);
+        if (!match) {
+          const seenOrdinals = matches.map((occ) => occ.ordinal).join(",");
+          return {
+            assertion,
+            passed: false,
+            reason: `expected "${assertion.name}" at ordinal ${assertion.order}, matching calls at [${seenOrdinals}]`,
+          };
+        }
+      }
+      return { assertion, passed: true };
+    }
+    case "no_tool_called": {
+      const matches = toolCalls.filter(
+        (occ) => occ.toolCall.name === assertion.name,
+      );
+      if (matches.length > 0) {
+        return {
+          assertion,
+          passed: false,
+          reason: `expected no call to "${assertion.name}", saw ${matches.length}`,
+        };
+      }
+      return { assertion, passed: true };
+    }
+    case "message_contains_substring": {
+      for (const turn of turns) {
+        if (turn.role !== assertion.role) continue;
+        const content = (turn as AssistantTurn | { content: unknown[] }).content;
+        if (!Array.isArray(content)) continue;
+        for (const block of content) {
+          if (
+            block &&
+            typeof block === "object" &&
+            (block as { type?: string }).type === "text" &&
+            typeof (block as { text?: unknown }).text === "string" &&
+            ((block as { text: string }).text).includes(assertion.substring)
+          ) {
+            return { assertion, passed: true };
+          }
+        }
+      }
+      return {
+        assertion,
+        passed: false,
+        reason: `no ${assertion.role} message contained substring "${assertion.substring}"`,
+      };
+    }
+    case "tool_call_arg_matches": {
+      const matches = toolCalls.filter(
+        (occ) => occ.toolCall.name === assertion.name,
+      );
+      if (matches.length === 0) {
+        return {
+          assertion,
+          passed: false,
+          reason: `no tool call named "${assertion.name}" to match arg against`,
+        };
+      }
+      let regex: RegExp;
+      try {
+        regex = new RegExp(assertion.matches_regex);
+      } catch (error) {
+        const detail = error instanceof Error ? error.message : String(error);
+        return {
+          assertion,
+          passed: false,
+          reason: `invalid regex "${assertion.matches_regex}": ${detail}`,
+        };
+      }
+      for (const occurrence of matches) {
+        const value = resolveArgPath(occurrence.toolCall.arguments, assertion.arg_path);
+        if (typeof value === "string" && regex.test(value)) {
+          return { assertion, passed: true };
+        }
+      }
+      return {
+        assertion,
+        passed: false,
+        reason: `no "${assertion.name}" call had arg "${assertion.arg_path}" matching /${assertion.matches_regex}/`,
+      };
+    }
+    case "trace_max_turns": {
+      if (turns.length > assertion.max) {
+        return {
+          assertion,
+          passed: false,
+          reason: `trace has ${turns.length} turns, max allowed is ${assertion.max}`,
+        };
+      }
+      return { assertion, passed: true };
+    }
+  }
+}
+
+export function judgeAssertions(
+  assertions: Assertion[],
+  turns: TraceTurn[],
+): AssertionResult[] {
+  const toolCalls = collectToolCalls(turns);
+  return assertions.map((assertion) => evaluateOne(assertion, turns, toolCalls));
+}

tests/eval/eval-runner.ts

@@ -0,0 +1,55 @@
+/**
+ * Eval runner for B5 harness.
+ *
+ * Given a parsed `FixtureFile`, produces an `EvalResult` by:
+ *  1. Summing the token ledger across all recorded assistant turns.
+ *  2. Running the assertion judge against the recorded turns.
+ *  3. Measuring wall-clock duration of steps (1)–(2).
+ *
+ * Tonight's scope deliberately evaluates assertions against the RECORDED
+ * trace, not a live replay-driven agent loop. That keeps the harness
+ * useful as a snapshot-regression tool the moment the first fixture lands,
+ * without wiring in the full `pi-agent-core` Agent loop (which requires
+ * tools + hooks and is better done incrementally once we have fixtures).
+ *
+ * The replay-client + fixture-loader are written now so the live-replay
+ * tier can be added in B5.1 without reshaping types.
+ */
+
+import { judgeAssertions } from "./assertion-judge.js";
+import type { EvalResult, FixtureFile, TokenLedger } from "./types.js";
+
+const ZERO_LEDGER: TokenLedger = {
+  input: 0,
+  cache_read: 0,
+  cache_creation: 0,
+  output: 0,
+};
+
+function sumLedger(fixture: FixtureFile): TokenLedger {
+  return fixture.turns.reduce<TokenLedger>((acc, turn) => {
+    if (turn.role !== "assistant") return acc;
+    return {
+      input: acc.input + turn.tokenLedger.input,
+      cache_read: acc.cache_read + turn.tokenLedger.cache_read,
+      cache_creation: acc.cache_creation + turn.tokenLedger.cache_creation,
+      output: acc.output + turn.tokenLedger.output,
+    };
+  }, ZERO_LEDGER);
+}
+
+export function runEval(fixture: FixtureFile): EvalResult {
+  const start = Date.now();
+  const results = judgeAssertions(fixture.assertions, fixture.turns);
+  const tokenLedger = sumLedger(fixture);
+  const durationMs = Date.now() - start;
+  const passed = results.every((result) => result.passed);
+
+  return {
+    fixtureId: fixture.meta.fixtureId,
+    passed,
+    assertions: results,
+    tokenLedger,
+    durationMs,
+  };
+}

tests/eval/fixture-loader.ts

@@ -0,0 +1,109 @@
+/**
+ * Fixture loader for B5 eval harness.
+ *
+ * Reads a JSONL file and parses it into a strongly-typed `FixtureFile`.
+ *
+ * Non-negotiable invariants:
+ * - The first non-empty line MUST be a `meta` record. Otherwise the loader
+ *   throws — no synthesized default metadata.
+ * - The last non-empty line MUST be an `assertions` record. Otherwise throw.
+ * - Turn `turnIndex` must be strictly increasing starting at 0. Gaps or
+ *   duplicates throw.
+ * - Unknown `type` values throw. We do not silently ignore junk lines.
+ *
+ * The loader does no I/O beyond `readFile`. It does not tolerate partial or
+ * corrupt fixtures by "best-effort parsing" — Karpathy rule: bad data must
+ * crash loud so the user fixes the fixture, not the harness.
+ */
+
+import { readFile } from "node:fs/promises";
+import type {
+  AssertionsBlock,
+  FixtureFile,
+  FixtureLine,
+  FixtureMeta,
+  TraceTurn,
+} from "./types.js";
+
+function parseLine(raw: string, lineNumber: number, source: string): FixtureLine {
+  try {
+    return JSON.parse(raw) as FixtureLine;
+  } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
+    throw new Error(
+      `fixture ${source}: line ${lineNumber} is not valid JSON: ${detail}`,
+    );
+  }
+}
+
+function isMeta(value: FixtureLine): value is FixtureMeta {
+  return value.type === "meta";
+}
+
+function isAssertions(value: FixtureLine): value is AssertionsBlock {
+  return value.type === "assertions";
+}
+
+function isTurn(value: FixtureLine): value is TraceTurn {
+  return value.type === "turn";
+}
+
+export async function loadFixture(path: string): Promise<FixtureFile> {
+  const raw = await readFile(path, "utf8");
+  const lines = raw
+    .split("\n")
+    .map((line, idx) => ({ line: line.trim(), lineNumber: idx + 1 }))
+    .filter((entry) => entry.line.length > 0);
+
+  if (lines.length < 3) {
+    throw new Error(
+      `fixture ${path}: must contain at least a meta record, one turn, and an assertions record`,
+    );
+  }
+
+  const parsed: { record: FixtureLine; lineNumber: number }[] = lines.map(
+    ({ line, lineNumber }) => ({
+      record: parseLine(line, lineNumber, path),
+      lineNumber,
+    }),
+  );
+
+  const first = parsed[0];
+  if (!first || !isMeta(first.record)) {
+    throw new Error(
+      `fixture ${path}: first line must be a {"type":"meta"} record`,
+    );
+  }
+  const last = parsed[parsed.length - 1];
+  if (!last || !isAssertions(last.record)) {
+    throw new Error(
+      `fixture ${path}: last line must be a {"type":"assertions"} record`,
+    );
+  }
+
+  const turns: TraceTurn[] = [];
+  for (let i = 1; i < parsed.length - 1; i += 1) {
+    const entry = parsed[i];
+    if (!entry) {
+      throw new Error(`fixture ${path}: internal index error`);
+    }
+    if (!isTurn(entry.record)) {
+      throw new Error(
+        `fixture ${path}: line ${entry.lineNumber} has unexpected type "${entry.record.type}" (only turn records allowed between meta and assertions)`,
+      );
+    }
+    const expectedIndex = turns.length;
+    if (entry.record.turnIndex !== expectedIndex) {
+      throw new Error(
+        `fixture ${path}: line ${entry.lineNumber} has turnIndex=${entry.record.turnIndex}, expected ${expectedIndex}`,
+      );
+    }
+    turns.push(entry.record);
+  }
+
+  return {
+    meta: first.record,
+    turns,
+    assertions: last.record.assertions,
+  };
+}

tests/eval/fixtures/README.md

@@ -0,0 +1,44 @@
+# Eval fixtures
+
+Every file in this directory is a **recorded trace** of a real AgentCompany
+turn against a real LLM provider. Fixtures are recorded-truth, not
+synthesized mock data — the distinction is what keeps the B5 harness on the
+"fixture, not fallback" side of the roundtable line.
+
+## File format
+
+Each fixture is a `.jsonl` file with one JSON object per line:
+
+1. First line: a `{ "type": "meta", ... }` record (see `FixtureMeta` in
+   `../types.ts`).
+2. Middle lines: one `{ "type": "turn", "turnIndex": N, ... }` record per
+   conversation turn, in order. `turnIndex` is strictly increasing from `0`.
+3. Last line: a `{ "type": "assertions", "assertions": [...] }` record with
+   the expected behavior contract.
+
+The loader rejects files that do not match this shape. No silent skipping.
+
+## Recording workflow
+
+```bash
+pnpm eval:record -- \
+  --fixture-id ceo-team-building-empty \
+  --scenario "Empty company, CEO bootstrap" \
+  --out tests/eval/fixtures/ceo-team-building-empty.jsonl \
+  --system-prompt-preview "$(cat /tmp/current-system-prompt.txt)"
+```
+
+The current CLI writes a stub meta + empty assertions. The full recorder
+(which drives `pi-agent-core` against a real provider using the
+AgentCompany worker's system prompt and tool registry) lands in a
+follow-up commit — see `../README.md` for the B5 roadmap.
+
+## Committing fixtures
+
+Fixtures belong in git. They are small, deterministic, and the whole point
+of the eval tier is to diff behavior against recorded truth. Do NOT add
+`tests/eval/fixtures/` to `.gitignore`.
+
+Fixtures are excluded from the electron-builder production bundle; see
+`electron-builder.yml` (the `files` glob only picks `out/**`, so `tests/`
+never enters the packaged `.asar`).

tests/eval/harness.test.ts

@@ -0,0 +1,49 @@
+/**
+ * B5 eval harness — driver spec.
+ *
+ * This spec discovers every `*.jsonl` fixture under `tests/eval/fixtures/`
+ * and runs it through the assertion judge. If there are zero fixtures, the
+ * spec file has zero tests — vitest is invoked with `--passWithNoTests`
+ * from the `test:eval` script so this is a green no-op until the user
+ * records real fixtures.
+ *
+ * The harness does NOT synthesize placeholder fixtures under any
+ * circumstance. An empty `fixtures/` directory is a real state — the tier
+ * is simply not yet exercising anything — and the green run reflects that
+ * honestly.
+ */
+
+import { readdir } from "node:fs/promises";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+import { loadFixture } from "./fixture-loader.js";
+import { runEval } from "./eval-runner.js";
+
+const FIXTURE_DIR = join(__dirname, "fixtures");
+
+async function listFixtures(): Promise<string[]> {
+  const entries = await readdir(FIXTURE_DIR, { withFileTypes: true });
+  return entries
+    .filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl"))
+    .map((entry) => join(FIXTURE_DIR, entry.name))
+    .sort();
+}
+
+const fixturePaths = await listFixtures();
+
+describe.each(fixturePaths)("eval fixture %s", (fixturePath) => {
+  it("passes all assertions", async () => {
+    const fixture = await loadFixture(fixturePath);
+    const result = runEval(fixture);
+    if (!result.passed) {
+      const failures = result.assertions
+        .filter((assertion) => !assertion.passed)
+        .map((assertion) => `- ${assertion.assertion.kind}: ${assertion.reason ?? "<no reason>"}`)
+        .join("\n");
+      throw new Error(
+        `fixture ${result.fixtureId} failed:\n${failures}\ntoken ledger: ${JSON.stringify(result.tokenLedger)}`,
+      );
+    }
+    expect(result.passed).toBe(true);
+  });
+});

tests/eval/record-fixture.ts

@@ -0,0 +1,111 @@
+/**
+ * Fixture recorder CLI for B5 eval harness.
+ *
+ * Entrypoint: `pnpm eval:record -- --out tests/eval/fixtures/<id>.jsonl`
+ *
+ * This CLI is a *scaffold*: it accepts a scenario config path and an output
+ * path, validates the config shape, and writes a stub fixture containing
+ * only the meta header plus an empty assertions block. The real recording
+ * loop — which drives `pi-agent-core` against a real provider with the
+ * AgentCompany system prompt and tool catalogue — lives in a follow-up
+ * commit. Tonight's scope is the harness skeleton; the user populates
+ * `tests/eval/fixtures/` by running the full recorder once it is wired.
+ *
+ * Why a stub and not a full recording loop tonight:
+ * - Wiring the full loop requires importing the AgentCompany worker's
+ *   system prompt builder and tool registry. Those modules live in
+ *   `src/main/` and depend on the Electron runtime; importing them from a
+ *   Node CLI would require either a recorder-only entrypoint or a vite
+ *   build step. Both are bigger than tonight's surgical scope.
+ * - Keeping the CLI as a stub today forces a deliberate follow-up rather
+ *   than silently recording fixtures from the wrong context — a Karpathy
+ *   "let it crash" moment.
+ *
+ * Usage:
+ *   tsx tests/eval/record-fixture.ts \
+ *     --fixture-id ceo-team-building-empty \
+ *     --scenario "Empty company, CEO bootstrap" \
+ *     --out tests/eval/fixtures/ceo-team-building-empty.jsonl
+ */
+
+import { createHash } from "node:crypto";
+import { writeFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { createRequire } from "node:module";
+import type { FixtureMeta } from "./types.js";
+
+interface CliArgs {
+  fixtureId: string;
+  scenario: string;
+  out: string;
+  systemPromptPreview: string;
+}
+
+function readArg(args: string[], flag: string, required: boolean): string | undefined {
+  const idx = args.indexOf(flag);
+  if (idx === -1) {
+    if (required) {
+      throw new Error(`record-fixture: missing required flag ${flag}`);
+    }
+    return undefined;
+  }
+  const value = args[idx + 1];
+  if (typeof value !== "string" || value.startsWith("--")) {
+    throw new Error(`record-fixture: flag ${flag} requires a value`);
+  }
+  return value;
+}
+
+function parseArgs(argv: string[]): CliArgs {
+  const args = argv.slice(2);
+  const fixtureId = readArg(args, "--fixture-id", true)!;
+  const scenario = readArg(args, "--scenario", true)!;
+  const out = readArg(args, "--out", true)!;
+  const systemPromptPreview = readArg(args, "--system-prompt-preview", false) ?? "";
+  return { fixtureId, scenario, out, systemPromptPreview };
+}
+
+function resolvePiAgentCoreVersion(): string {
+  const require = createRequire(import.meta.url);
+  // Narrow shape of the pi-agent-core package.json read at runtime.
+  const pkg = require("@mariozechner/pi-agent-core/package.json") as { version?: unknown };
+  if (typeof pkg.version !== "string") {
+    throw new Error(
+      "record-fixture: could not resolve @mariozechner/pi-agent-core version",
+    );
+  }
+  return pkg.version;
+}
+
+function buildMeta(args: CliArgs): FixtureMeta {
+  const hash = createHash("sha256").update(args.systemPromptPreview).digest("hex");
+  return {
+    type: "meta",
+    fixtureId: args.fixtureId,
+    piAgentCoreVersion: resolvePiAgentCoreVersion(),
+    recordedAt: new Date().toISOString(),
+    scenario: args.scenario,
+    systemPromptHash: `sha256:${hash}`,
+  };
+}
+
+async function main(): Promise<void> {
+  const args = parseArgs(process.argv);
+  const meta = buildMeta(args);
+  const lines: string[] = [
+    JSON.stringify(meta),
+    JSON.stringify({ type: "assertions", assertions: [] }),
+  ];
+  const outPath = resolve(args.out);
+  await writeFile(outPath, `${lines.join("\n")}\n`, "utf8");
+  process.stdout.write(
+    `record-fixture: wrote stub fixture ${args.fixtureId} to ${outPath}\n` +
+      `record-fixture: NEXT STEP — replace the stub with recorded turns by running the full recorder (see tests/eval/README.md).\n`,
+  );
+}
+
+main().catch((error) => {
+  const detail = error instanceof Error ? error.message : String(error);
+  process.stderr.write(`record-fixture: ${detail}\n`);
+  process.exit(1);
+});

tests/eval/replay-client.ts

@@ -0,0 +1,110 @@
+/**
+ * Replay client for B5 eval harness.
+ *
+ * Swaps in for `streamSimple` (the pi-ai LLM call) during eval runs. Instead
+ * of hitting a real provider, it replays the assistant turns from the fixture
+ * in order, emitting the `AssistantMessageEvent` protocol expected by
+ * `pi-agent-core`.
+ *
+ * Hard contract (Karpathy rule: no fallback):
+ * - If the agent loop asks for the Nth assistant turn and the fixture has
+ *   fewer assistant turns, the replay client THROWS. It does not return a
+ *   placeholder response, does not return an empty assistant message, and
+ *   does not invent an `errorMessage`. Missing fixture = bug in fixture, and
+ *   the runner must surface that loud.
+ * - The client ignores the incoming `context.messages` because the fixture
+ *   is the source of truth. A future harness tier (B5.1 or later) can diff
+ *   the incoming messages against the fixture's recorded prior messages to
+ *   detect context drift — out of scope tonight.
+ */
+
+import type { AssistantMessage } from "@mariozechner/pi-ai";
+import {
+  AssistantMessageEventStream,
+  createAssistantMessageEventStream,
+} from "@mariozechner/pi-ai";
+import type { StreamFn } from "@mariozechner/pi-agent-core";
+import type { AssistantTurn, FixtureFile } from "./types.js";
+
+function collectAssistantTurns(fixture: FixtureFile): AssistantTurn[] {
+  return fixture.turns.filter(
+    (turn): turn is AssistantTurn => turn.role === "assistant",
+  );
+}
+
+function buildAssistantMessage(
+  turn: AssistantTurn,
+  modelId: string,
+  provider: string,
+  api: string,
+): AssistantMessage {
+  const ledger = turn.tokenLedger;
+  const totalTokens =
+    ledger.input + ledger.cache_read + ledger.cache_creation + ledger.output;
+  const hasToolCall = turn.content.some((block) => block.type === "toolCall");
+  return {
+    role: "assistant",
+    content: turn.content,
+    api,
+    provider,
+    model: modelId,
+    usage: {
+      input: ledger.input,
+      output: ledger.output,
+      cacheRead: ledger.cache_read,
+      cacheWrite: ledger.cache_creation,
+      totalTokens,
+      cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
+    },
+    stopReason: hasToolCall ? "toolUse" : "stop",
+    timestamp: Date.now(),
+  };
+}
+
+function pushRecordedMessage(
+  stream: AssistantMessageEventStream,
+  message: AssistantMessage,
+): void {
+  stream.push({ type: "start", partial: message });
+  const reason = (message.stopReason === "toolUse" || message.stopReason === "length"
+    ? message.stopReason
+    : "stop") as "stop" | "toolUse" | "length";
+  stream.push({ type: "done", reason, message });
+  stream.end(message);
+}
+
+/**
+ * Create a replay stream function bound to a fixture.
+ *
+ * The returned function tracks its own cursor. Each invocation consumes the
+ * next assistant turn. If the caller exhausts the fixture, the next
+ * invocation throws.
+ */
+export function createReplayStreamFn(fixture: FixtureFile): StreamFn {
+  const assistantTurns = collectAssistantTurns(fixture);
+  let cursor = 0;
+
+  return ((model, _context, _options) => {
+    if (cursor >= assistantTurns.length) {
+      throw new Error(
+        `replay-client: fixture "${fixture.meta.fixtureId}" exhausted after ${assistantTurns.length} assistant turns; agent requested turn ${cursor + 1}`,
+      );
+    }
+    const turn = assistantTurns[cursor];
+    cursor += 1;
+    if (!turn) {
+      throw new Error(
+        `replay-client: internal cursor error at position ${cursor - 1}`,
+      );
+    }
+    const message = buildAssistantMessage(
+      turn,
+      model.id,
+      model.provider,
+      model.api,
+    );
+    const stream = createAssistantMessageEventStream();
+    pushRecordedMessage(stream, message);
+    return stream;
+  }) satisfies StreamFn;
+}

tests/eval/types.ts

@@ -0,0 +1,162 @@
+/**
+ * Type definitions for the B5 minimal eval harness.
+ *
+ * Design invariants:
+ * - Fixtures are recorded LLM traces (history real runs), not synthesized mocks.
+ *   A fixture is the serialized truth of a turn that actually happened against
+ *   a real provider; replaying it tells us whether the *harness* still reaches
+ *   the same tool-call shape given the same prompt/context.
+ * - Every type here reuses `@mariozechner/pi-ai` primitives verbatim. Shadow
+ *   types are forbidden; if pi-ai changes its `AssistantMessage` shape, this
+ *   harness breaks loud — that is the intent.
+ * - No runtime field here is optional for "fallback reasons". If a field is
+ *   optional, it encodes real optionality of the source data (e.g. a turn
+ *   record that does not carry usage because it is a user turn).
+ */
+
+import type {
+  AssistantMessage,
+  ToolResultMessage,
+  UserMessage,
+} from "@mariozechner/pi-ai";
+
+/**
+ * Top-level fixture file meta header.
+ *
+ * Every JSONL fixture begins with exactly one `meta` record. The runner will
+ * throw if the first line is not a meta record — no fallback default metadata
+ * is synthesized.
+ */
+export interface FixtureMeta {
+  type: "meta";
+  /** Stable fixture identifier; used as the scenario name in eval output. */
+  fixtureId: string;
+  /** pi-agent-core version used to record this fixture. */
+  piAgentCoreVersion: string;
+  /** ISO-8601 timestamp when the fixture was recorded against a real provider. */
+  recordedAt: string;
+  /** Short human description of what the scenario exercises. */
+  scenario: string;
+  /** sha256 of the system prompt used at recording time. Used to detect drift. */
+  systemPromptHash: string;
+}
+
+/**
+ * Token ledger captured for assistant turns.
+ *
+ * Shape mirrors `AssistantMessage.usage` but uses snake_case fields that match
+ * the JSONL spec printed in `README.md`. The mapping is 1:1 with pi-ai's
+ * `Usage` primitives (`input`, `output`, `cacheRead`, `cacheWrite`) — renamed
+ * for the on-disk format to stay provider-neutral.
+ */
+export interface TokenLedger {
+  input: number;
+  cache_read: number;
+  cache_creation: number;
+  output: number;
+}
+
+export interface UserTurn {
+  type: "turn";
+  turnIndex: number;
+  role: "user";
+  /** Verbatim user content blocks from the real recording. */
+  content: UserMessage["content"];
+}
+
+export interface AssistantTurn {
+  type: "turn";
+  turnIndex: number;
+  role: "assistant";
+  /** Verbatim assistant content blocks (text / thinking / toolCall). */
+  content: AssistantMessage["content"];
+  /** Token ledger recorded for this assistant turn. Required on assistant turns. */
+  tokenLedger: TokenLedger;
+}
+
+export interface ToolResultTurn {
+  type: "turn";
+  turnIndex: number;
+  role: "toolResult";
+  toolUseId: string;
+  content: ToolResultMessage["content"];
+}
+
+export type TraceTurn = UserTurn | AssistantTurn | ToolResultTurn;
+
+/**
+ * Structural assertions that can be evaluated against a replayed trace.
+ *
+ * Deliberately minimal: assertions express tool-call shape, order, count, and
+ * textual substrings. Semantic quality judgments (tone, plan soundness) are
+ * out of scope for this tier and will be handled by the LLM-as-judge tier
+ * described in B5.1.
+ */
+export type Assertion =
+  | {
+      kind: "tool_called";
+      name: string;
+      min_count?: number;
+      max_count?: number;
+      /** Ordinal position in the trace (0-based) among all tool calls. */
+      order?: number;
+    }
+  | { kind: "no_tool_called"; name: string }
+  | {
+      kind: "message_contains_substring";
+      role: "assistant" | "user";
+      substring: string;
+    }
+  | {
+      kind: "tool_call_arg_matches";
+      name: string;
+      /** Dotted JSON path into the tool call's arguments object, e.g. "agent.role". */
+      arg_path: string;
+      /** ECMAScript regex source evaluated against the argument string. */
+      matches_regex: string;
+    }
+  | { kind: "trace_max_turns"; max: number };
+
+export interface AssertionsBlock {
+  type: "assertions";
+  assertions: Assertion[];
+}
+
+/**
+ * A JSONL fixture file is parsed into exactly one of these records per line.
+ * The loader rejects files that do not begin with `meta` and end with
+ * `assertions`.
+ */
+export type FixtureLine = FixtureMeta | TraceTurn | AssertionsBlock;
+
+/**
+ * Parsed fixture file ready to be replayed by the runner.
+ */
+export interface FixtureFile {
+  meta: FixtureMeta;
+  turns: TraceTurn[];
+  assertions: Assertion[];
+}
+
+/**
+ * Per-assertion outcome within a single eval run.
+ */
+export interface AssertionResult {
+  assertion: Assertion;
+  passed: boolean;
+  /** Populated when `passed === false`; concise human-readable explanation. */
+  reason?: string;
+}
+
+/**
+ * Result of evaluating one fixture against the current code path.
+ */
+export interface EvalResult {
+  fixtureId: string;
+  passed: boolean;
+  assertions: AssertionResult[];
+  /** Aggregated token ledger summed across all assistant turns in the fixture. */
+  tokenLedger: TokenLedger;
+  /** Wall-clock duration of the replay, in milliseconds. */
+  durationMs: number;
+}