fix(eval): run replay harness in verify

package.json

@@ -26,11 +26,11 @@
     "typecheck": "tsc --noEmit",
     "test:unit": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/unit",
     "test:integration": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/integration",
-    "test:eval": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/eval --passWithNoTests",
+    "test:eval": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/eval",
     "eval:record": "node --disable-warning=ExperimentalWarning --experimental-strip-types tests/eval/record-fixture.ts",
     "test:e2e": "node --no-warnings ./node_modules/@playwright/test/cli.js test -c tests/e2e/playwright.config.ts",
-    "test:run": "pnpm test:unit && pnpm test:integration && pnpm test:e2e",
-    "verify": "pnpm typecheck && pnpm lint && pnpm test:unit && pnpm test:integration && pnpm test:e2e",
+    "test:run": "pnpm test:unit && pnpm test:integration && pnpm test:eval && pnpm test:e2e",
+    "verify": "pnpm typecheck && pnpm lint && pnpm test:unit && pnpm test:integration && pnpm test:eval && pnpm test:e2e",
     "verify:release": "pnpm verify && pnpm build:dir",
     "postinstall": "node scripts/postinstall.mjs"
   },

tests/eval/README.md

@@ -1,10 +1,10 @@
-# B5 eval harness (minimal scaffold)
+# B5 eval harness
 
-This directory holds the minimal LLM behavior eval tier for AgentCompany.
-It replays recorded LLM traces (fixtures) and checks structural assertions
-against them — tool names, call counts, argument shapes, message
-substrings, and turn caps. It does *not* perform semantic quality judgment
-yet; that tier is proposed in
+This directory holds the LLM behavior eval tier for AgentCompany. It replays
+recorded LLM traces (fixtures) through `pi-agent-core`'s Agent loop, feeds
+recorded tool results back through replay tools, and checks structural
+assertions against the replayed transcript: tool names, call counts, argument
+shapes, message substrings, and turn caps. Semantic quality judgment belongs in
 `docs/proposals/B5.1-llm-as-judge-followup.md`.
 
 ## File layout
@@ -13,9 +13,11 @@ yet; that tier is proposed in
 |---|---|
 | `types.ts` | Wire types — fixtures, assertions, results. Reuses `@mariozechner/pi-ai` primitives. |
 | `fixture-loader.ts` | JSONL → `FixtureFile`. Rejects malformed or missing meta/assertions blocks. |
-| `replay-client.ts` | Swaps in for `pi-ai`'s `streamSimple`; replays recorded assistant turns. Used by the live-replay tier in B5.1. |
+| `replay-client.ts` | Swaps in for `pi-ai`'s `streamSimple`; replays recorded assistant turns. |
+| `tool-loop-replay.ts` | Runs a fixture through `pi-agent-core` Agent with recorded tool-result executors. |
+| `scenario-contract.ts` | Validates scenario prompt/user files, prompt hashes, and real prompt-builder SOP coverage. |
 | `assertion-judge.ts` | Structural assertions over recorded trace. No semantic judgment. |
-| `eval-runner.ts` | Parse fixture → judge assertions → produce `EvalResult`. |
+| `eval-runner.ts` | Parse fixture → Agent replay → judge assertions → produce `EvalResult`. |
 | `record-fixture.ts` | CLI for writing fixture meta headers. Full record loop lands in B5.1. |
 | `harness.test.ts` | vitest driver that walks `fixtures/` and asserts every fixture passes. |
 | `fixtures/` | Recorded JSONL traces. Git-tracked. Not bundled into production (`electron-builder.yml` only ships `out/**`). |
@@ -23,7 +25,7 @@ yet; that tier is proposed in
 ## Running
 
 ```bash
-# Assertion-only eval (no fixtures needed; passes on empty fixture dir)
+# Agent-loop replay eval
 pnpm test:eval
 ```
 
@@ -119,10 +121,10 @@ See `types.ts` for the exhaustive schema.
 pnpm test:eval --run
 ```
 
-A green run means: given the recorded assistant turn, every assertion
-you wrote passes. A red run means either the assertion is wrong or the
-assistant did not produce the behaviour you expected — inspect the
-fixture and decide which.
+A green run means the scenario prompt hash matches, the first user message
+matches, the recorded transcript replays through the Agent loop, and every
+assertion passes. A red run points at the broken contract: scenario drift,
+tool-loop shape, replay failure, or assertion mismatch.
 
 ### Re-recording
 
@@ -131,18 +133,12 @@ header surfaces that). To re-record, delete the old fixture and run the
 recorder again. Do **not** hand-edit the `turn` records; that defeats
 the point of recording against a real provider.
 
-### What the recorder does NOT do
+### Recorder boundary
 
-- It does not loop tool calls back to the provider. Today's recorder
-  captures one turn: user message in, assistant reply out. Multi-turn
-  with tool execution is B5.1's scope.
-- It does not retry on provider errors. A 529 or a 5xx surfaces as a
-  non-zero exit and nothing is written. Re-run manually if you think
-  the failure was transient.
-- It does not reach into AgentCompany's `src/main/` to import the
-  worker's system prompt builder or tool catalogue. You bring your own
-  prompt and message text on the CLI. Importing the worker would
-  require the Electron runtime; the CLI is node-only on purpose.
+The recorder captures the live provider response for the system prompt and
+user message on disk. Tool-call replay happens during `pnpm test:eval`, where
+recorded assistant turns run through the real Agent loop and recorded tool
+results are returned by deterministic replay tools.
 
 ## Fixture JSONL format
 
@@ -187,24 +183,17 @@ Anything that requires semantic judgment (tone, strategy, plan quality)
 belongs in the LLM-as-judge tier. See
 `docs/proposals/B5.1-llm-as-judge-followup.md`.
 
-## What this scaffold does NOT do tonight
+## Hard contracts
 
-- **No synthesized fixtures.** `fixtures/` ships empty. Fixtures arrive
-  by running the recorder against a live provider.
-- **No multi-turn tool-use recording.** The recorder captures one
-  user→assistant turn. Multi-turn runs with real tool execution are
-  B5.1's scope.
-- **No live-replay.** `replay-client.ts` is ready to swap into the agent
-  loop; wiring it into a real `pi-agent-core` Agent instance is also
-  B5.1. Tonight the runner asserts against recorded turns directly.
-- **No LLM-as-judge.** Assertions are structural only. Semantic judgment
-  tier is proposed separately.
-- **No logging / telemetry.** Results are returned from `runEval` as
-  plain values; the vitest reporter is the only output surface.
-
-Each omission is deliberate and documented. Nothing here silently degrades
-to a placeholder — missing fixture = hard error, malformed fixture = hard
-error, exhausted fixture = hard error.
+- Fixtures are recorded traces from live provider calls.
+- `fixtures/` belongs in git and participates in `pnpm verify`.
+- `scenarios/<fixtureId>/system.txt` must hash to
+  `meta.systemPromptHash`.
+- `scenarios/<fixtureId>/user.txt` must match the first user turn exactly.
+- Structured assistant `toolCall` blocks require matching `toolResult` turns
+  before the next assistant or user turn.
+- Replay exhaustion, malformed fixtures, and scenario drift throw hard errors.
+- Assertions stay structural; semantic judgment lives in the follow-up tier.
 
 ## Production bundle posture
 

tests/eval/eval-runner.ts

@@ -1,23 +1,20 @@
 /**
- * Eval runner for B5 harness.
+ * Eval runner for the recorded-fixture harness.
  *
- * Given a parsed `FixtureFile`, produces an `EvalResult` by:
- *  1. Summing the token ledger across all recorded assistant turns.
- *  2. Running the assertion judge against the recorded turns.
- *  3. Measuring wall-clock duration of steps (1)–(2).
- *
- * Tonight's scope deliberately evaluates assertions against the RECORDED
- * trace, not a live replay-driven agent loop. That keeps the harness
- * useful as a snapshot-regression tool the moment the first fixture lands,
- * without wiring in the full `pi-agent-core` Agent loop (which requires
- * tools + hooks and is better done incrementally once we have fixtures).
- *
- * The replay-client + fixture-loader are written now so the live-replay
- * tier can be added in B5.1 without reshaping types.
+ * A fixture now runs through the real `pi-agent-core` Agent loop with a
+ * replay stream and recorded tool-result executors. Assertions are judged
+ * against the replayed transcript that the Agent produced, so tool-call
+ * ordering, tool-result handoff, token ledger aggregation, and assertion
+ * evaluation share the same path the worker uses in production.
  */
 
 import { judgeAssertions } from "./assertion-judge.js";
-import type { EvalResult, FixtureFile, TokenLedger } from "./types.js";
+import { replayFixtureThroughAgent } from "./tool-loop-replay.js";
+import type { EvalResult, FixtureFile, TokenLedger, TraceTurn } from "./types.js";
+
+export interface RunEvalOptions {
+  systemPrompt?: string;
+}
 
 const ZERO_LEDGER: TokenLedger = {
   input: 0,
@@ -26,8 +23,8 @@ const ZERO_LEDGER: TokenLedger = {
   output: 0,
 };
 
-function sumLedger(fixture: FixtureFile): TokenLedger {
-  return fixture.turns.reduce<TokenLedger>((acc, turn) => {
+function sumLedger(turns: TraceTurn[]): TokenLedger {
+  return turns.reduce<TokenLedger>((acc, turn) => {
     if (turn.role !== "assistant") return acc;
     return {
       input: acc.input + turn.tokenLedger.input,
@@ -38,10 +35,17 @@ function sumLedger(fixture: FixtureFile): TokenLedger {
   }, ZERO_LEDGER);
 }
 
-export function runEval(fixture: FixtureFile): EvalResult {
+export async function runEval(
+  fixture: FixtureFile,
+  options: RunEvalOptions = {},
+): Promise<EvalResult> {
   const start = Date.now();
-  const results = judgeAssertions(fixture.assertions, fixture.turns);
-  const tokenLedger = sumLedger(fixture);
+  const replayedTurns = await replayFixtureThroughAgent(
+    fixture,
+    options.systemPrompt ?? "",
+  );
+  const results = judgeAssertions(fixture.assertions, replayedTurns);
+  const tokenLedger = sumLedger(replayedTurns);
   const durationMs = Date.now() - start;
   const passed = results.every((result) => result.passed);
 

tests/eval/fixtures/README.md

@@ -28,10 +28,10 @@ pnpm eval:record -- \
   --system-prompt-preview "$(cat /tmp/current-system-prompt.txt)"
 ```
 
-The current CLI writes a stub meta + empty assertions. The full recorder
-(which drives `pi-agent-core` against a real provider using the
-AgentCompany worker's system prompt and tool registry) lands in a
-follow-up commit — see `../README.md` for the B5 roadmap.
+The current CLI records the provider assistant turn for a scenario prompt and
+user message. `pnpm test:eval` runs the fixture through `pi-agent-core` with
+recorded tool-result replay and validates the scenario prompt hash before
+assertions run.
 
 ## Committing fixtures
 

tests/eval/harness.test.ts

@@ -26,10 +26,17 @@ import { describe, expect, it } from "vitest";
 import { judgeAssertions } from "./assertion-judge.js";
 import { runEval } from "./eval-runner.js";
 import { loadFixture } from "./fixture-loader.js";
+import {
+  EVAL_SCENARIO_MATRIX,
+  assertScenarioMatchesFixture,
+  buildMatrixPrompt,
+  loadScenarioContract,
+} from "./scenario-contract.js";
 import type {
   Assertion,
   AssistantTurn,
   FixtureFile,
+  ToolResultTurn,
   UserTurn,
 } from "./types.js";
 
@@ -71,6 +78,25 @@ function buildInMemoryFixture(assertions: Assertion[]): FixtureFile {
       output: 60,
     },
   };
+  const toolResultTurn: ToolResultTurn = {
+    type: "turn",
+    turnIndex: 2,
+    role: "toolResult",
+    toolUseId: "call_001",
+    content: [{ type: "text", text: "hired agent-42" }],
+  };
+  const finalAssistantTurn: AssistantTurn = {
+    type: "turn",
+    turnIndex: 3,
+    role: "assistant",
+    content: [{ type: "text", text: "Writer hired." }],
+    tokenLedger: {
+      input: 0,
+      cache_read: 0,
+      cache_creation: 0,
+      output: 0,
+    },
+  };
   return {
     meta: {
       type: "meta",
@@ -80,13 +106,13 @@ function buildInMemoryFixture(assertions: Assertion[]): FixtureFile {
       scenario: "in-memory self-test fixture",
       systemPromptHash: "sha256:self-test",
     },
-    turns: [userTurn, assistantTurn],
+    turns: [userTurn, assistantTurn, toolResultTurn, finalAssistantTurn],
     assertions,
   };
 }
 
 describe("harness self-test (in-memory fixture)", () => {
-  it("passes when every assertion matches the recorded trace", () => {
+  it("passes when every assertion matches the recorded trace", async () => {
     const fixture = buildInMemoryFixture([
       { kind: "tool_called", name: "hire_agent", min_count: 1 },
       {
@@ -103,7 +129,7 @@ describe("harness self-test (in-memory fixture)", () => {
       },
       { kind: "trace_max_turns", max: 5 },
     ]);
-    const result = runEval(fixture);
+    const result = await runEval(fixture, { systemPrompt: "Self-test prompt." });
     expect(result.passed).toBe(true);
     expect(result.fixtureId).toBe("self-test/in-memory");
     expect(result.assertions).toHaveLength(5);
@@ -117,21 +143,21 @@ describe("harness self-test (in-memory fixture)", () => {
     expect(result.durationMs).toBeGreaterThanOrEqual(0);
   });
 
-  it("fails with a precise reason when an assertion misses", () => {
+  it("fails with a precise reason when an assertion misses", async () => {
     const fixture = buildInMemoryFixture([
       { kind: "tool_called", name: "write_plan", min_count: 1 },
       { kind: "no_tool_called", name: "hire_agent" },
       { kind: "trace_max_turns", max: 1 },
     ]);
-    const result = runEval(fixture);
+    const result = await runEval(fixture, { systemPrompt: "Self-test prompt." });
     expect(result.passed).toBe(false);
     const reasons = result.assertions.map((r) => r.reason ?? "");
     expect(reasons[0]).toContain('expected at least 1 call(s) to "write_plan"');
     expect(reasons[1]).toContain('expected no call to "hire_agent"');
-    expect(reasons[2]).toContain("trace has 2 turns, max allowed is 1");
+    expect(reasons[2]).toContain("trace has 4 turns, max allowed is 1");
   });
 
-  it("message_does_not_contain_substring passes when substring is absent", () => {
+  it("message_does_not_contain_substring passes when substring is absent", async () => {
     const fixture = buildInMemoryFixture([
       {
         kind: "message_does_not_contain_substring",
@@ -139,12 +165,12 @@ describe("harness self-test (in-memory fixture)", () => {
         substring: "node -e",
       },
     ]);
-    const result = runEval(fixture);
+    const result = await runEval(fixture, { systemPrompt: "Self-test prompt." });
     expect(result.passed).toBe(true);
     expect(result.assertions[0]?.passed).toBe(true);
   });
 
-  it("message_does_not_contain_substring fails when substring is present", () => {
+  it("message_does_not_contain_substring fails when substring is present", async () => {
     const fixture = buildInMemoryFixture([
       {
         kind: "message_does_not_contain_substring",
@@ -152,7 +178,7 @@ describe("harness self-test (in-memory fixture)", () => {
         substring: "Hiring a writer",
       },
     ]);
-    const result = runEval(fixture);
+    const result = await runEval(fixture, { systemPrompt: "Self-test prompt." });
     expect(result.passed).toBe(false);
     expect(result.assertions[0]?.reason).toContain(
       'a assistant message contained forbidden substring "Hiring a writer"',
@@ -174,12 +200,31 @@ describe("harness self-test (in-memory fixture)", () => {
   });
 });
 
+describe("eval SOP coverage matrix", () => {
+  it.each(EVAL_SCENARIO_MATRIX)(
+    "covers real prompt SOPs for $fixtureId",
+    (entry) => {
+      const prompt = buildMatrixPrompt(entry);
+      for (const sop of entry.requiredSops) {
+        expect(prompt).toContain(sop);
+      }
+      for (const sop of entry.forbiddenSops ?? []) {
+        expect(prompt).not.toContain(sop);
+      }
+    },
+  );
+});
+
 const fixturePaths = await listFixtures();
 
 describe.each(fixturePaths)("eval fixture %s", (fixturePath) => {
   it("passes all assertions", async () => {
     const fixture = await loadFixture(fixturePath);
-    const result = runEval(fixture);
+    const scenario = await loadScenarioContract(fixture.meta.fixtureId);
+    assertScenarioMatchesFixture(fixture, scenario);
+    const result = await runEval(fixture, {
+      systemPrompt: scenario.systemPrompt,
+    });
     if (!result.passed) {
       const failures = result.assertions
         .filter((assertion) => !assertion.passed)

tests/eval/scenario-contract.ts

@@ -0,0 +1,237 @@
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+  buildTaskPrompt,
+  type PromptContext,
+} from "../../src/main/prompt-builder/index";
+import type { AgentRecord } from "../../src/shared/types";
+import type { FixtureFile, UserTurn } from "./types.js";
+
+const EVAL_DIR = dirname(fileURLToPath(import.meta.url));
+const SCENARIO_DIR = join(EVAL_DIR, "scenarios");
+
+export type EvalPromptAudience =
+  | "executive"
+  | "manager"
+  | "individual_contributor";
+
+export interface EvalScenarioMatrixEntry {
+  fixtureId: string;
+  audience: EvalPromptAudience;
+  requiredSops: string[];
+  forbiddenSops?: string[];
+}
+
+export interface LoadedScenarioContract {
+  fixtureId: string;
+  systemPrompt: string;
+  userMessage: string;
+}
+
+export const EVAL_SCENARIO_MATRIX: EvalScenarioMatrixEntry[] = [
+  {
+    fixtureId: "ceo-board-message-reply",
+    audience: "executive",
+    requiredSops: [
+      "SOP-001: Daily Standup Protocol",
+      "SOP-003: Delegation Protocol",
+      "SOP-007: Inter-Agent Communication",
+    ],
+  },
+  {
+    fixtureId: "ceo-bootstrap-empty",
+    audience: "executive",
+    requiredSops: [
+      "SOP-003: Delegation Protocol",
+      "SOP-005: Hiring Protocol",
+      "SOP-014: Hiring Pipeline",
+    ],
+  },
+  {
+    fixtureId: "ic-task-execute",
+    audience: "individual_contributor",
+    requiredSops: [
+      "SOP-002: Task Lifecycle",
+      "SOP-011: Performance & Quality Standards",
+    ],
+    forbiddenSops: [
+      "SOP-005: Hiring Protocol",
+      "SOP-010: Budget & Financial Protocol",
+      "SOP-014: Hiring Pipeline",
+    ],
+  },
+  {
+    fixtureId: "manager-task-handoff",
+    audience: "manager",
+    requiredSops: [
+      "SOP-002: Task Lifecycle",
+      "SOP-003: Delegation Protocol",
+      "SOP-004: Code Review Protocol",
+    ],
+  },
+  {
+    fixtureId: "non-ceo-dm-reply",
+    audience: "manager",
+    requiredSops: [
+      "SOP-001: Daily Standup Protocol",
+      "SOP-007: Inter-Agent Communication",
+      "SOP-008: Cross-Department Collaboration",
+    ],
+  },
+];
+
+function createAgent(overrides: Partial<AgentRecord>): AgentRecord {
+  return {
+    id: "agent-1",
+    companyId: "company-1",
+    name: "Agent",
+    role: "Engineer",
+    title: "Engineer",
+    roleLevel: "ic",
+    department: "engineering",
+    status: "active",
+    reportsTo: "manager-1",
+    connectorId: "openai-codex",
+    workspaceId: null,
+    model: null,
+    capabilities: "Build production software",
+    budgetMonthlyUsd: 200,
+    spentMonthlyUsd: 0,
+    metadataJson: "{}",
+    heartbeatEnabled: true,
+    heartbeatIntervalSec: 120,
+    lastHeartbeatAt: null,
+    createdAt: "2026-04-20T00:00:00.000Z",
+    updatedAt: "2026-04-20T00:00:00.000Z",
+    ...overrides,
+  };
+}
+
+function buildPromptContext(audience: EvalPromptAudience): PromptContext {
+  const ceo = createAgent({
+    id: "ceo-1",
+    name: "CEO",
+    role: "CEO",
+    title: "Chief Executive Officer",
+    roleLevel: "c_suite",
+    department: "executive",
+    reportsTo: null,
+    capabilities: "Lead the company",
+  });
+  const manager = createAgent({
+    id: "manager-1",
+    name: "VP Engineering",
+    role: "VP Engineering",
+    title: "VP Engineering",
+    roleLevel: "vp",
+    department: "engineering",
+    reportsTo: ceo.id,
+    capabilities: "Run engineering delivery",
+  });
+  const ic = createAgent({
+    id: "ic-1",
+    name: "Backend IC",
+    role: "Backend Engineer",
+    title: "Backend Engineer",
+    roleLevel: "ic",
+    department: "engineering",
+    reportsTo: manager.id,
+    capabilities: "Implement backend tasks",
+  });
+
+  const agent =
+    audience === "executive" ? ceo : audience === "manager" ? manager : ic;
+  const directReports =
+    audience === "executive" ? [manager] : audience === "manager" ? [ic] : [];
+  const chainOfCommand =
+    audience === "executive" ? [] : audience === "manager" ? [ceo] : [manager, ceo];
+
+  return {
+    agent,
+    apiUrl: "http://127.0.0.1:3000",
+    apiKey: "eval-token",
+    runId: "eval-run",
+    task: null,
+    companyName: "AgentCompany Eval",
+    companyDescription: "Recorded eval prompt coverage",
+    goals: [],
+    projects: [],
+    directReports,
+    chainOfCommand,
+    assignedTasks: [],
+    allAgents: [ceo, manager, ic],
+    pendingApprovals: [],
+    tasksAwaitingReview: [],
+    wakeReason: null,
+    allTasks: [],
+    recentComments: [],
+    socialAccounts: [],
+  };
+}
+
+function hashSystemPrompt(systemPrompt: string): string {
+  return `sha256:${createHash("sha256").update(systemPrompt).digest("hex")}`;
+}
+
+function getFirstUserText(fixture: FixtureFile): string {
+  const first = fixture.turns[0];
+  if (!first || first.role !== "user") {
+    throw new Error(
+      `eval scenario: fixture "${fixture.meta.fixtureId}" must start with a user turn`,
+    );
+  }
+  const userTurn = first as UserTurn;
+  if (!Array.isArray(userTurn.content)) {
+    throw new Error(
+      `eval scenario: fixture "${fixture.meta.fixtureId}" first user turn must use content blocks`,
+    );
+  }
+  return userTurn.content
+    .map((block) => (block.type === "text" ? block.text : ""))
+    .join("");
+}
+
+export async function loadScenarioContract(
+  fixtureId: string,
+): Promise<LoadedScenarioContract> {
+  const root = join(SCENARIO_DIR, fixtureId);
+  const [systemPrompt, userMessage] = await Promise.all([
+    readFile(join(root, "system.txt"), "utf8"),
+    readFile(join(root, "user.txt"), "utf8"),
+  ]);
+  if (systemPrompt.length === 0) {
+    throw new Error(`eval scenario: ${fixtureId}/system.txt is empty`);
+  }
+  if (userMessage.length === 0) {
+    throw new Error(`eval scenario: ${fixtureId}/user.txt is empty`);
+  }
+  return {
+    fixtureId,
+    systemPrompt,
+    userMessage,
+  };
+}
+
+export function assertScenarioMatchesFixture(
+  fixture: FixtureFile,
+  scenario: LoadedScenarioContract,
+): void {
+  const expectedHash = hashSystemPrompt(scenario.systemPrompt);
+  if (fixture.meta.systemPromptHash !== expectedHash) {
+    throw new Error(
+      `eval scenario: fixture "${fixture.meta.fixtureId}" has systemPromptHash ${fixture.meta.systemPromptHash}, expected ${expectedHash}`,
+    );
+  }
+  const fixtureUserText = getFirstUserText(fixture);
+  if (fixtureUserText !== scenario.userMessage) {
+    throw new Error(
+      `eval scenario: fixture "${fixture.meta.fixtureId}" first user turn does not match scenarios/${scenario.fixtureId}/user.txt`,
+    );
+  }
+}
+
+export function buildMatrixPrompt(entry: EvalScenarioMatrixEntry): string {
+  return buildTaskPrompt(buildPromptContext(entry.audience));
+}

tests/eval/tool-loop-replay.ts

@@ -0,0 +1,258 @@
+import {
+  Agent,
+  type AgentMessage,
+  type AgentTool,
+} from "@mariozechner/pi-agent-core";
+import {
+  Type,
+  type Api,
+  type AssistantMessage,
+  type Model,
+  type ToolCall,
+  type UserMessage,
+} from "@mariozechner/pi-ai";
+import { createReplayStreamFn } from "./replay-client.js";
+import type {
+  AssistantTurn,
+  FixtureFile,
+  TokenLedger,
+  ToolResultTurn,
+  TraceTurn,
+  UserTurn,
+} from "./types.js";
+
+const REPLAY_TOOL_PARAMETERS = Type.Record(Type.String(), Type.Any());
+
+const ZERO_COST = {
+  input: 0,
+  output: 0,
+  cacheRead: 0,
+  cacheWrite: 0,
+};
+
+function createReplayModel(): Model<Api> {
+  return {
+    id: "eval-replay",
+    name: "Eval Replay",
+    api: "eval-replay",
+    provider: "eval-replay",
+    baseUrl: "https://example.invalid",
+    reasoning: false,
+    input: ["text"],
+    cost: ZERO_COST,
+    contextWindow: 128_000,
+    maxTokens: 8192,
+  };
+}
+
+function collectAssistantToolCalls(fixture: FixtureFile): ToolCall[] {
+  const calls: ToolCall[] = [];
+  for (const turn of fixture.turns) {
+    if (turn.role !== "assistant") continue;
+    for (const block of turn.content) {
+      if (block.type === "toolCall") calls.push(block);
+    }
+  }
+  return calls;
+}
+
+function validateToolLoopShape(fixture: FixtureFile): void {
+  const pending = new Set<string>();
+  const seen = new Set<string>();
+
+  for (const turn of fixture.turns) {
+    if (turn.role === "assistant") {
+      if (pending.size > 0) {
+        throw new Error(
+          `eval replay: fixture "${fixture.meta.fixtureId}" has assistant turn ${turn.turnIndex} before tool result(s): ${Array.from(pending).join(", ")}`,
+        );
+      }
+      for (const block of turn.content) {
+        if (block.type !== "toolCall") continue;
+        if (seen.has(block.id)) {
+          throw new Error(
+            `eval replay: fixture "${fixture.meta.fixtureId}" repeats tool call id "${block.id}"`,
+          );
+        }
+        seen.add(block.id);
+        pending.add(block.id);
+      }
+      continue;
+    }
+
+    if (turn.role === "toolResult") {
+      if (!pending.has(turn.toolUseId)) {
+        throw new Error(
+          `eval replay: fixture "${fixture.meta.fixtureId}" has tool result "${turn.toolUseId}" without a pending tool call`,
+        );
+      }
+      pending.delete(turn.toolUseId);
+      continue;
+    }
+
+    if (turn.role === "user" && pending.size > 0) {
+      throw new Error(
+        `eval replay: fixture "${fixture.meta.fixtureId}" has user turn ${turn.turnIndex} before tool result(s): ${Array.from(pending).join(", ")}`,
+      );
+    }
+  }
+
+  if (pending.size > 0) {
+    throw new Error(
+      `eval replay: fixture "${fixture.meta.fixtureId}" ended before tool result(s): ${Array.from(pending).join(", ")}`,
+    );
+  }
+}
+
+function collectRecordedToolResults(
+  fixture: FixtureFile,
+): Map<string, ToolResultTurn> {
+  const results = new Map<string, ToolResultTurn>();
+  for (const turn of fixture.turns) {
+    if (turn.role !== "toolResult") continue;
+    if (results.has(turn.toolUseId)) {
+      throw new Error(
+        `eval replay: fixture "${fixture.meta.fixtureId}" repeats tool result for "${turn.toolUseId}"`,
+      );
+    }
+    results.set(turn.toolUseId, turn);
+  }
+  return results;
+}
+
+function createReplayTools(fixture: FixtureFile): AgentTool[] {
+  const results = collectRecordedToolResults(fixture);
+  const toolNames = Array.from(
+    new Set(collectAssistantToolCalls(fixture).map((call) => call.name)),
+  ).sort();
+
+  return toolNames.map((name) => ({
+    name,
+    label: `eval replay ${name}`,
+    description:
+      "Replay a recorded eval tool result. The fixture owns the content and the call id is the lookup key.",
+    parameters: REPLAY_TOOL_PARAMETERS,
+    async execute(toolCallId, params) {
+      const recorded = results.get(toolCallId);
+      if (!recorded) {
+        throw new Error(
+          `eval replay: fixture "${fixture.meta.fixtureId}" has no recorded tool result for "${toolCallId}"`,
+        );
+      }
+      return {
+        content: recorded.content,
+        details: {
+          fixtureId: fixture.meta.fixtureId,
+          toolCallId,
+          toolName: name,
+          arguments: params,
+        },
+      };
+    },
+    executionMode: "sequential",
+  }));
+}
+
+function asUserMessage(turn: UserTurn): UserMessage {
+  return {
+    role: "user",
+    content: turn.content,
+    timestamp: Date.now(),
+  };
+}
+
+function usageToLedger(message: AssistantMessage): TokenLedger {
+  return {
+    input: message.usage.input,
+    cache_read: message.usage.cacheRead,
+    cache_creation: message.usage.cacheWrite,
+    output: message.usage.output,
+  };
+}
+
+function agentMessageToTraceTurn(
+  message: AgentMessage,
+  turnIndex: number,
+): TraceTurn | null {
+  if (message.role === "user") {
+    const content = typeof message.content === "string"
+      ? [{ type: "text" as const, text: message.content }]
+      : message.content;
+    return {
+      type: "turn",
+      turnIndex,
+      role: "user",
+      content,
+    };
+  }
+
+  if (message.role === "assistant") {
+    return {
+      type: "turn",
+      turnIndex,
+      role: "assistant",
+      content: message.content,
+      tokenLedger: usageToLedger(message),
+    } satisfies AssistantTurn;
+  }
+
+  if (message.role === "toolResult") {
+    return {
+      type: "turn",
+      turnIndex,
+      role: "toolResult",
+      toolUseId: message.toolCallId,
+      content: message.content,
+    };
+  }
+
+  return null;
+}
+
+function collectUserTurns(fixture: FixtureFile): UserTurn[] {
+  return fixture.turns.filter((turn): turn is UserTurn => turn.role === "user");
+}
+
+export async function replayFixtureThroughAgent(
+  fixture: FixtureFile,
+  systemPrompt: string,
+): Promise<TraceTurn[]> {
+  validateToolLoopShape(fixture);
+
+  const userTurns = collectUserTurns(fixture);
+  const firstUserTurn = userTurns[0];
+  if (!firstUserTurn || fixture.turns[0]?.role !== "user") {
+    throw new Error(
+      `eval replay: fixture "${fixture.meta.fixtureId}" must start with a user turn`,
+    );
+  }
+
+  const agent = new Agent({
+    initialState: {
+      systemPrompt,
+      model: createReplayModel(),
+      tools: createReplayTools(fixture),
+      messages: [],
+      thinkingLevel: "off",
+    },
+    sessionId: `eval:${fixture.meta.fixtureId}`,
+    streamFn: createReplayStreamFn(fixture),
+    toolExecution: "sequential",
+  });
+
+  for (const followUp of userTurns.slice(1)) {
+    agent.followUp(asUserMessage(followUp));
+  }
+
+  await agent.prompt(asUserMessage(firstUserTurn));
+
+  if (agent.state.errorMessage) {
+    throw new Error(
+      `eval replay: fixture "${fixture.meta.fixtureId}" failed inside pi Agent loop: ${agent.state.errorMessage}`,
+    );
+  }
+
+  return agent.state.messages
+    .map((message, idx) => agentMessageToTraceTurn(message, idx))
+    .filter((turn): turn is TraceTurn => turn !== null);
+}