From cb315f308d94ec054d84960bfc06c61ec294905a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Wed, 22 Apr 2026 14:47:31 -0400 Subject: [PATCH] fix(eval): run replay harness in verify --- package.json | 6 +- tests/eval/README.md | 71 ++++----- tests/eval/eval-runner.ts | 44 +++--- tests/eval/fixtures/README.md | 8 +- tests/eval/harness.test.ts | 67 +++++++-- tests/eval/scenario-contract.ts | 237 +++++++++++++++++++++++++++++ tests/eval/tool-loop-replay.ts | 258 ++++++++++++++++++++++++++++++++ 7 files changed, 612 insertions(+), 79 deletions(-) create mode 100644 tests/eval/scenario-contract.ts create mode 100644 tests/eval/tool-loop-replay.ts diff --git a/package.json b/package.json index 287d109e..19fc023e 100644 --- a/package.json +++ b/package.json @@ -26,11 +26,11 @@ "typecheck": "tsc --noEmit", "test:unit": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/unit", "test:integration": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/integration", - "test:eval": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/eval --passWithNoTests", + "test:eval": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/eval", "eval:record": "node --disable-warning=ExperimentalWarning --experimental-strip-types tests/eval/record-fixture.ts", "test:e2e": "node --no-warnings ./node_modules/@playwright/test/cli.js test -c tests/e2e/playwright.config.ts", - "test:run": "pnpm test:unit && pnpm test:integration && pnpm test:e2e", - "verify": "pnpm typecheck && pnpm lint && pnpm test:unit && pnpm test:integration && pnpm test:e2e", + "test:run": "pnpm test:unit && pnpm test:integration && pnpm test:eval && pnpm test:e2e", + "verify": "pnpm typecheck && pnpm lint && pnpm test:unit && pnpm test:integration && pnpm test:eval && pnpm test:e2e", "verify:release": "pnpm verify && pnpm build:dir", "postinstall": "node scripts/postinstall.mjs" }, diff --git a/tests/eval/README.md b/tests/eval/README.md index 4af2c133..84e1526b 100644 --- a/tests/eval/README.md +++ b/tests/eval/README.md @@ -1,10 +1,10 @@ -# B5 eval harness (minimal scaffold) +# B5 eval harness -This directory holds the minimal LLM behavior eval tier for AgentCompany. -It replays recorded LLM traces (fixtures) and checks structural assertions -against them — tool names, call counts, argument shapes, message -substrings, and turn caps. It does *not* perform semantic quality judgment -yet; that tier is proposed in +This directory holds the LLM behavior eval tier for AgentCompany. It replays +recorded LLM traces (fixtures) through `pi-agent-core`'s Agent loop, feeds +recorded tool results back through replay tools, and checks structural +assertions against the replayed transcript: tool names, call counts, argument +shapes, message substrings, and turn caps. Semantic quality judgment belongs in `docs/proposals/B5.1-llm-as-judge-followup.md`. ## File layout @@ -13,9 +13,11 @@ yet; that tier is proposed in |---|---| | `types.ts` | Wire types — fixtures, assertions, results. Reuses `@mariozechner/pi-ai` primitives. | | `fixture-loader.ts` | JSONL → `FixtureFile`. Rejects malformed or missing meta/assertions blocks. | -| `replay-client.ts` | Swaps in for `pi-ai`'s `streamSimple`; replays recorded assistant turns. Used by the live-replay tier in B5.1. | +| `replay-client.ts` | Swaps in for `pi-ai`'s `streamSimple`; replays recorded assistant turns. | +| `tool-loop-replay.ts` | Runs a fixture through `pi-agent-core` Agent with recorded tool-result executors. | +| `scenario-contract.ts` | Validates scenario prompt/user files, prompt hashes, and real prompt-builder SOP coverage. | | `assertion-judge.ts` | Structural assertions over recorded trace. No semantic judgment. | -| `eval-runner.ts` | Parse fixture → judge assertions → produce `EvalResult`. | +| `eval-runner.ts` | Parse fixture → Agent replay → judge assertions → produce `EvalResult`. | | `record-fixture.ts` | CLI for writing fixture meta headers. Full record loop lands in B5.1. | | `harness.test.ts` | vitest driver that walks `fixtures/` and asserts every fixture passes. | | `fixtures/` | Recorded JSONL traces. Git-tracked. Not bundled into production (`electron-builder.yml` only ships `out/**`). | @@ -23,7 +25,7 @@ yet; that tier is proposed in ## Running ```bash -# Assertion-only eval (no fixtures needed; passes on empty fixture dir) +# Agent-loop replay eval pnpm test:eval ``` @@ -119,10 +121,10 @@ See `types.ts` for the exhaustive schema. pnpm test:eval --run ``` -A green run means: given the recorded assistant turn, every assertion -you wrote passes. A red run means either the assertion is wrong or the -assistant did not produce the behaviour you expected — inspect the -fixture and decide which. +A green run means the scenario prompt hash matches, the first user message +matches, the recorded transcript replays through the Agent loop, and every +assertion passes. A red run points at the broken contract: scenario drift, +tool-loop shape, replay failure, or assertion mismatch. ### Re-recording @@ -131,18 +133,12 @@ header surfaces that). To re-record, delete the old fixture and run the recorder again. Do **not** hand-edit the `turn` records; that defeats the point of recording against a real provider. -### What the recorder does NOT do +### Recorder boundary -- It does not loop tool calls back to the provider. Today's recorder - captures one turn: user message in, assistant reply out. Multi-turn - with tool execution is B5.1's scope. -- It does not retry on provider errors. A 529 or a 5xx surfaces as a - non-zero exit and nothing is written. Re-run manually if you think - the failure was transient. -- It does not reach into AgentCompany's `src/main/` to import the - worker's system prompt builder or tool catalogue. You bring your own - prompt and message text on the CLI. Importing the worker would - require the Electron runtime; the CLI is node-only on purpose. +The recorder captures the live provider response for the system prompt and +user message on disk. Tool-call replay happens during `pnpm test:eval`, where +recorded assistant turns run through the real Agent loop and recorded tool +results are returned by deterministic replay tools. ## Fixture JSONL format @@ -187,24 +183,17 @@ Anything that requires semantic judgment (tone, strategy, plan quality) belongs in the LLM-as-judge tier. See `docs/proposals/B5.1-llm-as-judge-followup.md`. -## What this scaffold does NOT do tonight +## Hard contracts -- **No synthesized fixtures.** `fixtures/` ships empty. Fixtures arrive - by running the recorder against a live provider. -- **No multi-turn tool-use recording.** The recorder captures one - user→assistant turn. Multi-turn runs with real tool execution are - B5.1's scope. -- **No live-replay.** `replay-client.ts` is ready to swap into the agent - loop; wiring it into a real `pi-agent-core` Agent instance is also - B5.1. Tonight the runner asserts against recorded turns directly. -- **No LLM-as-judge.** Assertions are structural only. Semantic judgment - tier is proposed separately. -- **No logging / telemetry.** Results are returned from `runEval` as - plain values; the vitest reporter is the only output surface. - -Each omission is deliberate and documented. Nothing here silently degrades -to a placeholder — missing fixture = hard error, malformed fixture = hard -error, exhausted fixture = hard error. +- Fixtures are recorded traces from live provider calls. +- `fixtures/` belongs in git and participates in `pnpm verify`. +- `scenarios//system.txt` must hash to + `meta.systemPromptHash`. +- `scenarios//user.txt` must match the first user turn exactly. +- Structured assistant `toolCall` blocks require matching `toolResult` turns + before the next assistant or user turn. +- Replay exhaustion, malformed fixtures, and scenario drift throw hard errors. +- Assertions stay structural; semantic judgment lives in the follow-up tier. ## Production bundle posture diff --git a/tests/eval/eval-runner.ts b/tests/eval/eval-runner.ts index 922bcdd8..d2e49b3e 100644 --- a/tests/eval/eval-runner.ts +++ b/tests/eval/eval-runner.ts @@ -1,23 +1,20 @@ /** - * Eval runner for B5 harness. + * Eval runner for the recorded-fixture harness. * - * Given a parsed `FixtureFile`, produces an `EvalResult` by: - * 1. Summing the token ledger across all recorded assistant turns. - * 2. Running the assertion judge against the recorded turns. - * 3. Measuring wall-clock duration of steps (1)–(2). - * - * Tonight's scope deliberately evaluates assertions against the RECORDED - * trace, not a live replay-driven agent loop. That keeps the harness - * useful as a snapshot-regression tool the moment the first fixture lands, - * without wiring in the full `pi-agent-core` Agent loop (which requires - * tools + hooks and is better done incrementally once we have fixtures). - * - * The replay-client + fixture-loader are written now so the live-replay - * tier can be added in B5.1 without reshaping types. + * A fixture now runs through the real `pi-agent-core` Agent loop with a + * replay stream and recorded tool-result executors. Assertions are judged + * against the replayed transcript that the Agent produced, so tool-call + * ordering, tool-result handoff, token ledger aggregation, and assertion + * evaluation share the same path the worker uses in production. */ import { judgeAssertions } from "./assertion-judge.js"; -import type { EvalResult, FixtureFile, TokenLedger } from "./types.js"; +import { replayFixtureThroughAgent } from "./tool-loop-replay.js"; +import type { EvalResult, FixtureFile, TokenLedger, TraceTurn } from "./types.js"; + +export interface RunEvalOptions { + systemPrompt?: string; +} const ZERO_LEDGER: TokenLedger = { input: 0, @@ -26,8 +23,8 @@ const ZERO_LEDGER: TokenLedger = { output: 0, }; -function sumLedger(fixture: FixtureFile): TokenLedger { - return fixture.turns.reduce((acc, turn) => { +function sumLedger(turns: TraceTurn[]): TokenLedger { + return turns.reduce((acc, turn) => { if (turn.role !== "assistant") return acc; return { input: acc.input + turn.tokenLedger.input, @@ -38,10 +35,17 @@ function sumLedger(fixture: FixtureFile): TokenLedger { }, ZERO_LEDGER); } -export function runEval(fixture: FixtureFile): EvalResult { +export async function runEval( + fixture: FixtureFile, + options: RunEvalOptions = {}, +): Promise { const start = Date.now(); - const results = judgeAssertions(fixture.assertions, fixture.turns); - const tokenLedger = sumLedger(fixture); + const replayedTurns = await replayFixtureThroughAgent( + fixture, + options.systemPrompt ?? "", + ); + const results = judgeAssertions(fixture.assertions, replayedTurns); + const tokenLedger = sumLedger(replayedTurns); const durationMs = Date.now() - start; const passed = results.every((result) => result.passed); diff --git a/tests/eval/fixtures/README.md b/tests/eval/fixtures/README.md index 787ef46d..513c2620 100644 --- a/tests/eval/fixtures/README.md +++ b/tests/eval/fixtures/README.md @@ -28,10 +28,10 @@ pnpm eval:record -- \ --system-prompt-preview "$(cat /tmp/current-system-prompt.txt)" ``` -The current CLI writes a stub meta + empty assertions. The full recorder -(which drives `pi-agent-core` against a real provider using the -AgentCompany worker's system prompt and tool registry) lands in a -follow-up commit — see `../README.md` for the B5 roadmap. +The current CLI records the provider assistant turn for a scenario prompt and +user message. `pnpm test:eval` runs the fixture through `pi-agent-core` with +recorded tool-result replay and validates the scenario prompt hash before +assertions run. ## Committing fixtures diff --git a/tests/eval/harness.test.ts b/tests/eval/harness.test.ts index 22cd3dd3..a5d3d345 100644 --- a/tests/eval/harness.test.ts +++ b/tests/eval/harness.test.ts @@ -26,10 +26,17 @@ import { describe, expect, it } from "vitest"; import { judgeAssertions } from "./assertion-judge.js"; import { runEval } from "./eval-runner.js"; import { loadFixture } from "./fixture-loader.js"; +import { + EVAL_SCENARIO_MATRIX, + assertScenarioMatchesFixture, + buildMatrixPrompt, + loadScenarioContract, +} from "./scenario-contract.js"; import type { Assertion, AssistantTurn, FixtureFile, + ToolResultTurn, UserTurn, } from "./types.js"; @@ -71,6 +78,25 @@ function buildInMemoryFixture(assertions: Assertion[]): FixtureFile { output: 60, }, }; + const toolResultTurn: ToolResultTurn = { + type: "turn", + turnIndex: 2, + role: "toolResult", + toolUseId: "call_001", + content: [{ type: "text", text: "hired agent-42" }], + }; + const finalAssistantTurn: AssistantTurn = { + type: "turn", + turnIndex: 3, + role: "assistant", + content: [{ type: "text", text: "Writer hired." }], + tokenLedger: { + input: 0, + cache_read: 0, + cache_creation: 0, + output: 0, + }, + }; return { meta: { type: "meta", @@ -80,13 +106,13 @@ function buildInMemoryFixture(assertions: Assertion[]): FixtureFile { scenario: "in-memory self-test fixture", systemPromptHash: "sha256:self-test", }, - turns: [userTurn, assistantTurn], + turns: [userTurn, assistantTurn, toolResultTurn, finalAssistantTurn], assertions, }; } describe("harness self-test (in-memory fixture)", () => { - it("passes when every assertion matches the recorded trace", () => { + it("passes when every assertion matches the recorded trace", async () => { const fixture = buildInMemoryFixture([ { kind: "tool_called", name: "hire_agent", min_count: 1 }, { @@ -103,7 +129,7 @@ describe("harness self-test (in-memory fixture)", () => { }, { kind: "trace_max_turns", max: 5 }, ]); - const result = runEval(fixture); + const result = await runEval(fixture, { systemPrompt: "Self-test prompt." }); expect(result.passed).toBe(true); expect(result.fixtureId).toBe("self-test/in-memory"); expect(result.assertions).toHaveLength(5); @@ -117,21 +143,21 @@ describe("harness self-test (in-memory fixture)", () => { expect(result.durationMs).toBeGreaterThanOrEqual(0); }); - it("fails with a precise reason when an assertion misses", () => { + it("fails with a precise reason when an assertion misses", async () => { const fixture = buildInMemoryFixture([ { kind: "tool_called", name: "write_plan", min_count: 1 }, { kind: "no_tool_called", name: "hire_agent" }, { kind: "trace_max_turns", max: 1 }, ]); - const result = runEval(fixture); + const result = await runEval(fixture, { systemPrompt: "Self-test prompt." }); expect(result.passed).toBe(false); const reasons = result.assertions.map((r) => r.reason ?? ""); expect(reasons[0]).toContain('expected at least 1 call(s) to "write_plan"'); expect(reasons[1]).toContain('expected no call to "hire_agent"'); - expect(reasons[2]).toContain("trace has 2 turns, max allowed is 1"); + expect(reasons[2]).toContain("trace has 4 turns, max allowed is 1"); }); - it("message_does_not_contain_substring passes when substring is absent", () => { + it("message_does_not_contain_substring passes when substring is absent", async () => { const fixture = buildInMemoryFixture([ { kind: "message_does_not_contain_substring", @@ -139,12 +165,12 @@ describe("harness self-test (in-memory fixture)", () => { substring: "node -e", }, ]); - const result = runEval(fixture); + const result = await runEval(fixture, { systemPrompt: "Self-test prompt." }); expect(result.passed).toBe(true); expect(result.assertions[0]?.passed).toBe(true); }); - it("message_does_not_contain_substring fails when substring is present", () => { + it("message_does_not_contain_substring fails when substring is present", async () => { const fixture = buildInMemoryFixture([ { kind: "message_does_not_contain_substring", @@ -152,7 +178,7 @@ describe("harness self-test (in-memory fixture)", () => { substring: "Hiring a writer", }, ]); - const result = runEval(fixture); + const result = await runEval(fixture, { systemPrompt: "Self-test prompt." }); expect(result.passed).toBe(false); expect(result.assertions[0]?.reason).toContain( 'a assistant message contained forbidden substring "Hiring a writer"', @@ -174,12 +200,31 @@ describe("harness self-test (in-memory fixture)", () => { }); }); +describe("eval SOP coverage matrix", () => { + it.each(EVAL_SCENARIO_MATRIX)( + "covers real prompt SOPs for $fixtureId", + (entry) => { + const prompt = buildMatrixPrompt(entry); + for (const sop of entry.requiredSops) { + expect(prompt).toContain(sop); + } + for (const sop of entry.forbiddenSops ?? []) { + expect(prompt).not.toContain(sop); + } + }, + ); +}); + const fixturePaths = await listFixtures(); describe.each(fixturePaths)("eval fixture %s", (fixturePath) => { it("passes all assertions", async () => { const fixture = await loadFixture(fixturePath); - const result = runEval(fixture); + const scenario = await loadScenarioContract(fixture.meta.fixtureId); + assertScenarioMatchesFixture(fixture, scenario); + const result = await runEval(fixture, { + systemPrompt: scenario.systemPrompt, + }); if (!result.passed) { const failures = result.assertions .filter((assertion) => !assertion.passed) diff --git a/tests/eval/scenario-contract.ts b/tests/eval/scenario-contract.ts new file mode 100644 index 00000000..6c94aff0 --- /dev/null +++ b/tests/eval/scenario-contract.ts @@ -0,0 +1,237 @@ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + buildTaskPrompt, + type PromptContext, +} from "../../src/main/prompt-builder/index"; +import type { AgentRecord } from "../../src/shared/types"; +import type { FixtureFile, UserTurn } from "./types.js"; + +const EVAL_DIR = dirname(fileURLToPath(import.meta.url)); +const SCENARIO_DIR = join(EVAL_DIR, "scenarios"); + +export type EvalPromptAudience = + | "executive" + | "manager" + | "individual_contributor"; + +export interface EvalScenarioMatrixEntry { + fixtureId: string; + audience: EvalPromptAudience; + requiredSops: string[]; + forbiddenSops?: string[]; +} + +export interface LoadedScenarioContract { + fixtureId: string; + systemPrompt: string; + userMessage: string; +} + +export const EVAL_SCENARIO_MATRIX: EvalScenarioMatrixEntry[] = [ + { + fixtureId: "ceo-board-message-reply", + audience: "executive", + requiredSops: [ + "SOP-001: Daily Standup Protocol", + "SOP-003: Delegation Protocol", + "SOP-007: Inter-Agent Communication", + ], + }, + { + fixtureId: "ceo-bootstrap-empty", + audience: "executive", + requiredSops: [ + "SOP-003: Delegation Protocol", + "SOP-005: Hiring Protocol", + "SOP-014: Hiring Pipeline", + ], + }, + { + fixtureId: "ic-task-execute", + audience: "individual_contributor", + requiredSops: [ + "SOP-002: Task Lifecycle", + "SOP-011: Performance & Quality Standards", + ], + forbiddenSops: [ + "SOP-005: Hiring Protocol", + "SOP-010: Budget & Financial Protocol", + "SOP-014: Hiring Pipeline", + ], + }, + { + fixtureId: "manager-task-handoff", + audience: "manager", + requiredSops: [ + "SOP-002: Task Lifecycle", + "SOP-003: Delegation Protocol", + "SOP-004: Code Review Protocol", + ], + }, + { + fixtureId: "non-ceo-dm-reply", + audience: "manager", + requiredSops: [ + "SOP-001: Daily Standup Protocol", + "SOP-007: Inter-Agent Communication", + "SOP-008: Cross-Department Collaboration", + ], + }, +]; + +function createAgent(overrides: Partial): AgentRecord { + return { + id: "agent-1", + companyId: "company-1", + name: "Agent", + role: "Engineer", + title: "Engineer", + roleLevel: "ic", + department: "engineering", + status: "active", + reportsTo: "manager-1", + connectorId: "openai-codex", + workspaceId: null, + model: null, + capabilities: "Build production software", + budgetMonthlyUsd: 200, + spentMonthlyUsd: 0, + metadataJson: "{}", + heartbeatEnabled: true, + heartbeatIntervalSec: 120, + lastHeartbeatAt: null, + createdAt: "2026-04-20T00:00:00.000Z", + updatedAt: "2026-04-20T00:00:00.000Z", + ...overrides, + }; +} + +function buildPromptContext(audience: EvalPromptAudience): PromptContext { + const ceo = createAgent({ + id: "ceo-1", + name: "CEO", + role: "CEO", + title: "Chief Executive Officer", + roleLevel: "c_suite", + department: "executive", + reportsTo: null, + capabilities: "Lead the company", + }); + const manager = createAgent({ + id: "manager-1", + name: "VP Engineering", + role: "VP Engineering", + title: "VP Engineering", + roleLevel: "vp", + department: "engineering", + reportsTo: ceo.id, + capabilities: "Run engineering delivery", + }); + const ic = createAgent({ + id: "ic-1", + name: "Backend IC", + role: "Backend Engineer", + title: "Backend Engineer", + roleLevel: "ic", + department: "engineering", + reportsTo: manager.id, + capabilities: "Implement backend tasks", + }); + + const agent = + audience === "executive" ? ceo : audience === "manager" ? manager : ic; + const directReports = + audience === "executive" ? [manager] : audience === "manager" ? [ic] : []; + const chainOfCommand = + audience === "executive" ? [] : audience === "manager" ? [ceo] : [manager, ceo]; + + return { + agent, + apiUrl: "http://127.0.0.1:3000", + apiKey: "eval-token", + runId: "eval-run", + task: null, + companyName: "AgentCompany Eval", + companyDescription: "Recorded eval prompt coverage", + goals: [], + projects: [], + directReports, + chainOfCommand, + assignedTasks: [], + allAgents: [ceo, manager, ic], + pendingApprovals: [], + tasksAwaitingReview: [], + wakeReason: null, + allTasks: [], + recentComments: [], + socialAccounts: [], + }; +} + +function hashSystemPrompt(systemPrompt: string): string { + return `sha256:${createHash("sha256").update(systemPrompt).digest("hex")}`; +} + +function getFirstUserText(fixture: FixtureFile): string { + const first = fixture.turns[0]; + if (!first || first.role !== "user") { + throw new Error( + `eval scenario: fixture "${fixture.meta.fixtureId}" must start with a user turn`, + ); + } + const userTurn = first as UserTurn; + if (!Array.isArray(userTurn.content)) { + throw new Error( + `eval scenario: fixture "${fixture.meta.fixtureId}" first user turn must use content blocks`, + ); + } + return userTurn.content + .map((block) => (block.type === "text" ? block.text : "")) + .join(""); +} + +export async function loadScenarioContract( + fixtureId: string, +): Promise { + const root = join(SCENARIO_DIR, fixtureId); + const [systemPrompt, userMessage] = await Promise.all([ + readFile(join(root, "system.txt"), "utf8"), + readFile(join(root, "user.txt"), "utf8"), + ]); + if (systemPrompt.length === 0) { + throw new Error(`eval scenario: ${fixtureId}/system.txt is empty`); + } + if (userMessage.length === 0) { + throw new Error(`eval scenario: ${fixtureId}/user.txt is empty`); + } + return { + fixtureId, + systemPrompt, + userMessage, + }; +} + +export function assertScenarioMatchesFixture( + fixture: FixtureFile, + scenario: LoadedScenarioContract, +): void { + const expectedHash = hashSystemPrompt(scenario.systemPrompt); + if (fixture.meta.systemPromptHash !== expectedHash) { + throw new Error( + `eval scenario: fixture "${fixture.meta.fixtureId}" has systemPromptHash ${fixture.meta.systemPromptHash}, expected ${expectedHash}`, + ); + } + const fixtureUserText = getFirstUserText(fixture); + if (fixtureUserText !== scenario.userMessage) { + throw new Error( + `eval scenario: fixture "${fixture.meta.fixtureId}" first user turn does not match scenarios/${scenario.fixtureId}/user.txt`, + ); + } +} + +export function buildMatrixPrompt(entry: EvalScenarioMatrixEntry): string { + return buildTaskPrompt(buildPromptContext(entry.audience)); +} diff --git a/tests/eval/tool-loop-replay.ts b/tests/eval/tool-loop-replay.ts new file mode 100644 index 00000000..d44dc198 --- /dev/null +++ b/tests/eval/tool-loop-replay.ts @@ -0,0 +1,258 @@ +import { + Agent, + type AgentMessage, + type AgentTool, +} from "@mariozechner/pi-agent-core"; +import { + Type, + type Api, + type AssistantMessage, + type Model, + type ToolCall, + type UserMessage, +} from "@mariozechner/pi-ai"; +import { createReplayStreamFn } from "./replay-client.js"; +import type { + AssistantTurn, + FixtureFile, + TokenLedger, + ToolResultTurn, + TraceTurn, + UserTurn, +} from "./types.js"; + +const REPLAY_TOOL_PARAMETERS = Type.Record(Type.String(), Type.Any()); + +const ZERO_COST = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, +}; + +function createReplayModel(): Model { + return { + id: "eval-replay", + name: "Eval Replay", + api: "eval-replay", + provider: "eval-replay", + baseUrl: "https://example.invalid", + reasoning: false, + input: ["text"], + cost: ZERO_COST, + contextWindow: 128_000, + maxTokens: 8192, + }; +} + +function collectAssistantToolCalls(fixture: FixtureFile): ToolCall[] { + const calls: ToolCall[] = []; + for (const turn of fixture.turns) { + if (turn.role !== "assistant") continue; + for (const block of turn.content) { + if (block.type === "toolCall") calls.push(block); + } + } + return calls; +} + +function validateToolLoopShape(fixture: FixtureFile): void { + const pending = new Set(); + const seen = new Set(); + + for (const turn of fixture.turns) { + if (turn.role === "assistant") { + if (pending.size > 0) { + throw new Error( + `eval replay: fixture "${fixture.meta.fixtureId}" has assistant turn ${turn.turnIndex} before tool result(s): ${Array.from(pending).join(", ")}`, + ); + } + for (const block of turn.content) { + if (block.type !== "toolCall") continue; + if (seen.has(block.id)) { + throw new Error( + `eval replay: fixture "${fixture.meta.fixtureId}" repeats tool call id "${block.id}"`, + ); + } + seen.add(block.id); + pending.add(block.id); + } + continue; + } + + if (turn.role === "toolResult") { + if (!pending.has(turn.toolUseId)) { + throw new Error( + `eval replay: fixture "${fixture.meta.fixtureId}" has tool result "${turn.toolUseId}" without a pending tool call`, + ); + } + pending.delete(turn.toolUseId); + continue; + } + + if (turn.role === "user" && pending.size > 0) { + throw new Error( + `eval replay: fixture "${fixture.meta.fixtureId}" has user turn ${turn.turnIndex} before tool result(s): ${Array.from(pending).join(", ")}`, + ); + } + } + + if (pending.size > 0) { + throw new Error( + `eval replay: fixture "${fixture.meta.fixtureId}" ended before tool result(s): ${Array.from(pending).join(", ")}`, + ); + } +} + +function collectRecordedToolResults( + fixture: FixtureFile, +): Map { + const results = new Map(); + for (const turn of fixture.turns) { + if (turn.role !== "toolResult") continue; + if (results.has(turn.toolUseId)) { + throw new Error( + `eval replay: fixture "${fixture.meta.fixtureId}" repeats tool result for "${turn.toolUseId}"`, + ); + } + results.set(turn.toolUseId, turn); + } + return results; +} + +function createReplayTools(fixture: FixtureFile): AgentTool[] { + const results = collectRecordedToolResults(fixture); + const toolNames = Array.from( + new Set(collectAssistantToolCalls(fixture).map((call) => call.name)), + ).sort(); + + return toolNames.map((name) => ({ + name, + label: `eval replay ${name}`, + description: + "Replay a recorded eval tool result. The fixture owns the content and the call id is the lookup key.", + parameters: REPLAY_TOOL_PARAMETERS, + async execute(toolCallId, params) { + const recorded = results.get(toolCallId); + if (!recorded) { + throw new Error( + `eval replay: fixture "${fixture.meta.fixtureId}" has no recorded tool result for "${toolCallId}"`, + ); + } + return { + content: recorded.content, + details: { + fixtureId: fixture.meta.fixtureId, + toolCallId, + toolName: name, + arguments: params, + }, + }; + }, + executionMode: "sequential", + })); +} + +function asUserMessage(turn: UserTurn): UserMessage { + return { + role: "user", + content: turn.content, + timestamp: Date.now(), + }; +} + +function usageToLedger(message: AssistantMessage): TokenLedger { + return { + input: message.usage.input, + cache_read: message.usage.cacheRead, + cache_creation: message.usage.cacheWrite, + output: message.usage.output, + }; +} + +function agentMessageToTraceTurn( + message: AgentMessage, + turnIndex: number, +): TraceTurn | null { + if (message.role === "user") { + const content = typeof message.content === "string" + ? [{ type: "text" as const, text: message.content }] + : message.content; + return { + type: "turn", + turnIndex, + role: "user", + content, + }; + } + + if (message.role === "assistant") { + return { + type: "turn", + turnIndex, + role: "assistant", + content: message.content, + tokenLedger: usageToLedger(message), + } satisfies AssistantTurn; + } + + if (message.role === "toolResult") { + return { + type: "turn", + turnIndex, + role: "toolResult", + toolUseId: message.toolCallId, + content: message.content, + }; + } + + return null; +} + +function collectUserTurns(fixture: FixtureFile): UserTurn[] { + return fixture.turns.filter((turn): turn is UserTurn => turn.role === "user"); +} + +export async function replayFixtureThroughAgent( + fixture: FixtureFile, + systemPrompt: string, +): Promise { + validateToolLoopShape(fixture); + + const userTurns = collectUserTurns(fixture); + const firstUserTurn = userTurns[0]; + if (!firstUserTurn || fixture.turns[0]?.role !== "user") { + throw new Error( + `eval replay: fixture "${fixture.meta.fixtureId}" must start with a user turn`, + ); + } + + const agent = new Agent({ + initialState: { + systemPrompt, + model: createReplayModel(), + tools: createReplayTools(fixture), + messages: [], + thinkingLevel: "off", + }, + sessionId: `eval:${fixture.meta.fixtureId}`, + streamFn: createReplayStreamFn(fixture), + toolExecution: "sequential", + }); + + for (const followUp of userTurns.slice(1)) { + agent.followUp(asUserMessage(followUp)); + } + + await agent.prompt(asUserMessage(firstUserTurn)); + + if (agent.state.errorMessage) { + throw new Error( + `eval replay: fixture "${fixture.meta.fixtureId}" failed inside pi Agent loop: ${agent.state.errorMessage}`, + ); + } + + return agent.state.messages + .map((message, idx) => agentMessageToTraceTurn(message, idx)) + .filter((turn): turn is TraceTurn => turn !== null); +}