fix(eval): run replay harness in verify

This commit is contained in:
ZacharyZhang-NY committed 2026-04-22 14:47:31 -04:00
1 parent f90fb037f9
commit cb315f308d
7 files changed
+612 -79

No files matched your search

+3 -3
View File
@@ -26,11 +26,11 @@
"typecheck": "tsc --noEmit", "typecheck": "tsc --noEmit",
"test:unit": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/unit", "test:unit": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/unit",
"test:integration": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/integration", "test:integration": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/integration",
"test:eval": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/eval --passWithNoTests", "test:eval": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/eval",
"eval:record": "node --disable-warning=ExperimentalWarning --experimental-strip-types tests/eval/record-fixture.ts", "eval:record": "node --disable-warning=ExperimentalWarning --experimental-strip-types tests/eval/record-fixture.ts",
"test:e2e": "node --no-warnings ./node_modules/@playwright/test/cli.js test -c tests/e2e/playwright.config.ts", "test:e2e": "node --no-warnings ./node_modules/@playwright/test/cli.js test -c tests/e2e/playwright.config.ts",
"test:run": "pnpm test:unit && pnpm test:integration && pnpm test:e2e", "test:run": "pnpm test:unit && pnpm test:integration && pnpm test:eval && pnpm test:e2e",
"verify": "pnpm typecheck && pnpm lint && pnpm test:unit && pnpm test:integration && pnpm test:e2e", "verify": "pnpm typecheck && pnpm lint && pnpm test:unit && pnpm test:integration && pnpm test:eval && pnpm test:e2e",
"verify:release": "pnpm verify && pnpm build:dir", "verify:release": "pnpm verify && pnpm build:dir",
"postinstall": "node scripts/postinstall.mjs" "postinstall": "node scripts/postinstall.mjs"
}, },
+30 -41
View File
@@ -1,10 +1,10 @@
# B5 eval harness (minimal scaffold) # B5 eval harness
This directory holds the minimal LLM behavior eval tier for AgentCompany. This directory holds the LLM behavior eval tier for AgentCompany. It replays
It replays recorded LLM traces (fixtures) and checks structural assertions recorded LLM traces (fixtures) through `pi-agent-core`'s Agent loop, feeds
against them — tool names, call counts, argument shapes, message recorded tool results back through replay tools, and checks structural
substrings, and turn caps. It does *not* perform semantic quality judgment assertions against the replayed transcript: tool names, call counts, argument
yet; that tier is proposed in shapes, message substrings, and turn caps. Semantic quality judgment belongs in
`docs/proposals/B5.1-llm-as-judge-followup.md`. `docs/proposals/B5.1-llm-as-judge-followup.md`.
## File layout ## File layout
@@ -13,9 +13,11 @@ yet; that tier is proposed in
|---|---| |---|---|
| `types.ts` | Wire types — fixtures, assertions, results. Reuses `@mariozechner/pi-ai` primitives. | | `types.ts` | Wire types — fixtures, assertions, results. Reuses `@mariozechner/pi-ai` primitives. |
| `fixture-loader.ts` | JSONL → `FixtureFile`. Rejects malformed or missing meta/assertions blocks. | | `fixture-loader.ts` | JSONL → `FixtureFile`. Rejects malformed or missing meta/assertions blocks. |
| `replay-client.ts` | Swaps in for `pi-ai`'s `streamSimple`; replays recorded assistant turns. Used by the live-replay tier in B5.1. | | `replay-client.ts` | Swaps in for `pi-ai`'s `streamSimple`; replays recorded assistant turns. |
| `tool-loop-replay.ts` | Runs a fixture through `pi-agent-core` Agent with recorded tool-result executors. |
| `scenario-contract.ts` | Validates scenario prompt/user files, prompt hashes, and real prompt-builder SOP coverage. |
| `assertion-judge.ts` | Structural assertions over recorded trace. No semantic judgment. | | `assertion-judge.ts` | Structural assertions over recorded trace. No semantic judgment. |
| `eval-runner.ts` | Parse fixture → judge assertions → produce `EvalResult`. | | `eval-runner.ts` | Parse fixture → Agent replay → judge assertions → produce `EvalResult`. |
| `record-fixture.ts` | CLI for writing fixture meta headers. Full record loop lands in B5.1. | | `record-fixture.ts` | CLI for writing fixture meta headers. Full record loop lands in B5.1. |
| `harness.test.ts` | vitest driver that walks `fixtures/` and asserts every fixture passes. | | `harness.test.ts` | vitest driver that walks `fixtures/` and asserts every fixture passes. |
| `fixtures/` | Recorded JSONL traces. Git-tracked. Not bundled into production (`electron-builder.yml` only ships `out/**`). | | `fixtures/` | Recorded JSONL traces. Git-tracked. Not bundled into production (`electron-builder.yml` only ships `out/**`). |
@@ -23,7 +25,7 @@ yet; that tier is proposed in
## Running ## Running
```bash ```bash
# Assertion-only eval (no fixtures needed; passes on empty fixture dir) # Agent-loop replay eval
pnpm test:eval pnpm test:eval
``` ```
@@ -119,10 +121,10 @@ See `types.ts` for the exhaustive schema.
pnpm test:eval --run pnpm test:eval --run
``` ```
A green run means: given the recorded assistant turn, every assertion A green run means the scenario prompt hash matches, the first user message
you wrote passes. A red run means either the assertion is wrong or the matches, the recorded transcript replays through the Agent loop, and every
assistant did not produce the behaviour you expected — inspect the assertion passes. A red run points at the broken contract: scenario drift,
fixture and decide which. tool-loop shape, replay failure, or assertion mismatch.
### Re-recording ### Re-recording
@@ -131,18 +133,12 @@ header surfaces that). To re-record, delete the old fixture and run the
recorder again. Do **not** hand-edit the `turn` records; that defeats recorder again. Do **not** hand-edit the `turn` records; that defeats
the point of recording against a real provider. the point of recording against a real provider.
### What the recorder does NOT do ### Recorder boundary
- It does not loop tool calls back to the provider. Today's recorder The recorder captures the live provider response for the system prompt and
captures one turn: user message in, assistant reply out. Multi-turn user message on disk. Tool-call replay happens during `pnpm test:eval`, where
with tool execution is B5.1's scope. recorded assistant turns run through the real Agent loop and recorded tool
- It does not retry on provider errors. A 529 or a 5xx surfaces as a results are returned by deterministic replay tools.
non-zero exit and nothing is written. Re-run manually if you think
the failure was transient.
- It does not reach into AgentCompany's `src/main/` to import the
worker's system prompt builder or tool catalogue. You bring your own
prompt and message text on the CLI. Importing the worker would
require the Electron runtime; the CLI is node-only on purpose.
## Fixture JSONL format ## Fixture JSONL format
@@ -187,24 +183,17 @@ Anything that requires semantic judgment (tone, strategy, plan quality)
belongs in the LLM-as-judge tier. See belongs in the LLM-as-judge tier. See
`docs/proposals/B5.1-llm-as-judge-followup.md`. `docs/proposals/B5.1-llm-as-judge-followup.md`.
## What this scaffold does NOT do tonight ## Hard contracts
- **No synthesized fixtures.** `fixtures/` ships empty. Fixtures arrive - Fixtures are recorded traces from live provider calls.
by running the recorder against a live provider. - `fixtures/` belongs in git and participates in `pnpm verify`.
- **No multi-turn tool-use recording.** The recorder captures one - `scenarios/<fixtureId>/system.txt` must hash to
user→assistant turn. Multi-turn runs with real tool execution are `meta.systemPromptHash`.
B5.1's scope. - `scenarios/<fixtureId>/user.txt` must match the first user turn exactly.
- **No live-replay.** `replay-client.ts` is ready to swap into the agent - Structured assistant `toolCall` blocks require matching `toolResult` turns
loop; wiring it into a real `pi-agent-core` Agent instance is also before the next assistant or user turn.
B5.1. Tonight the runner asserts against recorded turns directly. - Replay exhaustion, malformed fixtures, and scenario drift throw hard errors.
- **No LLM-as-judge.** Assertions are structural only. Semantic judgment - Assertions stay structural; semantic judgment lives in the follow-up tier.
tier is proposed separately.
- **No logging / telemetry.** Results are returned from `runEval` as
plain values; the vitest reporter is the only output surface.
Each omission is deliberate and documented. Nothing here silently degrades
to a placeholder — missing fixture = hard error, malformed fixture = hard
error, exhausted fixture = hard error.
## Production bundle posture ## Production bundle posture
+24 -20
View File
@@ -1,23 +1,20 @@
/** /**
* Eval runner for B5 harness. * Eval runner for the recorded-fixture harness.
* *
* Given a parsed `FixtureFile`, produces an `EvalResult` by: * A fixture now runs through the real `pi-agent-core` Agent loop with a
* 1. Summing the token ledger across all recorded assistant turns. * replay stream and recorded tool-result executors. Assertions are judged
* 2. Running the assertion judge against the recorded turns. * against the replayed transcript that the Agent produced, so tool-call
* 3. Measuring wall-clock duration of steps (1)–(2). * ordering, tool-result handoff, token ledger aggregation, and assertion
* * evaluation share the same path the worker uses in production.
* Tonight's scope deliberately evaluates assertions against the RECORDED
* trace, not a live replay-driven agent loop. That keeps the harness
* useful as a snapshot-regression tool the moment the first fixture lands,
* without wiring in the full `pi-agent-core` Agent loop (which requires
* tools + hooks and is better done incrementally once we have fixtures).
*
* The replay-client + fixture-loader are written now so the live-replay
* tier can be added in B5.1 without reshaping types.
*/ */
import { judgeAssertions } from "./assertion-judge.js"; import { judgeAssertions } from "./assertion-judge.js";
import type { EvalResult, FixtureFile, TokenLedger } from "./types.js"; import { replayFixtureThroughAgent } from "./tool-loop-replay.js";
import type { EvalResult, FixtureFile, TokenLedger, TraceTurn } from "./types.js";
export interface RunEvalOptions {
systemPrompt?: string;
}
const ZERO_LEDGER: TokenLedger = { const ZERO_LEDGER: TokenLedger = {
input: 0, input: 0,
@@ -26,8 +23,8 @@ const ZERO_LEDGER: TokenLedger = {
output: 0, output: 0,
}; };
function sumLedger(fixture: FixtureFile): TokenLedger { function sumLedger(turns: TraceTurn[]): TokenLedger {
return fixture.turns.reduce<TokenLedger>((acc, turn) => { return turns.reduce<TokenLedger>((acc, turn) => {
if (turn.role !== "assistant") return acc; if (turn.role !== "assistant") return acc;
return { return {
input: acc.input + turn.tokenLedger.input, input: acc.input + turn.tokenLedger.input,
@@ -38,10 +35,17 @@ function sumLedger(fixture: FixtureFile): TokenLedger {
}, ZERO_LEDGER); }, ZERO_LEDGER);
} }
export function runEval(fixture: FixtureFile): EvalResult { export async function runEval(
fixture: FixtureFile,
options: RunEvalOptions = {},
): Promise<EvalResult> {
const start = Date.now(); const start = Date.now();
const results = judgeAssertions(fixture.assertions, fixture.turns); const replayedTurns = await replayFixtureThroughAgent(
const tokenLedger = sumLedger(fixture); fixture,
options.systemPrompt ?? "",
);
const results = judgeAssertions(fixture.assertions, replayedTurns);
const tokenLedger = sumLedger(replayedTurns);
const durationMs = Date.now() - start; const durationMs = Date.now() - start;
const passed = results.every((result) => result.passed); const passed = results.every((result) => result.passed);
+4 -4
View File
@@ -28,10 +28,10 @@ pnpm eval:record -- \
--system-prompt-preview "$(cat /tmp/current-system-prompt.txt)" --system-prompt-preview "$(cat /tmp/current-system-prompt.txt)"
``` ```
The current CLI writes a stub meta + empty assertions. The full recorder The current CLI records the provider assistant turn for a scenario prompt and
(which drives `pi-agent-core` against a real provider using the user message. `pnpm test:eval` runs the fixture through `pi-agent-core` with
AgentCompany worker's system prompt and tool registry) lands in a recorded tool-result replay and validates the scenario prompt hash before
follow-up commit — see `../README.md` for the B5 roadmap. assertions run.
## Committing fixtures ## Committing fixtures
+56 -11
View File
@@ -26,10 +26,17 @@ import { describe, expect, it } from "vitest";
import { judgeAssertions } from "./assertion-judge.js"; import { judgeAssertions } from "./assertion-judge.js";
import { runEval } from "./eval-runner.js"; import { runEval } from "./eval-runner.js";
import { loadFixture } from "./fixture-loader.js"; import { loadFixture } from "./fixture-loader.js";
import {
EVAL_SCENARIO_MATRIX,
assertScenarioMatchesFixture,
buildMatrixPrompt,
loadScenarioContract,
} from "./scenario-contract.js";
import type { import type {
Assertion, Assertion,
AssistantTurn, AssistantTurn,
FixtureFile, FixtureFile,
ToolResultTurn,
UserTurn, UserTurn,
} from "./types.js"; } from "./types.js";
@@ -71,6 +78,25 @@ function buildInMemoryFixture(assertions: Assertion[]): FixtureFile {
output: 60, output: 60,
}, },
}; };
const toolResultTurn: ToolResultTurn = {
type: "turn",
turnIndex: 2,
role: "toolResult",
toolUseId: "call_001",
content: [{ type: "text", text: "hired agent-42" }],
};
const finalAssistantTurn: AssistantTurn = {
type: "turn",
turnIndex: 3,
role: "assistant",
content: [{ type: "text", text: "Writer hired." }],
tokenLedger: {
input: 0,
cache_read: 0,
cache_creation: 0,
output: 0,
},
};
return { return {
meta: { meta: {
type: "meta", type: "meta",
@@ -80,13 +106,13 @@ function buildInMemoryFixture(assertions: Assertion[]): FixtureFile {
scenario: "in-memory self-test fixture", scenario: "in-memory self-test fixture",
systemPromptHash: "sha256:self-test", systemPromptHash: "sha256:self-test",
}, },
turns: [userTurn, assistantTurn], turns: [userTurn, assistantTurn, toolResultTurn, finalAssistantTurn],
assertions, assertions,
}; };
} }
describe("harness self-test (in-memory fixture)", () => { describe("harness self-test (in-memory fixture)", () => {
it("passes when every assertion matches the recorded trace", () => { it("passes when every assertion matches the recorded trace", async () => {
const fixture = buildInMemoryFixture([ const fixture = buildInMemoryFixture([
{ kind: "tool_called", name: "hire_agent", min_count: 1 }, { kind: "tool_called", name: "hire_agent", min_count: 1 },
{ {
@@ -103,7 +129,7 @@ describe("harness self-test (in-memory fixture)", () => {
}, },
{ kind: "trace_max_turns", max: 5 }, { kind: "trace_max_turns", max: 5 },
]); ]);
const result = runEval(fixture); const result = await runEval(fixture, { systemPrompt: "Self-test prompt." });
expect(result.passed).toBe(true); expect(result.passed).toBe(true);
expect(result.fixtureId).toBe("self-test/in-memory"); expect(result.fixtureId).toBe("self-test/in-memory");
expect(result.assertions).toHaveLength(5); expect(result.assertions).toHaveLength(5);
@@ -117,21 +143,21 @@ describe("harness self-test (in-memory fixture)", () => {
expect(result.durationMs).toBeGreaterThanOrEqual(0); expect(result.durationMs).toBeGreaterThanOrEqual(0);
}); });
it("fails with a precise reason when an assertion misses", () => { it("fails with a precise reason when an assertion misses", async () => {
const fixture = buildInMemoryFixture([ const fixture = buildInMemoryFixture([
{ kind: "tool_called", name: "write_plan", min_count: 1 }, { kind: "tool_called", name: "write_plan", min_count: 1 },
{ kind: "no_tool_called", name: "hire_agent" }, { kind: "no_tool_called", name: "hire_agent" },
{ kind: "trace_max_turns", max: 1 }, { kind: "trace_max_turns", max: 1 },
]); ]);
const result = runEval(fixture); const result = await runEval(fixture, { systemPrompt: "Self-test prompt." });
expect(result.passed).toBe(false); expect(result.passed).toBe(false);
const reasons = result.assertions.map((r) => r.reason ?? ""); const reasons = result.assertions.map((r) => r.reason ?? "");
expect(reasons[0]).toContain('expected at least 1 call(s) to "write_plan"'); expect(reasons[0]).toContain('expected at least 1 call(s) to "write_plan"');
expect(reasons[1]).toContain('expected no call to "hire_agent"'); expect(reasons[1]).toContain('expected no call to "hire_agent"');
expect(reasons[2]).toContain("trace has 2 turns, max allowed is 1"); expect(reasons[2]).toContain("trace has 4 turns, max allowed is 1");
}); });
it("message_does_not_contain_substring passes when substring is absent", () => { it("message_does_not_contain_substring passes when substring is absent", async () => {
const fixture = buildInMemoryFixture([ const fixture = buildInMemoryFixture([
{ {
kind: "message_does_not_contain_substring", kind: "message_does_not_contain_substring",
@@ -139,12 +165,12 @@ describe("harness self-test (in-memory fixture)", () => {
substring: "node -e", substring: "node -e",
}, },
]); ]);
const result = runEval(fixture); const result = await runEval(fixture, { systemPrompt: "Self-test prompt." });
expect(result.passed).toBe(true); expect(result.passed).toBe(true);
expect(result.assertions[0]?.passed).toBe(true); expect(result.assertions[0]?.passed).toBe(true);
}); });
it("message_does_not_contain_substring fails when substring is present", () => { it("message_does_not_contain_substring fails when substring is present", async () => {
const fixture = buildInMemoryFixture([ const fixture = buildInMemoryFixture([
{ {
kind: "message_does_not_contain_substring", kind: "message_does_not_contain_substring",
@@ -152,7 +178,7 @@ describe("harness self-test (in-memory fixture)", () => {
substring: "Hiring a writer", substring: "Hiring a writer",
}, },
]); ]);
const result = runEval(fixture); const result = await runEval(fixture, { systemPrompt: "Self-test prompt." });
expect(result.passed).toBe(false); expect(result.passed).toBe(false);
expect(result.assertions[0]?.reason).toContain( expect(result.assertions[0]?.reason).toContain(
'a assistant message contained forbidden substring "Hiring a writer"', 'a assistant message contained forbidden substring "Hiring a writer"',
@@ -174,12 +200,31 @@ describe("harness self-test (in-memory fixture)", () => {
}); });
}); });
describe("eval SOP coverage matrix", () => {
it.each(EVAL_SCENARIO_MATRIX)(
"covers real prompt SOPs for $fixtureId",
(entry) => {
const prompt = buildMatrixPrompt(entry);
for (const sop of entry.requiredSops) {
expect(prompt).toContain(sop);
}
for (const sop of entry.forbiddenSops ?? []) {
expect(prompt).not.toContain(sop);
}
},
);
});
const fixturePaths = await listFixtures(); const fixturePaths = await listFixtures();
describe.each(fixturePaths)("eval fixture %s", (fixturePath) => { describe.each(fixturePaths)("eval fixture %s", (fixturePath) => {
it("passes all assertions", async () => { it("passes all assertions", async () => {
const fixture = await loadFixture(fixturePath); const fixture = await loadFixture(fixturePath);
const result = runEval(fixture); const scenario = await loadScenarioContract(fixture.meta.fixtureId);
assertScenarioMatchesFixture(fixture, scenario);
const result = await runEval(fixture, {
systemPrompt: scenario.systemPrompt,
});
if (!result.passed) { if (!result.passed) {
const failures = result.assertions const failures = result.assertions
.filter((assertion) => !assertion.passed) .filter((assertion) => !assertion.passed)
+237
View File
@@ -0,0 +1,237 @@
import { createHash } from "node:crypto";
import { readFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import {
buildTaskPrompt,
type PromptContext,
} from "../../src/main/prompt-builder/index";
import type { AgentRecord } from "../../src/shared/types";
import type { FixtureFile, UserTurn } from "./types.js";
const EVAL_DIR = dirname(fileURLToPath(import.meta.url));
const SCENARIO_DIR = join(EVAL_DIR, "scenarios");
export type EvalPromptAudience =
| "executive"
| "manager"
| "individual_contributor";
export interface EvalScenarioMatrixEntry {
fixtureId: string;
audience: EvalPromptAudience;
requiredSops: string[];
forbiddenSops?: string[];
}
export interface LoadedScenarioContract {
fixtureId: string;
systemPrompt: string;
userMessage: string;
}
export const EVAL_SCENARIO_MATRIX: EvalScenarioMatrixEntry[] = [
{
fixtureId: "ceo-board-message-reply",
audience: "executive",
requiredSops: [
"SOP-001: Daily Standup Protocol",
"SOP-003: Delegation Protocol",
"SOP-007: Inter-Agent Communication",
],
},
{
fixtureId: "ceo-bootstrap-empty",
audience: "executive",
requiredSops: [
"SOP-003: Delegation Protocol",
"SOP-005: Hiring Protocol",
"SOP-014: Hiring Pipeline",
],
},
{
fixtureId: "ic-task-execute",
audience: "individual_contributor",
requiredSops: [
"SOP-002: Task Lifecycle",
"SOP-011: Performance & Quality Standards",
],
forbiddenSops: [
"SOP-005: Hiring Protocol",
"SOP-010: Budget & Financial Protocol",
"SOP-014: Hiring Pipeline",
],
},
{
fixtureId: "manager-task-handoff",
audience: "manager",
requiredSops: [
"SOP-002: Task Lifecycle",
"SOP-003: Delegation Protocol",
"SOP-004: Code Review Protocol",
],
},
{
fixtureId: "non-ceo-dm-reply",
audience: "manager",
requiredSops: [
"SOP-001: Daily Standup Protocol",
"SOP-007: Inter-Agent Communication",
"SOP-008: Cross-Department Collaboration",
],
},
];
function createAgent(overrides: Partial<AgentRecord>): AgentRecord {
return {
id: "agent-1",
companyId: "company-1",
name: "Agent",
role: "Engineer",
title: "Engineer",
roleLevel: "ic",
department: "engineering",
status: "active",
reportsTo: "manager-1",
connectorId: "openai-codex",
workspaceId: null,
model: null,
capabilities: "Build production software",
budgetMonthlyUsd: 200,
spentMonthlyUsd: 0,
metadataJson: "{}",
heartbeatEnabled: true,
heartbeatIntervalSec: 120,
lastHeartbeatAt: null,
createdAt: "2026-04-20T00:00:00.000Z",
updatedAt: "2026-04-20T00:00:00.000Z",
...overrides,
};
}
function buildPromptContext(audience: EvalPromptAudience): PromptContext {
const ceo = createAgent({
id: "ceo-1",
name: "CEO",
role: "CEO",
title: "Chief Executive Officer",
roleLevel: "c_suite",
department: "executive",
reportsTo: null,
capabilities: "Lead the company",
});
const manager = createAgent({
id: "manager-1",
name: "VP Engineering",
role: "VP Engineering",
title: "VP Engineering",
roleLevel: "vp",
department: "engineering",
reportsTo: ceo.id,
capabilities: "Run engineering delivery",
});
const ic = createAgent({
id: "ic-1",
name: "Backend IC",
role: "Backend Engineer",
title: "Backend Engineer",
roleLevel: "ic",
department: "engineering",
reportsTo: manager.id,
capabilities: "Implement backend tasks",
});
const agent =
audience === "executive" ? ceo : audience === "manager" ? manager : ic;
const directReports =
audience === "executive" ? [manager] : audience === "manager" ? [ic] : [];
const chainOfCommand =
audience === "executive" ? [] : audience === "manager" ? [ceo] : [manager, ceo];
return {
agent,
apiUrl: "http://127.0.0.1:3000",
apiKey: "eval-token",
runId: "eval-run",
task: null,
companyName: "AgentCompany Eval",
companyDescription: "Recorded eval prompt coverage",
goals: [],
projects: [],
directReports,
chainOfCommand,
assignedTasks: [],
allAgents: [ceo, manager, ic],
pendingApprovals: [],
tasksAwaitingReview: [],
wakeReason: null,
allTasks: [],
recentComments: [],
socialAccounts: [],
};
}
function hashSystemPrompt(systemPrompt: string): string {
return `sha256:${createHash("sha256").update(systemPrompt).digest("hex")}`;
}
function getFirstUserText(fixture: FixtureFile): string {
const first = fixture.turns[0];
if (!first || first.role !== "user") {
throw new Error(
`eval scenario: fixture "${fixture.meta.fixtureId}" must start with a user turn`,
);
}
const userTurn = first as UserTurn;
if (!Array.isArray(userTurn.content)) {
throw new Error(
`eval scenario: fixture "${fixture.meta.fixtureId}" first user turn must use content blocks`,
);
}
return userTurn.content
.map((block) => (block.type === "text" ? block.text : ""))
.join("");
}
export async function loadScenarioContract(
fixtureId: string,
): Promise<LoadedScenarioContract> {
const root = join(SCENARIO_DIR, fixtureId);
const [systemPrompt, userMessage] = await Promise.all([
readFile(join(root, "system.txt"), "utf8"),
readFile(join(root, "user.txt"), "utf8"),
]);
if (systemPrompt.length === 0) {
throw new Error(`eval scenario: ${fixtureId}/system.txt is empty`);
}
if (userMessage.length === 0) {
throw new Error(`eval scenario: ${fixtureId}/user.txt is empty`);
}
return {
fixtureId,
systemPrompt,
userMessage,
};
}
export function assertScenarioMatchesFixture(
fixture: FixtureFile,
scenario: LoadedScenarioContract,
): void {
const expectedHash = hashSystemPrompt(scenario.systemPrompt);
if (fixture.meta.systemPromptHash !== expectedHash) {
throw new Error(
`eval scenario: fixture "${fixture.meta.fixtureId}" has systemPromptHash ${fixture.meta.systemPromptHash}, expected ${expectedHash}`,
);
}
const fixtureUserText = getFirstUserText(fixture);
if (fixtureUserText !== scenario.userMessage) {
throw new Error(
`eval scenario: fixture "${fixture.meta.fixtureId}" first user turn does not match scenarios/${scenario.fixtureId}/user.txt`,
);
}
}
export function buildMatrixPrompt(entry: EvalScenarioMatrixEntry): string {
return buildTaskPrompt(buildPromptContext(entry.audience));
}
+258
View File
@@ -0,0 +1,258 @@
import {
Agent,
type AgentMessage,
type AgentTool,
} from "@mariozechner/pi-agent-core";
import {
Type,
type Api,
type AssistantMessage,
type Model,
type ToolCall,
type UserMessage,
} from "@mariozechner/pi-ai";
import { createReplayStreamFn } from "./replay-client.js";
import type {
AssistantTurn,
FixtureFile,
TokenLedger,
ToolResultTurn,
TraceTurn,
UserTurn,
} from "./types.js";
const REPLAY_TOOL_PARAMETERS = Type.Record(Type.String(), Type.Any());
const ZERO_COST = {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
};
function createReplayModel(): Model<Api> {
return {
id: "eval-replay",
name: "Eval Replay",
api: "eval-replay",
provider: "eval-replay",
baseUrl: "https://example.invalid",
reasoning: false,
input: ["text"],
cost: ZERO_COST,
contextWindow: 128_000,
maxTokens: 8192,
};
}
function collectAssistantToolCalls(fixture: FixtureFile): ToolCall[] {
const calls: ToolCall[] = [];
for (const turn of fixture.turns) {
if (turn.role !== "assistant") continue;
for (const block of turn.content) {
if (block.type === "toolCall") calls.push(block);
}
}
return calls;
}
function validateToolLoopShape(fixture: FixtureFile): void {
const pending = new Set<string>();
const seen = new Set<string>();
for (const turn of fixture.turns) {
if (turn.role === "assistant") {
if (pending.size > 0) {
throw new Error(
`eval replay: fixture "${fixture.meta.fixtureId}" has assistant turn ${turn.turnIndex} before tool result(s): ${Array.from(pending).join(", ")}`,
);
}
for (const block of turn.content) {
if (block.type !== "toolCall") continue;
if (seen.has(block.id)) {
throw new Error(
`eval replay: fixture "${fixture.meta.fixtureId}" repeats tool call id "${block.id}"`,
);
}
seen.add(block.id);
pending.add(block.id);
}
continue;
}
if (turn.role === "toolResult") {
if (!pending.has(turn.toolUseId)) {
throw new Error(
`eval replay: fixture "${fixture.meta.fixtureId}" has tool result "${turn.toolUseId}" without a pending tool call`,
);
}
pending.delete(turn.toolUseId);
continue;
}
if (turn.role === "user" && pending.size > 0) {
throw new Error(
`eval replay: fixture "${fixture.meta.fixtureId}" has user turn ${turn.turnIndex} before tool result(s): ${Array.from(pending).join(", ")}`,
);
}
}
if (pending.size > 0) {
throw new Error(
`eval replay: fixture "${fixture.meta.fixtureId}" ended before tool result(s): ${Array.from(pending).join(", ")}`,
);
}
}
function collectRecordedToolResults(
fixture: FixtureFile,
): Map<string, ToolResultTurn> {
const results = new Map<string, ToolResultTurn>();
for (const turn of fixture.turns) {
if (turn.role !== "toolResult") continue;
if (results.has(turn.toolUseId)) {
throw new Error(
`eval replay: fixture "${fixture.meta.fixtureId}" repeats tool result for "${turn.toolUseId}"`,
);
}
results.set(turn.toolUseId, turn);
}
return results;
}
function createReplayTools(fixture: FixtureFile): AgentTool[] {
const results = collectRecordedToolResults(fixture);
const toolNames = Array.from(
new Set(collectAssistantToolCalls(fixture).map((call) => call.name)),
).sort();
return toolNames.map((name) => ({
name,
label: `eval replay ${name}`,
description:
"Replay a recorded eval tool result. The fixture owns the content and the call id is the lookup key.",
parameters: REPLAY_TOOL_PARAMETERS,
async execute(toolCallId, params) {
const recorded = results.get(toolCallId);
if (!recorded) {
throw new Error(
`eval replay: fixture "${fixture.meta.fixtureId}" has no recorded tool result for "${toolCallId}"`,
);
}
return {
content: recorded.content,
details: {
fixtureId: fixture.meta.fixtureId,
toolCallId,
toolName: name,
arguments: params,
},
};
},
executionMode: "sequential",
}));
}
function asUserMessage(turn: UserTurn): UserMessage {
return {
role: "user",
content: turn.content,
timestamp: Date.now(),
};
}
function usageToLedger(message: AssistantMessage): TokenLedger {
return {
input: message.usage.input,
cache_read: message.usage.cacheRead,
cache_creation: message.usage.cacheWrite,
output: message.usage.output,
};
}
function agentMessageToTraceTurn(
message: AgentMessage,
turnIndex: number,
): TraceTurn | null {
if (message.role === "user") {
const content = typeof message.content === "string"
? [{ type: "text" as const, text: message.content }]
: message.content;
return {
type: "turn",
turnIndex,
role: "user",
content,
};
}
if (message.role === "assistant") {
return {
type: "turn",
turnIndex,
role: "assistant",
content: message.content,
tokenLedger: usageToLedger(message),
} satisfies AssistantTurn;
}
if (message.role === "toolResult") {
return {
type: "turn",
turnIndex,
role: "toolResult",
toolUseId: message.toolCallId,
content: message.content,
};
}
return null;
}
function collectUserTurns(fixture: FixtureFile): UserTurn[] {
return fixture.turns.filter((turn): turn is UserTurn => turn.role === "user");
}
export async function replayFixtureThroughAgent(
fixture: FixtureFile,
systemPrompt: string,
): Promise<TraceTurn[]> {
validateToolLoopShape(fixture);
const userTurns = collectUserTurns(fixture);
const firstUserTurn = userTurns[0];
if (!firstUserTurn || fixture.turns[0]?.role !== "user") {
throw new Error(
`eval replay: fixture "${fixture.meta.fixtureId}" must start with a user turn`,
);
}
const agent = new Agent({
initialState: {
systemPrompt,
model: createReplayModel(),
tools: createReplayTools(fixture),
messages: [],
thinkingLevel: "off",
},
sessionId: `eval:${fixture.meta.fixtureId}`,
streamFn: createReplayStreamFn(fixture),
toolExecution: "sequential",
});
for (const followUp of userTurns.slice(1)) {
agent.followUp(asUserMessage(followUp));
}
await agent.prompt(asUserMessage(firstUserTurn));
if (agent.state.errorMessage) {
throw new Error(
`eval replay: fixture "${fixture.meta.fixtureId}" failed inside pi Agent loop: ${agent.state.errorMessage}`,
);
}
return agent.state.messages
.map((message, idx) => agentMessageToTraceTurn(message, idx))
.filter((turn): turn is TraceTurn => turn !== null);
}