fix(eval): run replay harness in verify
This commit is contained in:
1 parent
f90fb037f9
commit
cb315f308d
7 files changed
+612
-79
No files matched your search
+3
-3
@@ -26,11 +26,11 @@
|
|||||||
"typecheck": "tsc --noEmit",
|
"typecheck": "tsc --noEmit",
|
||||||
"test:unit": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/unit",
|
"test:unit": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/unit",
|
||||||
"test:integration": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/integration",
|
"test:integration": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/integration",
|
||||||
"test:eval": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/eval --passWithNoTests",
|
"test:eval": "node --disable-warning=ExperimentalWarning ./node_modules/vitest/vitest.mjs run --config vitest.config.ts tests/eval",
|
||||||
"eval:record": "node --disable-warning=ExperimentalWarning --experimental-strip-types tests/eval/record-fixture.ts",
|
"eval:record": "node --disable-warning=ExperimentalWarning --experimental-strip-types tests/eval/record-fixture.ts",
|
||||||
"test:e2e": "node --no-warnings ./node_modules/@playwright/test/cli.js test -c tests/e2e/playwright.config.ts",
|
"test:e2e": "node --no-warnings ./node_modules/@playwright/test/cli.js test -c tests/e2e/playwright.config.ts",
|
||||||
"test:run": "pnpm test:unit && pnpm test:integration && pnpm test:e2e",
|
"test:run": "pnpm test:unit && pnpm test:integration && pnpm test:eval && pnpm test:e2e",
|
||||||
"verify": "pnpm typecheck && pnpm lint && pnpm test:unit && pnpm test:integration && pnpm test:e2e",
|
"verify": "pnpm typecheck && pnpm lint && pnpm test:unit && pnpm test:integration && pnpm test:eval && pnpm test:e2e",
|
||||||
"verify:release": "pnpm verify && pnpm build:dir",
|
"verify:release": "pnpm verify && pnpm build:dir",
|
||||||
"postinstall": "node scripts/postinstall.mjs"
|
"postinstall": "node scripts/postinstall.mjs"
|
||||||
},
|
},
|
||||||
|
|||||||
+30
-41
@@ -1,10 +1,10 @@
|
|||||||
# B5 eval harness (minimal scaffold)
|
# B5 eval harness
|
||||||
|
|
||||||
This directory holds the minimal LLM behavior eval tier for AgentCompany.
|
This directory holds the LLM behavior eval tier for AgentCompany. It replays
|
||||||
It replays recorded LLM traces (fixtures) and checks structural assertions
|
recorded LLM traces (fixtures) through `pi-agent-core`'s Agent loop, feeds
|
||||||
against them — tool names, call counts, argument shapes, message
|
recorded tool results back through replay tools, and checks structural
|
||||||
substrings, and turn caps. It does *not* perform semantic quality judgment
|
assertions against the replayed transcript: tool names, call counts, argument
|
||||||
yet; that tier is proposed in
|
shapes, message substrings, and turn caps. Semantic quality judgment belongs in
|
||||||
`docs/proposals/B5.1-llm-as-judge-followup.md`.
|
`docs/proposals/B5.1-llm-as-judge-followup.md`.
|
||||||
|
|
||||||
## File layout
|
## File layout
|
||||||
@@ -13,9 +13,11 @@ yet; that tier is proposed in
|
|||||||
|---|---|
|
|---|---|
|
||||||
| `types.ts` | Wire types — fixtures, assertions, results. Reuses `@mariozechner/pi-ai` primitives. |
|
| `types.ts` | Wire types — fixtures, assertions, results. Reuses `@mariozechner/pi-ai` primitives. |
|
||||||
| `fixture-loader.ts` | JSONL → `FixtureFile`. Rejects malformed or missing meta/assertions blocks. |
|
| `fixture-loader.ts` | JSONL → `FixtureFile`. Rejects malformed or missing meta/assertions blocks. |
|
||||||
| `replay-client.ts` | Swaps in for `pi-ai`'s `streamSimple`; replays recorded assistant turns. Used by the live-replay tier in B5.1. |
|
| `replay-client.ts` | Swaps in for `pi-ai`'s `streamSimple`; replays recorded assistant turns. |
|
||||||
|
| `tool-loop-replay.ts` | Runs a fixture through `pi-agent-core` Agent with recorded tool-result executors. |
|
||||||
|
| `scenario-contract.ts` | Validates scenario prompt/user files, prompt hashes, and real prompt-builder SOP coverage. |
|
||||||
| `assertion-judge.ts` | Structural assertions over recorded trace. No semantic judgment. |
|
| `assertion-judge.ts` | Structural assertions over recorded trace. No semantic judgment. |
|
||||||
| `eval-runner.ts` | Parse fixture → judge assertions → produce `EvalResult`. |
|
| `eval-runner.ts` | Parse fixture → Agent replay → judge assertions → produce `EvalResult`. |
|
||||||
| `record-fixture.ts` | CLI for writing fixture meta headers. Full record loop lands in B5.1. |
|
| `record-fixture.ts` | CLI for writing fixture meta headers. Full record loop lands in B5.1. |
|
||||||
| `harness.test.ts` | vitest driver that walks `fixtures/` and asserts every fixture passes. |
|
| `harness.test.ts` | vitest driver that walks `fixtures/` and asserts every fixture passes. |
|
||||||
| `fixtures/` | Recorded JSONL traces. Git-tracked. Not bundled into production (`electron-builder.yml` only ships `out/**`). |
|
| `fixtures/` | Recorded JSONL traces. Git-tracked. Not bundled into production (`electron-builder.yml` only ships `out/**`). |
|
||||||
@@ -23,7 +25,7 @@ yet; that tier is proposed in
|
|||||||
## Running
|
## Running
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Assertion-only eval (no fixtures needed; passes on empty fixture dir)
|
# Agent-loop replay eval
|
||||||
pnpm test:eval
|
pnpm test:eval
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -119,10 +121,10 @@ See `types.ts` for the exhaustive schema.
|
|||||||
pnpm test:eval --run
|
pnpm test:eval --run
|
||||||
```
|
```
|
||||||
|
|
||||||
A green run means: given the recorded assistant turn, every assertion
|
A green run means the scenario prompt hash matches, the first user message
|
||||||
you wrote passes. A red run means either the assertion is wrong or the
|
matches, the recorded transcript replays through the Agent loop, and every
|
||||||
assistant did not produce the behaviour you expected — inspect the
|
assertion passes. A red run points at the broken contract: scenario drift,
|
||||||
fixture and decide which.
|
tool-loop shape, replay failure, or assertion mismatch.
|
||||||
|
|
||||||
### Re-recording
|
### Re-recording
|
||||||
|
|
||||||
@@ -131,18 +133,12 @@ header surfaces that). To re-record, delete the old fixture and run the
|
|||||||
recorder again. Do **not** hand-edit the `turn` records; that defeats
|
recorder again. Do **not** hand-edit the `turn` records; that defeats
|
||||||
the point of recording against a real provider.
|
the point of recording against a real provider.
|
||||||
|
|
||||||
### What the recorder does NOT do
|
### Recorder boundary
|
||||||
|
|
||||||
- It does not loop tool calls back to the provider. Today's recorder
|
The recorder captures the live provider response for the system prompt and
|
||||||
captures one turn: user message in, assistant reply out. Multi-turn
|
user message on disk. Tool-call replay happens during `pnpm test:eval`, where
|
||||||
with tool execution is B5.1's scope.
|
recorded assistant turns run through the real Agent loop and recorded tool
|
||||||
- It does not retry on provider errors. A 529 or a 5xx surfaces as a
|
results are returned by deterministic replay tools.
|
||||||
non-zero exit and nothing is written. Re-run manually if you think
|
|
||||||
the failure was transient.
|
|
||||||
- It does not reach into AgentCompany's `src/main/` to import the
|
|
||||||
worker's system prompt builder or tool catalogue. You bring your own
|
|
||||||
prompt and message text on the CLI. Importing the worker would
|
|
||||||
require the Electron runtime; the CLI is node-only on purpose.
|
|
||||||
|
|
||||||
## Fixture JSONL format
|
## Fixture JSONL format
|
||||||
|
|
||||||
@@ -187,24 +183,17 @@ Anything that requires semantic judgment (tone, strategy, plan quality)
|
|||||||
belongs in the LLM-as-judge tier. See
|
belongs in the LLM-as-judge tier. See
|
||||||
`docs/proposals/B5.1-llm-as-judge-followup.md`.
|
`docs/proposals/B5.1-llm-as-judge-followup.md`.
|
||||||
|
|
||||||
## What this scaffold does NOT do tonight
|
## Hard contracts
|
||||||
|
|
||||||
- **No synthesized fixtures.** `fixtures/` ships empty. Fixtures arrive
|
- Fixtures are recorded traces from live provider calls.
|
||||||
by running the recorder against a live provider.
|
- `fixtures/` belongs in git and participates in `pnpm verify`.
|
||||||
- **No multi-turn tool-use recording.** The recorder captures one
|
- `scenarios/<fixtureId>/system.txt` must hash to
|
||||||
user→assistant turn. Multi-turn runs with real tool execution are
|
`meta.systemPromptHash`.
|
||||||
B5.1's scope.
|
- `scenarios/<fixtureId>/user.txt` must match the first user turn exactly.
|
||||||
- **No live-replay.** `replay-client.ts` is ready to swap into the agent
|
- Structured assistant `toolCall` blocks require matching `toolResult` turns
|
||||||
loop; wiring it into a real `pi-agent-core` Agent instance is also
|
before the next assistant or user turn.
|
||||||
B5.1. Tonight the runner asserts against recorded turns directly.
|
- Replay exhaustion, malformed fixtures, and scenario drift throw hard errors.
|
||||||
- **No LLM-as-judge.** Assertions are structural only. Semantic judgment
|
- Assertions stay structural; semantic judgment lives in the follow-up tier.
|
||||||
tier is proposed separately.
|
|
||||||
- **No logging / telemetry.** Results are returned from `runEval` as
|
|
||||||
plain values; the vitest reporter is the only output surface.
|
|
||||||
|
|
||||||
Each omission is deliberate and documented. Nothing here silently degrades
|
|
||||||
to a placeholder — missing fixture = hard error, malformed fixture = hard
|
|
||||||
error, exhausted fixture = hard error.
|
|
||||||
|
|
||||||
## Production bundle posture
|
## Production bundle posture
|
||||||
|
|
||||||
|
|||||||
+24
-20
@@ -1,23 +1,20 @@
|
|||||||
/**
|
/**
|
||||||
* Eval runner for B5 harness.
|
* Eval runner for the recorded-fixture harness.
|
||||||
*
|
*
|
||||||
* Given a parsed `FixtureFile`, produces an `EvalResult` by:
|
* A fixture now runs through the real `pi-agent-core` Agent loop with a
|
||||||
* 1. Summing the token ledger across all recorded assistant turns.
|
* replay stream and recorded tool-result executors. Assertions are judged
|
||||||
* 2. Running the assertion judge against the recorded turns.
|
* against the replayed transcript that the Agent produced, so tool-call
|
||||||
* 3. Measuring wall-clock duration of steps (1)–(2).
|
* ordering, tool-result handoff, token ledger aggregation, and assertion
|
||||||
*
|
* evaluation share the same path the worker uses in production.
|
||||||
* Tonight's scope deliberately evaluates assertions against the RECORDED
|
|
||||||
* trace, not a live replay-driven agent loop. That keeps the harness
|
|
||||||
* useful as a snapshot-regression tool the moment the first fixture lands,
|
|
||||||
* without wiring in the full `pi-agent-core` Agent loop (which requires
|
|
||||||
* tools + hooks and is better done incrementally once we have fixtures).
|
|
||||||
*
|
|
||||||
* The replay-client + fixture-loader are written now so the live-replay
|
|
||||||
* tier can be added in B5.1 without reshaping types.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { judgeAssertions } from "./assertion-judge.js";
|
import { judgeAssertions } from "./assertion-judge.js";
|
||||||
import type { EvalResult, FixtureFile, TokenLedger } from "./types.js";
|
import { replayFixtureThroughAgent } from "./tool-loop-replay.js";
|
||||||
|
import type { EvalResult, FixtureFile, TokenLedger, TraceTurn } from "./types.js";
|
||||||
|
|
||||||
|
export interface RunEvalOptions {
|
||||||
|
systemPrompt?: string;
|
||||||
|
}
|
||||||
|
|
||||||
const ZERO_LEDGER: TokenLedger = {
|
const ZERO_LEDGER: TokenLedger = {
|
||||||
input: 0,
|
input: 0,
|
||||||
@@ -26,8 +23,8 @@ const ZERO_LEDGER: TokenLedger = {
|
|||||||
output: 0,
|
output: 0,
|
||||||
};
|
};
|
||||||
|
|
||||||
function sumLedger(fixture: FixtureFile): TokenLedger {
|
function sumLedger(turns: TraceTurn[]): TokenLedger {
|
||||||
return fixture.turns.reduce<TokenLedger>((acc, turn) => {
|
return turns.reduce<TokenLedger>((acc, turn) => {
|
||||||
if (turn.role !== "assistant") return acc;
|
if (turn.role !== "assistant") return acc;
|
||||||
return {
|
return {
|
||||||
input: acc.input + turn.tokenLedger.input,
|
input: acc.input + turn.tokenLedger.input,
|
||||||
@@ -38,10 +35,17 @@ function sumLedger(fixture: FixtureFile): TokenLedger {
|
|||||||
}, ZERO_LEDGER);
|
}, ZERO_LEDGER);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function runEval(fixture: FixtureFile): EvalResult {
|
export async function runEval(
|
||||||
|
fixture: FixtureFile,
|
||||||
|
options: RunEvalOptions = {},
|
||||||
|
): Promise<EvalResult> {
|
||||||
const start = Date.now();
|
const start = Date.now();
|
||||||
const results = judgeAssertions(fixture.assertions, fixture.turns);
|
const replayedTurns = await replayFixtureThroughAgent(
|
||||||
const tokenLedger = sumLedger(fixture);
|
fixture,
|
||||||
|
options.systemPrompt ?? "",
|
||||||
|
);
|
||||||
|
const results = judgeAssertions(fixture.assertions, replayedTurns);
|
||||||
|
const tokenLedger = sumLedger(replayedTurns);
|
||||||
const durationMs = Date.now() - start;
|
const durationMs = Date.now() - start;
|
||||||
const passed = results.every((result) => result.passed);
|
const passed = results.every((result) => result.passed);
|
||||||
|
|
||||||
|
|||||||
@@ -28,10 +28,10 @@ pnpm eval:record -- \
|
|||||||
--system-prompt-preview "$(cat /tmp/current-system-prompt.txt)"
|
--system-prompt-preview "$(cat /tmp/current-system-prompt.txt)"
|
||||||
```
|
```
|
||||||
|
|
||||||
The current CLI writes a stub meta + empty assertions. The full recorder
|
The current CLI records the provider assistant turn for a scenario prompt and
|
||||||
(which drives `pi-agent-core` against a real provider using the
|
user message. `pnpm test:eval` runs the fixture through `pi-agent-core` with
|
||||||
AgentCompany worker's system prompt and tool registry) lands in a
|
recorded tool-result replay and validates the scenario prompt hash before
|
||||||
follow-up commit — see `../README.md` for the B5 roadmap.
|
assertions run.
|
||||||
|
|
||||||
## Committing fixtures
|
## Committing fixtures
|
||||||
|
|
||||||
|
|||||||
+56
-11
@@ -26,10 +26,17 @@ import { describe, expect, it } from "vitest";
|
|||||||
import { judgeAssertions } from "./assertion-judge.js";
|
import { judgeAssertions } from "./assertion-judge.js";
|
||||||
import { runEval } from "./eval-runner.js";
|
import { runEval } from "./eval-runner.js";
|
||||||
import { loadFixture } from "./fixture-loader.js";
|
import { loadFixture } from "./fixture-loader.js";
|
||||||
|
import {
|
||||||
|
EVAL_SCENARIO_MATRIX,
|
||||||
|
assertScenarioMatchesFixture,
|
||||||
|
buildMatrixPrompt,
|
||||||
|
loadScenarioContract,
|
||||||
|
} from "./scenario-contract.js";
|
||||||
import type {
|
import type {
|
||||||
Assertion,
|
Assertion,
|
||||||
AssistantTurn,
|
AssistantTurn,
|
||||||
FixtureFile,
|
FixtureFile,
|
||||||
|
ToolResultTurn,
|
||||||
UserTurn,
|
UserTurn,
|
||||||
} from "./types.js";
|
} from "./types.js";
|
||||||
|
|
||||||
@@ -71,6 +78,25 @@ function buildInMemoryFixture(assertions: Assertion[]): FixtureFile {
|
|||||||
output: 60,
|
output: 60,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
const toolResultTurn: ToolResultTurn = {
|
||||||
|
type: "turn",
|
||||||
|
turnIndex: 2,
|
||||||
|
role: "toolResult",
|
||||||
|
toolUseId: "call_001",
|
||||||
|
content: [{ type: "text", text: "hired agent-42" }],
|
||||||
|
};
|
||||||
|
const finalAssistantTurn: AssistantTurn = {
|
||||||
|
type: "turn",
|
||||||
|
turnIndex: 3,
|
||||||
|
role: "assistant",
|
||||||
|
content: [{ type: "text", text: "Writer hired." }],
|
||||||
|
tokenLedger: {
|
||||||
|
input: 0,
|
||||||
|
cache_read: 0,
|
||||||
|
cache_creation: 0,
|
||||||
|
output: 0,
|
||||||
|
},
|
||||||
|
};
|
||||||
return {
|
return {
|
||||||
meta: {
|
meta: {
|
||||||
type: "meta",
|
type: "meta",
|
||||||
@@ -80,13 +106,13 @@ function buildInMemoryFixture(assertions: Assertion[]): FixtureFile {
|
|||||||
scenario: "in-memory self-test fixture",
|
scenario: "in-memory self-test fixture",
|
||||||
systemPromptHash: "sha256:self-test",
|
systemPromptHash: "sha256:self-test",
|
||||||
},
|
},
|
||||||
turns: [userTurn, assistantTurn],
|
turns: [userTurn, assistantTurn, toolResultTurn, finalAssistantTurn],
|
||||||
assertions,
|
assertions,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("harness self-test (in-memory fixture)", () => {
|
describe("harness self-test (in-memory fixture)", () => {
|
||||||
it("passes when every assertion matches the recorded trace", () => {
|
it("passes when every assertion matches the recorded trace", async () => {
|
||||||
const fixture = buildInMemoryFixture([
|
const fixture = buildInMemoryFixture([
|
||||||
{ kind: "tool_called", name: "hire_agent", min_count: 1 },
|
{ kind: "tool_called", name: "hire_agent", min_count: 1 },
|
||||||
{
|
{
|
||||||
@@ -103,7 +129,7 @@ describe("harness self-test (in-memory fixture)", () => {
|
|||||||
},
|
},
|
||||||
{ kind: "trace_max_turns", max: 5 },
|
{ kind: "trace_max_turns", max: 5 },
|
||||||
]);
|
]);
|
||||||
const result = runEval(fixture);
|
const result = await runEval(fixture, { systemPrompt: "Self-test prompt." });
|
||||||
expect(result.passed).toBe(true);
|
expect(result.passed).toBe(true);
|
||||||
expect(result.fixtureId).toBe("self-test/in-memory");
|
expect(result.fixtureId).toBe("self-test/in-memory");
|
||||||
expect(result.assertions).toHaveLength(5);
|
expect(result.assertions).toHaveLength(5);
|
||||||
@@ -117,21 +143,21 @@ describe("harness self-test (in-memory fixture)", () => {
|
|||||||
expect(result.durationMs).toBeGreaterThanOrEqual(0);
|
expect(result.durationMs).toBeGreaterThanOrEqual(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("fails with a precise reason when an assertion misses", () => {
|
it("fails with a precise reason when an assertion misses", async () => {
|
||||||
const fixture = buildInMemoryFixture([
|
const fixture = buildInMemoryFixture([
|
||||||
{ kind: "tool_called", name: "write_plan", min_count: 1 },
|
{ kind: "tool_called", name: "write_plan", min_count: 1 },
|
||||||
{ kind: "no_tool_called", name: "hire_agent" },
|
{ kind: "no_tool_called", name: "hire_agent" },
|
||||||
{ kind: "trace_max_turns", max: 1 },
|
{ kind: "trace_max_turns", max: 1 },
|
||||||
]);
|
]);
|
||||||
const result = runEval(fixture);
|
const result = await runEval(fixture, { systemPrompt: "Self-test prompt." });
|
||||||
expect(result.passed).toBe(false);
|
expect(result.passed).toBe(false);
|
||||||
const reasons = result.assertions.map((r) => r.reason ?? "");
|
const reasons = result.assertions.map((r) => r.reason ?? "");
|
||||||
expect(reasons[0]).toContain('expected at least 1 call(s) to "write_plan"');
|
expect(reasons[0]).toContain('expected at least 1 call(s) to "write_plan"');
|
||||||
expect(reasons[1]).toContain('expected no call to "hire_agent"');
|
expect(reasons[1]).toContain('expected no call to "hire_agent"');
|
||||||
expect(reasons[2]).toContain("trace has 2 turns, max allowed is 1");
|
expect(reasons[2]).toContain("trace has 4 turns, max allowed is 1");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("message_does_not_contain_substring passes when substring is absent", () => {
|
it("message_does_not_contain_substring passes when substring is absent", async () => {
|
||||||
const fixture = buildInMemoryFixture([
|
const fixture = buildInMemoryFixture([
|
||||||
{
|
{
|
||||||
kind: "message_does_not_contain_substring",
|
kind: "message_does_not_contain_substring",
|
||||||
@@ -139,12 +165,12 @@ describe("harness self-test (in-memory fixture)", () => {
|
|||||||
substring: "node -e",
|
substring: "node -e",
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
const result = runEval(fixture);
|
const result = await runEval(fixture, { systemPrompt: "Self-test prompt." });
|
||||||
expect(result.passed).toBe(true);
|
expect(result.passed).toBe(true);
|
||||||
expect(result.assertions[0]?.passed).toBe(true);
|
expect(result.assertions[0]?.passed).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("message_does_not_contain_substring fails when substring is present", () => {
|
it("message_does_not_contain_substring fails when substring is present", async () => {
|
||||||
const fixture = buildInMemoryFixture([
|
const fixture = buildInMemoryFixture([
|
||||||
{
|
{
|
||||||
kind: "message_does_not_contain_substring",
|
kind: "message_does_not_contain_substring",
|
||||||
@@ -152,7 +178,7 @@ describe("harness self-test (in-memory fixture)", () => {
|
|||||||
substring: "Hiring a writer",
|
substring: "Hiring a writer",
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
const result = runEval(fixture);
|
const result = await runEval(fixture, { systemPrompt: "Self-test prompt." });
|
||||||
expect(result.passed).toBe(false);
|
expect(result.passed).toBe(false);
|
||||||
expect(result.assertions[0]?.reason).toContain(
|
expect(result.assertions[0]?.reason).toContain(
|
||||||
'a assistant message contained forbidden substring "Hiring a writer"',
|
'a assistant message contained forbidden substring "Hiring a writer"',
|
||||||
@@ -174,12 +200,31 @@ describe("harness self-test (in-memory fixture)", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("eval SOP coverage matrix", () => {
|
||||||
|
it.each(EVAL_SCENARIO_MATRIX)(
|
||||||
|
"covers real prompt SOPs for $fixtureId",
|
||||||
|
(entry) => {
|
||||||
|
const prompt = buildMatrixPrompt(entry);
|
||||||
|
for (const sop of entry.requiredSops) {
|
||||||
|
expect(prompt).toContain(sop);
|
||||||
|
}
|
||||||
|
for (const sop of entry.forbiddenSops ?? []) {
|
||||||
|
expect(prompt).not.toContain(sop);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
const fixturePaths = await listFixtures();
|
const fixturePaths = await listFixtures();
|
||||||
|
|
||||||
describe.each(fixturePaths)("eval fixture %s", (fixturePath) => {
|
describe.each(fixturePaths)("eval fixture %s", (fixturePath) => {
|
||||||
it("passes all assertions", async () => {
|
it("passes all assertions", async () => {
|
||||||
const fixture = await loadFixture(fixturePath);
|
const fixture = await loadFixture(fixturePath);
|
||||||
const result = runEval(fixture);
|
const scenario = await loadScenarioContract(fixture.meta.fixtureId);
|
||||||
|
assertScenarioMatchesFixture(fixture, scenario);
|
||||||
|
const result = await runEval(fixture, {
|
||||||
|
systemPrompt: scenario.systemPrompt,
|
||||||
|
});
|
||||||
if (!result.passed) {
|
if (!result.passed) {
|
||||||
const failures = result.assertions
|
const failures = result.assertions
|
||||||
.filter((assertion) => !assertion.passed)
|
.filter((assertion) => !assertion.passed)
|
||||||
|
|||||||
@@ -0,0 +1,237 @@
|
|||||||
|
import { createHash } from "node:crypto";
|
||||||
|
import { readFile } from "node:fs/promises";
|
||||||
|
import { dirname, join } from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
import {
|
||||||
|
buildTaskPrompt,
|
||||||
|
type PromptContext,
|
||||||
|
} from "../../src/main/prompt-builder/index";
|
||||||
|
import type { AgentRecord } from "../../src/shared/types";
|
||||||
|
import type { FixtureFile, UserTurn } from "./types.js";
|
||||||
|
|
||||||
|
const EVAL_DIR = dirname(fileURLToPath(import.meta.url));
|
||||||
|
const SCENARIO_DIR = join(EVAL_DIR, "scenarios");
|
||||||
|
|
||||||
|
export type EvalPromptAudience =
|
||||||
|
| "executive"
|
||||||
|
| "manager"
|
||||||
|
| "individual_contributor";
|
||||||
|
|
||||||
|
export interface EvalScenarioMatrixEntry {
|
||||||
|
fixtureId: string;
|
||||||
|
audience: EvalPromptAudience;
|
||||||
|
requiredSops: string[];
|
||||||
|
forbiddenSops?: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LoadedScenarioContract {
|
||||||
|
fixtureId: string;
|
||||||
|
systemPrompt: string;
|
||||||
|
userMessage: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const EVAL_SCENARIO_MATRIX: EvalScenarioMatrixEntry[] = [
|
||||||
|
{
|
||||||
|
fixtureId: "ceo-board-message-reply",
|
||||||
|
audience: "executive",
|
||||||
|
requiredSops: [
|
||||||
|
"SOP-001: Daily Standup Protocol",
|
||||||
|
"SOP-003: Delegation Protocol",
|
||||||
|
"SOP-007: Inter-Agent Communication",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
fixtureId: "ceo-bootstrap-empty",
|
||||||
|
audience: "executive",
|
||||||
|
requiredSops: [
|
||||||
|
"SOP-003: Delegation Protocol",
|
||||||
|
"SOP-005: Hiring Protocol",
|
||||||
|
"SOP-014: Hiring Pipeline",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
fixtureId: "ic-task-execute",
|
||||||
|
audience: "individual_contributor",
|
||||||
|
requiredSops: [
|
||||||
|
"SOP-002: Task Lifecycle",
|
||||||
|
"SOP-011: Performance & Quality Standards",
|
||||||
|
],
|
||||||
|
forbiddenSops: [
|
||||||
|
"SOP-005: Hiring Protocol",
|
||||||
|
"SOP-010: Budget & Financial Protocol",
|
||||||
|
"SOP-014: Hiring Pipeline",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
fixtureId: "manager-task-handoff",
|
||||||
|
audience: "manager",
|
||||||
|
requiredSops: [
|
||||||
|
"SOP-002: Task Lifecycle",
|
||||||
|
"SOP-003: Delegation Protocol",
|
||||||
|
"SOP-004: Code Review Protocol",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
fixtureId: "non-ceo-dm-reply",
|
||||||
|
audience: "manager",
|
||||||
|
requiredSops: [
|
||||||
|
"SOP-001: Daily Standup Protocol",
|
||||||
|
"SOP-007: Inter-Agent Communication",
|
||||||
|
"SOP-008: Cross-Department Collaboration",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
function createAgent(overrides: Partial<AgentRecord>): AgentRecord {
|
||||||
|
return {
|
||||||
|
id: "agent-1",
|
||||||
|
companyId: "company-1",
|
||||||
|
name: "Agent",
|
||||||
|
role: "Engineer",
|
||||||
|
title: "Engineer",
|
||||||
|
roleLevel: "ic",
|
||||||
|
department: "engineering",
|
||||||
|
status: "active",
|
||||||
|
reportsTo: "manager-1",
|
||||||
|
connectorId: "openai-codex",
|
||||||
|
workspaceId: null,
|
||||||
|
model: null,
|
||||||
|
capabilities: "Build production software",
|
||||||
|
budgetMonthlyUsd: 200,
|
||||||
|
spentMonthlyUsd: 0,
|
||||||
|
metadataJson: "{}",
|
||||||
|
heartbeatEnabled: true,
|
||||||
|
heartbeatIntervalSec: 120,
|
||||||
|
lastHeartbeatAt: null,
|
||||||
|
createdAt: "2026-04-20T00:00:00.000Z",
|
||||||
|
updatedAt: "2026-04-20T00:00:00.000Z",
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildPromptContext(audience: EvalPromptAudience): PromptContext {
|
||||||
|
const ceo = createAgent({
|
||||||
|
id: "ceo-1",
|
||||||
|
name: "CEO",
|
||||||
|
role: "CEO",
|
||||||
|
title: "Chief Executive Officer",
|
||||||
|
roleLevel: "c_suite",
|
||||||
|
department: "executive",
|
||||||
|
reportsTo: null,
|
||||||
|
capabilities: "Lead the company",
|
||||||
|
});
|
||||||
|
const manager = createAgent({
|
||||||
|
id: "manager-1",
|
||||||
|
name: "VP Engineering",
|
||||||
|
role: "VP Engineering",
|
||||||
|
title: "VP Engineering",
|
||||||
|
roleLevel: "vp",
|
||||||
|
department: "engineering",
|
||||||
|
reportsTo: ceo.id,
|
||||||
|
capabilities: "Run engineering delivery",
|
||||||
|
});
|
||||||
|
const ic = createAgent({
|
||||||
|
id: "ic-1",
|
||||||
|
name: "Backend IC",
|
||||||
|
role: "Backend Engineer",
|
||||||
|
title: "Backend Engineer",
|
||||||
|
roleLevel: "ic",
|
||||||
|
department: "engineering",
|
||||||
|
reportsTo: manager.id,
|
||||||
|
capabilities: "Implement backend tasks",
|
||||||
|
});
|
||||||
|
|
||||||
|
const agent =
|
||||||
|
audience === "executive" ? ceo : audience === "manager" ? manager : ic;
|
||||||
|
const directReports =
|
||||||
|
audience === "executive" ? [manager] : audience === "manager" ? [ic] : [];
|
||||||
|
const chainOfCommand =
|
||||||
|
audience === "executive" ? [] : audience === "manager" ? [ceo] : [manager, ceo];
|
||||||
|
|
||||||
|
return {
|
||||||
|
agent,
|
||||||
|
apiUrl: "http://127.0.0.1:3000",
|
||||||
|
apiKey: "eval-token",
|
||||||
|
runId: "eval-run",
|
||||||
|
task: null,
|
||||||
|
companyName: "AgentCompany Eval",
|
||||||
|
companyDescription: "Recorded eval prompt coverage",
|
||||||
|
goals: [],
|
||||||
|
projects: [],
|
||||||
|
directReports,
|
||||||
|
chainOfCommand,
|
||||||
|
assignedTasks: [],
|
||||||
|
allAgents: [ceo, manager, ic],
|
||||||
|
pendingApprovals: [],
|
||||||
|
tasksAwaitingReview: [],
|
||||||
|
wakeReason: null,
|
||||||
|
allTasks: [],
|
||||||
|
recentComments: [],
|
||||||
|
socialAccounts: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function hashSystemPrompt(systemPrompt: string): string {
|
||||||
|
return `sha256:${createHash("sha256").update(systemPrompt).digest("hex")}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getFirstUserText(fixture: FixtureFile): string {
|
||||||
|
const first = fixture.turns[0];
|
||||||
|
if (!first || first.role !== "user") {
|
||||||
|
throw new Error(
|
||||||
|
`eval scenario: fixture "${fixture.meta.fixtureId}" must start with a user turn`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const userTurn = first as UserTurn;
|
||||||
|
if (!Array.isArray(userTurn.content)) {
|
||||||
|
throw new Error(
|
||||||
|
`eval scenario: fixture "${fixture.meta.fixtureId}" first user turn must use content blocks`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return userTurn.content
|
||||||
|
.map((block) => (block.type === "text" ? block.text : ""))
|
||||||
|
.join("");
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function loadScenarioContract(
|
||||||
|
fixtureId: string,
|
||||||
|
): Promise<LoadedScenarioContract> {
|
||||||
|
const root = join(SCENARIO_DIR, fixtureId);
|
||||||
|
const [systemPrompt, userMessage] = await Promise.all([
|
||||||
|
readFile(join(root, "system.txt"), "utf8"),
|
||||||
|
readFile(join(root, "user.txt"), "utf8"),
|
||||||
|
]);
|
||||||
|
if (systemPrompt.length === 0) {
|
||||||
|
throw new Error(`eval scenario: ${fixtureId}/system.txt is empty`);
|
||||||
|
}
|
||||||
|
if (userMessage.length === 0) {
|
||||||
|
throw new Error(`eval scenario: ${fixtureId}/user.txt is empty`);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
fixtureId,
|
||||||
|
systemPrompt,
|
||||||
|
userMessage,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function assertScenarioMatchesFixture(
|
||||||
|
fixture: FixtureFile,
|
||||||
|
scenario: LoadedScenarioContract,
|
||||||
|
): void {
|
||||||
|
const expectedHash = hashSystemPrompt(scenario.systemPrompt);
|
||||||
|
if (fixture.meta.systemPromptHash !== expectedHash) {
|
||||||
|
throw new Error(
|
||||||
|
`eval scenario: fixture "${fixture.meta.fixtureId}" has systemPromptHash ${fixture.meta.systemPromptHash}, expected ${expectedHash}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const fixtureUserText = getFirstUserText(fixture);
|
||||||
|
if (fixtureUserText !== scenario.userMessage) {
|
||||||
|
throw new Error(
|
||||||
|
`eval scenario: fixture "${fixture.meta.fixtureId}" first user turn does not match scenarios/${scenario.fixtureId}/user.txt`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildMatrixPrompt(entry: EvalScenarioMatrixEntry): string {
|
||||||
|
return buildTaskPrompt(buildPromptContext(entry.audience));
|
||||||
|
}
|
||||||
@@ -0,0 +1,258 @@
|
|||||||
|
import {
|
||||||
|
Agent,
|
||||||
|
type AgentMessage,
|
||||||
|
type AgentTool,
|
||||||
|
} from "@mariozechner/pi-agent-core";
|
||||||
|
import {
|
||||||
|
Type,
|
||||||
|
type Api,
|
||||||
|
type AssistantMessage,
|
||||||
|
type Model,
|
||||||
|
type ToolCall,
|
||||||
|
type UserMessage,
|
||||||
|
} from "@mariozechner/pi-ai";
|
||||||
|
import { createReplayStreamFn } from "./replay-client.js";
|
||||||
|
import type {
|
||||||
|
AssistantTurn,
|
||||||
|
FixtureFile,
|
||||||
|
TokenLedger,
|
||||||
|
ToolResultTurn,
|
||||||
|
TraceTurn,
|
||||||
|
UserTurn,
|
||||||
|
} from "./types.js";
|
||||||
|
|
||||||
|
const REPLAY_TOOL_PARAMETERS = Type.Record(Type.String(), Type.Any());
|
||||||
|
|
||||||
|
const ZERO_COST = {
|
||||||
|
input: 0,
|
||||||
|
output: 0,
|
||||||
|
cacheRead: 0,
|
||||||
|
cacheWrite: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
function createReplayModel(): Model<Api> {
|
||||||
|
return {
|
||||||
|
id: "eval-replay",
|
||||||
|
name: "Eval Replay",
|
||||||
|
api: "eval-replay",
|
||||||
|
provider: "eval-replay",
|
||||||
|
baseUrl: "https://example.invalid",
|
||||||
|
reasoning: false,
|
||||||
|
input: ["text"],
|
||||||
|
cost: ZERO_COST,
|
||||||
|
contextWindow: 128_000,
|
||||||
|
maxTokens: 8192,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectAssistantToolCalls(fixture: FixtureFile): ToolCall[] {
|
||||||
|
const calls: ToolCall[] = [];
|
||||||
|
for (const turn of fixture.turns) {
|
||||||
|
if (turn.role !== "assistant") continue;
|
||||||
|
for (const block of turn.content) {
|
||||||
|
if (block.type === "toolCall") calls.push(block);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return calls;
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateToolLoopShape(fixture: FixtureFile): void {
|
||||||
|
const pending = new Set<string>();
|
||||||
|
const seen = new Set<string>();
|
||||||
|
|
||||||
|
for (const turn of fixture.turns) {
|
||||||
|
if (turn.role === "assistant") {
|
||||||
|
if (pending.size > 0) {
|
||||||
|
throw new Error(
|
||||||
|
`eval replay: fixture "${fixture.meta.fixtureId}" has assistant turn ${turn.turnIndex} before tool result(s): ${Array.from(pending).join(", ")}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for (const block of turn.content) {
|
||||||
|
if (block.type !== "toolCall") continue;
|
||||||
|
if (seen.has(block.id)) {
|
||||||
|
throw new Error(
|
||||||
|
`eval replay: fixture "${fixture.meta.fixtureId}" repeats tool call id "${block.id}"`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
seen.add(block.id);
|
||||||
|
pending.add(block.id);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (turn.role === "toolResult") {
|
||||||
|
if (!pending.has(turn.toolUseId)) {
|
||||||
|
throw new Error(
|
||||||
|
`eval replay: fixture "${fixture.meta.fixtureId}" has tool result "${turn.toolUseId}" without a pending tool call`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
pending.delete(turn.toolUseId);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (turn.role === "user" && pending.size > 0) {
|
||||||
|
throw new Error(
|
||||||
|
`eval replay: fixture "${fixture.meta.fixtureId}" has user turn ${turn.turnIndex} before tool result(s): ${Array.from(pending).join(", ")}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (pending.size > 0) {
|
||||||
|
throw new Error(
|
||||||
|
`eval replay: fixture "${fixture.meta.fixtureId}" ended before tool result(s): ${Array.from(pending).join(", ")}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectRecordedToolResults(
|
||||||
|
fixture: FixtureFile,
|
||||||
|
): Map<string, ToolResultTurn> {
|
||||||
|
const results = new Map<string, ToolResultTurn>();
|
||||||
|
for (const turn of fixture.turns) {
|
||||||
|
if (turn.role !== "toolResult") continue;
|
||||||
|
if (results.has(turn.toolUseId)) {
|
||||||
|
throw new Error(
|
||||||
|
`eval replay: fixture "${fixture.meta.fixtureId}" repeats tool result for "${turn.toolUseId}"`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
results.set(turn.toolUseId, turn);
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
function createReplayTools(fixture: FixtureFile): AgentTool[] {
|
||||||
|
const results = collectRecordedToolResults(fixture);
|
||||||
|
const toolNames = Array.from(
|
||||||
|
new Set(collectAssistantToolCalls(fixture).map((call) => call.name)),
|
||||||
|
).sort();
|
||||||
|
|
||||||
|
return toolNames.map((name) => ({
|
||||||
|
name,
|
||||||
|
label: `eval replay ${name}`,
|
||||||
|
description:
|
||||||
|
"Replay a recorded eval tool result. The fixture owns the content and the call id is the lookup key.",
|
||||||
|
parameters: REPLAY_TOOL_PARAMETERS,
|
||||||
|
async execute(toolCallId, params) {
|
||||||
|
const recorded = results.get(toolCallId);
|
||||||
|
if (!recorded) {
|
||||||
|
throw new Error(
|
||||||
|
`eval replay: fixture "${fixture.meta.fixtureId}" has no recorded tool result for "${toolCallId}"`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
content: recorded.content,
|
||||||
|
details: {
|
||||||
|
fixtureId: fixture.meta.fixtureId,
|
||||||
|
toolCallId,
|
||||||
|
toolName: name,
|
||||||
|
arguments: params,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
executionMode: "sequential",
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
function asUserMessage(turn: UserTurn): UserMessage {
|
||||||
|
return {
|
||||||
|
role: "user",
|
||||||
|
content: turn.content,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function usageToLedger(message: AssistantMessage): TokenLedger {
|
||||||
|
return {
|
||||||
|
input: message.usage.input,
|
||||||
|
cache_read: message.usage.cacheRead,
|
||||||
|
cache_creation: message.usage.cacheWrite,
|
||||||
|
output: message.usage.output,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function agentMessageToTraceTurn(
|
||||||
|
message: AgentMessage,
|
||||||
|
turnIndex: number,
|
||||||
|
): TraceTurn | null {
|
||||||
|
if (message.role === "user") {
|
||||||
|
const content = typeof message.content === "string"
|
||||||
|
? [{ type: "text" as const, text: message.content }]
|
||||||
|
: message.content;
|
||||||
|
return {
|
||||||
|
type: "turn",
|
||||||
|
turnIndex,
|
||||||
|
role: "user",
|
||||||
|
content,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (message.role === "assistant") {
|
||||||
|
return {
|
||||||
|
type: "turn",
|
||||||
|
turnIndex,
|
||||||
|
role: "assistant",
|
||||||
|
content: message.content,
|
||||||
|
tokenLedger: usageToLedger(message),
|
||||||
|
} satisfies AssistantTurn;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (message.role === "toolResult") {
|
||||||
|
return {
|
||||||
|
type: "turn",
|
||||||
|
turnIndex,
|
||||||
|
role: "toolResult",
|
||||||
|
toolUseId: message.toolCallId,
|
||||||
|
content: message.content,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectUserTurns(fixture: FixtureFile): UserTurn[] {
|
||||||
|
return fixture.turns.filter((turn): turn is UserTurn => turn.role === "user");
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function replayFixtureThroughAgent(
|
||||||
|
fixture: FixtureFile,
|
||||||
|
systemPrompt: string,
|
||||||
|
): Promise<TraceTurn[]> {
|
||||||
|
validateToolLoopShape(fixture);
|
||||||
|
|
||||||
|
const userTurns = collectUserTurns(fixture);
|
||||||
|
const firstUserTurn = userTurns[0];
|
||||||
|
if (!firstUserTurn || fixture.turns[0]?.role !== "user") {
|
||||||
|
throw new Error(
|
||||||
|
`eval replay: fixture "${fixture.meta.fixtureId}" must start with a user turn`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const agent = new Agent({
|
||||||
|
initialState: {
|
||||||
|
systemPrompt,
|
||||||
|
model: createReplayModel(),
|
||||||
|
tools: createReplayTools(fixture),
|
||||||
|
messages: [],
|
||||||
|
thinkingLevel: "off",
|
||||||
|
},
|
||||||
|
sessionId: `eval:${fixture.meta.fixtureId}`,
|
||||||
|
streamFn: createReplayStreamFn(fixture),
|
||||||
|
toolExecution: "sequential",
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const followUp of userTurns.slice(1)) {
|
||||||
|
agent.followUp(asUserMessage(followUp));
|
||||||
|
}
|
||||||
|
|
||||||
|
await agent.prompt(asUserMessage(firstUserTurn));
|
||||||
|
|
||||||
|
if (agent.state.errorMessage) {
|
||||||
|
throw new Error(
|
||||||
|
`eval replay: fixture "${fixture.meta.fixtureId}" failed inside pi Agent loop: ${agent.state.errorMessage}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return agent.state.messages
|
||||||
|
.map((message, idx) => agentMessageToTraceTurn(message, idx))
|
||||||
|
.filter((turn): turn is TraceTurn => turn !== null);
|
||||||
|
}
|
||||||
Reference in new issue
Block a user