ZacharyZhang-NY
e7f879bed0
Land the B5 harness skeleton: fixture loader (strict JSONL contract with meta header + ordered turn records + assertions tail), replay-client that swaps in for pi-ai's streamSimple by replaying recorded assistant turns, structural assertion judge (tool_called / no_tool_called / message_contains_substring / tool_call_arg_matches / trace_max_turns), an eval-runner that sums the token ledger and produces an EvalResult, a vitest driver (tests/eval/harness.test.ts) that discovers *.jsonl fixtures, and a record-fixture CLI that writes fixture meta headers. Hard contracts: - No fallback: replay-client and fixture-loader throw on exhaustion / malformed input rather than synthesizing defaults. - No mock: types reuse @mariozechner/pi-ai AgentMessage / ToolCall verbatim; shadow types are forbidden. - No fake fixture data: tests/eval/fixtures/ ships only README + .gitkeep. Real traces are recorded by the user against live providers. package.json adds test:eval (vitest --passWithNoTests so an empty fixture dir is green) and eval:record (CLI entrypoint). electron-builder already excludes tests/ via its files: glob — fixtures never enter the shipped .asar.
56 lines
1.9 KiB
TypeScript
56 lines
1.9 KiB
TypeScript
/**
|
||
* Eval runner for B5 harness.
|
||
*
|
||
* Given a parsed `FixtureFile`, produces an `EvalResult` by:
|
||
* 1. Summing the token ledger across all recorded assistant turns.
|
||
* 2. Running the assertion judge against the recorded turns.
|
||
* 3. Measuring wall-clock duration of steps (1)–(2).
|
||
*
|
||
* Tonight's scope deliberately evaluates assertions against the RECORDED
|
||
* trace, not a live replay-driven agent loop. That keeps the harness
|
||
* useful as a snapshot-regression tool the moment the first fixture lands,
|
||
* without wiring in the full `pi-agent-core` Agent loop (which requires
|
||
* tools + hooks and is better done incrementally once we have fixtures).
|
||
*
|
||
* The replay-client + fixture-loader are written now so the live-replay
|
||
* tier can be added in B5.1 without reshaping types.
|
||
*/
|
||
|
||
import { judgeAssertions } from "./assertion-judge.js";
|
||
import type { EvalResult, FixtureFile, TokenLedger } from "./types.js";
|
||
|
||
const ZERO_LEDGER: TokenLedger = {
|
||
input: 0,
|
||
cache_read: 0,
|
||
cache_creation: 0,
|
||
output: 0,
|
||
};
|
||
|
||
function sumLedger(fixture: FixtureFile): TokenLedger {
|
||
return fixture.turns.reduce<TokenLedger>((acc, turn) => {
|
||
if (turn.role !== "assistant") return acc;
|
||
return {
|
||
input: acc.input + turn.tokenLedger.input,
|
||
cache_read: acc.cache_read + turn.tokenLedger.cache_read,
|
||
cache_creation: acc.cache_creation + turn.tokenLedger.cache_creation,
|
||
output: acc.output + turn.tokenLedger.output,
|
||
};
|
||
}, ZERO_LEDGER);
|
||
}
|
||
|
||
export function runEval(fixture: FixtureFile): EvalResult {
|
||
const start = Date.now();
|
||
const results = judgeAssertions(fixture.assertions, fixture.turns);
|
||
const tokenLedger = sumLedger(fixture);
|
||
const durationMs = Date.now() - start;
|
||
const passed = results.every((result) => result.passed);
|
||
|
||
return {
|
||
fixtureId: fixture.meta.fixtureId,
|
||
passed,
|
||
assertions: results,
|
||
tokenLedger,
|
||
durationMs,
|
||
};
|
||
}
|