ZacharyZhang-NY
e7f879bed0
Land the B5 harness skeleton: fixture loader (strict JSONL contract with meta header + ordered turn records + assertions tail), replay-client that swaps in for pi-ai's streamSimple by replaying recorded assistant turns, structural assertion judge (tool_called / no_tool_called / message_contains_substring / tool_call_arg_matches / trace_max_turns), an eval-runner that sums the token ledger and produces an EvalResult, a vitest driver (tests/eval/harness.test.ts) that discovers *.jsonl fixtures, and a record-fixture CLI that writes fixture meta headers. Hard contracts: - No fallback: replay-client and fixture-loader throw on exhaustion / malformed input rather than synthesizing defaults. - No mock: types reuse @mariozechner/pi-ai AgentMessage / ToolCall verbatim; shadow types are forbidden. - No fake fixture data: tests/eval/fixtures/ ships only README + .gitkeep. Real traces are recorded by the user against live providers. package.json adds test:eval (vitest --passWithNoTests so an empty fixture dir is green) and eval:record (CLI entrypoint). electron-builder already excludes tests/ via its files: glob — fixtures never enter the shipped .asar.
111 lines
3.6 KiB
TypeScript
111 lines
3.6 KiB
TypeScript
/**
|
|
* Replay client for B5 eval harness.
|
|
*
|
|
* Swaps in for `streamSimple` (the pi-ai LLM call) during eval runs. Instead
|
|
* of hitting a real provider, it replays the assistant turns from the fixture
|
|
* in order, emitting the `AssistantMessageEvent` protocol expected by
|
|
* `pi-agent-core`.
|
|
*
|
|
* Hard contract (Karpathy rule: no fallback):
|
|
* - If the agent loop asks for the Nth assistant turn and the fixture has
|
|
* fewer assistant turns, the replay client THROWS. It does not return a
|
|
* placeholder response, does not return an empty assistant message, and
|
|
* does not invent an `errorMessage`. Missing fixture = bug in fixture, and
|
|
* the runner must surface that loud.
|
|
* - The client ignores the incoming `context.messages` because the fixture
|
|
* is the source of truth. A future harness tier (B5.1 or later) can diff
|
|
* the incoming messages against the fixture's recorded prior messages to
|
|
* detect context drift — out of scope tonight.
|
|
*/
|
|
|
|
import type { AssistantMessage } from "@mariozechner/pi-ai";
|
|
import {
|
|
AssistantMessageEventStream,
|
|
createAssistantMessageEventStream,
|
|
} from "@mariozechner/pi-ai";
|
|
import type { StreamFn } from "@mariozechner/pi-agent-core";
|
|
import type { AssistantTurn, FixtureFile } from "./types.js";
|
|
|
|
function collectAssistantTurns(fixture: FixtureFile): AssistantTurn[] {
|
|
return fixture.turns.filter(
|
|
(turn): turn is AssistantTurn => turn.role === "assistant",
|
|
);
|
|
}
|
|
|
|
function buildAssistantMessage(
|
|
turn: AssistantTurn,
|
|
modelId: string,
|
|
provider: string,
|
|
api: string,
|
|
): AssistantMessage {
|
|
const ledger = turn.tokenLedger;
|
|
const totalTokens =
|
|
ledger.input + ledger.cache_read + ledger.cache_creation + ledger.output;
|
|
const hasToolCall = turn.content.some((block) => block.type === "toolCall");
|
|
return {
|
|
role: "assistant",
|
|
content: turn.content,
|
|
api,
|
|
provider,
|
|
model: modelId,
|
|
usage: {
|
|
input: ledger.input,
|
|
output: ledger.output,
|
|
cacheRead: ledger.cache_read,
|
|
cacheWrite: ledger.cache_creation,
|
|
totalTokens,
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
},
|
|
stopReason: hasToolCall ? "toolUse" : "stop",
|
|
timestamp: Date.now(),
|
|
};
|
|
}
|
|
|
|
function pushRecordedMessage(
|
|
stream: AssistantMessageEventStream,
|
|
message: AssistantMessage,
|
|
): void {
|
|
stream.push({ type: "start", partial: message });
|
|
const reason = (message.stopReason === "toolUse" || message.stopReason === "length"
|
|
? message.stopReason
|
|
: "stop") as "stop" | "toolUse" | "length";
|
|
stream.push({ type: "done", reason, message });
|
|
stream.end(message);
|
|
}
|
|
|
|
/**
|
|
* Create a replay stream function bound to a fixture.
|
|
*
|
|
* The returned function tracks its own cursor. Each invocation consumes the
|
|
* next assistant turn. If the caller exhausts the fixture, the next
|
|
* invocation throws.
|
|
*/
|
|
export function createReplayStreamFn(fixture: FixtureFile): StreamFn {
|
|
const assistantTurns = collectAssistantTurns(fixture);
|
|
let cursor = 0;
|
|
|
|
return ((model, _context, _options) => {
|
|
if (cursor >= assistantTurns.length) {
|
|
throw new Error(
|
|
`replay-client: fixture "${fixture.meta.fixtureId}" exhausted after ${assistantTurns.length} assistant turns; agent requested turn ${cursor + 1}`,
|
|
);
|
|
}
|
|
const turn = assistantTurns[cursor];
|
|
cursor += 1;
|
|
if (!turn) {
|
|
throw new Error(
|
|
`replay-client: internal cursor error at position ${cursor - 1}`,
|
|
);
|
|
}
|
|
const message = buildAssistantMessage(
|
|
turn,
|
|
model.id,
|
|
model.provider,
|
|
model.api,
|
|
);
|
|
const stream = createAssistantMessageEventStream();
|
|
pushRecordedMessage(stream, message);
|
|
return stream;
|
|
}) satisfies StreamFn;
|
|
}
|