Build minimal LLM eval harness scaffold (B5) — fixtures recorded separately
Land the B5 harness skeleton: fixture loader (strict JSONL contract with meta header + ordered turn records + assertions tail), replay-client that swaps in for pi-ai's streamSimple by replaying recorded assistant turns, structural assertion judge (tool_called / no_tool_called / message_contains_substring / tool_call_arg_matches / trace_max_turns), an eval-runner that sums the token ledger and produces an EvalResult, a vitest driver (tests/eval/harness.test.ts) that discovers *.jsonl fixtures, and a record-fixture CLI that writes fixture meta headers. Hard contracts: - No fallback: replay-client and fixture-loader throw on exhaustion / malformed input rather than synthesizing defaults. - No mock: types reuse @mariozechner/pi-ai AgentMessage / ToolCall verbatim; shadow types are forbidden. - No fake fixture data: tests/eval/fixtures/ ships only README + .gitkeep. Real traces are recorded by the user against live providers. package.json adds test:eval (vitest --passWithNoTests so an empty fixture dir is green) and eval:record (CLI entrypoint). electron-builder already excludes tests/ via its files: glob — fixtures never enter the shipped .asar.
This commit is contained in:
1 parent
76d9031a41
commit
e7f879bed0
12 files changed
+931
No files matched your search
@@ -0,0 +1,111 @@
|
||||
/**
|
||||
* Fixture recorder CLI for B5 eval harness.
|
||||
*
|
||||
* Entrypoint: `pnpm eval:record -- --out tests/eval/fixtures/<id>.jsonl`
|
||||
*
|
||||
* This CLI is a *scaffold*: it accepts a scenario config path and an output
|
||||
* path, validates the config shape, and writes a stub fixture containing
|
||||
* only the meta header plus an empty assertions block. The real recording
|
||||
* loop — which drives `pi-agent-core` against a real provider with the
|
||||
* AgentCompany system prompt and tool catalogue — lives in a follow-up
|
||||
* commit. Tonight's scope is the harness skeleton; the user populates
|
||||
* `tests/eval/fixtures/` by running the full recorder once it is wired.
|
||||
*
|
||||
* Why a stub and not a full recording loop tonight:
|
||||
* - Wiring the full loop requires importing the AgentCompany worker's
|
||||
* system prompt builder and tool registry. Those modules live in
|
||||
* `src/main/` and depend on the Electron runtime; importing them from a
|
||||
* Node CLI would require either a recorder-only entrypoint or a vite
|
||||
* build step. Both are bigger than tonight's surgical scope.
|
||||
* - Keeping the CLI as a stub today forces a deliberate follow-up rather
|
||||
* than silently recording fixtures from the wrong context — a Karpathy
|
||||
* "let it crash" moment.
|
||||
*
|
||||
* Usage:
|
||||
* tsx tests/eval/record-fixture.ts \
|
||||
* --fixture-id ceo-team-building-empty \
|
||||
* --scenario "Empty company, CEO bootstrap" \
|
||||
* --out tests/eval/fixtures/ceo-team-building-empty.jsonl
|
||||
*/
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
import { createRequire } from "node:module";
|
||||
import type { FixtureMeta } from "./types.js";
|
||||
|
||||
interface CliArgs {
|
||||
fixtureId: string;
|
||||
scenario: string;
|
||||
out: string;
|
||||
systemPromptPreview: string;
|
||||
}
|
||||
|
||||
function readArg(args: string[], flag: string, required: boolean): string | undefined {
|
||||
const idx = args.indexOf(flag);
|
||||
if (idx === -1) {
|
||||
if (required) {
|
||||
throw new Error(`record-fixture: missing required flag ${flag}`);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
const value = args[idx + 1];
|
||||
if (typeof value !== "string" || value.startsWith("--")) {
|
||||
throw new Error(`record-fixture: flag ${flag} requires a value`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): CliArgs {
|
||||
const args = argv.slice(2);
|
||||
const fixtureId = readArg(args, "--fixture-id", true)!;
|
||||
const scenario = readArg(args, "--scenario", true)!;
|
||||
const out = readArg(args, "--out", true)!;
|
||||
const systemPromptPreview = readArg(args, "--system-prompt-preview", false) ?? "";
|
||||
return { fixtureId, scenario, out, systemPromptPreview };
|
||||
}
|
||||
|
||||
function resolvePiAgentCoreVersion(): string {
|
||||
const require = createRequire(import.meta.url);
|
||||
// Narrow shape of the pi-agent-core package.json read at runtime.
|
||||
const pkg = require("@mariozechner/pi-agent-core/package.json") as { version?: unknown };
|
||||
if (typeof pkg.version !== "string") {
|
||||
throw new Error(
|
||||
"record-fixture: could not resolve @mariozechner/pi-agent-core version",
|
||||
);
|
||||
}
|
||||
return pkg.version;
|
||||
}
|
||||
|
||||
function buildMeta(args: CliArgs): FixtureMeta {
|
||||
const hash = createHash("sha256").update(args.systemPromptPreview).digest("hex");
|
||||
return {
|
||||
type: "meta",
|
||||
fixtureId: args.fixtureId,
|
||||
piAgentCoreVersion: resolvePiAgentCoreVersion(),
|
||||
recordedAt: new Date().toISOString(),
|
||||
scenario: args.scenario,
|
||||
systemPromptHash: `sha256:${hash}`,
|
||||
};
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs(process.argv);
|
||||
const meta = buildMeta(args);
|
||||
const lines: string[] = [
|
||||
JSON.stringify(meta),
|
||||
JSON.stringify({ type: "assertions", assertions: [] }),
|
||||
];
|
||||
const outPath = resolve(args.out);
|
||||
await writeFile(outPath, `${lines.join("\n")}\n`, "utf8");
|
||||
process.stdout.write(
|
||||
`record-fixture: wrote stub fixture ${args.fixtureId} to ${outPath}\n` +
|
||||
`record-fixture: NEXT STEP — replace the stub with recorded turns by running the full recorder (see tests/eval/README.md).\n`,
|
||||
);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
const detail = error instanceof Error ? error.message : String(error);
|
||||
process.stderr.write(`record-fixture: ${detail}\n`);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in new issue
Block a user