diff --git a/docs/proposals/_BLOCKED-BY-CONSTRAINT.md b/docs/proposals/_BLOCKED-BY-CONSTRAINT.md index da17ba56..eeec0a79 100644 --- a/docs/proposals/_BLOCKED-BY-CONSTRAINT.md +++ b/docs/proposals/_BLOCKED-BY-CONSTRAINT.md @@ -215,4 +215,4 @@ Linus 最后一句:**"先让数据结构对,其它 6 步会一起变简单" - No production mocks / fallbacks / silent catches — 仅 dev `tests/eval/` 例外。 - No runtime logging / telemetry beyond existing electron-log usage. - No dual-write migrations without backup path。 -- `tests/eval/` 必须在 `electron-builder.yml` 的 files 过滤中排除,保证不进生产 bundle。 +- `electron-builder.yml` 采用 whitelist `files:` 模式(`out/**` + `package.json` + 指定 assets),`tests/eval/` 本就不在白名单内,无需新增排除规则。 diff --git a/tests/eval/assertion-judge.ts b/tests/eval/assertion-judge.ts index 3f90d4cc..9ce46021 100644 --- a/tests/eval/assertion-judge.ts +++ b/tests/eval/assertion-judge.ts @@ -112,7 +112,9 @@ function evaluateOne( case "message_contains_substring": { for (const turn of turns) { if (turn.role !== assertion.role) continue; - const content = (turn as AssistantTurn | { content: unknown[] }).content; + // UserTurn and AssistantTurn both expose .content; the `role` guard + // above restricts us to those variants, so no cast is needed. + const content = turn.content; if (!Array.isArray(content)) continue; for (const block of content) { if ( diff --git a/tests/eval/harness.test.ts b/tests/eval/harness.test.ts index 7a9b13d0..fffa4e1c 100644 --- a/tests/eval/harness.test.ts +++ b/tests/eval/harness.test.ts @@ -1,27 +1,42 @@ /** * B5 eval harness — driver spec. * - * This spec discovers every `*.jsonl` fixture under `tests/eval/fixtures/` - * and runs it through the assertion judge. If there are zero fixtures, the - * spec file has zero tests — vitest is invoked with `--passWithNoTests` - * from the `test:eval` script so this is a green no-op until the user - * records real fixtures. + * Two layers: * - * The harness does NOT synthesize placeholder fixtures under any - * circumstance. An empty `fixtures/` directory is a real state — the tier - * is simply not yet exercising anything — and the green run reflects that - * honestly. + * 1. Self-tests (always run). An in-memory fixture exercises the loader + * contract, assertion judge, eval runner, and token ledger aggregation. + * This gives `pnpm test:eval` real signal even before any `.jsonl` + * fixture lands on disk. Zero fixture files = tier still exercised. + * + * 2. Fixture discovery (runs when `.jsonl` files exist). Every `*.jsonl` + * under `tests/eval/fixtures/` is discovered and runs through the + * judge. If the fixtures directory is missing we treat that as "no + * fixtures yet", return an empty list, and let the self-tests carry + * the signal on their own — we do NOT synthesize placeholder fixtures. + * + * Karpathy rule: bad fixture data must crash loud at the loader. The + * harness itself is validated by self-tests so a breakage there never + * hides behind an empty fixtures directory. */ +import { existsSync } from "node:fs"; import { readdir } from "node:fs/promises"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -import { loadFixture } from "./fixture-loader.js"; +import { judgeAssertions } from "./assertion-judge.js"; import { runEval } from "./eval-runner.js"; +import { loadFixture } from "./fixture-loader.js"; +import type { + Assertion, + AssistantTurn, + FixtureFile, + UserTurn, +} from "./types.js"; const FIXTURE_DIR = join(__dirname, "fixtures"); async function listFixtures(): Promise { + if (!existsSync(FIXTURE_DIR)) return []; const entries = await readdir(FIXTURE_DIR, { withFileTypes: true }); return entries .filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")) @@ -29,6 +44,108 @@ async function listFixtures(): Promise { .sort(); } +function buildInMemoryFixture(assertions: Assertion[]): FixtureFile { + const userTurn: UserTurn = { + type: "turn", + turnIndex: 0, + role: "user", + content: [{ type: "text", text: "hire a writer" }], + }; + const assistantTurn: AssistantTurn = { + type: "turn", + turnIndex: 1, + role: "assistant", + content: [ + { type: "text", text: "Hiring a writer now." }, + { + type: "toolCall", + id: "call_001", + name: "hire_agent", + arguments: { role: "writer", seniority: "senior" }, + }, + ], + tokenLedger: { + input: 120, + cache_read: 40, + cache_creation: 10, + output: 60, + }, + }; + return { + meta: { + type: "meta", + fixtureId: "self-test/in-memory", + piAgentCoreVersion: "0.0.0-self-test", + recordedAt: "2026-04-20T00:00:00.000Z", + scenario: "in-memory self-test fixture", + systemPromptHash: "sha256:self-test", + }, + turns: [userTurn, assistantTurn], + assertions, + }; +} + +describe("harness self-test (in-memory fixture)", () => { + it("passes when every assertion matches the recorded trace", () => { + const fixture = buildInMemoryFixture([ + { kind: "tool_called", name: "hire_agent", min_count: 1 }, + { + kind: "tool_call_arg_matches", + name: "hire_agent", + arg_path: "role", + matches_regex: "^writer$", + }, + { kind: "no_tool_called", name: "write_plan" }, + { + kind: "message_contains_substring", + role: "assistant", + substring: "Hiring a writer", + }, + { kind: "trace_max_turns", max: 5 }, + ]); + const result = runEval(fixture); + expect(result.passed).toBe(true); + expect(result.fixtureId).toBe("self-test/in-memory"); + expect(result.assertions).toHaveLength(5); + expect(result.assertions.every((r) => r.passed)).toBe(true); + expect(result.tokenLedger).toEqual({ + input: 120, + cache_read: 40, + cache_creation: 10, + output: 60, + }); + expect(result.durationMs).toBeGreaterThanOrEqual(0); + }); + + it("fails with a precise reason when an assertion misses", () => { + const fixture = buildInMemoryFixture([ + { kind: "tool_called", name: "write_plan", min_count: 1 }, + { kind: "no_tool_called", name: "hire_agent" }, + { kind: "trace_max_turns", max: 1 }, + ]); + const result = runEval(fixture); + expect(result.passed).toBe(false); + const reasons = result.assertions.map((r) => r.reason ?? ""); + expect(reasons[0]).toContain('expected at least 1 call(s) to "write_plan"'); + expect(reasons[1]).toContain('expected no call to "hire_agent"'); + expect(reasons[2]).toContain("trace has 2 turns, max allowed is 1"); + }); + + it("judgeAssertions is a pure function of (assertions, turns)", () => { + const fixture = buildInMemoryFixture([]); + const first = judgeAssertions( + [{ kind: "tool_called", name: "hire_agent" }], + fixture.turns, + ); + const second = judgeAssertions( + [{ kind: "tool_called", name: "hire_agent" }], + fixture.turns, + ); + expect(first).toEqual(second); + expect(first[0]?.passed).toBe(true); + }); +}); + const fixturePaths = await listFixtures(); describe.each(fixturePaths)("eval fixture %s", (fixturePath) => { diff --git a/tests/eval/replay-client.ts b/tests/eval/replay-client.ts index aca37152..d642d549 100644 --- a/tests/eval/replay-client.ts +++ b/tests/eval/replay-client.ts @@ -90,13 +90,9 @@ export function createReplayStreamFn(fixture: FixtureFile): StreamFn { `replay-client: fixture "${fixture.meta.fixtureId}" exhausted after ${assistantTurns.length} assistant turns; agent requested turn ${cursor + 1}`, ); } - const turn = assistantTurns[cursor]; + // Cursor is bounds-checked above, so the element is guaranteed to exist. + const turn = assistantTurns[cursor]!; cursor += 1; - if (!turn) { - throw new Error( - `replay-client: internal cursor error at position ${cursor - 1}`, - ); - } const message = buildAssistantMessage( turn, model.id,