B5 harness iteration: self-test + assertion-judge cleanup + dead branch fix + _BLOCKED timeline correction

- harness.test.ts: add in-memory self-test layer (pass + fail + purity), guard
  fixtures dir with existsSync so absent dir yields empty list without
  crashing the runner. Empty-fixture pnpm test:eval now carries real signal.
- replay-client.ts: remove dead if (!turn) branch; cursor bounds-check above
  already guarantees the element. Use non-null assertion with a comment.
- assertion-judge.ts: drop unnecessary (turn as AssistantTurn | { content:
  unknown[] }) cast. turn.role === assertion.role already narrows to
  UserTurn | AssistantTurn and both expose .content.
- _BLOCKED-BY-CONSTRAINT.md: correct electron-builder note. files: is a
  whitelist, so tests/eval is automatically excluded and no new rule is
  required.
This commit is contained in:
ZacharyZhang-NY committed 2026-04-20 11:48:55 -04:00
1 parent cb2a0a0693
commit f849efadf5
4 files changed
+133 -18

No files matched your search

+1 -1
View File
@@ -215,4 +215,4 @@ Linus 最后一句:**"先让数据结构对,其它 6 步会一起变简单"
- No production mocks / fallbacks / silent catches — 仅 dev `tests/eval/` 例外。
- No runtime logging / telemetry beyond existing electron-log usage.
- No dual-write migrations without backup path。
- `tests/eval/` 必须在 `electron-builder.yml` 的 files 过滤中排除,保证不进生产 bundle。
- `electron-builder.yml` 采用 whitelist `files:` 模式(`out/**` + `package.json` + 指定 assets),`tests/eval/` 本就不在白名单内,无需新增排除规则。
+3 -1
View File
@@ -112,7 +112,9 @@ function evaluateOne(
case "message_contains_substring": {
for (const turn of turns) {
if (turn.role !== assertion.role) continue;
const content = (turn as AssistantTurn | { content: unknown[] }).content;
// UserTurn and AssistantTurn both expose .content; the `role` guard
// above restricts us to those variants, so no cast is needed.
const content = turn.content;
if (!Array.isArray(content)) continue;
for (const block of content) {
if (
+127 -10
View File
@@ -1,27 +1,42 @@
/**
* B5 eval harness — driver spec.
*
* This spec discovers every `*.jsonl` fixture under `tests/eval/fixtures/`
* and runs it through the assertion judge. If there are zero fixtures, the
* spec file has zero tests — vitest is invoked with `--passWithNoTests`
* from the `test:eval` script so this is a green no-op until the user
* records real fixtures.
* Two layers:
*
* The harness does NOT synthesize placeholder fixtures under any
* circumstance. An empty `fixtures/` directory is a real state — the tier
* is simply not yet exercising anything — and the green run reflects that
* honestly.
* 1. Self-tests (always run). An in-memory fixture exercises the loader
* contract, assertion judge, eval runner, and token ledger aggregation.
* This gives `pnpm test:eval` real signal even before any `.jsonl`
* fixture lands on disk. Zero fixture files = tier still exercised.
*
* 2. Fixture discovery (runs when `.jsonl` files exist). Every `*.jsonl`
* under `tests/eval/fixtures/` is discovered and runs through the
* judge. If the fixtures directory is missing we treat that as "no
* fixtures yet", return an empty list, and let the self-tests carry
* the signal on their own — we do NOT synthesize placeholder fixtures.
*
* Karpathy rule: bad fixture data must crash loud at the loader. The
* harness itself is validated by self-tests so a breakage there never
* hides behind an empty fixtures directory.
*/
import { existsSync } from "node:fs";
import { readdir } from "node:fs/promises";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { loadFixture } from "./fixture-loader.js";
import { judgeAssertions } from "./assertion-judge.js";
import { runEval } from "./eval-runner.js";
import { loadFixture } from "./fixture-loader.js";
import type {
Assertion,
AssistantTurn,
FixtureFile,
UserTurn,
} from "./types.js";
const FIXTURE_DIR = join(__dirname, "fixtures");
async function listFixtures(): Promise<string[]> {
if (!existsSync(FIXTURE_DIR)) return [];
const entries = await readdir(FIXTURE_DIR, { withFileTypes: true });
return entries
.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl"))
@@ -29,6 +44,108 @@ async function listFixtures(): Promise<string[]> {
.sort();
}
function buildInMemoryFixture(assertions: Assertion[]): FixtureFile {
const userTurn: UserTurn = {
type: "turn",
turnIndex: 0,
role: "user",
content: [{ type: "text", text: "hire a writer" }],
};
const assistantTurn: AssistantTurn = {
type: "turn",
turnIndex: 1,
role: "assistant",
content: [
{ type: "text", text: "Hiring a writer now." },
{
type: "toolCall",
id: "call_001",
name: "hire_agent",
arguments: { role: "writer", seniority: "senior" },
},
],
tokenLedger: {
input: 120,
cache_read: 40,
cache_creation: 10,
output: 60,
},
};
return {
meta: {
type: "meta",
fixtureId: "self-test/in-memory",
piAgentCoreVersion: "0.0.0-self-test",
recordedAt: "2026-04-20T00:00:00.000Z",
scenario: "in-memory self-test fixture",
systemPromptHash: "sha256:self-test",
},
turns: [userTurn, assistantTurn],
assertions,
};
}
describe("harness self-test (in-memory fixture)", () => {
it("passes when every assertion matches the recorded trace", () => {
const fixture = buildInMemoryFixture([
{ kind: "tool_called", name: "hire_agent", min_count: 1 },
{
kind: "tool_call_arg_matches",
name: "hire_agent",
arg_path: "role",
matches_regex: "^writer$",
},
{ kind: "no_tool_called", name: "write_plan" },
{
kind: "message_contains_substring",
role: "assistant",
substring: "Hiring a writer",
},
{ kind: "trace_max_turns", max: 5 },
]);
const result = runEval(fixture);
expect(result.passed).toBe(true);
expect(result.fixtureId).toBe("self-test/in-memory");
expect(result.assertions).toHaveLength(5);
expect(result.assertions.every((r) => r.passed)).toBe(true);
expect(result.tokenLedger).toEqual({
input: 120,
cache_read: 40,
cache_creation: 10,
output: 60,
});
expect(result.durationMs).toBeGreaterThanOrEqual(0);
});
it("fails with a precise reason when an assertion misses", () => {
const fixture = buildInMemoryFixture([
{ kind: "tool_called", name: "write_plan", min_count: 1 },
{ kind: "no_tool_called", name: "hire_agent" },
{ kind: "trace_max_turns", max: 1 },
]);
const result = runEval(fixture);
expect(result.passed).toBe(false);
const reasons = result.assertions.map((r) => r.reason ?? "");
expect(reasons[0]).toContain('expected at least 1 call(s) to "write_plan"');
expect(reasons[1]).toContain('expected no call to "hire_agent"');
expect(reasons[2]).toContain("trace has 2 turns, max allowed is 1");
});
it("judgeAssertions is a pure function of (assertions, turns)", () => {
const fixture = buildInMemoryFixture([]);
const first = judgeAssertions(
[{ kind: "tool_called", name: "hire_agent" }],
fixture.turns,
);
const second = judgeAssertions(
[{ kind: "tool_called", name: "hire_agent" }],
fixture.turns,
);
expect(first).toEqual(second);
expect(first[0]?.passed).toBe(true);
});
});
const fixturePaths = await listFixtures();
describe.each(fixturePaths)("eval fixture %s", (fixturePath) => {
+2 -6
View File
@@ -90,13 +90,9 @@ export function createReplayStreamFn(fixture: FixtureFile): StreamFn {
`replay-client: fixture "${fixture.meta.fixtureId}" exhausted after ${assistantTurns.length} assistant turns; agent requested turn ${cursor + 1}`,
);
}
const turn = assistantTurns[cursor];
// Cursor is bounds-checked above, so the element is guaranteed to exist.
const turn = assistantTurns[cursor]!;
cursor += 1;
if (!turn) {
throw new Error(
`replay-client: internal cursor error at position ${cursor - 1}`,
);
}
const message = buildAssistantMessage(
turn,
model.id,