B5 harness iteration: self-test + assertion-judge cleanup + dead branch fix + _BLOCKED timeline correction
- harness.test.ts: add in-memory self-test layer (pass + fail + purity), guard
fixtures dir with existsSync so absent dir yields empty list without
crashing the runner. Empty-fixture pnpm test:eval now carries real signal.
- replay-client.ts: remove dead if (!turn) branch; cursor bounds-check above
already guarantees the element. Use non-null assertion with a comment.
- assertion-judge.ts: drop unnecessary (turn as AssistantTurn | { content:
unknown[] }) cast. turn.role === assertion.role already narrows to
UserTurn | AssistantTurn and both expose .content.
- _BLOCKED-BY-CONSTRAINT.md: correct electron-builder note. files: is a
whitelist, so tests/eval is automatically excluded and no new rule is
required.
This commit is contained in:
1 parent
cb2a0a0693
commit
f849efadf5
4 files changed
+133
-18
No files matched your search
@@ -215,4 +215,4 @@ Linus 最后一句:**"先让数据结构对,其它 6 步会一起变简单"
|
||||
- No production mocks / fallbacks / silent catches — 仅 dev `tests/eval/` 例外。
|
||||
- No runtime logging / telemetry beyond existing electron-log usage.
|
||||
- No dual-write migrations without backup path。
|
||||
- `tests/eval/` 必须在 `electron-builder.yml` 的 files 过滤中排除,保证不进生产 bundle。
|
||||
- `electron-builder.yml` 采用 whitelist `files:` 模式(`out/**` + `package.json` + 指定 assets),`tests/eval/` 本就不在白名单内,无需新增排除规则。
|
||||
@@ -112,7 +112,9 @@ function evaluateOne(
|
||||
case "message_contains_substring": {
|
||||
for (const turn of turns) {
|
||||
if (turn.role !== assertion.role) continue;
|
||||
const content = (turn as AssistantTurn | { content: unknown[] }).content;
|
||||
// UserTurn and AssistantTurn both expose .content; the `role` guard
|
||||
// above restricts us to those variants, so no cast is needed.
|
||||
const content = turn.content;
|
||||
if (!Array.isArray(content)) continue;
|
||||
for (const block of content) {
|
||||
if (
|
||||
|
||||
+127
-10
@@ -1,27 +1,42 @@
|
||||
/**
|
||||
* B5 eval harness — driver spec.
|
||||
*
|
||||
* This spec discovers every `*.jsonl` fixture under `tests/eval/fixtures/`
|
||||
* and runs it through the assertion judge. If there are zero fixtures, the
|
||||
* spec file has zero tests — vitest is invoked with `--passWithNoTests`
|
||||
* from the `test:eval` script so this is a green no-op until the user
|
||||
* records real fixtures.
|
||||
* Two layers:
|
||||
*
|
||||
* The harness does NOT synthesize placeholder fixtures under any
|
||||
* circumstance. An empty `fixtures/` directory is a real state — the tier
|
||||
* is simply not yet exercising anything — and the green run reflects that
|
||||
* honestly.
|
||||
* 1. Self-tests (always run). An in-memory fixture exercises the loader
|
||||
* contract, assertion judge, eval runner, and token ledger aggregation.
|
||||
* This gives `pnpm test:eval` real signal even before any `.jsonl`
|
||||
* fixture lands on disk. Zero fixture files = tier still exercised.
|
||||
*
|
||||
* 2. Fixture discovery (runs when `.jsonl` files exist). Every `*.jsonl`
|
||||
* under `tests/eval/fixtures/` is discovered and runs through the
|
||||
* judge. If the fixtures directory is missing we treat that as "no
|
||||
* fixtures yet", return an empty list, and let the self-tests carry
|
||||
* the signal on their own — we do NOT synthesize placeholder fixtures.
|
||||
*
|
||||
* Karpathy rule: bad fixture data must crash loud at the loader. The
|
||||
* harness itself is validated by self-tests so a breakage there never
|
||||
* hides behind an empty fixtures directory.
|
||||
*/
|
||||
|
||||
import { existsSync } from "node:fs";
|
||||
import { readdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { loadFixture } from "./fixture-loader.js";
|
||||
import { judgeAssertions } from "./assertion-judge.js";
|
||||
import { runEval } from "./eval-runner.js";
|
||||
import { loadFixture } from "./fixture-loader.js";
|
||||
import type {
|
||||
Assertion,
|
||||
AssistantTurn,
|
||||
FixtureFile,
|
||||
UserTurn,
|
||||
} from "./types.js";
|
||||
|
||||
const FIXTURE_DIR = join(__dirname, "fixtures");
|
||||
|
||||
async function listFixtures(): Promise<string[]> {
|
||||
if (!existsSync(FIXTURE_DIR)) return [];
|
||||
const entries = await readdir(FIXTURE_DIR, { withFileTypes: true });
|
||||
return entries
|
||||
.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl"))
|
||||
@@ -29,6 +44,108 @@ async function listFixtures(): Promise<string[]> {
|
||||
.sort();
|
||||
}
|
||||
|
||||
function buildInMemoryFixture(assertions: Assertion[]): FixtureFile {
|
||||
const userTurn: UserTurn = {
|
||||
type: "turn",
|
||||
turnIndex: 0,
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "hire a writer" }],
|
||||
};
|
||||
const assistantTurn: AssistantTurn = {
|
||||
type: "turn",
|
||||
turnIndex: 1,
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "text", text: "Hiring a writer now." },
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "call_001",
|
||||
name: "hire_agent",
|
||||
arguments: { role: "writer", seniority: "senior" },
|
||||
},
|
||||
],
|
||||
tokenLedger: {
|
||||
input: 120,
|
||||
cache_read: 40,
|
||||
cache_creation: 10,
|
||||
output: 60,
|
||||
},
|
||||
};
|
||||
return {
|
||||
meta: {
|
||||
type: "meta",
|
||||
fixtureId: "self-test/in-memory",
|
||||
piAgentCoreVersion: "0.0.0-self-test",
|
||||
recordedAt: "2026-04-20T00:00:00.000Z",
|
||||
scenario: "in-memory self-test fixture",
|
||||
systemPromptHash: "sha256:self-test",
|
||||
},
|
||||
turns: [userTurn, assistantTurn],
|
||||
assertions,
|
||||
};
|
||||
}
|
||||
|
||||
describe("harness self-test (in-memory fixture)", () => {
|
||||
it("passes when every assertion matches the recorded trace", () => {
|
||||
const fixture = buildInMemoryFixture([
|
||||
{ kind: "tool_called", name: "hire_agent", min_count: 1 },
|
||||
{
|
||||
kind: "tool_call_arg_matches",
|
||||
name: "hire_agent",
|
||||
arg_path: "role",
|
||||
matches_regex: "^writer$",
|
||||
},
|
||||
{ kind: "no_tool_called", name: "write_plan" },
|
||||
{
|
||||
kind: "message_contains_substring",
|
||||
role: "assistant",
|
||||
substring: "Hiring a writer",
|
||||
},
|
||||
{ kind: "trace_max_turns", max: 5 },
|
||||
]);
|
||||
const result = runEval(fixture);
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.fixtureId).toBe("self-test/in-memory");
|
||||
expect(result.assertions).toHaveLength(5);
|
||||
expect(result.assertions.every((r) => r.passed)).toBe(true);
|
||||
expect(result.tokenLedger).toEqual({
|
||||
input: 120,
|
||||
cache_read: 40,
|
||||
cache_creation: 10,
|
||||
output: 60,
|
||||
});
|
||||
expect(result.durationMs).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
it("fails with a precise reason when an assertion misses", () => {
|
||||
const fixture = buildInMemoryFixture([
|
||||
{ kind: "tool_called", name: "write_plan", min_count: 1 },
|
||||
{ kind: "no_tool_called", name: "hire_agent" },
|
||||
{ kind: "trace_max_turns", max: 1 },
|
||||
]);
|
||||
const result = runEval(fixture);
|
||||
expect(result.passed).toBe(false);
|
||||
const reasons = result.assertions.map((r) => r.reason ?? "");
|
||||
expect(reasons[0]).toContain('expected at least 1 call(s) to "write_plan"');
|
||||
expect(reasons[1]).toContain('expected no call to "hire_agent"');
|
||||
expect(reasons[2]).toContain("trace has 2 turns, max allowed is 1");
|
||||
});
|
||||
|
||||
it("judgeAssertions is a pure function of (assertions, turns)", () => {
|
||||
const fixture = buildInMemoryFixture([]);
|
||||
const first = judgeAssertions(
|
||||
[{ kind: "tool_called", name: "hire_agent" }],
|
||||
fixture.turns,
|
||||
);
|
||||
const second = judgeAssertions(
|
||||
[{ kind: "tool_called", name: "hire_agent" }],
|
||||
fixture.turns,
|
||||
);
|
||||
expect(first).toEqual(second);
|
||||
expect(first[0]?.passed).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
const fixturePaths = await listFixtures();
|
||||
|
||||
describe.each(fixturePaths)("eval fixture %s", (fixturePath) => {
|
||||
|
||||
@@ -90,13 +90,9 @@ export function createReplayStreamFn(fixture: FixtureFile): StreamFn {
|
||||
`replay-client: fixture "${fixture.meta.fixtureId}" exhausted after ${assistantTurns.length} assistant turns; agent requested turn ${cursor + 1}`,
|
||||
);
|
||||
}
|
||||
const turn = assistantTurns[cursor];
|
||||
// Cursor is bounds-checked above, so the element is guaranteed to exist.
|
||||
const turn = assistantTurns[cursor]!;
|
||||
cursor += 1;
|
||||
if (!turn) {
|
||||
throw new Error(
|
||||
`replay-client: internal cursor error at position ${cursor - 1}`,
|
||||
);
|
||||
}
|
||||
const message = buildAssistantMessage(
|
||||
turn,
|
||||
model.id,
|
||||
|
||||
Reference in new issue
Block a user