From 58a4be7fe7a6457948d242986594330cb00e0785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Wed, 22 Apr 2026 16:28:50 -0400 Subject: [PATCH] test(eval): enforce SOP prompt coverage --- tests/eval/README.md | 13 +++++++ tests/eval/harness.test.ts | 63 +++++++++++++++++++++++++++++++-- tests/eval/scenario-contract.ts | 32 +++++++++++++---- 3 files changed, 99 insertions(+), 9 deletions(-) diff --git a/tests/eval/README.md b/tests/eval/README.md index 84e1526b..24ffe1fe 100644 --- a/tests/eval/README.md +++ b/tests/eval/README.md @@ -22,6 +22,17 @@ shapes, message substrings, and turn caps. Semantic quality judgment belongs in | `harness.test.ts` | vitest driver that walks `fixtures/` and asserts every fixture passes. | | `fixtures/` | Recorded JSONL traces. Git-tracked. Not bundled into production (`electron-builder.yml` only ships `out/**`). | +## SOP coverage gate + +`scenario-contract.ts` owns the prompt SOP coverage matrix. Each row renders a +real `buildTaskPrompt` context for one audience and lists the SOP headings that +must be present or absent for that scenario. + +`harness.test.ts` reads every source file under +`src/main/prompt-builder/sops/*.en.md` and requires each `SOP-###` heading to be +claimed by at least one matrix scenario. Adding, deleting, or renaming a SOP +therefore changes `pnpm test:eval` directly. + ## Running ```bash @@ -187,6 +198,8 @@ belongs in the LLM-as-judge tier. See - Fixtures are recorded traces from live provider calls. - `fixtures/` belongs in git and participates in `pnpm verify`. +- Every source SOP under `src/main/prompt-builder/sops/` belongs to at least + one `EVAL_SCENARIO_MATRIX.requiredSops` entry. - `scenarios//system.txt` must hash to `meta.systemPromptHash`. - `scenarios//user.txt` must match the first user turn exactly. diff --git a/tests/eval/harness.test.ts b/tests/eval/harness.test.ts index a5d3d345..5db4724f 100644 --- a/tests/eval/harness.test.ts +++ b/tests/eval/harness.test.ts @@ -20,8 +20,8 @@ */ import { existsSync } from "node:fs"; -import { readdir } from "node:fs/promises"; -import { join } from "node:path"; +import { readFile, readdir } from "node:fs/promises"; +import { basename, join } from "node:path"; import { describe, expect, it } from "vitest"; import { judgeAssertions } from "./assertion-judge.js"; import { runEval } from "./eval-runner.js"; @@ -41,6 +41,9 @@ import type { } from "./types.js"; const FIXTURE_DIR = join(__dirname, "fixtures"); +const SOP_DIR = join(__dirname, "../../src/main/prompt-builder/sops"); +const SOP_ID_PATTERN = /^(SOP-\d{3}):/; +const SOP_HEADING_PATTERN = /^## (SOP-\d{3}: .+)$/m; async function listFixtures(): Promise { if (!existsSync(FIXTURE_DIR)) return []; @@ -51,6 +54,37 @@ async function listFixtures(): Promise { .sort(); } +async function listSourceSops(): Promise> { + const entries = await readdir(SOP_DIR, { withFileTypes: true }); + const sopFiles = entries + .filter((entry) => entry.isFile() && entry.name.endsWith(".en.md")) + .map((entry) => join(SOP_DIR, entry.name)) + .sort(); + + const sops: Array<{ id: string; heading: string; fileName: string }> = []; + for (const filePath of sopFiles) { + const content = await readFile(filePath, "utf8"); + const heading = content.match(SOP_HEADING_PATTERN)?.[1]; + if (!heading) { + throw new Error(`eval SOP coverage: ${basename(filePath)} is missing a level-2 SOP heading`); + } + const id = heading.match(SOP_ID_PATTERN)?.[1]; + if (!id) { + throw new Error(`eval SOP coverage: ${basename(filePath)} has malformed heading "${heading}"`); + } + sops.push({ id, heading, fileName: basename(filePath) }); + } + return sops; +} + +function extractCoveredSopId(sop: string): string { + const id = sop.match(SOP_ID_PATTERN)?.[1]; + if (!id) { + throw new Error(`eval SOP coverage: matrix entry has malformed SOP label "${sop}"`); + } + return id; +} + function buildInMemoryFixture(assertions: Assertion[]): FixtureFile { const userTurn: UserTurn = { type: "turn", @@ -202,7 +236,7 @@ describe("harness self-test (in-memory fixture)", () => { describe("eval SOP coverage matrix", () => { it.each(EVAL_SCENARIO_MATRIX)( - "covers real prompt SOPs for $fixtureId", + "covers real prompt SOPs for $scenarioId", (entry) => { const prompt = buildMatrixPrompt(entry); for (const sop of entry.requiredSops) { @@ -213,6 +247,29 @@ describe("eval SOP coverage matrix", () => { } }, ); + + it("pairs every source SOP file with at least one eval matrix scenario", async () => { + const sourceSops = await listSourceSops(); + const coveredSopIds = new Set( + EVAL_SCENARIO_MATRIX.flatMap((entry) => entry.requiredSops.map(extractCoveredSopId)), + ); + + const missing = sourceSops + .filter((sop) => !coveredSopIds.has(sop.id)) + .map((sop) => `${sop.id} ${sop.fileName}`); + + expect(missing).toEqual([]); + }); + + it("keeps eval matrix SOP labels tied to source SOP files", async () => { + const sourceSops = await listSourceSops(); + const sourceSopIds = new Set(sourceSops.map((sop) => sop.id)); + const unknown = EVAL_SCENARIO_MATRIX + .flatMap((entry) => entry.requiredSops.map(extractCoveredSopId)) + .filter((sopId) => !sourceSopIds.has(sopId)); + + expect(unknown).toEqual([]); + }); }); const fixturePaths = await listFixtures(); diff --git a/tests/eval/scenario-contract.ts b/tests/eval/scenario-contract.ts index 6c94aff0..69599b71 100644 --- a/tests/eval/scenario-contract.ts +++ b/tests/eval/scenario-contract.ts @@ -18,7 +18,7 @@ export type EvalPromptAudience = | "individual_contributor"; export interface EvalScenarioMatrixEntry { - fixtureId: string; + scenarioId: string; audience: EvalPromptAudience; requiredSops: string[]; forbiddenSops?: string[]; @@ -32,7 +32,7 @@ export interface LoadedScenarioContract { export const EVAL_SCENARIO_MATRIX: EvalScenarioMatrixEntry[] = [ { - fixtureId: "ceo-board-message-reply", + scenarioId: "ceo-board-message-reply", audience: "executive", requiredSops: [ "SOP-001: Daily Standup Protocol", @@ -41,7 +41,7 @@ export const EVAL_SCENARIO_MATRIX: EvalScenarioMatrixEntry[] = [ ], }, { - fixtureId: "ceo-bootstrap-empty", + scenarioId: "ceo-bootstrap-empty", audience: "executive", requiredSops: [ "SOP-003: Delegation Protocol", @@ -50,7 +50,7 @@ export const EVAL_SCENARIO_MATRIX: EvalScenarioMatrixEntry[] = [ ], }, { - fixtureId: "ic-task-execute", + scenarioId: "ic-task-execute", audience: "individual_contributor", requiredSops: [ "SOP-002: Task Lifecycle", @@ -63,7 +63,7 @@ export const EVAL_SCENARIO_MATRIX: EvalScenarioMatrixEntry[] = [ ], }, { - fixtureId: "manager-task-handoff", + scenarioId: "manager-task-handoff", audience: "manager", requiredSops: [ "SOP-002: Task Lifecycle", @@ -72,7 +72,7 @@ export const EVAL_SCENARIO_MATRIX: EvalScenarioMatrixEntry[] = [ ], }, { - fixtureId: "non-ceo-dm-reply", + scenarioId: "non-ceo-dm-reply", audience: "manager", requiredSops: [ "SOP-001: Daily Standup Protocol", @@ -80,6 +80,26 @@ export const EVAL_SCENARIO_MATRIX: EvalScenarioMatrixEntry[] = [ "SOP-008: Cross-Department Collaboration", ], }, + { + scenarioId: "ceo-budget-cadence-incident", + audience: "executive", + requiredSops: [ + "SOP-006: Escalation Chain", + "SOP-010: Budget & Financial Protocol", + "SOP-012: Sprint & Cadence Management", + "SOP-013: Incident Response Protocol", + ], + }, + { + scenarioId: "manager-docs-knowledge-handoff", + audience: "manager", + requiredSops: [ + "SOP-006: Escalation Chain", + "SOP-009: Document Management", + "SOP-013: Incident Response Protocol", + "SOP-015: Knowledge Management", + ], + }, ]; function createAgent(overrides: Partial): AgentRecord {