test(eval): enforce SOP prompt coverage

tests/eval/README.md

@@ -22,6 +22,17 @@ shapes, message substrings, and turn caps. Semantic quality judgment belongs in
 | `harness.test.ts` | vitest driver that walks `fixtures/` and asserts every fixture passes. |
 | `fixtures/` | Recorded JSONL traces. Git-tracked. Not bundled into production (`electron-builder.yml` only ships `out/**`). |
 
+## SOP coverage gate
+
+`scenario-contract.ts` owns the prompt SOP coverage matrix. Each row renders a
+real `buildTaskPrompt` context for one audience and lists the SOP headings that
+must be present or absent for that scenario.
+
+`harness.test.ts` reads every source file under
+`src/main/prompt-builder/sops/*.en.md` and requires each `SOP-###` heading to be
+claimed by at least one matrix scenario. Adding, deleting, or renaming a SOP
+therefore changes `pnpm test:eval` directly.
+
 ## Running
 
 ```bash
@@ -187,6 +198,8 @@ belongs in the LLM-as-judge tier. See
 
 - Fixtures are recorded traces from live provider calls.
 - `fixtures/` belongs in git and participates in `pnpm verify`.
+- Every source SOP under `src/main/prompt-builder/sops/` belongs to at least
+  one `EVAL_SCENARIO_MATRIX.requiredSops` entry.
 - `scenarios/<fixtureId>/system.txt` must hash to
   `meta.systemPromptHash`.
 - `scenarios/<fixtureId>/user.txt` must match the first user turn exactly.

tests/eval/harness.test.ts

@@ -20,8 +20,8 @@
  */
 
 import { existsSync } from "node:fs";
-import { readdir } from "node:fs/promises";
-import { join } from "node:path";
+import { readFile, readdir } from "node:fs/promises";
+import { basename, join } from "node:path";
 import { describe, expect, it } from "vitest";
 import { judgeAssertions } from "./assertion-judge.js";
 import { runEval } from "./eval-runner.js";
@@ -41,6 +41,9 @@ import type {
 } from "./types.js";
 
 const FIXTURE_DIR = join(__dirname, "fixtures");
+const SOP_DIR = join(__dirname, "../../src/main/prompt-builder/sops");
+const SOP_ID_PATTERN = /^(SOP-\d{3}):/;
+const SOP_HEADING_PATTERN = /^## (SOP-\d{3}: .+)$/m;
 
 async function listFixtures(): Promise<string[]> {
   if (!existsSync(FIXTURE_DIR)) return [];
@@ -51,6 +54,37 @@ async function listFixtures(): Promise<string[]> {
     .sort();
 }
 
+async function listSourceSops(): Promise<Array<{ id: string; heading: string; fileName: string }>> {
+  const entries = await readdir(SOP_DIR, { withFileTypes: true });
+  const sopFiles = entries
+    .filter((entry) => entry.isFile() && entry.name.endsWith(".en.md"))
+    .map((entry) => join(SOP_DIR, entry.name))
+    .sort();
+
+  const sops: Array<{ id: string; heading: string; fileName: string }> = [];
+  for (const filePath of sopFiles) {
+    const content = await readFile(filePath, "utf8");
+    const heading = content.match(SOP_HEADING_PATTERN)?.[1];
+    if (!heading) {
+      throw new Error(`eval SOP coverage: ${basename(filePath)} is missing a level-2 SOP heading`);
+    }
+    const id = heading.match(SOP_ID_PATTERN)?.[1];
+    if (!id) {
+      throw new Error(`eval SOP coverage: ${basename(filePath)} has malformed heading "${heading}"`);
+    }
+    sops.push({ id, heading, fileName: basename(filePath) });
+  }
+  return sops;
+}
+
+function extractCoveredSopId(sop: string): string {
+  const id = sop.match(SOP_ID_PATTERN)?.[1];
+  if (!id) {
+    throw new Error(`eval SOP coverage: matrix entry has malformed SOP label "${sop}"`);
+  }
+  return id;
+}
+
 function buildInMemoryFixture(assertions: Assertion[]): FixtureFile {
   const userTurn: UserTurn = {
     type: "turn",
@@ -202,7 +236,7 @@ describe("harness self-test (in-memory fixture)", () => {
 
 describe("eval SOP coverage matrix", () => {
   it.each(EVAL_SCENARIO_MATRIX)(
-    "covers real prompt SOPs for $fixtureId",
+    "covers real prompt SOPs for $scenarioId",
     (entry) => {
       const prompt = buildMatrixPrompt(entry);
       for (const sop of entry.requiredSops) {
@@ -213,6 +247,29 @@ describe("eval SOP coverage matrix", () => {
       }
     },
   );
+
+  it("pairs every source SOP file with at least one eval matrix scenario", async () => {
+    const sourceSops = await listSourceSops();
+    const coveredSopIds = new Set(
+      EVAL_SCENARIO_MATRIX.flatMap((entry) => entry.requiredSops.map(extractCoveredSopId)),
+    );
+
+    const missing = sourceSops
+      .filter((sop) => !coveredSopIds.has(sop.id))
+      .map((sop) => `${sop.id} ${sop.fileName}`);
+
+    expect(missing).toEqual([]);
+  });
+
+  it("keeps eval matrix SOP labels tied to source SOP files", async () => {
+    const sourceSops = await listSourceSops();
+    const sourceSopIds = new Set(sourceSops.map((sop) => sop.id));
+    const unknown = EVAL_SCENARIO_MATRIX
+      .flatMap((entry) => entry.requiredSops.map(extractCoveredSopId))
+      .filter((sopId) => !sourceSopIds.has(sopId));
+
+    expect(unknown).toEqual([]);
+  });
 });
 
 const fixturePaths = await listFixtures();

tests/eval/scenario-contract.ts

@@ -18,7 +18,7 @@ export type EvalPromptAudience =
   | "individual_contributor";
 
 export interface EvalScenarioMatrixEntry {
-  fixtureId: string;
+  scenarioId: string;
   audience: EvalPromptAudience;
   requiredSops: string[];
   forbiddenSops?: string[];
@@ -32,7 +32,7 @@ export interface LoadedScenarioContract {
 
 export const EVAL_SCENARIO_MATRIX: EvalScenarioMatrixEntry[] = [
   {
-    fixtureId: "ceo-board-message-reply",
+    scenarioId: "ceo-board-message-reply",
     audience: "executive",
     requiredSops: [
       "SOP-001: Daily Standup Protocol",
@@ -41,7 +41,7 @@ export const EVAL_SCENARIO_MATRIX: EvalScenarioMatrixEntry[] = [
     ],
   },
   {
-    fixtureId: "ceo-bootstrap-empty",
+    scenarioId: "ceo-bootstrap-empty",
     audience: "executive",
     requiredSops: [
       "SOP-003: Delegation Protocol",
@@ -50,7 +50,7 @@ export const EVAL_SCENARIO_MATRIX: EvalScenarioMatrixEntry[] = [
     ],
   },
   {
-    fixtureId: "ic-task-execute",
+    scenarioId: "ic-task-execute",
     audience: "individual_contributor",
     requiredSops: [
       "SOP-002: Task Lifecycle",
@@ -63,7 +63,7 @@ export const EVAL_SCENARIO_MATRIX: EvalScenarioMatrixEntry[] = [
     ],
   },
   {
-    fixtureId: "manager-task-handoff",
+    scenarioId: "manager-task-handoff",
     audience: "manager",
     requiredSops: [
       "SOP-002: Task Lifecycle",
@@ -72,7 +72,7 @@ export const EVAL_SCENARIO_MATRIX: EvalScenarioMatrixEntry[] = [
     ],
   },
   {
-    fixtureId: "non-ceo-dm-reply",
+    scenarioId: "non-ceo-dm-reply",
     audience: "manager",
     requiredSops: [
       "SOP-001: Daily Standup Protocol",
@@ -80,6 +80,26 @@ export const EVAL_SCENARIO_MATRIX: EvalScenarioMatrixEntry[] = [
       "SOP-008: Cross-Department Collaboration",
     ],
   },
+  {
+    scenarioId: "ceo-budget-cadence-incident",
+    audience: "executive",
+    requiredSops: [
+      "SOP-006: Escalation Chain",
+      "SOP-010: Budget & Financial Protocol",
+      "SOP-012: Sprint & Cadence Management",
+      "SOP-013: Incident Response Protocol",
+    ],
+  },
+  {
+    scenarioId: "manager-docs-knowledge-handoff",
+    audience: "manager",
+    requiredSops: [
+      "SOP-006: Escalation Chain",
+      "SOP-009: Document Management",
+      "SOP-013: Incident Response Protocol",
+      "SOP-015: Knowledge Management",
+    ],
+  },
 ];
 
 function createAgent(overrides: Partial<AgentRecord>): AgentRecord {