test(eval): add task-level prompt section harness

tests/eval/README.md

@@ -19,6 +19,8 @@ the eval matrix also pins the AI Co-Founder human-review prompt contract.
 | `replay-client.ts` | Swaps in for `pi-ai`'s `streamSimple`; replays recorded assistant turns. |
 | `tool-loop-replay.ts` | Runs a fixture through `pi-agent-core` Agent with recorded tool-result executors. |
 | `scenario-contract.ts` | Validates scenario prompt/user files, prompt hashes, and real prompt-builder SOP coverage. |
+| `task-level-prompt-eval.ts` | Renders task-level prompt scenarios, records section token/byte metrics, and gates prompt section removal. |
+| `task-level-prompt-cases.ts` | Task-level prompt matrix used by the section gate. |
 | `assertion-judge.ts` | Structural assertions over recorded trace. No semantic judgment. |
 | `semantic-rubric-judge.ts` | Semantic rubric dimensions over assistant transcript evidence. |
 | `eval-runner.ts` | Parse fixture → Agent replay → judge assertions → produce `EvalResult`. |
@@ -37,6 +39,25 @@ must be present or absent for that scenario.
 claimed by at least one matrix scenario. Adding, deleting, or renaming a SOP
 therefore changes `pnpm test:eval` directly.
 
+## Task-level prompt section gate
+
+`task-level-prompt-eval.ts` adds a task-level prompt matrix above the recorded
+trace replay layer. Each case renders a real `PromptContext` through
+`buildTaskPrompt`, extracts Markdown sections, records token and byte counts,
+and checks required / forbidden headings and fragments.
+
+This gate is the ablation ledger for future prompt cleanup. A prompt section can
+be removed only after the required-heading usage map shows no task-level eval
+case depends on it, or after the case itself is re-recorded and reviewed with a
+new behavioral fixture. The current matrix covers:
+
+| Case | Contract |
+|---|---|
+| `ceo-bootstrap-hiring` | CEO cold start exposes team-building + connectors and suppresses normal operating mode. |
+| `manager-review-handoff` | Manager review work exposes review queue, team workload, and manager SOPs. |
+| `ic-implementation-task` | IC execution exposes checkout, quality, assigned-task, and current-task sections. |
+| `cofounder-plan-review-gate` | AI Co-Founder mode exposes human-review policy and pending approval context. |
+
 ## Running
 
 ```bash

tests/eval/task-level-prompt-cases.ts

@@ -0,0 +1,345 @@
+import type { PromptContext } from "../../src/main/prompt-builder/index";
+import type {
+  AgentRecord,
+  ApprovalRecord,
+  GoalRecord,
+  ProjectRecord,
+  TaskRecord,
+} from "../../src/shared/types";
+import type { TaskPromptEvalCase } from "./task-level-prompt-eval.js";
+
+const EVAL_NOW = "2026-04-20T00:00:00.000Z";
+
+function createAgent(overrides: Partial<AgentRecord>): AgentRecord {
+  return {
+    id: "agent-1",
+    companyId: "company-1",
+    name: "Agent",
+    role: "Engineer",
+    title: "Engineer",
+    roleLevel: "ic",
+    department: "engineering",
+    status: "active",
+    reportsTo: "manager-1",
+    connectorId: "openai-codex",
+    workspaceId: null,
+    model: "gpt-5.4",
+    capabilities: "Build production software",
+    budgetMonthlyUsd: 200,
+    spentMonthlyUsd: 0,
+    metadataJson: "{}",
+    heartbeatEnabled: true,
+    heartbeatIntervalSec: 120,
+    lastHeartbeatAt: null,
+    createdAt: EVAL_NOW,
+    updatedAt: EVAL_NOW,
+    ...overrides,
+  };
+}
+
+function createTask(overrides: Partial<TaskRecord>): TaskRecord {
+  return {
+    id: "task-1",
+    companyId: "company-1",
+    projectId: null,
+    goalId: null,
+    parentId: null,
+    title: "Execute assigned work",
+    description: "Complete the assigned production task with verification.",
+    assigneeAgentId: "agent-1",
+    workspaceId: null,
+    status: "in_progress",
+    priority: "high",
+    activeRunId: null,
+    costUsd: 0,
+    taskType: null,
+    requiresUserReview: false,
+    requiresFilesystem: true,
+    startedAt: null,
+    completedAt: null,
+    cancelledAt: null,
+    createdAt: EVAL_NOW,
+    updatedAt: EVAL_NOW,
+    ...overrides,
+  };
+}
+
+function createGoal(overrides: Partial<GoalRecord>): GoalRecord {
+  return {
+    id: "goal-1",
+    companyId: "company-1",
+    title: "Launch reliable agent operations",
+    description: "Build a dependable automated company operating loop.",
+    parentId: null,
+    ownerAgentId: "ceo-1",
+    status: "active",
+    createdAt: EVAL_NOW,
+    updatedAt: EVAL_NOW,
+    ...overrides,
+  };
+}
+
+function createProject(overrides: Partial<ProjectRecord>): ProjectRecord {
+  return {
+    id: "project-1",
+    companyId: "company-1",
+    goalId: "goal-1",
+    name: "Reliability hardening",
+    description: "Reduce operational regressions across runtime paths.",
+    leadAgentId: "manager-1",
+    status: "in_progress",
+    targetDate: null,
+    createdAt: EVAL_NOW,
+    updatedAt: EVAL_NOW,
+    ...overrides,
+  };
+}
+
+function createApproval(overrides: Partial<ApprovalRecord>): ApprovalRecord {
+  return {
+    id: "approval-1",
+    companyId: "company-1",
+    relatedTaskId: "task-1",
+    relatedAgentId: null,
+    requestedByAgentId: "agent-1",
+    type: "plan_review",
+    payloadSummary: "Human review plan before implementation.",
+    impactSummary: "Blocks execution until the co-founder approves scope.",
+    payloadJson: null,
+    decisionNote: "",
+    state: "pending",
+    createdAt: EVAL_NOW,
+    updatedAt: EVAL_NOW,
+    ...overrides,
+  };
+}
+
+const ceo = createAgent({
+  id: "ceo-1",
+  name: "CEO",
+  role: "CEO",
+  title: "Chief Executive Officer",
+  roleLevel: "c_suite",
+  department: "executive",
+  reportsTo: null,
+  capabilities: "Lead the company",
+});
+const manager = createAgent({
+  id: "manager-1",
+  name: "VP Engineering",
+  role: "VP Engineering",
+  title: "VP Engineering",
+  roleLevel: "vp",
+  reportsTo: ceo.id,
+  capabilities: "Run engineering delivery",
+});
+const ic = createAgent({
+  id: "ic-1",
+  name: "Backend IC",
+  role: "Backend Engineer",
+  title: "Backend Engineer",
+  reportsTo: manager.id,
+  capabilities: "Implement backend tasks",
+});
+
+function buildContext(overrides: Partial<PromptContext>): PromptContext {
+  return {
+    agent: ic,
+    apiUrl: "http://127.0.0.1:3000",
+    apiKey: "eval-token",
+    runId: "eval-run",
+    task: null,
+    companyName: "AgentCompany Eval",
+    companyDescription: "Task-level prompt eval coverage",
+    goals: [createGoal({})],
+    projects: [createProject({})],
+    directReports: [],
+    chainOfCommand: [manager, ceo],
+    assignedTasks: [],
+    allAgents: [ceo, manager, ic],
+    pendingApprovals: [],
+    tasksAwaitingReview: [],
+    wakeReason: "assignment",
+    allTasks: [],
+    recentComments: [],
+    socialAccounts: [],
+    allConnectors: [
+      { id: "openai-apikey", label: "OpenAI API", status: "ready", enabledModels: ["gpt-5.4"] },
+      { id: "anthropic-oauth", label: "Anthropic OAuth", status: "ready", enabledModels: ["claude-sonnet-4-5"] },
+    ],
+    ...overrides,
+  };
+}
+
+const reviewTask = createTask({
+  id: "task-review-1",
+  title: "Review /api/companies p99 fix",
+  description: "Review the submitted database query fix and approve or request changes.",
+  assigneeAgentId: ic.id,
+  status: "in_review",
+});
+
+export const TASK_LEVEL_PROMPT_EVAL_CASES: TaskPromptEvalCase[] = [
+  {
+    caseId: "ceo-bootstrap-hiring",
+    taskLevel: "executive",
+    description: "CEO cold-start task must expose hiring mode and suppress normal operating mode.",
+    context: buildContext({
+      agent: ceo,
+      runId: "eval-run-ceo-bootstrap",
+      task: createTask({
+        id: "task-ceo-bootstrap",
+        title: "Bootstrap the leadership team",
+        description: "Create the first leadership hires and file the organization strategy.",
+        assigneeAgentId: ceo.id,
+      }),
+      directReports: [],
+      chainOfCommand: [],
+      allAgents: [ceo],
+    }),
+    requiredHeadings: [
+      "Agent Identity",
+      "Your Leadership Style",
+      "Position",
+      "Active Goals",
+      "Operating Instructions (CEO — Team Building Phase)",
+      "Available Connectors",
+      "Current Task",
+      "Context",
+    ],
+    forbiddenHeadings: [
+      "Operating Instructions (Executive / CEO)",
+      "Proactive Behaviors (Things to Do Without Being Asked)",
+      "Tasks Awaiting Your Review (1 pending)",
+    ],
+    requiredFragments: [
+      "file 3–5 `hire_agent` tool calls",
+      "Current Run ID: eval-run-ceo-bootstrap",
+    ],
+    forbiddenFragments: ["Goal Decomposition"],
+  },
+  {
+    caseId: "manager-review-handoff",
+    taskLevel: "manager",
+    description: "Manager review task must expose review queue, workload, and delegation guidance.",
+    context: buildContext({
+      agent: manager,
+      runId: "eval-run-manager-review",
+      task: createTask({
+        id: "task-manager-review",
+        title: "Review latency fix",
+        description: "Review the submitted p99 fix and unblock the release.",
+        assigneeAgentId: manager.id,
+      }),
+      directReports: [ic],
+      chainOfCommand: [ceo],
+      allAgents: [ceo, manager, ic],
+      allTasks: [reviewTask],
+      tasksAwaitingReview: [reviewTask],
+      recentComments: [{
+        taskId: reviewTask.id,
+        taskTitle: reviewTask.title,
+        authorName: ic.name,
+        body: "Implemented the index-backed query path and attached benchmark output.",
+        createdAt: EVAL_NOW,
+      }],
+    }),
+    requiredHeadings: [
+      "Agent Identity",
+      "Your Management Style",
+      "Your Direct Reports",
+      "Operating Instructions (Manager)",
+      "Team Workload",
+      "Current Task",
+      "Tasks Awaiting Your Review (1 pending)",
+      "Context",
+    ],
+    forbiddenHeadings: [
+      "Operating Instructions (Individual Contributor)",
+      "Operating Instructions (CEO — Team Building Phase)",
+    ],
+    requiredFragments: ["1 in review", "REVIEW WORKFLOW", "Review /api/companies p99 fix"],
+    forbiddenFragments: ["file 3–5 `hire_agent` tool calls"],
+  },
+  {
+    caseId: "ic-implementation-task",
+    taskLevel: "individual_contributor",
+    description: "IC implementation task must expose checkout, quality, and current task instructions.",
+    context: buildContext({
+      agent: ic,
+      runId: "eval-run-ic-implementation",
+      task: createTask({
+        id: "task-ic-implementation",
+        title: "Fix pagination boundary",
+        description: "Patch src/utils/paginate.ts and prove tests pass.",
+        assigneeAgentId: ic.id,
+      }),
+      directReports: [],
+      chainOfCommand: [manager, ceo],
+      allAgents: [ceo, manager, ic],
+      assignedTasks: [createTask({
+        id: "task-next",
+        title: "Document pagination invariant",
+        assigneeAgentId: ic.id,
+        status: "todo",
+      })],
+    }),
+    requiredHeadings: [
+      "Agent Identity",
+      "Your Work Philosophy",
+      "Operating Instructions (Individual Contributor)",
+      "Task Execution Workflow (SOP-002)",
+      "Quality Standards",
+      "Current Task",
+      "Your Assigned Tasks",
+      "Context",
+    ],
+    forbiddenHeadings: [
+      "Operating Instructions (Manager)",
+      "Operating Instructions (CEO — Team Building Phase)",
+      "Tasks Awaiting Your Review (1 pending)",
+    ],
+    requiredFragments: ["Checkout", "set to \"in_review\"", "task-ic-implementation"],
+  },
+  {
+    caseId: "cofounder-plan-review-gate",
+    taskLevel: "manager",
+    description: "AI cofounder task must expose human review gate and pending approval context.",
+    context: buildContext({
+      agent: manager,
+      runId: "eval-run-cofounder-review",
+      executionMode: "ai_cofounder",
+      task: createTask({
+        id: "task-cofounder-plan",
+        title: "Plan checkout recovery refactor",
+        description: "Prepare a plan before modifying task checkout recovery.",
+        assigneeAgentId: manager.id,
+        requiresUserReview: true,
+      }),
+      directReports: [ic],
+      chainOfCommand: [ceo],
+      allAgents: [ceo, manager, ic],
+      pendingApprovals: [createApproval({
+        id: "approval-plan-1",
+        relatedTaskId: "task-cofounder-plan",
+        requestedByAgentId: manager.id,
+      })],
+    }),
+    requiredHeadings: [
+      "Execution Mode: AI Co-Founder (Human-Reviewed)",
+      "Operating Instructions (Manager)",
+      "Current Task",
+      "Pending Approvals",
+      "Context",
+    ],
+    forbiddenHeadings: [
+      "Operating Instructions (Executive / CEO)",
+      "Operating Instructions (CEO — Team Building Phase)",
+    ],
+    requiredFragments: [
+      "Treat pending approvals as blocking",
+      "plan_review: Human review plan before implementation.",
+      "approval-plan-1",
+    ],
+  },
+];

tests/eval/task-level-prompt-eval.test.ts

@@ -0,0 +1,116 @@
+import { describe, expect, it, vi } from "vitest";
+import { getEncoding } from "js-tiktoken";
+import {
+  collectRequiredHeadingUsage,
+  evaluateTaskPromptCase,
+  runTaskPromptEvalCase,
+} from "./task-level-prompt-eval.js";
+import { TASK_LEVEL_PROMPT_EVAL_CASES } from "./task-level-prompt-cases.js";
+import { buildStableSystemPrompt, buildTaskPrompt } from "../../src/main/prompt-builder/index";
+
+const TEST_ENCODING = getEncoding("cl100k_base");
+
+describe("task-level prompt eval harness", () => {
+  it.each(TASK_LEVEL_PROMPT_EVAL_CASES)(
+    "passes task-level prompt section contract for $caseId",
+    (testCase) => {
+      vi.useFakeTimers();
+      vi.setSystemTime(new Date("2026-04-20T12:00:00.000Z"));
+      const result = runTaskPromptEvalCase(testCase);
+      vi.useRealTimers();
+
+      if (!result.passed) {
+        throw new Error(`${result.caseId} failed:\n${result.failures.join("\n")}`);
+      }
+
+      expect(result.promptHash).toMatch(/^sha256:[a-f0-9]{64}$/);
+      expect(result.totalTokens).toBeGreaterThan(0);
+      expect(result.stablePrefixTokens).toBeGreaterThan(0);
+      expect(result.volatileTailTokens).toBeGreaterThan(0);
+      expect(result.sections.length).toBeGreaterThan(testCase.requiredHeadings.length);
+      for (const heading of testCase.requiredHeadings) {
+        const section = result.sections.find((candidate) => candidate.heading === heading);
+        expect(section?.tokenCount).toBeGreaterThan(0);
+        expect(section?.byteLength).toBeGreaterThan(0);
+      }
+    },
+  );
+
+  it("builds the heading usage ledger that gates future prompt section removal", () => {
+    const usage = collectRequiredHeadingUsage(TASK_LEVEL_PROMPT_EVAL_CASES);
+
+    expect(usage.get("Current Task")).toEqual([
+      "ceo-bootstrap-hiring",
+      "manager-review-handoff",
+      "ic-implementation-task",
+      "cofounder-plan-review-gate",
+    ]);
+    expect(usage.get("Operating Instructions (CEO — Team Building Phase)")).toEqual([
+      "ceo-bootstrap-hiring",
+    ]);
+    expect(usage.get("Tasks Awaiting Your Review (1 pending)")).toEqual([
+      "manager-review-handoff",
+    ]);
+    expect(usage.get("Execution Mode: AI Co-Founder (Human-Reviewed)")).toEqual([
+      "cofounder-plan-review-gate",
+    ]);
+  });
+
+  it("counts volatile tail tokens from the actual prompt boundary slice", () => {
+    const stablePrefix = "AgentCompany API\n";
+    const volatileTail = "\n# Current Task";
+    const prompt = `${stablePrefix}${volatileTail}`;
+    const result = evaluateTaskPromptCase({
+      caseId: "token-boundary",
+      taskLevel: "individual_contributor",
+      description: "BPE boundary regression case.",
+      context: TASK_LEVEL_PROMPT_EVAL_CASES[0]!.context,
+      requiredHeadings: ["Current Task"],
+      forbiddenHeadings: [],
+    }, prompt, stablePrefix);
+
+    expect(result.volatileTailTokens).toBe(TEST_ENCODING.encode(volatileTail).length);
+    expect(result.volatileTailTokens).not.toBe(result.totalTokens - result.stablePrefixTokens);
+  });
+
+  it("fails when CEO bootstrap prompt receives the operating proactive footer", () => {
+    const testCase = TASK_LEVEL_PROMPT_EVAL_CASES.find(
+      (candidate) => candidate.caseId === "ceo-bootstrap-hiring",
+    );
+    if (!testCase) throw new Error("missing ceo-bootstrap-hiring eval case");
+
+    vi.useFakeTimers();
+    vi.setSystemTime(new Date("2026-04-20T12:00:00.000Z"));
+    const stablePrefix = buildStableSystemPrompt(testCase.context);
+    const prompt = `${buildTaskPrompt(testCase.context)}
+# Proactive Behaviors (Things to Do Without Being Asked)
+- If a department has no active tasks: create strategic tasks.`;
+    const result = evaluateTaskPromptCase(testCase, prompt, stablePrefix);
+    vi.useRealTimers();
+
+    expect(result.passed).toBe(false);
+    expect(result.failures).toContain(
+      'forbidden heading "Proactive Behaviors (Things to Do Without Being Asked)" is present',
+    );
+  });
+
+  it("fails loud when a required prompt section is removed", () => {
+    const testCase = TASK_LEVEL_PROMPT_EVAL_CASES.find(
+      (candidate) => candidate.caseId === "ic-implementation-task",
+    );
+    if (!testCase) throw new Error("missing ic-implementation-task eval case");
+
+    vi.useFakeTimers();
+    vi.setSystemTime(new Date("2026-04-20T12:00:00.000Z"));
+    const stablePrefix = buildStableSystemPrompt(testCase.context);
+    const prompt = buildTaskPrompt(testCase.context).replace(
+      /^# Current Task\n[\s\S]*?(?=^# Your Assigned Tasks|^# Context)/m,
+      "",
+    );
+    const result = evaluateTaskPromptCase(testCase, prompt, stablePrefix);
+    vi.useRealTimers();
+
+    expect(result.passed).toBe(false);
+    expect(result.failures).toContain('missing required heading "Current Task"');
+  });
+});

tests/eval/task-level-prompt-eval.ts

@@ -0,0 +1,140 @@
+import { createHash } from "node:crypto";
+import { getEncoding } from "js-tiktoken";
+import {
+  buildStableSystemPrompt,
+  buildTaskPrompt,
+  type PromptContext,
+} from "../../src/main/prompt-builder/index";
+
+const PROMPT_ENCODING = getEncoding("cl100k_base");
+
+export interface TaskPromptEvalCase {
+  caseId: string;
+  taskLevel: "executive" | "manager" | "individual_contributor";
+  description: string;
+  context: PromptContext;
+  requiredHeadings: string[];
+  forbiddenHeadings: string[];
+  requiredFragments?: string[];
+  forbiddenFragments?: string[];
+}
+
+export interface PromptSectionMetric {
+  heading: string;
+  level: number;
+  byteLength: number;
+  tokenCount: number;
+  inStablePrefix: boolean;
+}
+
+export interface TaskPromptEvalResult {
+  caseId: string;
+  passed: boolean;
+  promptHash: string;
+  totalTokens: number;
+  stablePrefixTokens: number;
+  volatileTailTokens: number;
+  sections: PromptSectionMetric[];
+  failures: string[];
+}
+
+function tokenCount(value: string): number {
+  return PROMPT_ENCODING.encode(value).length;
+}
+
+function hashPrompt(prompt: string): string {
+  return `sha256:${createHash("sha256").update(prompt).digest("hex")}`;
+}
+
+export function extractPromptSections(prompt: string, stablePrefixLength: number): PromptSectionMetric[] {
+  const matches = Array.from(prompt.matchAll(/^(#{1,6})\s+(.+)$/gm));
+  return matches.map((match, index) => {
+    if (match.index === undefined || match[1] === undefined || match[2] === undefined) {
+      throw new Error("task prompt eval: malformed markdown heading match");
+    }
+    const start = match.index;
+    const next = matches[index + 1];
+    const end = next?.index ?? prompt.length;
+    const text = prompt.slice(start, end);
+    return {
+      heading: match[2].trim(),
+      level: match[1].length,
+      byteLength: Buffer.byteLength(text, "utf8"),
+      tokenCount: tokenCount(text),
+      inStablePrefix: start < stablePrefixLength,
+    };
+  });
+}
+
+export function evaluateTaskPromptCase(
+  testCase: TaskPromptEvalCase,
+  prompt: string,
+  stablePrefix: string,
+): TaskPromptEvalResult {
+  const failures: string[] = [];
+  if (!prompt.startsWith(stablePrefix)) {
+    failures.push("full prompt does not begin with the stable system prefix");
+  }
+
+  const sections = extractPromptSections(prompt, stablePrefix.length);
+  const headings = new Set(sections.map((section) => section.heading));
+
+  for (const heading of testCase.requiredHeadings) {
+    if (!headings.has(heading)) {
+      failures.push(`missing required heading "${heading}"`);
+    }
+  }
+
+  for (const heading of testCase.forbiddenHeadings) {
+    if (headings.has(heading)) {
+      failures.push(`forbidden heading "${heading}" is present`);
+    }
+  }
+
+  for (const fragment of testCase.requiredFragments ?? []) {
+    if (!prompt.includes(fragment)) {
+      failures.push(`missing required fragment "${fragment}"`);
+    }
+  }
+
+  for (const fragment of testCase.forbiddenFragments ?? []) {
+    if (prompt.includes(fragment)) {
+      failures.push(`forbidden fragment "${fragment}" is present`);
+    }
+  }
+
+  const totalTokens = tokenCount(prompt);
+  const stablePrefixTokens = tokenCount(stablePrefix);
+  const volatileTailTokens = tokenCount(prompt.slice(stablePrefix.length));
+
+  return {
+    caseId: testCase.caseId,
+    passed: failures.length === 0,
+    promptHash: hashPrompt(prompt),
+    totalTokens,
+    stablePrefixTokens,
+    volatileTailTokens,
+    sections,
+    failures,
+  };
+}
+
+export function runTaskPromptEvalCase(testCase: TaskPromptEvalCase): TaskPromptEvalResult {
+  return evaluateTaskPromptCase(
+    testCase,
+    buildTaskPrompt(testCase.context),
+    buildStableSystemPrompt(testCase.context),
+  );
+}
+
+export function collectRequiredHeadingUsage(cases: TaskPromptEvalCase[]): Map<string, string[]> {
+  const usage = new Map<string, string[]>();
+  for (const testCase of cases) {
+    for (const heading of testCase.requiredHeadings) {
+      const caseIds = usage.get(heading) ?? [];
+      caseIds.push(testCase.caseId);
+      usage.set(heading, caseIds);
+    }
+  }
+  return usage;
+}