diff --git a/tests/eval/README.md b/tests/eval/README.md index 438226af..d00c6c74 100644 --- a/tests/eval/README.md +++ b/tests/eval/README.md @@ -19,6 +19,8 @@ the eval matrix also pins the AI Co-Founder human-review prompt contract. | `replay-client.ts` | Swaps in for `pi-ai`'s `streamSimple`; replays recorded assistant turns. | | `tool-loop-replay.ts` | Runs a fixture through `pi-agent-core` Agent with recorded tool-result executors. | | `scenario-contract.ts` | Validates scenario prompt/user files, prompt hashes, and real prompt-builder SOP coverage. | +| `task-level-prompt-eval.ts` | Renders task-level prompt scenarios, records section token/byte metrics, and gates prompt section removal. | +| `task-level-prompt-cases.ts` | Task-level prompt matrix used by the section gate. | | `assertion-judge.ts` | Structural assertions over recorded trace. No semantic judgment. | | `semantic-rubric-judge.ts` | Semantic rubric dimensions over assistant transcript evidence. | | `eval-runner.ts` | Parse fixture → Agent replay → judge assertions → produce `EvalResult`. | @@ -37,6 +39,25 @@ must be present or absent for that scenario. claimed by at least one matrix scenario. Adding, deleting, or renaming a SOP therefore changes `pnpm test:eval` directly. +## Task-level prompt section gate + +`task-level-prompt-eval.ts` adds a task-level prompt matrix above the recorded +trace replay layer. Each case renders a real `PromptContext` through +`buildTaskPrompt`, extracts Markdown sections, records token and byte counts, +and checks required / forbidden headings and fragments. + +This gate is the ablation ledger for future prompt cleanup. A prompt section can +be removed only after the required-heading usage map shows no task-level eval +case depends on it, or after the case itself is re-recorded and reviewed with a +new behavioral fixture. The current matrix covers: + +| Case | Contract | +|---|---| +| `ceo-bootstrap-hiring` | CEO cold start exposes team-building + connectors and suppresses normal operating mode. | +| `manager-review-handoff` | Manager review work exposes review queue, team workload, and manager SOPs. | +| `ic-implementation-task` | IC execution exposes checkout, quality, assigned-task, and current-task sections. | +| `cofounder-plan-review-gate` | AI Co-Founder mode exposes human-review policy and pending approval context. | + ## Running ```bash diff --git a/tests/eval/task-level-prompt-cases.ts b/tests/eval/task-level-prompt-cases.ts new file mode 100644 index 00000000..926ce6fa --- /dev/null +++ b/tests/eval/task-level-prompt-cases.ts @@ -0,0 +1,345 @@ +import type { PromptContext } from "../../src/main/prompt-builder/index"; +import type { + AgentRecord, + ApprovalRecord, + GoalRecord, + ProjectRecord, + TaskRecord, +} from "../../src/shared/types"; +import type { TaskPromptEvalCase } from "./task-level-prompt-eval.js"; + +const EVAL_NOW = "2026-04-20T00:00:00.000Z"; + +function createAgent(overrides: Partial): AgentRecord { + return { + id: "agent-1", + companyId: "company-1", + name: "Agent", + role: "Engineer", + title: "Engineer", + roleLevel: "ic", + department: "engineering", + status: "active", + reportsTo: "manager-1", + connectorId: "openai-codex", + workspaceId: null, + model: "gpt-5.4", + capabilities: "Build production software", + budgetMonthlyUsd: 200, + spentMonthlyUsd: 0, + metadataJson: "{}", + heartbeatEnabled: true, + heartbeatIntervalSec: 120, + lastHeartbeatAt: null, + createdAt: EVAL_NOW, + updatedAt: EVAL_NOW, + ...overrides, + }; +} + +function createTask(overrides: Partial): TaskRecord { + return { + id: "task-1", + companyId: "company-1", + projectId: null, + goalId: null, + parentId: null, + title: "Execute assigned work", + description: "Complete the assigned production task with verification.", + assigneeAgentId: "agent-1", + workspaceId: null, + status: "in_progress", + priority: "high", + activeRunId: null, + costUsd: 0, + taskType: null, + requiresUserReview: false, + requiresFilesystem: true, + startedAt: null, + completedAt: null, + cancelledAt: null, + createdAt: EVAL_NOW, + updatedAt: EVAL_NOW, + ...overrides, + }; +} + +function createGoal(overrides: Partial): GoalRecord { + return { + id: "goal-1", + companyId: "company-1", + title: "Launch reliable agent operations", + description: "Build a dependable automated company operating loop.", + parentId: null, + ownerAgentId: "ceo-1", + status: "active", + createdAt: EVAL_NOW, + updatedAt: EVAL_NOW, + ...overrides, + }; +} + +function createProject(overrides: Partial): ProjectRecord { + return { + id: "project-1", + companyId: "company-1", + goalId: "goal-1", + name: "Reliability hardening", + description: "Reduce operational regressions across runtime paths.", + leadAgentId: "manager-1", + status: "in_progress", + targetDate: null, + createdAt: EVAL_NOW, + updatedAt: EVAL_NOW, + ...overrides, + }; +} + +function createApproval(overrides: Partial): ApprovalRecord { + return { + id: "approval-1", + companyId: "company-1", + relatedTaskId: "task-1", + relatedAgentId: null, + requestedByAgentId: "agent-1", + type: "plan_review", + payloadSummary: "Human review plan before implementation.", + impactSummary: "Blocks execution until the co-founder approves scope.", + payloadJson: null, + decisionNote: "", + state: "pending", + createdAt: EVAL_NOW, + updatedAt: EVAL_NOW, + ...overrides, + }; +} + +const ceo = createAgent({ + id: "ceo-1", + name: "CEO", + role: "CEO", + title: "Chief Executive Officer", + roleLevel: "c_suite", + department: "executive", + reportsTo: null, + capabilities: "Lead the company", +}); +const manager = createAgent({ + id: "manager-1", + name: "VP Engineering", + role: "VP Engineering", + title: "VP Engineering", + roleLevel: "vp", + reportsTo: ceo.id, + capabilities: "Run engineering delivery", +}); +const ic = createAgent({ + id: "ic-1", + name: "Backend IC", + role: "Backend Engineer", + title: "Backend Engineer", + reportsTo: manager.id, + capabilities: "Implement backend tasks", +}); + +function buildContext(overrides: Partial): PromptContext { + return { + agent: ic, + apiUrl: "http://127.0.0.1:3000", + apiKey: "eval-token", + runId: "eval-run", + task: null, + companyName: "AgentCompany Eval", + companyDescription: "Task-level prompt eval coverage", + goals: [createGoal({})], + projects: [createProject({})], + directReports: [], + chainOfCommand: [manager, ceo], + assignedTasks: [], + allAgents: [ceo, manager, ic], + pendingApprovals: [], + tasksAwaitingReview: [], + wakeReason: "assignment", + allTasks: [], + recentComments: [], + socialAccounts: [], + allConnectors: [ + { id: "openai-apikey", label: "OpenAI API", status: "ready", enabledModels: ["gpt-5.4"] }, + { id: "anthropic-oauth", label: "Anthropic OAuth", status: "ready", enabledModels: ["claude-sonnet-4-5"] }, + ], + ...overrides, + }; +} + +const reviewTask = createTask({ + id: "task-review-1", + title: "Review /api/companies p99 fix", + description: "Review the submitted database query fix and approve or request changes.", + assigneeAgentId: ic.id, + status: "in_review", +}); + +export const TASK_LEVEL_PROMPT_EVAL_CASES: TaskPromptEvalCase[] = [ + { + caseId: "ceo-bootstrap-hiring", + taskLevel: "executive", + description: "CEO cold-start task must expose hiring mode and suppress normal operating mode.", + context: buildContext({ + agent: ceo, + runId: "eval-run-ceo-bootstrap", + task: createTask({ + id: "task-ceo-bootstrap", + title: "Bootstrap the leadership team", + description: "Create the first leadership hires and file the organization strategy.", + assigneeAgentId: ceo.id, + }), + directReports: [], + chainOfCommand: [], + allAgents: [ceo], + }), + requiredHeadings: [ + "Agent Identity", + "Your Leadership Style", + "Position", + "Active Goals", + "Operating Instructions (CEO — Team Building Phase)", + "Available Connectors", + "Current Task", + "Context", + ], + forbiddenHeadings: [ + "Operating Instructions (Executive / CEO)", + "Proactive Behaviors (Things to Do Without Being Asked)", + "Tasks Awaiting Your Review (1 pending)", + ], + requiredFragments: [ + "file 3–5 `hire_agent` tool calls", + "Current Run ID: eval-run-ceo-bootstrap", + ], + forbiddenFragments: ["Goal Decomposition"], + }, + { + caseId: "manager-review-handoff", + taskLevel: "manager", + description: "Manager review task must expose review queue, workload, and delegation guidance.", + context: buildContext({ + agent: manager, + runId: "eval-run-manager-review", + task: createTask({ + id: "task-manager-review", + title: "Review latency fix", + description: "Review the submitted p99 fix and unblock the release.", + assigneeAgentId: manager.id, + }), + directReports: [ic], + chainOfCommand: [ceo], + allAgents: [ceo, manager, ic], + allTasks: [reviewTask], + tasksAwaitingReview: [reviewTask], + recentComments: [{ + taskId: reviewTask.id, + taskTitle: reviewTask.title, + authorName: ic.name, + body: "Implemented the index-backed query path and attached benchmark output.", + createdAt: EVAL_NOW, + }], + }), + requiredHeadings: [ + "Agent Identity", + "Your Management Style", + "Your Direct Reports", + "Operating Instructions (Manager)", + "Team Workload", + "Current Task", + "Tasks Awaiting Your Review (1 pending)", + "Context", + ], + forbiddenHeadings: [ + "Operating Instructions (Individual Contributor)", + "Operating Instructions (CEO — Team Building Phase)", + ], + requiredFragments: ["1 in review", "REVIEW WORKFLOW", "Review /api/companies p99 fix"], + forbiddenFragments: ["file 3–5 `hire_agent` tool calls"], + }, + { + caseId: "ic-implementation-task", + taskLevel: "individual_contributor", + description: "IC implementation task must expose checkout, quality, and current task instructions.", + context: buildContext({ + agent: ic, + runId: "eval-run-ic-implementation", + task: createTask({ + id: "task-ic-implementation", + title: "Fix pagination boundary", + description: "Patch src/utils/paginate.ts and prove tests pass.", + assigneeAgentId: ic.id, + }), + directReports: [], + chainOfCommand: [manager, ceo], + allAgents: [ceo, manager, ic], + assignedTasks: [createTask({ + id: "task-next", + title: "Document pagination invariant", + assigneeAgentId: ic.id, + status: "todo", + })], + }), + requiredHeadings: [ + "Agent Identity", + "Your Work Philosophy", + "Operating Instructions (Individual Contributor)", + "Task Execution Workflow (SOP-002)", + "Quality Standards", + "Current Task", + "Your Assigned Tasks", + "Context", + ], + forbiddenHeadings: [ + "Operating Instructions (Manager)", + "Operating Instructions (CEO — Team Building Phase)", + "Tasks Awaiting Your Review (1 pending)", + ], + requiredFragments: ["Checkout", "set to \"in_review\"", "task-ic-implementation"], + }, + { + caseId: "cofounder-plan-review-gate", + taskLevel: "manager", + description: "AI cofounder task must expose human review gate and pending approval context.", + context: buildContext({ + agent: manager, + runId: "eval-run-cofounder-review", + executionMode: "ai_cofounder", + task: createTask({ + id: "task-cofounder-plan", + title: "Plan checkout recovery refactor", + description: "Prepare a plan before modifying task checkout recovery.", + assigneeAgentId: manager.id, + requiresUserReview: true, + }), + directReports: [ic], + chainOfCommand: [ceo], + allAgents: [ceo, manager, ic], + pendingApprovals: [createApproval({ + id: "approval-plan-1", + relatedTaskId: "task-cofounder-plan", + requestedByAgentId: manager.id, + })], + }), + requiredHeadings: [ + "Execution Mode: AI Co-Founder (Human-Reviewed)", + "Operating Instructions (Manager)", + "Current Task", + "Pending Approvals", + "Context", + ], + forbiddenHeadings: [ + "Operating Instructions (Executive / CEO)", + "Operating Instructions (CEO — Team Building Phase)", + ], + requiredFragments: [ + "Treat pending approvals as blocking", + "plan_review: Human review plan before implementation.", + "approval-plan-1", + ], + }, +]; diff --git a/tests/eval/task-level-prompt-eval.test.ts b/tests/eval/task-level-prompt-eval.test.ts new file mode 100644 index 00000000..fe15ff2c --- /dev/null +++ b/tests/eval/task-level-prompt-eval.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it, vi } from "vitest"; +import { getEncoding } from "js-tiktoken"; +import { + collectRequiredHeadingUsage, + evaluateTaskPromptCase, + runTaskPromptEvalCase, +} from "./task-level-prompt-eval.js"; +import { TASK_LEVEL_PROMPT_EVAL_CASES } from "./task-level-prompt-cases.js"; +import { buildStableSystemPrompt, buildTaskPrompt } from "../../src/main/prompt-builder/index"; + +const TEST_ENCODING = getEncoding("cl100k_base"); + +describe("task-level prompt eval harness", () => { + it.each(TASK_LEVEL_PROMPT_EVAL_CASES)( + "passes task-level prompt section contract for $caseId", + (testCase) => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-04-20T12:00:00.000Z")); + const result = runTaskPromptEvalCase(testCase); + vi.useRealTimers(); + + if (!result.passed) { + throw new Error(`${result.caseId} failed:\n${result.failures.join("\n")}`); + } + + expect(result.promptHash).toMatch(/^sha256:[a-f0-9]{64}$/); + expect(result.totalTokens).toBeGreaterThan(0); + expect(result.stablePrefixTokens).toBeGreaterThan(0); + expect(result.volatileTailTokens).toBeGreaterThan(0); + expect(result.sections.length).toBeGreaterThan(testCase.requiredHeadings.length); + for (const heading of testCase.requiredHeadings) { + const section = result.sections.find((candidate) => candidate.heading === heading); + expect(section?.tokenCount).toBeGreaterThan(0); + expect(section?.byteLength).toBeGreaterThan(0); + } + }, + ); + + it("builds the heading usage ledger that gates future prompt section removal", () => { + const usage = collectRequiredHeadingUsage(TASK_LEVEL_PROMPT_EVAL_CASES); + + expect(usage.get("Current Task")).toEqual([ + "ceo-bootstrap-hiring", + "manager-review-handoff", + "ic-implementation-task", + "cofounder-plan-review-gate", + ]); + expect(usage.get("Operating Instructions (CEO — Team Building Phase)")).toEqual([ + "ceo-bootstrap-hiring", + ]); + expect(usage.get("Tasks Awaiting Your Review (1 pending)")).toEqual([ + "manager-review-handoff", + ]); + expect(usage.get("Execution Mode: AI Co-Founder (Human-Reviewed)")).toEqual([ + "cofounder-plan-review-gate", + ]); + }); + + it("counts volatile tail tokens from the actual prompt boundary slice", () => { + const stablePrefix = "AgentCompany API\n"; + const volatileTail = "\n# Current Task"; + const prompt = `${stablePrefix}${volatileTail}`; + const result = evaluateTaskPromptCase({ + caseId: "token-boundary", + taskLevel: "individual_contributor", + description: "BPE boundary regression case.", + context: TASK_LEVEL_PROMPT_EVAL_CASES[0]!.context, + requiredHeadings: ["Current Task"], + forbiddenHeadings: [], + }, prompt, stablePrefix); + + expect(result.volatileTailTokens).toBe(TEST_ENCODING.encode(volatileTail).length); + expect(result.volatileTailTokens).not.toBe(result.totalTokens - result.stablePrefixTokens); + }); + + it("fails when CEO bootstrap prompt receives the operating proactive footer", () => { + const testCase = TASK_LEVEL_PROMPT_EVAL_CASES.find( + (candidate) => candidate.caseId === "ceo-bootstrap-hiring", + ); + if (!testCase) throw new Error("missing ceo-bootstrap-hiring eval case"); + + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-04-20T12:00:00.000Z")); + const stablePrefix = buildStableSystemPrompt(testCase.context); + const prompt = `${buildTaskPrompt(testCase.context)} +# Proactive Behaviors (Things to Do Without Being Asked) +- If a department has no active tasks: create strategic tasks.`; + const result = evaluateTaskPromptCase(testCase, prompt, stablePrefix); + vi.useRealTimers(); + + expect(result.passed).toBe(false); + expect(result.failures).toContain( + 'forbidden heading "Proactive Behaviors (Things to Do Without Being Asked)" is present', + ); + }); + + it("fails loud when a required prompt section is removed", () => { + const testCase = TASK_LEVEL_PROMPT_EVAL_CASES.find( + (candidate) => candidate.caseId === "ic-implementation-task", + ); + if (!testCase) throw new Error("missing ic-implementation-task eval case"); + + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-04-20T12:00:00.000Z")); + const stablePrefix = buildStableSystemPrompt(testCase.context); + const prompt = buildTaskPrompt(testCase.context).replace( + /^# Current Task\n[\s\S]*?(?=^# Your Assigned Tasks|^# Context)/m, + "", + ); + const result = evaluateTaskPromptCase(testCase, prompt, stablePrefix); + vi.useRealTimers(); + + expect(result.passed).toBe(false); + expect(result.failures).toContain('missing required heading "Current Task"'); + }); +}); diff --git a/tests/eval/task-level-prompt-eval.ts b/tests/eval/task-level-prompt-eval.ts new file mode 100644 index 00000000..b8a1bb36 --- /dev/null +++ b/tests/eval/task-level-prompt-eval.ts @@ -0,0 +1,140 @@ +import { createHash } from "node:crypto"; +import { getEncoding } from "js-tiktoken"; +import { + buildStableSystemPrompt, + buildTaskPrompt, + type PromptContext, +} from "../../src/main/prompt-builder/index"; + +const PROMPT_ENCODING = getEncoding("cl100k_base"); + +export interface TaskPromptEvalCase { + caseId: string; + taskLevel: "executive" | "manager" | "individual_contributor"; + description: string; + context: PromptContext; + requiredHeadings: string[]; + forbiddenHeadings: string[]; + requiredFragments?: string[]; + forbiddenFragments?: string[]; +} + +export interface PromptSectionMetric { + heading: string; + level: number; + byteLength: number; + tokenCount: number; + inStablePrefix: boolean; +} + +export interface TaskPromptEvalResult { + caseId: string; + passed: boolean; + promptHash: string; + totalTokens: number; + stablePrefixTokens: number; + volatileTailTokens: number; + sections: PromptSectionMetric[]; + failures: string[]; +} + +function tokenCount(value: string): number { + return PROMPT_ENCODING.encode(value).length; +} + +function hashPrompt(prompt: string): string { + return `sha256:${createHash("sha256").update(prompt).digest("hex")}`; +} + +export function extractPromptSections(prompt: string, stablePrefixLength: number): PromptSectionMetric[] { + const matches = Array.from(prompt.matchAll(/^(#{1,6})\s+(.+)$/gm)); + return matches.map((match, index) => { + if (match.index === undefined || match[1] === undefined || match[2] === undefined) { + throw new Error("task prompt eval: malformed markdown heading match"); + } + const start = match.index; + const next = matches[index + 1]; + const end = next?.index ?? prompt.length; + const text = prompt.slice(start, end); + return { + heading: match[2].trim(), + level: match[1].length, + byteLength: Buffer.byteLength(text, "utf8"), + tokenCount: tokenCount(text), + inStablePrefix: start < stablePrefixLength, + }; + }); +} + +export function evaluateTaskPromptCase( + testCase: TaskPromptEvalCase, + prompt: string, + stablePrefix: string, +): TaskPromptEvalResult { + const failures: string[] = []; + if (!prompt.startsWith(stablePrefix)) { + failures.push("full prompt does not begin with the stable system prefix"); + } + + const sections = extractPromptSections(prompt, stablePrefix.length); + const headings = new Set(sections.map((section) => section.heading)); + + for (const heading of testCase.requiredHeadings) { + if (!headings.has(heading)) { + failures.push(`missing required heading "${heading}"`); + } + } + + for (const heading of testCase.forbiddenHeadings) { + if (headings.has(heading)) { + failures.push(`forbidden heading "${heading}" is present`); + } + } + + for (const fragment of testCase.requiredFragments ?? []) { + if (!prompt.includes(fragment)) { + failures.push(`missing required fragment "${fragment}"`); + } + } + + for (const fragment of testCase.forbiddenFragments ?? []) { + if (prompt.includes(fragment)) { + failures.push(`forbidden fragment "${fragment}" is present`); + } + } + + const totalTokens = tokenCount(prompt); + const stablePrefixTokens = tokenCount(stablePrefix); + const volatileTailTokens = tokenCount(prompt.slice(stablePrefix.length)); + + return { + caseId: testCase.caseId, + passed: failures.length === 0, + promptHash: hashPrompt(prompt), + totalTokens, + stablePrefixTokens, + volatileTailTokens, + sections, + failures, + }; +} + +export function runTaskPromptEvalCase(testCase: TaskPromptEvalCase): TaskPromptEvalResult { + return evaluateTaskPromptCase( + testCase, + buildTaskPrompt(testCase.context), + buildStableSystemPrompt(testCase.context), + ); +} + +export function collectRequiredHeadingUsage(cases: TaskPromptEvalCase[]): Map { + const usage = new Map(); + for (const testCase of cases) { + for (const heading of testCase.requiredHeadings) { + const caseIds = usage.get(heading) ?? []; + caseIds.push(testCase.caseId); + usage.set(heading, caseIds); + } + } + return usage; +}