diff --git a/tests/eval/assertion-judge.ts b/tests/eval/assertion-judge.ts index 9ce46021..fc5a4ca6 100644 --- a/tests/eval/assertion-judge.ts +++ b/tests/eval/assertion-judge.ts @@ -134,6 +134,29 @@ function evaluateOne( reason: `no ${assertion.role} message contained substring "${assertion.substring}"`, }; } + case "message_does_not_contain_substring": { + for (const turn of turns) { + if (turn.role !== assertion.role) continue; + const content = turn.content; + if (!Array.isArray(content)) continue; + for (const block of content) { + if ( + block && + typeof block === "object" && + (block as { type?: string }).type === "text" && + typeof (block as { text?: unknown }).text === "string" && + ((block as { text: string }).text).includes(assertion.substring) + ) { + return { + assertion, + passed: false, + reason: `a ${assertion.role} message contained forbidden substring "${assertion.substring}"`, + }; + } + } + } + return { assertion, passed: true }; + } case "tool_call_arg_matches": { const matches = toolCalls.filter( (occ) => occ.toolCall.name === assertion.name, diff --git a/tests/eval/fixtures/ceo-bootstrap-empty.jsonl b/tests/eval/fixtures/ceo-bootstrap-empty.jsonl index fe2929b8..d8e3ccf8 100644 --- a/tests/eval/fixtures/ceo-bootstrap-empty.jsonl +++ b/tests/eval/fixtures/ceo-bootstrap-empty.jsonl @@ -1,4 +1,4 @@ {"type":"meta","fixtureId":"ceo-bootstrap-empty","piAgentCoreVersion":"0.67.68","recordedAt":"2026-04-20T18:30:58.219Z","scenario":"Post-C4 baseline: CEO bootstrap with hire_agent and file_approval tools available. Expected: model uses tool names (not bash/fetch/node -e) for the 3-5 hires + 1 approval.","systemPromptHash":"sha256:8125df952f7fb3b94e5e7086ce4a86938a5ef4c9d5c867feba76e480de2e95c4"} {"type":"turn","turnIndex":0,"role":"user","content":[{"type":"text","text":"Trigger: bootstrap wake. Zero active direct reports. Start Team Building now — file three to five hire_agent POSTs and one approve_ceo_strategy POST against the internal HTTP API. Do not write a plan document. Execute tool calls.\n"}]} {"type":"turn","turnIndex":1,"role":"assistant","content":[{"type":"text","text":"I'll hire a focused team of 3-5 agents to cover core company functions, then file the strategy approval.\n\n\nSarah Chen\nChief Technology Officer\nCTO\nEngineering\nanthropic-oauth\nclaude-sonnet-4-5\nCEO\nTechnical architecture, AI/ML infrastructure, engineering leadership, product development\n15000\n\n\n\nMarcus Williams\nChief Operating Officer\nCOO\nOperations\nanthropic-oauth\nclaude-sonnet-4-5\nCEO\nBusiness operations, process optimization, resource allocation, cross-functional coordination\n12000\n\n\n\nJessica Rodriguez\nChief Product Officer\nCPO\nProduct\nanthropic-oauth\nclaude-sonnet-4-5\nCEO\nProduct strategy, user research, roadmap planning, go-to-market execution\n12000\n\n\n\nDavid Park\nHead of Research\nHead of Research\nResearch\nopenai-apikey\ngpt-4o-mini\nCEO\nAI research, competitive analysis, innovation strategy, technical documentation\n8000\n\n\n\napprove_ceo_strategy\nFour C-level direct reports to CEO: Sarah Chen (CTO/Engineering), Marcus Williams (COO/Operations), Jessica Rodriguez (CPO/Product), David Park (Head of Research). Flat structure enables rapid decision-making.\nThis leadership team balances technical execution (CTO), operational efficiency (COO), market fit (CPO), and innovation (Research). Four direct reports keep communication tight while covering all critical functions for an AI agent company. Mix of Anthropic and OpenAI models optimizes cost vs capability. Total monthly budget $47K allows scaling sub-teams as needed.\n"}],"tokenLedger":{"input":509,"cache_read":0,"cache_creation":0,"output":731}} -{"type": "assertions", "assertions": [{"kind": "message_contains_substring", "role": "assistant", "substring": "hire_agent"}, {"kind": "message_contains_substring", "role": "assistant", "substring": "approve_ceo_strategy"}, {"kind": "message_contains_substring", "role": "assistant", "substring": "file_approval"}, {"kind": "trace_max_turns", "max": 3}]} +{"type": "assertions", "assertions": [{"kind": "message_contains_substring", "role": "assistant", "substring": "hire_agent"}, {"kind": "message_contains_substring", "role": "assistant", "substring": "approve_ceo_strategy"}, {"kind": "message_contains_substring", "role": "assistant", "substring": "file_approval"}, {"kind": "trace_max_turns", "max": 3}, {"kind": "message_does_not_contain_substring", "role": "assistant", "substring": "node -e"}, {"kind": "message_does_not_contain_substring", "role": "assistant", "substring": "/api/companies"}]} diff --git a/tests/eval/harness.test.ts b/tests/eval/harness.test.ts index fffa4e1c..22cd3dd3 100644 --- a/tests/eval/harness.test.ts +++ b/tests/eval/harness.test.ts @@ -131,6 +131,34 @@ describe("harness self-test (in-memory fixture)", () => { expect(reasons[2]).toContain("trace has 2 turns, max allowed is 1"); }); + it("message_does_not_contain_substring passes when substring is absent", () => { + const fixture = buildInMemoryFixture([ + { + kind: "message_does_not_contain_substring", + role: "assistant", + substring: "node -e", + }, + ]); + const result = runEval(fixture); + expect(result.passed).toBe(true); + expect(result.assertions[0]?.passed).toBe(true); + }); + + it("message_does_not_contain_substring fails when substring is present", () => { + const fixture = buildInMemoryFixture([ + { + kind: "message_does_not_contain_substring", + role: "assistant", + substring: "Hiring a writer", + }, + ]); + const result = runEval(fixture); + expect(result.passed).toBe(false); + expect(result.assertions[0]?.reason).toContain( + 'a assistant message contained forbidden substring "Hiring a writer"', + ); + }); + it("judgeAssertions is a pure function of (assertions, turns)", () => { const fixture = buildInMemoryFixture([]); const first = judgeAssertions( diff --git a/tests/eval/types.ts b/tests/eval/types.ts index 90b825ac..9749871d 100644 --- a/tests/eval/types.ts +++ b/tests/eval/types.ts @@ -107,6 +107,17 @@ export type Assertion = role: "assistant" | "user"; substring: string; } + | { + /** + * Negative substring check. Fails when *any* message of `role` contains + * `substring`; used to pin post-refactor invariants where a particular + * pattern must disappear from the trace (for example, asserting that a + * CEO bootstrap no longer emits `node -e` after C4). + */ + kind: "message_does_not_contain_substring"; + role: "assistant" | "user"; + substring: string; + } | { kind: "tool_call_arg_matches"; name: string;