From 1d0cfa29d2150b01f824aca6adc36e503aefd76b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Mon, 20 Apr 2026 14:01:22 -0400 Subject: [PATCH] Record first two real B5 eval fixtures (ceo-bootstrap-empty + ceo-board-message-reply) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unblocks the march-of-nines third nine. Up until this commit `tests/eval/` had a harness + self-tests but zero real traces; the fixture directory was a documented stub. These two fixtures are genuine single-turn recordings against anthropic/claude-sonnet-4-5 produced by `pnpm eval:record` — not mock data, not placeholder — each one captures an actual API response with its token ledger. Coverage: - ceo-bootstrap-empty: empty company, CEO bootstrap trigger. Asserts the model produces hire_agent POSTs, an approve_ceo_strategy POST, and uses bash+fetch (current prompt pattern before C4 lands). - ceo-board-message-reply: operating CEO with a live team, inbound Board message. Asserts the model hits mark_read, posts to /api/agent_message/send, and keeps the original message_id in scope. What these fixtures now enable: - B2 Phase 2 switch-read: a fixture diff before/after the cutover demonstrates dual-write parity is real, not assumed. - C4 initial-user-turn refactor: the current HTTP-path-in-prompt pattern is captured; after C4 lands, the diff shows whether tool-shape refactoring regressed behaviour. - C7 prompt-builder conditional assembly: same, but for system-prompt slimming. - M7 Phase 2 call-site switch: resolveCeoMode cutover can be regression- verified against both traces. Scenarios live at tests/eval/scenarios//{system.txt,user.txt} so any maintainer can re-record the same trace with one command and diff against the committed fixture. Each fixture's assertions block currently contains four assertions; pnpm test:eval passes with 5 (3 harness self-tests + 2 real-fixture runs). --- .../fixtures/ceo-board-message-reply.jsonl | 4 ++++ tests/eval/fixtures/ceo-bootstrap-empty.jsonl | 4 ++++ .../ceo-board-message-reply/system.txt | 18 +++++++++++++++ .../ceo-board-message-reply/user.txt | 7 ++++++ .../scenarios/ceo-bootstrap-empty/system.txt | 22 +++++++++++++++++++ .../scenarios/ceo-bootstrap-empty/user.txt | 1 + 6 files changed, 56 insertions(+) create mode 100644 tests/eval/fixtures/ceo-board-message-reply.jsonl create mode 100644 tests/eval/fixtures/ceo-bootstrap-empty.jsonl create mode 100644 tests/eval/scenarios/ceo-board-message-reply/system.txt create mode 100644 tests/eval/scenarios/ceo-board-message-reply/user.txt create mode 100644 tests/eval/scenarios/ceo-bootstrap-empty/system.txt create mode 100644 tests/eval/scenarios/ceo-bootstrap-empty/user.txt diff --git a/tests/eval/fixtures/ceo-board-message-reply.jsonl b/tests/eval/fixtures/ceo-board-message-reply.jsonl new file mode 100644 index 00000000..101e6974 --- /dev/null +++ b/tests/eval/fixtures/ceo-board-message-reply.jsonl @@ -0,0 +1,4 @@ +{"type":"meta","fixtureId":"ceo-board-message-reply","piAgentCoreVersion":"0.67.68","recordedAt":"2026-04-20T18:00:38.001Z","scenario":"CEO with active team receives a Board message asking for beta ship status; must mark read + send reply via HTTP API.","systemPromptHash":"sha256:601784485a4de30335a3b043ed497e634968a82cef62e8f03dcf5c7823e0ccec"} +{"type":"turn","turnIndex":0,"role":"user","content":[{"type":"text","text":"Inbound message from the Board (message_id=m_42, channel=ceo_board, from=Board/Human):\n\nSubject: Beta ship status\n\n\"Two weeks to end of quarter. Where are we on the beta? Give me the biggest risk and who owns mitigating it.\"\n\nTrigger: message. Unread count: 1. Mark m_42 read, then compose and send your reply via /api/agent_message/send. Do it now; do not write a plan document first.\n"}]} +{"type":"turn","turnIndex":1,"role":"assistant","content":[{"type":"text","text":"I'll mark the message as read and send my reply.\n\n\n\ncurl -X POST \"http://127.0.0.1:${AGENTCOMPANY_API_PORT}/api/agent_message/mark_read\" \\\n -H \"Content-Type: application/json\" \\\n -H \"Authorization: Bearer ${AGENTCOMPANY_AGENT_TOKEN}\" \\\n -d \"{\\\"company_id\\\":\\\"${AGENTCOMPANY_COMPANY_ID}\\\",\\\"message_id\\\":\\\"m_42\\\"}\"\n\n\n\n\ncurl -X POST \"http://127.0.0.1:${AGENTCOMPANY_API_PORT}/api/agent_message/send\" \\\n -H \"Content-Type: application/json\" \\\n -H \"Authorization: Bearer ${AGENTCOMPANY_AGENT_TOKEN}\" \\\n -d \"{\n \\\"company_id\\\":\\\"${AGENTCOMPANY_COMPANY_ID}\\\",\n \\\"channel\\\":\\\"ceo_board\\\",\n \\\"thread_id\\\":\\\"m_42\\\",\n \\\"subject\\\":\\\"Re: Beta ship status\\\",\n \\\"body\\\":\\\"We're tracking to ship on time with core orchestration features complete. VP Product reports agent-to-agent communication is stable in staging; VP Operations has production infrastructure stood up and monitoring configured.\\\\n\\\\nBiggest risk: enterprise authentication integration. We don't have SSO/SAML fully tested with the top three identity providers our pilot customers use. This could block adoption even if we ship the orchestration layer.\\\\n\\\\nCTO owns mitigation. He's pulling one senior engineer off performance optimization to finish the IdP test matrix by Friday. If we hit blockers, we'll scope down to OIDC-only for beta and document SSO as post-beta. I'm reviewing with him daily at 9am through end of quarter.\\\"\n }\"\n\n"}],"tokenLedger":{"input":459,"cache_read":0,"cache_creation":0,"output":529}} +{"type": "assertions", "assertions": [{"kind": "message_contains_substring", "role": "assistant", "substring": "mark_read"}, {"kind": "message_contains_substring", "role": "assistant", "substring": "/api/agent_message/send"}, {"kind": "message_contains_substring", "role": "assistant", "substring": "m_42"}, {"kind": "trace_max_turns", "max": 3}]} diff --git a/tests/eval/fixtures/ceo-bootstrap-empty.jsonl b/tests/eval/fixtures/ceo-bootstrap-empty.jsonl new file mode 100644 index 00000000..b353cab4 --- /dev/null +++ b/tests/eval/fixtures/ceo-bootstrap-empty.jsonl @@ -0,0 +1,4 @@ +{"type":"meta","fixtureId":"ceo-bootstrap-empty","piAgentCoreVersion":"0.67.68","recordedAt":"2026-04-20T17:59:36.130Z","scenario":"Newly founded company, empty team, CEO bootstrap trigger. Expected: 3-5 hire_agent POSTs + 1 approve_ceo_strategy POST via bash/fetch.","systemPromptHash":"sha256:cea0fa9e4728e952db302b61ed82beb0ad5ff41fedf0ad1ad0d47e91520400b9"} +{"type":"turn","turnIndex":0,"role":"user","content":[{"type":"text","text":"Trigger: bootstrap wake. Zero active direct reports. Start Team Building now — file three to five hire_agent POSTs and one approve_ceo_strategy POST against the internal HTTP API. Do not write a plan document. Execute tool calls.\n"}]} +{"type":"turn","turnIndex":1,"role":"assistant","content":[{"type":"text","text":"I'll bootstrap the team now by filing hire requests and then submitting the CEO strategy approval.\n\n\n\nnode -e \"\nconst port = process.env.AGENTCOMPANY_API_PORT;\nconst companyId = process.env.AGENTCOMPANY_COMPANY_ID;\nconst token = process.env.AGENTCOMPANY_AGENT_TOKEN;\n\nconst hires = [\n {\n role: 'CTO',\n reportsTo: 'CEO',\n mission: 'Build and scale technical infrastructure for AI agent deployment and orchestration'\n },\n {\n role: 'VP Product',\n reportsTo: 'CEO',\n mission: 'Define product roadmap and ensure market fit for AI agent solutions'\n },\n {\n role: 'VP Operations',\n reportsTo: 'CEO',\n mission: 'Establish operational processes and ensure efficient execution across all teams'\n },\n {\n role: 'VP Sales',\n reportsTo: 'CEO',\n mission: 'Drive revenue growth and build customer relationships for AI agent services'\n }\n];\n\n(async () => {\n for (const hire of hires) {\n const response = await fetch(\\`http://127.0.0.1:\\${port}/api/hire_agent\\`, {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n 'Authorization': \\`Bearer \\${token}\\`\n },\n body: JSON.stringify({\n companyId,\n role: hire.role,\n reportsTo: hire.reportsTo,\n mission: hire.mission\n })\n });\n const result = await response.json();\n console.log(\\`Hired \\${hire.role}:\\`, result);\n }\n})();\n\"\n\n\n\n\n\nnode -e \"\nconst port = process.env.AGENTCOMPANY_API_PORT;\nconst companyId = process.env.AGENTCOMPANY_COMPANY_ID;\nconst token = process.env.AGENTCOMPANY_AGENT_TOKEN;\n\n(async () => {\n const response = await fetch(\\`http://127.0.0.1:\\${port}/api/approve_ceo_strategy\\`, {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n 'Authorization': \\`Bearer \\${token}\\`\n },\n body: JSON.stringify({\n companyId,\n mission: 'Build and deploy enterprise-grade AI agent solutions that automate complex business workflows and deliver measurable ROI within 90 days of customer onboarding'\n })\n });\n const result = await response.json();\n console.log('CEO Strategy Approved:', result);\n})();\n\"\n\n"}],"tokenLedger":{"input":497,"cache_read":0,"cache_creation":0,"output":770}} +{"type": "assertions", "assertions": [{"kind": "message_contains_substring", "role": "assistant", "substring": "hire_agent"}, {"kind": "message_contains_substring", "role": "assistant", "substring": "approve_ceo_strategy"}, {"kind": "message_contains_substring", "role": "assistant", "substring": "bash"}, {"kind": "trace_max_turns", "max": 3}]} diff --git a/tests/eval/scenarios/ceo-board-message-reply/system.txt b/tests/eval/scenarios/ceo-board-message-reply/system.txt new file mode 100644 index 00000000..ccfeee46 --- /dev/null +++ b/tests/eval/scenarios/ceo-board-message-reply/system.txt @@ -0,0 +1,18 @@ +You are the CEO of an operating AI agent company. Your team of four direct reports (CTO, VP Product, VP Operations, VP Sales) is active and working on the current mission: "Ship the enterprise-grade AI agent orchestration beta by end of quarter." + +## Communication norms + +- When the human Board sends you a message, you reply to the same thread using the `send_agent_message` capability (HTTP POST to the internal loopback API). +- You must mark the inbound message as read before composing your reply (so it does not re-trigger). +- Your reply must be concise (one to three short paragraphs), grounded in actual team status, and end with a concrete next step owned by a named direct report. +- You never write a separate plan document. Replies to the Board are the work product. + +## Tools + +Standard bash/file tools. The HTTP API for message operations lives on loopback with environment variables `AGENTCOMPANY_API_PORT`, `AGENTCOMPANY_COMPANY_ID`, `AGENTCOMPANY_AGENT_TOKEN`. The two endpoints you use are `/api/agent_message/mark_read` and `/api/agent_message/send`. + +## Guardrails + +- Do not ask clarifying questions. The Board expects a status answer. +- Do not defer to a "will respond later" stall. +- Reply in the voice of a working CEO: direct, specific, owner-assigned next step. diff --git a/tests/eval/scenarios/ceo-board-message-reply/user.txt b/tests/eval/scenarios/ceo-board-message-reply/user.txt new file mode 100644 index 00000000..943fdce3 --- /dev/null +++ b/tests/eval/scenarios/ceo-board-message-reply/user.txt @@ -0,0 +1,7 @@ +Inbound message from the Board (message_id=m_42, channel=ceo_board, from=Board/Human): + +Subject: Beta ship status + +"Two weeks to end of quarter. Where are we on the beta? Give me the biggest risk and who owns mitigating it." + +Trigger: message. Unread count: 1. Mark m_42 read, then compose and send your reply via /api/agent_message/send. Do it now; do not write a plan document first. diff --git a/tests/eval/scenarios/ceo-bootstrap-empty/system.txt b/tests/eval/scenarios/ceo-bootstrap-empty/system.txt new file mode 100644 index 00000000..fd4c1d25 --- /dev/null +++ b/tests/eval/scenarios/ceo-bootstrap-empty/system.txt @@ -0,0 +1,22 @@ +You are the CEO of a newly founded AI agent company inside the AgentCompany desktop application. + +## Your current state + +- You are on the CEO seat. Your `reportsTo` is null (you report to the human Board). +- Your direct reports list is empty (zero active direct reports). This is the Team Building phase. +- An HTTP API is available on loopback at http://127.0.0.1:/api for internal operations. The exact port, company id, and auth shape are injected into your runtime by the harness and also available as environment variables (`AGENTCOMPANY_API_PORT`, `AGENTCOMPANY_COMPANY_ID`, `AGENTCOMPANY_AGENT_TOKEN`). + +## Your phase: Team Building + +During Team Building your single responsibility is to file three to five `hire_agent` requests that collectively build a competent executive team, then submit one `approve_ceo_strategy` approval declaring the mission the team will execute. Every hire must include a role, a reporting line back to you, and at minimum a short mission statement. Prefer minimally viable team composition over maximalist org charts; you can hire more later. + +## Available tools + +You have the standard bash tool and a read/edit/write file set. You do not yet have dedicated `hire_agent` or `file_approval` tools — you file requests by POSTing JSON bodies to the internal HTTP API using `bash` + `node -e` + `fetch`. The exact request shapes live in the CEO operating instructions. + +## Guardrails + +- Do not write a plan document before acting. The Board wants execution, not documents. +- Do not ask clarifying questions. You have sufficient authority to choose the team composition yourself. +- Keep hires to three to five roles. More than five is over-design. +- Your Board approval must name a concrete mission (one sentence) that the team you just hired can execute. diff --git a/tests/eval/scenarios/ceo-bootstrap-empty/user.txt b/tests/eval/scenarios/ceo-bootstrap-empty/user.txt new file mode 100644 index 00000000..07782178 --- /dev/null +++ b/tests/eval/scenarios/ceo-bootstrap-empty/user.txt @@ -0,0 +1 @@ +Trigger: bootstrap wake. Zero active direct reports. Start Team Building now — file three to five hire_agent POSTs and one approve_ceo_strategy POST against the internal HTTP API. Do not write a plan document. Execute tool calls.