From 0ae014af05b957658e4b9d997c7a3470ee90ff36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Thu, 8 Oct 2026 18:21:46 -0400 Subject: [PATCH] Q113: a goal audit approves only with the decision alone on its last line Co-Authored-By: Claude Opus 5.5 --- src/main/services/agent-goal-tools.ts | 12 ++++++---- tests/unit/agent-goal-tools.test.ts | 32 +++++++++++++++++++++++---- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/main/services/agent-goal-tools.ts b/src/main/services/agent-goal-tools.ts index d7daf7c1..e64e1b42 100644 --- a/src/main/services/agent-goal-tools.ts +++ b/src/main/services/agent-goal-tools.ts @@ -173,7 +173,7 @@ async function runAudit( "The executor's claim:", evidence, `Inspect the work before deciding: list_files and read_file for the workspace, list_documents and read_document for the library. An approval counts only after you have read some of it. ${canRun ? "run_command runs in a throwaway copy of the workspace: build and run the tests yourself instead of trusting claims that they pass; nothing you do there reaches the real workspace." : "You cannot run commands here, so a claim that something was run or passes is unverified unless the workspace holds its results."} Judge outcomes, not artifacts about outcomes: plans, specs, status reports, and skeletons do not make a goal achieved; working deliverables do. If the goal promises software, look for the actual implementation, not documents describing it.`, - "End your final message with exactly one marker: if the goal is genuinely achieved, or followed by one short sentence naming the decisive gap.", + "Keep your reasoning apart from your decision. End your final message with the decision alone on its last line: by itself if the goal is genuinely achieved, or followed by one short sentence naming the decisive gap. Use no marker anywhere else.", ].join("\n\n"), }]; const instructions = "You are the independent completion auditor. You did not do the work and owe its author nothing; your only loyalty is to what actually exists. Be strict: an unverifiable claim is a false claim."; @@ -225,10 +225,14 @@ async function runAuditTool(deps: AgentToolDeps, ctx: AgentToolContext, call: To } } +/** Approved only when the last line is alone and no other marker appears; quoted, indented or hedged markers do not count. */ function parseVerdict(text: string): AuditVerdict { - const reason = text.replace(/<\/?(approved|disapproved)\s*\/?>/gi, " ").replace(/\s+/g, " ").trim(); - if (//i.test(text)) return { approved: true, reason: reason || "no gaps found" }; - return { approved: false, reason: reason || "the auditor did not state an explicit approval" }; + const markers = text.match(/<\/?\s*(approved|disapproved)\s*\/?>/gi) ?? []; + const reason = text.replace(/<\/?\s*(approved|disapproved)\s*\/?>/gi, " ").replace(/\s+/g, " ").trim(); + const lastLine = text.split("\n").map((line) => line.trimEnd()).filter(Boolean).at(-1); + if (markers.length === 1 && lastLine === "") return { approved: true, reason: reason || "no gaps found" }; + if (markers.some((marker) => /disapproved/i.test(marker))) return { approved: false, reason: reason || "the auditor disapproved without a reason" }; + return { approved: false, reason: reason ? `${reason} (no alone on the last line)` : "the auditor did not end with alone on the last line" }; } function actorIsCeoOrGoalOwner(deps: AgentToolDeps, ctx: AgentToolContext, ownerAgentId: string | null): boolean { diff --git a/tests/unit/agent-goal-tools.test.ts b/tests/unit/agent-goal-tools.test.ts index d1725f98..14928947 100644 --- a/tests/unit/agent-goal-tools.test.ts +++ b/tests/unit/agent-goal-tools.test.ts @@ -97,7 +97,7 @@ describe("complete_goal with independent audit", () => { const readResult = input.messages.at(-1); expect(readResult?.role).toBe("tool"); expect(input.messages.at(-2)).toMatchObject({ role: "assistant", replay }); - return turnOf("The implementation exists and runs. "); + return turnOf("The implementation exists and runs.\n"); }; const readOnly = argsFor(h, h.ceoId, auditor, call({ evidence: "src/index.ts implements the app and runs." })); @@ -128,7 +128,7 @@ describe("complete_goal with independent audit", () => { ranIn = results[1].output; return turnOf("Reading.", [{ id: "c3", name: "read_file", arguments: { path: "built.txt" } }]); } - return turnOf("Tests pass when run. "); + return turnOf("Tests pass when run.\n"); }; const output = await completeGoalTool(argsFor(h, h.ceoId, auditor, call({ evidence: "All tests pass." }))); expect(output).toContain("achieved"); @@ -199,7 +199,7 @@ describe("complete_goal with independent audit", () => { const auditor = async (input: AuditorInput): Promise => { const last = input.messages.at(-1); prompts.push(last && "text" in last ? String(last.text) : ""); - return turnOf("Looks done. "); + return turnOf("Looks done.\n"); }; const output = await completeGoalTool(argsFor(h, h.ceoId, auditor, call({ evidence: "Trust me." }))); expect(output).toContain("audit rejected"); @@ -221,7 +221,7 @@ describe("complete_goal with independent audit", () => { turnOf("History only.", [{ id: "c3", name: "read_document", arguments: { doc_id: docId, history: true } }]), turnOf(""), turnOf("Reading the report.", [{ id: "c2", name: "read_document", arguments: { doc_id: docId } }]), - turnOf("The report shows customers. "), + turnOf("The report shows customers.\n"), ]; const seen: string[][] = []; const histories: AuditorInput["messages"][] = []; @@ -240,6 +240,30 @@ describe("complete_goal with independent audit", () => { h.connection.close(); }); + test("only a final message that ends in a single approval marker closes the goal", async () => { + const verdicts: Array<[string, string]> = [ + [" on style, but the tests fail.", "active"], + ["I would write if the tests passed. They do not.", "active"], + ["Everything checks out: ``", "active"], + [" ", "active"], + ["The example approval token is:\n> ", "active"], + ["Verified. ", "active"], + ["Example:\n ", "active"], + ["The build and tests pass.\n\n", "achieved"], + ]; + for (const [verdict, expected] of verdicts) { + const h = createHarness(); + let turns = 0; + const auditor = async (): Promise => { + turns += 1; + return turns === 1 ? turnOf("Reading.", [{ id: "c1", name: "read_file", arguments: { path: "src/index.ts" } }]) : turnOf(verdict); + }; + await completeGoalTool(argsFor(h, h.ceoId, auditor, call({ evidence: "src/index.ts works" }))); + expect([verdict, goalStatus(h)]).toEqual([verdict, expected]); + h.connection.close(); + } + }); + test("no explicit approval means not achieved", async () => { const h = createHarness(); const auditor = async (): Promise => turnOf("Looks plausible I guess.");