From 7e384145916e107e6cbae1ad824d79dc9d26604f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Thu, 8 Oct 2026 18:15:59 -0400 Subject: [PATCH] Q112: a goal auditor with approved file access builds and tests in a throwaway copy Co-Authored-By: Claude Opus 5.5 --- src/main/services/agent-goal-tools.ts | 44 +++++++++++++++--- tests/unit/agent-goal-tools.test.ts | 64 +++++++++++++++++++++++++-- 2 files changed, 97 insertions(+), 11 deletions(-) diff --git a/src/main/services/agent-goal-tools.ts b/src/main/services/agent-goal-tools.ts index 58166d9d..d7daf7c1 100644 --- a/src/main/services/agent-goal-tools.ts +++ b/src/main/services/agent-goal-tools.ts @@ -1,3 +1,6 @@ +import { cpSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { fitContextBudget } from "./context-budget"; import { AuditService } from "./audit-service"; import { executeWorkspaceTool, WORKSPACE_TOOLS } from "./agent-workspace-tools"; @@ -124,16 +127,43 @@ async function auditGoalCompletion( goal: { title: string; description: string }, evidence: string, ): Promise { - // Read-only inspection of the workspace and the library: the auditor may look at anything but change nothing. + const workspace = args.ctx.workspace; + // With the operator's file-access approval the auditor may build and test, but only in a throwaway copy. + const scratch = workspace?.writeAccess ? copyWorkspace(workspace.localPath) : null; + try { + const auditWorkspace = workspace ? { ...workspace, localPath: scratch ?? workspace.localPath, writeAccess: scratch !== null } : null; + return await runAudit(args, goal, evidence, auditWorkspace); + } finally { + if (scratch) rmSync(scratch, { recursive: true, force: true }); + } +} + +function copyWorkspace(root: string): string { + const scratch = mkdtempSync(join(tmpdir(), "ac-audit-")); + try { + cpSync(root, scratch, { recursive: true }); + } catch (error) { + rmSync(scratch, { recursive: true, force: true }); + throw error; + } + console.info(`[goal-audit] auditing in a copy of ${root} at ${scratch}`); + return scratch; +} + +async function runAudit( + args: GoalCompletionArgs, + goal: { title: string; description: string }, + evidence: string, + workspace: AgentToolContext["workspace"], +): Promise { + const canRun = workspace?.writeAccess === true; + // The workspace (or its throwaway copy) and the library; nothing the auditor does reaches the real work. const auditTools = [ - ...WORKSPACE_TOOLS.filter((tool) => tool.name !== "run_command"), + ...WORKSPACE_TOOLS.filter((tool) => canRun || tool.name !== "run_command"), ...DOCUMENT_TOOLS.filter((tool) => AUDIT_DOCUMENT_TOOLS.has(tool.name)), ]; let evidenceRead = false; - const auditCtx: AgentToolContext = { - ...args.ctx, - workspace: args.ctx.workspace ? { ...args.ctx.workspace, writeAccess: false } : null, - }; + const auditCtx: AgentToolContext = { ...args.ctx, workspace }; const messages: AgentTurnMessage[] = [{ role: "user", text: [ @@ -142,7 +172,7 @@ async function auditGoalCompletion( `Goal detail: ${goal.description}`, "The executor's claim:", evidence, - "Inspect the work before deciding: list_files and read_file for the workspace, list_documents and read_document for the library. An approval counts only after you have read some of it. Judge outcomes, not artifacts about outcomes: plans, specs, status reports, and skeletons do not make a goal achieved; working deliverables do. If the goal promises software, look for the actual implementation, not documents describing it.", + `Inspect the work before deciding: list_files and read_file for the workspace, list_documents and read_document for the library. An approval counts only after you have read some of it. ${canRun ? "run_command runs in a throwaway copy of the workspace: build and run the tests yourself instead of trusting claims that they pass; nothing you do there reaches the real workspace." : "You cannot run commands here, so a claim that something was run or passes is unverified unless the workspace holds its results."} Judge outcomes, not artifacts about outcomes: plans, specs, status reports, and skeletons do not make a goal achieved; working deliverables do. If the goal promises software, look for the actual implementation, not documents describing it.`, "End your final message with exactly one marker: if the goal is genuinely achieved, or followed by one short sentence naming the decisive gap.", ].join("\n\n"), }]; diff --git a/tests/unit/agent-goal-tools.test.ts b/tests/unit/agent-goal-tools.test.ts index 6ed55895..d1725f98 100644 --- a/tests/unit/agent-goal-tools.test.ts +++ b/tests/unit/agent-goal-tools.test.ts @@ -1,5 +1,5 @@ import { DocumentService } from "../../src/main/services/document-service"; -import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import { randomUUID } from "node:crypto"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -23,6 +23,7 @@ function turnOf(text: string, toolCalls: ToolCall[] = []): AuditorTurn { const workspaceRoot = join(tmpdir(), `ac-goal-audit-${randomUUID()}`); mkdirSync(join(workspaceRoot, "src"), { recursive: true }); writeFileSync(join(workspaceRoot, "src", "index.ts"), "export const app = () => 'running';\n"); +writeFileSync(join(workspaceRoot, "check.js"), "require('fs').writeFileSync('built.txt', 'x'); console.log('all 3 tests passed'); console.log('cwd=' + process.cwd());\n"); afterAll(() => { rmSync(workspaceRoot, { recursive: true, force: true }); @@ -82,7 +83,7 @@ function goalStatus(h: Harness): string { } describe("complete_goal with independent audit", () => { - test("an approving auditor closes the goal; the auditor gets read-only workspace access", async () => { + test("an approving auditor closes the goal; without approved file access the auditor only reads", async () => { const h = createHarness(); const seenTools: string[][] = []; const replay = { format: "anthropic" as const, content: [{ type: "thinking", thinking: "check", signature: "SIG" }] }; @@ -99,15 +100,70 @@ describe("complete_goal with independent audit", () => { return turnOf("The implementation exists and runs. "); }; - const output = await completeGoalTool(argsFor(h, h.ceoId, auditor, call({ evidence: "src/index.ts implements the app and runs." }))); + const readOnly = argsFor(h, h.ceoId, auditor, call({ evidence: "src/index.ts implements the app and runs." })); + readOnly.ctx.workspace = { ...readOnly.ctx.workspace!, writeAccess: false }; + const output = await completeGoalTool(readOnly); expect(output).toContain("achieved"); expect(goalStatus(h)).toBe("achieved"); - // The auditor never receives run_command — inspection is read-only. + // Without the operator's file-access approval the auditor gets no run_command. expect(seenTools.every((tools) => !tools.includes("run_command"))).toBe(true); expect(seenTools[0]).toContain("read_file"); h.connection.close(); }); + test("with approved file access the auditor builds and tests in a throwaway copy of the workspace", async () => { + const h = createHarness(); + let turns = 0; + let commandOutput = ""; + let ranIn = ""; + const auditor = async (input: AuditorInput): Promise => { + turns += 1; + if (turns === 1) { + expect(input.tools.map((tool) => tool.name)).toContain("run_command"); + return turnOf("Running the tests.", [{ id: "c1", name: "run_command", arguments: { command: "node check.js" } }, { id: "c2", name: "list_files", arguments: {} }]); + } + if (turns === 2) { + const results = (input.messages.at(-1) as { results: Array<{ output: string }> }).results; + commandOutput = results[0].output; + ranIn = results[1].output; + return turnOf("Reading.", [{ id: "c3", name: "read_file", arguments: { path: "built.txt" } }]); + } + return turnOf("Tests pass when run. "); + }; + const output = await completeGoalTool(argsFor(h, h.ceoId, auditor, call({ evidence: "All tests pass." }))); + expect(output).toContain("achieved"); + expect(commandOutput).toContain("all 3 tests passed"); + expect(ranIn).toContain("built.txt"); + // The real workspace is untouched, and no copy is left behind. + expect(existsSync(join(workspaceRoot, "built.txt"))).toBe(false); + const copy = /cwd=(.+)/.exec(commandOutput)![1].trim(); + expect(copy).not.toBe(workspaceRoot); + expect(existsSync(copy)).toBe(false); + h.connection.close(); + }); + + test("a workspace that cannot be copied fails the audit and leaves no copy behind", async () => { + const h = createHarness(); + const args = argsFor(h, h.ceoId, async () => turnOf(""), call({ evidence: "done" })); + args.ctx.workspace = { ...args.ctx.workspace!, localPath: join(tmpdir(), `ac-gone-${randomUUID()}`) }; + // A private temp folder for this audit, so the check sees only its own copies. + const base = mkdtempSync(join(tmpdir(), "ac-tmpbase-")); + const saved = { TMP: process.env.TMP, TEMP: process.env.TEMP, TMPDIR: process.env.TMPDIR }; + process.env.TMP = process.env.TEMP = process.env.TMPDIR = base; + try { + expect(await completeGoalTool(args)).toMatch(/^ERROR: /); + } finally { + for (const [name, value] of Object.entries(saved)) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + } + expect(goalStatus(h)).toBe("active"); + expect(readdirSync(base)).toEqual([]); + rmSync(base, { recursive: true, force: true }); + h.connection.close(); + }); + test("a disapproving auditor keeps the goal open and returns the gap to the caller", async () => { const h = createHarness(); const auditor = async (): Promise => turnOf(" Only planning documents exist; there is no implementation.");