Q112: a goal auditor with approved file access builds and tests in a throwaway copy

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/main/services/agent-goal-tools.ts

@@ -1,3 +1,6 @@
+import { cpSync, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { fitContextBudget } from "./context-budget";
 import { AuditService } from "./audit-service";
 import { executeWorkspaceTool, WORKSPACE_TOOLS } from "./agent-workspace-tools";
@@ -124,16 +127,43 @@ async function auditGoalCompletion(
   goal: { title: string; description: string },
   evidence: string,
 ): Promise<AuditVerdict> {
-  // Read-only inspection of the workspace and the library: the auditor may look at anything but change nothing.
+  const workspace = args.ctx.workspace;
+  // With the operator's file-access approval the auditor may build and test, but only in a throwaway copy.
+  const scratch = workspace?.writeAccess ? copyWorkspace(workspace.localPath) : null;
+  try {
+    const auditWorkspace = workspace ? { ...workspace, localPath: scratch ?? workspace.localPath, writeAccess: scratch !== null } : null;
+    return await runAudit(args, goal, evidence, auditWorkspace);
+  } finally {
+    if (scratch) rmSync(scratch, { recursive: true, force: true });
+  }
+}
+
+function copyWorkspace(root: string): string {
+  const scratch = mkdtempSync(join(tmpdir(), "ac-audit-"));
+  try {
+    cpSync(root, scratch, { recursive: true });
+  } catch (error) {
+    rmSync(scratch, { recursive: true, force: true });
+    throw error;
+  }
+  console.info(`[goal-audit] auditing in a copy of ${root} at ${scratch}`);
+  return scratch;
+}
+
+async function runAudit(
+  args: GoalCompletionArgs,
+  goal: { title: string; description: string },
+  evidence: string,
+  workspace: AgentToolContext["workspace"],
+): Promise<AuditVerdict> {
+  const canRun = workspace?.writeAccess === true;
+  // The workspace (or its throwaway copy) and the library; nothing the auditor does reaches the real work.
   const auditTools = [
-    ...WORKSPACE_TOOLS.filter((tool) => tool.name !== "run_command"),
+    ...WORKSPACE_TOOLS.filter((tool) => canRun || tool.name !== "run_command"),
     ...DOCUMENT_TOOLS.filter((tool) => AUDIT_DOCUMENT_TOOLS.has(tool.name)),
   ];
   let evidenceRead = false;
-  const auditCtx: AgentToolContext = {
-    ...args.ctx,
-    workspace: args.ctx.workspace ? { ...args.ctx.workspace, writeAccess: false } : null,
-  };
+  const auditCtx: AgentToolContext = { ...args.ctx, workspace };
   const messages: AgentTurnMessage[] = [{
     role: "user",
     text: [
@@ -142,7 +172,7 @@ async function auditGoalCompletion(
       `Goal detail: ${goal.description}`,
       "The executor's claim:",
       evidence,
-      "Inspect the work before deciding: list_files and read_file for the workspace, list_documents and read_document for the library. An approval counts only after you have read some of it. Judge outcomes, not artifacts about outcomes: plans, specs, status reports, and skeletons do not make a goal achieved; working deliverables do. If the goal promises software, look for the actual implementation, not documents describing it.",
+      `Inspect the work before deciding: list_files and read_file for the workspace, list_documents and read_document for the library. An approval counts only after you have read some of it. ${canRun ? "run_command runs in a throwaway copy of the workspace: build and run the tests yourself instead of trusting claims that they pass; nothing you do there reaches the real workspace." : "You cannot run commands here, so a claim that something was run or passes is unverified unless the workspace holds its results."} Judge outcomes, not artifacts about outcomes: plans, specs, status reports, and skeletons do not make a goal achieved; working deliverables do. If the goal promises software, look for the actual implementation, not documents describing it.`,
       "End your final message with exactly one marker: <approved/> if the goal is genuinely achieved, or <disapproved/> followed by one short sentence naming the decisive gap.",
     ].join("\n\n"),
   }];

tests/unit/agent-goal-tools.test.ts

@@ -1,5 +1,5 @@
 import { DocumentService } from "../../src/main/services/document-service";
-import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
 import { randomUUID } from "node:crypto";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
@@ -23,6 +23,7 @@ function turnOf(text: string, toolCalls: ToolCall[] = []): AuditorTurn {
 const workspaceRoot = join(tmpdir(), `ac-goal-audit-${randomUUID()}`);
 mkdirSync(join(workspaceRoot, "src"), { recursive: true });
 writeFileSync(join(workspaceRoot, "src", "index.ts"), "export const app = () => 'running';\n");
+writeFileSync(join(workspaceRoot, "check.js"), "require('fs').writeFileSync('built.txt', 'x'); console.log('all 3 tests passed'); console.log('cwd=' + process.cwd());\n");
 
 afterAll(() => {
   rmSync(workspaceRoot, { recursive: true, force: true });
@@ -82,7 +83,7 @@ function goalStatus(h: Harness): string {
 }
 
 describe("complete_goal with independent audit", () => {
-  test("an approving auditor closes the goal; the auditor gets read-only workspace access", async () => {
+  test("an approving auditor closes the goal; without approved file access the auditor only reads", async () => {
     const h = createHarness();
     const seenTools: string[][] = [];
     const replay = { format: "anthropic" as const, content: [{ type: "thinking", thinking: "check", signature: "SIG" }] };
@@ -99,15 +100,70 @@ describe("complete_goal with independent audit", () => {
       return turnOf("The implementation exists and runs. <approved/>");
     };
 
-    const output = await completeGoalTool(argsFor(h, h.ceoId, auditor, call({ evidence: "src/index.ts implements the app and runs." })));
+    const readOnly = argsFor(h, h.ceoId, auditor, call({ evidence: "src/index.ts implements the app and runs." }));
+    readOnly.ctx.workspace = { ...readOnly.ctx.workspace!, writeAccess: false };
+    const output = await completeGoalTool(readOnly);
     expect(output).toContain("achieved");
     expect(goalStatus(h)).toBe("achieved");
-    // The auditor never receives run_command — inspection is read-only.
+    // Without the operator's file-access approval the auditor gets no run_command.
     expect(seenTools.every((tools) => !tools.includes("run_command"))).toBe(true);
     expect(seenTools[0]).toContain("read_file");
     h.connection.close();
   });
 
+  test("with approved file access the auditor builds and tests in a throwaway copy of the workspace", async () => {
+    const h = createHarness();
+    let turns = 0;
+    let commandOutput = "";
+    let ranIn = "";
+    const auditor = async (input: AuditorInput): Promise<AuditorTurn> => {
+      turns += 1;
+      if (turns === 1) {
+        expect(input.tools.map((tool) => tool.name)).toContain("run_command");
+        return turnOf("Running the tests.", [{ id: "c1", name: "run_command", arguments: { command: "node check.js" } }, { id: "c2", name: "list_files", arguments: {} }]);
+      }
+      if (turns === 2) {
+        const results = (input.messages.at(-1) as { results: Array<{ output: string }> }).results;
+        commandOutput = results[0].output;
+        ranIn = results[1].output;
+        return turnOf("Reading.", [{ id: "c3", name: "read_file", arguments: { path: "built.txt" } }]);
+      }
+      return turnOf("Tests pass when run. <approved/>");
+    };
+    const output = await completeGoalTool(argsFor(h, h.ceoId, auditor, call({ evidence: "All tests pass." })));
+    expect(output).toContain("achieved");
+    expect(commandOutput).toContain("all 3 tests passed");
+    expect(ranIn).toContain("built.txt");
+    // The real workspace is untouched, and no copy is left behind.
+    expect(existsSync(join(workspaceRoot, "built.txt"))).toBe(false);
+    const copy = /cwd=(.+)/.exec(commandOutput)![1].trim();
+    expect(copy).not.toBe(workspaceRoot);
+    expect(existsSync(copy)).toBe(false);
+    h.connection.close();
+  });
+
+  test("a workspace that cannot be copied fails the audit and leaves no copy behind", async () => {
+    const h = createHarness();
+    const args = argsFor(h, h.ceoId, async () => turnOf("<approved/>"), call({ evidence: "done" }));
+    args.ctx.workspace = { ...args.ctx.workspace!, localPath: join(tmpdir(), `ac-gone-${randomUUID()}`) };
+    // A private temp folder for this audit, so the check sees only its own copies.
+    const base = mkdtempSync(join(tmpdir(), "ac-tmpbase-"));
+    const saved = { TMP: process.env.TMP, TEMP: process.env.TEMP, TMPDIR: process.env.TMPDIR };
+    process.env.TMP = process.env.TEMP = process.env.TMPDIR = base;
+    try {
+      expect(await completeGoalTool(args)).toMatch(/^ERROR: /);
+    } finally {
+      for (const [name, value] of Object.entries(saved)) {
+        if (value === undefined) delete process.env[name];
+        else process.env[name] = value;
+      }
+    }
+    expect(goalStatus(h)).toBe("active");
+    expect(readdirSync(base)).toEqual([]);
+    rmSync(base, { recursive: true, force: true });
+    h.connection.close();
+  });
+
   test("a disapproving auditor keeps the goal open and returns the gap to the caller", async () => {
     const h = createHarness();
     const auditor = async (): Promise<AuditorTurn> => turnOf("<disapproved/> Only planning documents exist; there is no implementation.");