Q114: goal audits use the org's own auditor model and see pointers, not the executor's account

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/main/app/ipc-handlers.ts

@@ -30,6 +30,7 @@ import { SprintService } from "../services/sprint-service";
 import { ComposioConnectionService } from "../services/composio/composio-connection-service";
 import { ConnectorGrantsService } from "../services/composio/connector-grants";
 import { approvalDrifts } from "../services/approval-drift";
+import { setGoalAuditor } from "../services/goal-audit-inputs";
 import { IntegrationKeyStore } from "../services/integration-key-store";
 import { importStagedSkills, stageBuiltinSkillsLibrary } from "../services/builtin-skills";
 import { coalesce } from "./coalesce";
@@ -224,6 +225,7 @@ export function registerIpcHandlers(deps: IpcDeps): void {
   });
   ipcMain.handle(IPC_CHANNELS.updateProviderPolicy, (_event, input) => mutate(() => providers.updatePolicy(input)));
   ipcMain.handle(IPC_CHANNELS.setProviderPrices, (_event, input) => mutate(() => providers.setPrices(input)));
+  ipcMain.handle(IPC_CHANNELS.setGoalAuditor, (_event, input) => mutate(() => setGoalAuditor(deps.connection.db, input)));
   ipcMain.handle(IPC_CHANNELS.checkProviderHealth, async (_event, input) => {
     try {
       await providers.checkHealth(input.orgId, input.providerId);

src/main/database/migrations/087-goal-auditor.ts

@@ -0,0 +1,7 @@
+export const goalAuditorSchema = `
+  alter table organizations add column audit_provider_id text;
+  alter table organizations add column audit_model_id text;
+
+  insert into schema_migrations(version, name, applied_at)
+  values (87, 'goal_auditor', datetime('now'));
+`;

src/main/database/schema.ts

@@ -85,6 +85,7 @@ import { connectorAccountLabelsSchema } from "./migrations/083-connector-account
 import { connectorGrantLimitsSchema } from "./migrations/084-connector-grant-limits";
 import { approvalSubjectSnapshotSchema } from "./migrations/085-approval-subject-snapshot";
 import { heldDeliverablesSchema } from "./migrations/086-held-deliverables";
+import { goalAuditorSchema } from "./migrations/087-goal-auditor";
 
 interface Migration {
   version: number;
@@ -179,6 +180,7 @@ const MIGRATIONS: Migration[] = [
   { version: 84, name: "connector_grant_limits", sql: connectorGrantLimitsSchema },
   { version: 85, name: "approval_subject_snapshot", sql: approvalSubjectSnapshotSchema },
   { version: 86, name: "held_deliverables", sql: heldDeliverablesSchema },
+  { version: 87, name: "goal_auditor", sql: goalAuditorSchema },
 ];
 
 const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version;

src/main/services/agent-goal-tools.ts

@@ -11,6 +11,7 @@ import type { ProviderService } from "./provider-service";
 import { unreadableArguments, type AgentTurnMessage, type ToolCall, type ToolDefinition } from "./provider-tool-turn";
 import { MAX_TOOL_CALLS_PER_TURN, throwIfRunStopped, untilStopped } from "./run-control";
 import { heldWorkTitles } from "./held-deliverables";
+import { goalAuditor, resultPointers, type GoalAuditor, type ResultPointers } from "./goal-audit-inputs";
 import { DOCUMENT_TOOLS, listDocumentsTool, readDocumentTool } from "./agent-document-tools";
 
 const AUDIT_MAX_ITERATIONS = 6;
@@ -47,12 +48,10 @@ export const GOAL_TOOLS: ToolDefinition[] = [
       type: "object",
       additionalProperties: false,
       properties: {
-        evidence: {
-          type: "string",
-          description: "The concrete evidence that the goal is achieved: what exists, where it lives in the workspace, and what verification passed.",
-        },
+        files: { type: "array", items: { type: "string" }, description: "Workspace paths of the files that hold the result." },
+        documents: { type: "array", items: { type: "string" }, description: "Doc ids of the documents that hold the result." },
       },
-      required: ["evidence"],
+      required: [],
     },
   },
 ];
@@ -63,7 +62,6 @@ export interface GoalCompletionArgs {
   providers: Pick<ProviderService, "runToolTurn">;
   providerId: string;
   workspaceId: string | null;
-  inputBudgetTokens: number;
   call: ToolCall;
 }
 
@@ -82,13 +80,14 @@ export async function completeGoalTool(args: GoalCompletionArgs): Promise<string
       .get(ctx.taskGoalId, ctx.orgId) as { id: string; title: string; description: string; status: string; owner_agent_id: string | null } | undefined;
     if (!goal) throw new Error("Goal was not found.");
     if (!actorIsCeoOrGoalOwner(deps, ctx, goal.owner_agent_id)) {
-      return "ERROR: Only the CEO or the goal's owner can complete a goal. Report your evidence to them instead.";
+      return "ERROR: Only the CEO or the goal's owner can complete a goal. Point them to your result instead.";
     }
     if (goal.status !== "active") return `Goal "${goal.title}" is already ${goal.status}.`;
     if (heldWorkTitles(deps.db, { goalId: goal.id }).length > 0) return heldGoalRefusal(deps, goal);
-    const evidence = requireString(call.arguments.evidence, "evidence");
+    const pointers = resultPointers(deps.db, ctx, call.arguments);
+    const auditor = goalAuditor(deps.db, ctx, args.providerId);
 
-    const verdict = await auditGoalCompletion(args, goal, evidence);
+    const verdict = await auditGoalCompletion(args, goal, pointers, auditor);
     if (!verdict.approved) {
       return `The independent audit rejected closing the goal: ${verdict.reason} The goal stays active — close that gap or create the tasks that will.`;
     }
@@ -125,14 +124,15 @@ interface AuditVerdict {
 async function auditGoalCompletion(
   args: GoalCompletionArgs,
   goal: { title: string; description: string },
-  evidence: string,
+  pointers: ResultPointers,
+  auditor: GoalAuditor,
 ): Promise<AuditVerdict> {
   const workspace = args.ctx.workspace;
   // With the operator's file-access approval the auditor may build and test, but only in a throwaway copy.
   const scratch = workspace?.writeAccess ? copyWorkspace(workspace.localPath) : null;
   try {
     const auditWorkspace = workspace ? { ...workspace, localPath: scratch ?? workspace.localPath, writeAccess: scratch !== null } : null;
-    return await runAudit(args, goal, evidence, auditWorkspace);
+    return await runAudit(args, goal, pointers, auditor, auditWorkspace);
   } finally {
     if (scratch) rmSync(scratch, { recursive: true, force: true });
   }
@@ -153,7 +153,8 @@ function copyWorkspace(root: string): string {
 async function runAudit(
   args: GoalCompletionArgs,
   goal: { title: string; description: string },
-  evidence: string,
+  pointers: ResultPointers,
+  auditor: GoalAuditor,
   workspace: AgentToolContext["workspace"],
 ): Promise<AuditVerdict> {
   const canRun = workspace?.writeAccess === true;
@@ -170,8 +171,10 @@ async function runAudit(
       "A goal is claimed complete. Decide whether the claim survives the evidence.",
       `Goal: ${goal.title}`,
       `Goal detail: ${goal.description}`,
-      "The executor's claim:",
-      evidence,
+      // The executor's own account of the work is never shown: only where the result lives.
+      "The executor points to these as the result. Verify them yourself:",
+      `Files: ${pointers.files.length > 0 ? pointers.files.join(", ") : "none"}`,
+      `Documents: ${pointers.documents.length > 0 ? pointers.documents.map((doc) => `${doc.title} (doc_id=${doc.id})`).join(", ") : "none"}`,
       `Inspect the work before deciding: list_files and read_file for the workspace, list_documents and read_document for the library. An approval counts only after you have read some of it. ${canRun ? "run_command runs in a throwaway copy of the workspace: build and run the tests yourself instead of trusting claims that they pass; nothing you do there reaches the real workspace." : "You cannot run commands here, so a claim that something was run or passes is unverified unless the workspace holds its results."} Judge outcomes, not artifacts about outcomes: plans, specs, status reports, and skeletons do not make a goal achieved; working deliverables do. If the goal promises software, look for the actual implementation, not documents describing it.`,
       "Keep your reasoning apart from your decision. End your final message with the decision alone on its last line: <approved/> by itself if the goal is genuinely achieved, or <disapproved/> followed by one short sentence naming the decisive gap. Use no marker anywhere else.",
     ].join("\n\n"),
@@ -179,11 +182,12 @@ async function runAudit(
   const instructions = "You are the independent completion auditor. You did not do the work and owe its author nothing; your only loyalty is to what actually exists. Be strict: an unverifiable claim is a false claim.";
 
   for (let iteration = 1; iteration <= AUDIT_MAX_ITERATIONS; iteration += 1) {
-    fitContextBudget(instructions, messages, auditTools, args.inputBudgetTokens, iteration);
+    fitContextBudget(instructions, messages, auditTools, auditor.inputBudgetTokens, iteration);
     throwIfRunStopped();
     const turn = await untilStopped(args.providers.runToolTurn({
       orgId: args.ctx.orgId,
-      providerId: args.providerId,
+      providerId: auditor.providerId,
+      model: auditor.model,
       workspaceId: args.workspaceId,
       instructions,
       messages,

src/main/services/agent-run-loop.ts

@@ -137,7 +137,6 @@ export async function runAgentLoop(deps: AgentRunLoopDeps, input: AgentRunLoopIn
           providers: deps.providers,
           providerId: input.providerId,
           workspaceId: input.workspaceId,
-          inputBudgetTokens: input.inputBudgetTokens,
           call,
         }));
       } else {

src/main/services/goal-audit-inputs.ts

@@ -0,0 +1,73 @@
+import { existsSync } from "node:fs";
+import type { DatabaseSync } from "node:sqlite";
+import { optionalStringArray } from "./agent-tool-args";
+import { inputBudgetTokens } from "./context-budget";
+import type { AgentToolContext } from "./agent-tools";
+import { resolveWorkspacePath } from "./files/workspace-path";
+import type { JsonRecord } from "./provider-http";
+import { currentRunId } from "./run-control";
+import { effectiveModel } from "./task-run-preparation";
+
+const AUDITOR_SETTING = "Settings > Advanced > Goal audit";
+
+export interface GoalAuditor {
+  providerId: string;
+  model: string;
+  inputBudgetTokens: number;
+}
+
+/** Where the result lives, as the executor points to it; the auditor gets these, never the executor's account of the work. */
+export interface ResultPointers {
+  files: string[];
+  documents: Array<{ id: string; title: string }>;
+}
+
+/** The model that audits goals: the org's own choice, never the executor's model, so the two do not share one model's blind spots. */
+export function goalAuditor(db: DatabaseSync, ctx: AgentToolContext, executorProviderId: string): GoalAuditor {
+  const org = db.prepare("select audit_provider_id, audit_model_id from organizations where id = ?")
+    .get(ctx.orgId) as { audit_provider_id: string | null; audit_model_id: string | null };
+  if (!org.audit_provider_id || !org.audit_model_id) {
+    throw new Error(`Goal audits need an auditor model of their own, different from the executor's. Ask the board to set one in ${AUDITOR_SETTING}.`);
+  }
+  const runId = currentRunId();
+  const pinned = runId ? (db.prepare("select run_model from task_runs where id = ?").get(runId) as { run_model: string | null } | undefined)?.run_model ?? null : null;
+  const providerModel = (db.prepare("select model from providers where id = ?").get(executorProviderId) as { model: string | null } | undefined)?.model ?? null;
+  const executorModel = pinned ?? effectiveModel(ctx.agent.modelId, providerModel);
+  if (org.audit_provider_id === executorProviderId && org.audit_model_id === executorModel) {
+    throw new Error(`The goal auditor is set to the executor's own model (${executorModel}); the board must choose a different one in ${AUDITOR_SETTING}.`);
+  }
+  const provider = db.prepare("select context_window_tokens from providers where id = ? and org_id = ?")
+    .get(org.audit_provider_id, ctx.orgId) as { context_window_tokens: number } | undefined;
+  if (!provider) throw new Error(`The goal auditor's provider no longer exists; the board must choose another in ${AUDITOR_SETTING}.`);
+  // The audit fits the auditor's own context window, not the executor's.
+  return { providerId: org.audit_provider_id, model: org.audit_model_id, inputBudgetTokens: inputBudgetTokens(provider.context_window_tokens) };
+}
+
+/** The board's choice of goal auditor: a provider of this org and a model name. */
+export function setGoalAuditor(db: DatabaseSync, input: { orgId: string; providerId: string; modelId: string }): void {
+  const provider = db.prepare("select id from providers where id = ? and org_id = ?").get(input.providerId, input.orgId);
+  if (!provider) throw new Error("The goal auditor's provider was not found.");
+  const model = input.modelId.trim();
+  if (!model) throw new Error("The goal auditor needs a model.");
+  db.prepare("update organizations set audit_provider_id = ?, audit_model_id = ? where id = ?").run(input.providerId, model, input.orgId);
+  console.info(`[goal-audit] org ${input.orgId} audits goals with ${model} on ${input.providerId}`);
+}
+
+/** Checks the executor's pointers before any audit spend: every file in the workspace and every document must exist. */
+export function resultPointers(db: DatabaseSync, ctx: AgentToolContext, args: JsonRecord): ResultPointers {
+  const files = optionalStringArray(args.files, "files");
+  const documentIds = optionalStringArray(args.documents, "documents");
+  if (files.length + documentIds.length === 0) {
+    throw new Error("Point to the result: files (workspace paths) and/or documents (doc ids) that hold it.");
+  }
+  for (const file of files) {
+    if (!ctx.workspace) throw new Error("No workspace is bound to this task, so files cannot be pointed to; point to documents instead.");
+    if (!existsSync(resolveWorkspacePath(ctx.workspace.localPath, file).absolute)) throw new Error(`There is no file ${file} in the workspace.`);
+  }
+  const documents = documentIds.map((id) => {
+    const document = db.prepare("select id, title from documents where id = ? and org_id = ?").get(id, ctx.orgId) as { id: string; title: string } | undefined;
+    if (!document) throw new Error(`There is no document ${id}.`);
+    return document;
+  });
+  return { files, documents };
+}

src/main/services/goal-review-service.ts

@@ -130,7 +130,7 @@ function goalReviewDescription(goalTitle: string): string {
     `Every task under the goal "${goalTitle.trim()}" is finished, but the goal itself is still open. Decide what happens next — this is a checkpoint, not a formality.`,
     "First gather evidence: list_files and read_file the workspace, and read what the team reported. Plans, specs, and status documents are not the product. If the goal promises working software, the proof is code in the workspace that builds and passes its tests (run_command).",
     "Then take exactly one of these actions before closing this task:",
-    "1. The goal is genuinely achieved -> call complete_goal with your evidence. An independent auditor inspects the workspace before the goal actually closes, so claims without substance will bounce.",
+    "1. The goal is genuinely achieved -> call complete_goal with the files and documents that hold the result. An independent auditor inspects the workspace before the goal actually closes, so claims without substance will bounce.",
     "2. It is not achieved -> create the next wave of tasks (create_task, assigned by name). Implementation work must set requires_filesystem: true so the assignee can change code and run commands. Documents alone must not close the next wave either.",
   ].join("\n\n");
 }

src/main/services/provider-response-runner.ts

@@ -114,6 +114,8 @@ export class ProviderResponseRunner {
     instructions: string | null;
     messages: AgentTurnMessage[];
     tools: ToolDefinition[];
+    /** A model chosen for this call (a goal auditor's), outside the session's pinned model and account. */
+    model?: string;
   }): Promise<AgentTurnResult & { requestId: string }> {
     const provider = loadConfiguredProvider(this.deps.db, input.orgId, input.providerId);
     const workspaceId = this.normalizeWorkspaceId(input.orgId, input.workspaceId ?? null);
@@ -128,12 +130,14 @@ export class ProviderResponseRunner {
     const pins = taskRunId
       ? this.deps.db.prepare("select run_model, run_credential_id from task_runs where id = ?").get(taskRunId) as { run_model: string | null; run_credential_id: string | null }
       : null;
-    if (pins?.run_credential_id && pins.run_credential_id !== provider.credential_id) {
+    const chosen = input.model ?? null;
+    if (!chosen && pins?.run_credential_id && pins.run_credential_id !== provider.credential_id) {
       throw new Error("The provider's account was reconnected during this session; the next session continues with the new account.");
     }
     // Every turn of a session calls the model the session started with (checked then), whatever the configuration says now.
-    if (!pins?.run_model) assertModelSelectable(provider.provider_key, agent?.model_id ?? null, provider.model);
-    const model = pins?.run_model ?? effectiveModel(agent?.model_id ?? null, provider.model);
+    if (chosen) assertModelSelectable(provider.provider_key, chosen, provider.model);
+    else if (!pins?.run_model) assertModelSelectable(provider.provider_key, agent?.model_id ?? null, provider.model);
+    const model = chosen ?? pins?.run_model ?? effectiveModel(agent?.model_id ?? null, provider.model);
     if (!model) throw new Error("Provider model is required.");
     const owner: ProviderRow & { model: string } = { ...provider, model };
     const turn = this.deps.toolTurn ?? selectToolTurn(provider.provider_key);