Q83: a model passes an agent check before it does agent work

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/helpers/model-check.ts

@@ -0,0 +1,22 @@
+import type { DatabaseSync } from "node:sqlite";
+import { MODEL_CHECK_TOOL } from "../../src/main/services/model-check";
+import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
+
+/** A scripted provider that also passes the agent check the way a capable model would. */
+export function answeringModelCheck<T extends (input: ProviderToolTurnInput) => Promise<AgentTurnResult>>(turn: T): T {
+  return (async (input: ProviderToolTurnInput) => {
+    if (!input.tools.some((tool) => tool.name === MODEL_CHECK_TOOL)) return turn(input);
+    const first = input.messages[0];
+    const code = first?.role === "user" ? /The code is (\d+)/.exec(first.text)?.[1] : undefined;
+    if (!code) throw new Error("The agent check sent no code.");
+    return { text: "", toolCalls: [{ id: "check", name: MODEL_CHECK_TOOL, arguments: { code } }], rawResponseJson: "{}" };
+  }) as T;
+}
+
+/** Records that a provider's model already passed the agent check, so a test sees only its own requests. */
+export function markModelChecked(db: DatabaseSync, providerId: string, model: string): void {
+  const provider = db.prepare("select org_id, credential_id from providers where id = ?").get(providerId) as { org_id: string; credential_id: string | null } | undefined;
+  if (!provider?.credential_id) throw new Error(`Provider ${providerId} has no credential to check a model against.`);
+  db.prepare("insert into model_checks(provider_id, model, credential_id, org_id, passed, detail, checked_at) values (?, ?, ?, ?, 1, null, ?)")
+    .run(providerId, model, provider.credential_id, provider.org_id, new Date().toISOString());
+}

tests/integration/agent-model.test.ts

@@ -9,6 +9,7 @@ import { providerAdapters } from "../../src/main/services/provider-adapters";
 import { TaskExecutionService } from "../../src/main/services/task-execution-service";
 import { RISK_LEVEL, TASK_PRIORITY, TASK_STATUS } from "../../src/shared/domain";
 import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -27,7 +28,7 @@ describe("agent model", () => {
     };
     const connection = new DatabaseConnection(join(tmpdir(), `ac-model-${randomUUID()}.db`));
     const organization = new OrganizationService(connection.db);
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";
     const tasks: Record<string, string> = {};
@@ -53,7 +54,7 @@ describe("agent model", () => {
     const turn = async (): Promise<AgentTurnResult> => ({ text: "done", toolCalls: [], rawResponseJson: "{}" });
     const connection = new DatabaseConnection(join(tmpdir(), `ac-model-az-${randomUUID()}.db`));
     const organization = new OrganizationService(connection.db);
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";
     let taskId = "";
@@ -83,7 +84,7 @@ describe("agent model", () => {
     };
     const connection = new DatabaseConnection(join(tmpdir(), `ac-model-pin-${randomUUID()}.db`));
     const organization = new OrganizationService(connection.db);
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";
     let taskId = "";

tests/integration/agent-reply.test.ts

@@ -11,6 +11,7 @@ import { SnapshotService } from "../../src/main/services/snapshot-service";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import { canonicalMentionHandle } from "../../src/shared/mention-handles";
 import type { AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -59,7 +60,7 @@ function scriptedTeam(): (input: ProviderToolTurnInput) => Promise<AgentTurnResu
 describe("inter-agent reply", () => {
   test("an @mention wakes the teammate, who replies in the same thread", async () => {
     const connection = new DatabaseConnection(join(tmpdir(), `ac-reply-${randomUUID()}.db`));
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, scriptedTeam());
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(scriptedTeam()));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const taskExecution = new TaskExecutionService(connection.db, providers);
     const snapshot = new SnapshotService(connection.db);

tests/integration/autonomous-run.test.ts

@@ -12,6 +12,7 @@ import { TaskExecutionService } from "../../src/main/services/task-execution-ser
 import { SnapshotService } from "../../src/main/services/snapshot-service";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import type { AgentTurnMessage, AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -89,7 +90,7 @@ describe("autonomous bootstrap run", () => {
     const workspacePath = join(tmpdir(), `ac-auto-ws-${randomUUID()}`);
     mkdirSync(workspacePath, { recursive: true });
 
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, scriptedCeo());
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(scriptedCeo()));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const snapshot = new SnapshotService(connection.db);
 
@@ -183,7 +184,7 @@ describe("autonomous bootstrap run", () => {
   test("a goal created after onboarding reaches the CEO and runs autonomously", async () => {
     const connection = new DatabaseConnection(join(tmpdir(), `ac-goal-${randomUUID()}.db`));
     // A minimal well-behaved agent: does its session, closes its task, then stops.
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, async (input: ProviderToolTurnInput) => {
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(async (input: ProviderToolTurnInput) => {
       const firstUser = input.messages.find((message) => message.role === "user");
       const taskId = firstUser && firstUser.role === "user" ? /Task id: (\S+)/.exec(firstUser.text)?.[1] : undefined;
       const alreadyClosed = input.messages.some(
@@ -201,7 +202,7 @@ describe("autonomous bootstrap run", () => {
         toolCalls: [],
         rawResponseJson: JSON.stringify({ id: "resp_goal" }),
       };
-    });
+    }));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const organization = new OrganizationService(connection.db);
     const taskExecution = new TaskExecutionService(connection.db, providers);

tests/integration/ceo-founder-chat.test.ts

@@ -11,6 +11,7 @@ import { ChatService } from "../../src/main/services/chat-service";
 import { ensureCeoDirectSpace } from "../../src/main/services/ceo-direct-space";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import type { AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -67,7 +68,7 @@ describe("founder⇄CEO chat", () => {
   test("a founder DM wakes the CEO, who sees who is asking and replies in the pinned conversation", async () => {
     const connection = new DatabaseConnection(join(tmpdir(), `ac-ceo-chat-${randomUUID()}.db`));
     const capturedWakePrompts: string[] = [];
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, scriptedCeo(capturedWakePrompts));
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(scriptedCeo(capturedWakePrompts)));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const taskExecution = new TaskExecutionService(connection.db, providers);
     const chat = new ChatService(connection.db);
@@ -143,7 +144,7 @@ describe("founder⇄CEO chat", () => {
 
   test("a CEO who answers in plain text still has the reply delivered into the founder's thread", async () => {
     const connection = new DatabaseConnection(join(tmpdir(), `ac-ceo-plain-${randomUUID()}.db`));
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, plainTextCeo());
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(plainTextCeo()));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const taskExecution = new TaskExecutionService(connection.db, providers);
     const chat = new ChatService(connection.db);

tests/integration/channel-anchor.test.ts

@@ -10,6 +10,7 @@ import { providerAdapters } from "../../src/main/services/provider-adapters";
 import { TaskExecutionService } from "../../src/main/services/task-execution-service";
 import { canonicalMentionHandle } from "../../src/shared/mention-handles";
 import type { AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -37,7 +38,7 @@ function scriptedCeoReply(): (input: ProviderToolTurnInput) => Promise<AgentTurn
 describe("channel anchor wake", () => {
   test("a plain user message in a channel wakes the most senior member, who replies", async () => {
     const connection = new DatabaseConnection(join(tmpdir(), `ac-anchor-${randomUUID()}.db`));
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, scriptedCeoReply());
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(scriptedCeoReply()));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const taskExecution = new TaskExecutionService(connection.db, providers);
     const chat = new ChatService(connection.db);

tests/integration/chat-context.test.ts

@@ -14,6 +14,7 @@ import { dispatchAgentTool, type AgentToolContext } from "../../src/main/service
 import { ensureCeoDirectSpace } from "../../src/main/services/ceo-direct-space";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -34,7 +35,7 @@ async function founderLine() {
     return { text: `Reply ${wakePrompts.length}.`, toolCalls: [], rawResponseJson: "{}" };
   };
   const connection = new DatabaseConnection(join(tmpdir(), `ac-chatctx-${randomUUID()}.db`));
-  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
   const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
   const execution = new TaskExecutionService(connection.db, providers);
   const chat = new ChatService(connection.db);
@@ -153,7 +154,7 @@ describe("chat wake context", () => {
       }
       return { text: "Switching to Postgres.", toolCalls: [], rawResponseJson: "{}" };
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const execution = new TaskExecutionService(connection.db, providers);
     let ceoId = "";

tests/integration/goal-changes.test.ts

@@ -9,6 +9,7 @@ import { providerAdapters } from "../../src/main/services/provider-adapters";
 import { TaskExecutionService } from "../../src/main/services/task-execution-service";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -17,7 +18,7 @@ const testVault = {
 
 function company(turn: (input: ProviderToolTurnInput, db: DatabaseConnection["db"]) => AgentTurnResult) {
   const connection = new DatabaseConnection(join(tmpdir(), `ac-goalchg-${randomUUID()}.db`));
-  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, async (input) => turn(input, connection.db));
+  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(async (input) => turn(input, connection.db)));
   const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
   const execution = new TaskExecutionService(connection.db, providers);
   let orgId = "";

tests/integration/interactive-lane.test.ts

@@ -12,6 +12,7 @@ import { TaskExecutionService } from "../../src/main/services/task-execution-ser
 import { dispatchAgentTool } from "../../src/main/services/agent-tools";
 import { claimNextQueuedRun, startTaskQueueWorker } from "../../src/main/app/task-queue-worker";
 import type { AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -75,7 +76,7 @@ describe("interactive execution lane", () => {
       }
       return turn("done");
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, script);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(script));
     providers.saveCredential({ orgId, providerId, secret: "sk-test-key", model: "gpt-5.4" });
     const taskExecution = new TaskExecutionService(connection.db, providers);
 
@@ -162,7 +163,7 @@ describe("interactive execution lane", () => {
       }
       return turn("done");
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, script);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(script));
     providers.saveCredential({ orgId, providerId, secret: "sk-test-key", model: "gpt-5.4" });
     const taskExecution = new TaskExecutionService(connection.db, providers);
 
@@ -227,7 +228,7 @@ describe("interactive execution lane", () => {
     const alwaysOverloaded = async (): Promise<AgentTurnResult> => {
       throw new Error("The engine is currently overloaded, please try again later");
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, alwaysOverloaded);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(alwaysOverloaded));
     providers.saveCredential({ orgId, providerId, secret: "sk-test-key", model: "gpt-5.4" });
     const taskExecution = new TaskExecutionService(connection.db, providers);
     const wakeDana = () => dispatchAgentTool({ db: connection.db, organization, chat }, {

tests/integration/run-checkpoint.test.ts

@@ -11,6 +11,7 @@ import { ChatService } from "../../src/main/services/chat-service";
 import { ensureCeoDirectSpace } from "../../src/main/services/ceo-direct-space";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import type { AgentTurnResult } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -24,7 +25,7 @@ describe("session checkpoints", () => {
       ? { text: "Audit done: all clean.", toolCalls: [], rawResponseJson: "{}" }
       : { text: "Looking into it.", toolCalls: [{ id: `c_${randomUUID()}`, name: "read_messages", arguments: {} }], rawResponseJson: "{}" };
     const connection = new DatabaseConnection(join(tmpdir(), `ac-checkpoint-${randomUUID()}.db`));
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";

tests/integration/run-journal.test.ts

@@ -9,6 +9,7 @@ import { providerAdapters } from "../../src/main/services/provider-adapters";
 import { TaskExecutionService } from "../../src/main/services/task-execution-service";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -41,7 +42,7 @@ describe("run journal", () => {
       throw new Error("Provider dropped the connection mid-run");
     };
     const connection = new DatabaseConnection(join(tmpdir(), `ac-journal-${randomUUID()}.db`));
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";

tests/integration/run-memory.test.ts

@@ -11,6 +11,7 @@ import { ChatService } from "../../src/main/services/chat-service";
 import { ensureCeoDirectSpace } from "../../src/main/services/ceo-direct-space";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -26,7 +27,7 @@ async function company(reply: (prompt: string) => AgentTurnResult) {
     return reply(prompt);
   };
   const connection = new DatabaseConnection(join(tmpdir(), `ac-memory-${randomUUID()}.db`));
-  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
   const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
   const execution = new TaskExecutionService(connection.db, providers);
   let orgId = "";

tests/integration/run-stop.test.ts

@@ -13,6 +13,7 @@ import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import { postJson } from "../../src/main/services/provider-http";
 import { RUN_STOPPED_MESSAGE, stopRun, withRunControl } from "../../src/main/services/run-control";
 import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -21,7 +22,7 @@ const testVault = {
 
 function company(turn: (input: ProviderToolTurnInput) => Promise<AgentTurnResult>) {
   const connection = new DatabaseConnection(join(tmpdir(), `ac-stop-${randomUUID()}.db`));
-  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
   const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
   const execution = new TaskExecutionService(connection.db, providers);
   let orgId = "";
@@ -44,7 +45,7 @@ describe("stopping runs", () => {
       return { text: "Announcing.", toolCalls: [{ id: "c1", name: "send_message", arguments: { channel: "general", body: "Should never be posted." } }], rawResponseJson: "{}" };
     };
     const connection = new DatabaseConnection(join(tmpdir(), `ac-stop-${randomUUID()}.db`));
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";
@@ -115,7 +116,7 @@ describe("stopping runs", () => {
       return { text: "done", toolCalls: [], rawResponseJson: "{}" };
     };
     const connection = new DatabaseConnection(join(tmpdir(), `ac-deadline-${randomUUID()}.db`));
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const execution = new TaskExecutionService(connection.db, providers, undefined, undefined, undefined, 200);
     let orgId = "";

tests/integration/spend-budget.test.ts

@@ -9,6 +9,7 @@ import { providerAdapters } from "../../src/main/services/provider-adapters";
 import { TaskExecutionService } from "../../src/main/services/task-execution-service";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import type { AgentTurnResult } from "../../src/main/services/provider-tool-turn";
+import { markModelChecked } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -39,6 +40,7 @@ describe("money budgets", () => {
       ceoId = summary.ceoAgentId;
     });
     const providerId = (connection.db.prepare("select model_provider_id as id from agents where id = ?").get(ceoId) as { id: string }).id;
+    markModelChecked(connection.db, providerId, "gpt-5.4");
     providers.setPrices({ orgId, providerId, inputDollarsPerMtok: 2.5, cachedInputDollarsPerMtok: null, outputDollarsPerMtok: 10 });
     connection.db.prepare("update agents set budget_cents = 2000 where id = ?").run(ceoId);
 

tests/integration/stale-queued-runs.test.ts

@@ -11,6 +11,7 @@ import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import { ChatService } from "../../src/main/services/chat-service";
 import { ensureCeoDirectSpace } from "../../src/main/services/ceo-direct-space";
 import type { AgentTurnResult } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -24,7 +25,7 @@ function company() {
     return { text: "done", toolCalls: [], rawResponseJson: "{}" };
   };
   const connection = new DatabaseConnection(join(tmpdir(), `ac-stale-${randomUUID()}.db`));
-  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
   const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
   const execution = new TaskExecutionService(connection.db, providers);
   let orgId = "";

tests/integration/task-context-scope.test.ts

@@ -13,6 +13,7 @@ import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import { canonicalMentionHandle } from "../../src/shared/mention-handles";
 import { RISK_LEVEL, TASK_PRIORITY, TASK_STATUS } from "../../src/shared/domain";
 import type { AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -49,7 +50,7 @@ describe("task prompt scope", () => {
       return { text: step.text, toolCalls: step.toolCalls, rawResponseJson: "{}" };
     };
     const connection = new DatabaseConnection(join(tmpdir(), `ac-scope-${randomUUID()}.db`));
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";
@@ -90,7 +91,7 @@ describe("task prompt scope", () => {
       calls += 1;
       return { text: "done", toolCalls: [], rawResponseJson: "{}" };
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const organization = new OrganizationService(connection.db);
     const execution = new TaskExecutionService(connection.db, providers);
@@ -125,7 +126,7 @@ describe("task prompt scope", () => {
   test("a manual write run waits while another run is changing the same workspace", async () => {
     const connection = new DatabaseConnection(join(tmpdir(), `ac-wsplock-${randomUUID()}.db`));
     const turn = async (): Promise<AgentTurnResult> => ({ text: "done", toolCalls: [], rawResponseJson: "{}" });
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const organization = new OrganizationService(connection.db);
     const execution = new TaskExecutionService(connection.db, providers);

tests/integration/task-dependencies.test.ts

@@ -13,6 +13,7 @@ import { dispatchAgentTool } from "../../src/main/services/agent-tools";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import { RISK_LEVEL, TASK_PRIORITY, TASK_STATUS } from "../../src/shared/domain";
 import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -28,7 +29,7 @@ function company() {
     prompts.push(first.text);
     return { text: "done", toolCalls: [], rawResponseJson: "{}" };
   };
-  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
   const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
   const execution = new TaskExecutionService(connection.db, providers);
   const organization = new OrganizationService(connection.db);

tests/unit/agent-instructions.test.ts

@@ -14,6 +14,7 @@ import { buildAgentInstructions, type TaskExecutionAgentRow } from "../../src/ma
 import { AGENT_TOOLS } from "../../src/main/services/agent-tools";
 import { goalKickoffDescription } from "../../src/main/services/goal-kickoff";
 import { CEO_FIRST_HIRES } from "../../src/main/services/org-building";
+import { answeringModelCheck } from "../helpers/model-check";
 
 function agentRow(overrides: Partial<TaskExecutionAgentRow>): TaskExecutionAgentRow {
   return {
@@ -134,7 +135,7 @@ describe("agent instruction wiring", () => {
       capturedInstructions = input.instructions ?? "";
       return { text: "done", toolCalls: [], rawResponseJson: JSON.stringify({ id: "resp-1" }) };
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn));
     const execution = new TaskExecutionService(connection.db, providers);
 
     let orgId = "";
@@ -189,7 +190,7 @@ describe("agent instruction wiring", () => {
       capturedPrompt = first.text;
       return { text: "done", toolCalls: [], rawResponseJson: JSON.stringify({ id: "resp-1" }) };
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn));
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";
     let taskId = "";

tests/unit/char-accounting.test.ts

@@ -7,6 +7,7 @@ import { OrganizationService } from "../../src/main/services/organization-servic
 import { ProviderService } from "../../src/main/services/provider-service";
 import { providerAdapters } from "../../src/main/services/provider-adapters";
 import type { AgentTurnResult, ToolDefinition } from "../../src/main/services/provider-tool-turn";
+import { markModelChecked } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -25,6 +26,7 @@ describe("character accounting", () => {
       orgId = new OrganizationService(connection.db).createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" });
       providerId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'openai'").get(orgId) as { id: string }).id;
       providers.saveCredential({ orgId, providerId, secret: "sk-test", model: "gpt-5.4" });
+      markModelChecked(connection.db, providerId, "gpt-5.4");
     });
     const instructions = "You run this company.";
     const messages = [{ role: "user" as const, text: "Kick off the launch." }];

tests/unit/domain-services.test.ts

@@ -24,7 +24,7 @@ describe("domain services", () => {
     const connection = createConnection();
 
     const rows = connection.db.prepare("select version from schema_migrations order by version asc").all() as Array<{ version: number }>;
-    expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76]);
+    expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77]);
     connection.close();
   });
 

tests/unit/model-check.test.ts

@@ -0,0 +1,84 @@
+import { randomUUID } from "node:crypto";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, test } from "vitest";
+import { DatabaseConnection } from "../../src/main/database/connection";
+import { OrganizationService } from "../../src/main/services/organization-service";
+import { ProviderService } from "../../src/main/services/provider-service";
+import { providerAdapters } from "../../src/main/services/provider-adapters";
+import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
+import { PROVIDER_DATA_POLICY } from "../../src/shared/domain";
+import { PROVIDER_MONTHLY_LIMIT_MESSAGE } from "../../src/shared/provider-runtime-domain";
+
+const testVault = {
+  encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
+  decryptSecret: (ciphertext: Buffer) => ciphertext.toString("utf8"),
+};
+
+const WORK = { workspaceId: null, instructions: "Do the work.", messages: [{ role: "user" as const, text: "go" }], tools: [] };
+const TEXT_ONLY = async (): Promise<AgentTurnResult> => ({ text: "Sure, the code is 123.", toolCalls: [], rawResponseJson: "{}" });
+
+function harness(turn: (input: ProviderToolTurnInput) => Promise<AgentTurnResult>) {
+  const connection = new DatabaseConnection(join(tmpdir(), `ac-model-check-${randomUUID()}.db`));
+  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+  let orgId = "";
+  let providerId = "";
+  connection.transaction(() => {
+    orgId = new OrganizationService(connection.db).createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" });
+    providerId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'openai'").get(orgId) as { id: string }).id;
+    providers.saveCredential({ orgId, providerId, secret: "sk-test", model: "gpt-5.4" });
+  });
+  const requests = () => (connection.db.prepare("select count(*) as n from provider_requests").get() as { n: number }).n;
+  return { connection, providers, orgId, providerId, requests };
+}
+
+describe("a model is checked for agent work before it does any", () => {
+  test("a model that answers in text instead of calling tools is refused, and the verdict is kept", async () => {
+    const h = harness(TEXT_ONLY);
+    await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK }))
+      .rejects.toThrow("gpt-5.4 on openai failed the agent check: it answered in text instead of calling the tool it was given");
+    expect(h.requests()).toBe(1);
+    await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("failed the agent check");
+    expect(h.requests()).toBe(1);
+    h.connection.close();
+  });
+
+  test("a capable model is probed once, then works; the probe is a request like any other and counts against the quota", async () => {
+    const seen: ProviderToolTurnInput[] = [];
+    const h = harness(answeringModelCheck(async (input) => { seen.push(input); return { text: "Done.", toolCalls: [], rawResponseJson: "{}" }; }));
+    expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
+    expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
+    expect([h.requests(), seen.length]).toEqual([3, 2]);
+
+    const tight = harness(answeringModelCheck(async () => ({ text: "Done.", toolCalls: [], rawResponseJson: "{}" })));
+    tight.providers.updatePolicy({ orgId: tight.orgId, providerId: tight.providerId, dataPolicy: PROVIDER_DATA_POLICY.STANDARD, backgroundEnabled: false, monthlyRequestLimit: 1, workspaceScope: "organization", contextWindowTokens: 128_000 });
+    await expect(tight.providers.runToolTurn({ orgId: tight.orgId, providerId: tight.providerId, ...WORK })).rejects.toThrow(PROVIDER_MONTHLY_LIMIT_MESSAGE);
+    expect(tight.requests()).toBe(1);
+    h.connection.close();
+    tight.connection.close();
+  });
+
+  test("a probe whose request fails records nothing; a new model gets its own probe; Settings can check again", async () => {
+    let mode: "down" | "text" | "capable" = "down";
+    const capable: (input: ProviderToolTurnInput) => Promise<AgentTurnResult> = answeringModelCheck(async () => ({ text: "ok", toolCalls: [], rawResponseJson: "{}" }));
+    const h = harness(async (input) => {
+      if (mode === "down") throw new Error("connect ECONNREFUSED");
+      return mode === "text" ? TEXT_ONLY() : capable(input);
+    });
+    await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("ECONNREFUSED");
+    expect(h.connection.db.prepare("select count(*) as n from model_checks").get()).toEqual({ n: 0 });
+
+    mode = "text";
+    expect(await h.providers.checkModel({ orgId: h.orgId, providerId: h.providerId, model: "gpt-5.4" }))
+      .toEqual({ passed: false, detail: "it answered in text instead of calling the tool it was given" });
+    mode = "capable";
+    expect(await h.providers.checkModel({ orgId: h.orgId, providerId: h.providerId, model: "gpt-5.4" })).toEqual({ passed: true });
+    expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("ok");
+
+    h.connection.db.prepare("update providers set model = 'gpt-5.5' where id = ?").run(h.providerId);
+    mode = "text";
+    await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("gpt-5.5 on openai failed the agent check");
+    h.connection.close();
+  });
+});

tests/unit/quota-reservation.test.ts

@@ -9,6 +9,7 @@ import { providerAdapters } from "../../src/main/services/provider-adapters";
 import { PROVIDER_MONTHLY_LIMIT_MESSAGE } from "../../src/shared/provider-runtime-domain";
 import { PROVIDER_DATA_POLICY } from "../../src/shared/domain";
 import type { AgentTurnResult } from "../../src/main/services/provider-tool-turn";
+import { markModelChecked } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -33,6 +34,7 @@ describe("monthly quota reservation", () => {
       orgId = new OrganizationService(connection.db).createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" });
       providerId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'openai'").get(orgId) as { id: string }).id;
       providers.saveCredential({ orgId, providerId, secret: "sk-test", model: "gpt-5.4" });
+      markModelChecked(connection.db, providerId, "gpt-5.4");
       providers.updatePolicy({ orgId, providerId, dataPolicy: PROVIDER_DATA_POLICY.STANDARD, backgroundEnabled: false, monthlyRequestLimit: 1, workspaceScope: "organization", contextWindowTokens: 128_000 });
     });
     const call = () => providers.runToolTurn({ orgId, providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] });

tests/unit/rate-limit-observation.test.ts

@@ -10,6 +10,7 @@ import { TaskExecutionService } from "../../src/main/services/task-execution-ser
 import { RISK_LEVEL, TASK_PRIORITY, TASK_STATUS } from "../../src/shared/domain";
 import type { AgentTurnResult } from "../../src/main/services/provider-tool-turn";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
+import { markModelChecked } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -33,6 +34,7 @@ describe("rate limits seen on successful calls", () => {
       orgId = organization.createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" });
       providerId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'openai'").get(orgId) as { id: string }).id;
       providers.saveCredential({ orgId, providerId, secret: "sk-test", model: "gpt-5.4" });
+      markModelChecked(connection.db, providerId, "gpt-5.4");
       const agentId = organization.createAgent({ orgId, name: "Ada", role: "Engineer", department: "Engineering", modelProviderId: providerId, modelId: "gpt-5.4", budgetCents: 1000, managerAgentId: null, heartbeatEnabled: true });
       organization.createTask({ orgId, title: "Work", description: "d", status: TASK_STATUS.BACKLOG, assigneeAgentId: agentId, risk: RISK_LEVEL.LOW, projectId: null, goalId: null, workspaceId: null, priority: TASK_PRIORITY.NORMAL, requiresFilesystem: false });
     });

tests/unit/task-execution-service.test.ts

@@ -14,6 +14,7 @@ import { SnapshotService } from "../../src/main/services/snapshot-service";
 import { TaskExecutionService } from "../../src/main/services/task-execution-service";
 import { APPROVAL_STATE, PROVIDER_DATA_POLICY, RISK_LEVEL, TASK_PRIORITY, TASK_STATUS } from "../../src/shared/domain";
 import { currentProviderUsageWindow, PROVIDER_MONTHLY_LIMIT_MESSAGE } from "../../src/shared/provider-runtime-domain";
+import { answeringModelCheck } from "../helpers/model-check";
 
 function createConnection(): DatabaseConnection {
   return new DatabaseConnection(join(tmpdir(), `agentcompany-new-task-execution-${randomUUID()}.db`));
@@ -40,7 +41,7 @@ describe("task execution service", () => {
       if (first?.role === "user") capturedPrompt = first.text;
       return { text: "executed:gpt-5.4", toolCalls: [], rawResponseJson: JSON.stringify({ id: "resp-task-1" }) };
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn));
     const skills = new SkillService(connection.db, join(tmpdir(), `agentcompany-task-skills-${randomUUID()}`));
     const execution = new TaskExecutionService(connection.db, providers);
     const skillDir = join(tmpdir(), `agentcompany-task-skill-${randomUUID()}`);
@@ -152,7 +153,7 @@ describe("task execution service", () => {
       toolCalls: [],
       rawResponseJson: JSON.stringify({ id: "resp-task-approved" }),
     });
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn));
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";
     let employeeId = "";
@@ -291,7 +292,7 @@ describe("task execution service", () => {
     const fakeToolTurn = async (): Promise<AgentTurnResult> => {
       throw new Error("provider rejected task");
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn));
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";
     let taskId = "";
@@ -349,7 +350,7 @@ describe("task execution service", () => {
       turns += 1;
       return { text: "done", toolCalls: [], rawResponseJson: JSON.stringify({ id: `resp-${turns}` }) };
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn));
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";
     let taskId = "";
@@ -848,7 +849,7 @@ describe("task execution service", () => {
     const fakeToolTurn = async (): Promise<AgentTurnResult> => {
       throw new Error("The usage limit has been reached");
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn));
     let nowMs = Date.now();
     const execution = new TaskExecutionService(connection.db, providers, undefined, undefined, () => nowMs);
     let orgId = "";