Q83: a model passes an agent check before it does agent work

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/main/app/ipc-handlers.ts

@@ -228,6 +228,13 @@ export function registerIpcHandlers(deps: IpcDeps): void {
       return fail("provider_models_failed", error instanceof Error ? error.message : String(error));
     }
   });
+  ipcMain.handle(IPC_CHANNELS.checkProviderModel, async (_event, input) => {
+    try {
+      return ok(await providers.checkModel(input));
+    } catch (error) {
+      return fail("provider_model_check_failed", error instanceof Error ? error.message : String(error));
+    }
+  });
   ipcMain.handle(IPC_CHANNELS.launchProviderSignIn, (_event, input) => {
     try {
       providers.launchSignInWindow(input);

src/main/database/migrations/077-model-checks.ts

@@ -0,0 +1,15 @@
+export const modelChecksSchema = `
+  create table model_checks (
+    provider_id text not null references providers(id) on delete cascade,
+    model text not null,
+    credential_id text not null,
+    org_id text not null,
+    passed integer not null,
+    detail text,
+    checked_at text not null,
+    primary key (provider_id, model, credential_id)
+  );
+
+  insert into schema_migrations(version, name, applied_at)
+  values (77, 'model_checks', datetime('now'));
+`;

src/main/database/schema.ts

@@ -75,6 +75,7 @@ import { usageWindowTokensSchema } from "./migrations/073-usage-window-tokens";
 import { runFailureKindSchema } from "./migrations/074-run-failure-kind";
 import { providerCooldownsSchema } from "./migrations/075-provider-cooldowns";
 import { runPinsSchema } from "./migrations/076-run-pins";
+import { modelChecksSchema } from "./migrations/077-model-checks";
 
 interface Migration {
   version: number;
@@ -159,6 +160,7 @@ const MIGRATIONS: Migration[] = [
   { version: 74, name: "run_failure_kind", sql: runFailureKindSchema },
   { version: 75, name: "provider_cooldowns", sql: providerCooldownsSchema },
   { version: 76, name: "run_pins", sql: runPinsSchema },
+  { version: 77, name: "model_checks", sql: modelChecksSchema },
 ];
 
 const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version;

src/main/services/model-check.ts

@@ -0,0 +1,58 @@
+import { randomInt } from "node:crypto";
+import type { DatabaseSync } from "node:sqlite";
+import type { AgentTurnMessage, AgentTurnResult, ToolDefinition } from "./provider-tool-turn";
+
+export const MODEL_CHECK_TOOL = "confirm_ready";
+
+export interface ModelCheckTurn {
+  code: string;
+  instructions: string;
+  messages: AgentTurnMessage[];
+  tools: ToolDefinition[];
+}
+
+/** One turn that needs what agent work needs: system instructions, a tool, and arguments taken from the conversation. */
+export function modelCheckTurn(): ModelCheckTurn {
+  const code = String(randomInt(100_000, 1_000_000));
+  return {
+    code,
+    instructions: `You are being checked for agent work. Answer only by calling ${MODEL_CHECK_TOOL} with the code from the user's message; never answer in text.`,
+    messages: [{ role: "user", text: `The code is ${code}.` }],
+    tools: [{
+      name: MODEL_CHECK_TOOL,
+      description: "Confirms the agent is ready by echoing the code.",
+      parameters: { type: "object", properties: { code: { type: "string", description: "The code from the user's message." } }, required: ["code"] },
+    }],
+  };
+}
+
+/** Why the answer shows the model cannot do agent work on this interface, or null when it passed. */
+export function modelCheckProblem(result: Pick<AgentTurnResult, "toolCalls">, code: string): string | null {
+  const call = result.toolCalls.find((candidate) => candidate.name === MODEL_CHECK_TOOL);
+  if (!call) return result.toolCalls.length > 0 ? `it called ${result.toolCalls[0].name}, a tool it was not given` : "it answered in text instead of calling the tool it was given";
+  if (call.argumentsError) return `its tool arguments were ${call.argumentsError}`;
+  if (String(call.arguments.code) !== code) return "it called the tool without the code its instructions asked for";
+  return null;
+}
+
+export type ModelCheck = { passed: true } | { passed: false; detail: string };
+
+export function readModelCheck(db: DatabaseSync, providerId: string, model: string, credentialId: string): ModelCheck | null {
+  const row = db.prepare("select passed, detail from model_checks where provider_id = ? and model = ? and credential_id = ?")
+    .get(providerId, model, credentialId) as { passed: number; detail: string | null } | undefined;
+  if (!row) return null;
+  if (row.passed === 1) return { passed: true };
+  if (!row.detail) throw new Error(`The failed agent check of ${model} has no reason recorded.`);
+  return { passed: false, detail: row.detail };
+}
+
+export function recordModelCheck(db: DatabaseSync, input: { orgId: string; providerId: string; model: string; credentialId: string; problem: string | null; now: Date }): void {
+  db.prepare(`
+    insert into model_checks(provider_id, model, credential_id, org_id, passed, detail, checked_at) values (?, ?, ?, ?, ?, ?, ?)
+    on conflict(provider_id, model, credential_id) do update set passed = excluded.passed, detail = excluded.detail, checked_at = excluded.checked_at
+  `).run(input.providerId, input.model, input.credentialId, input.orgId, input.problem ? 0 : 1, input.problem, input.now.toISOString());
+}
+
+export function modelCheckFailure(providerLabel: string, model: string, detail: string): string {
+  return `${model} on ${providerLabel} failed the agent check: ${detail}. Pick another model, or check it again in Settings.`;
+}

src/main/services/provider-response-runner.ts

@@ -18,6 +18,8 @@ import { resolveTurnCredential } from "./provider-tool-turns/credential-resolver
 import { currentRunId } from "./run-control";
 import { budgetBlock, isPriced, promptChars, providerPrices } from "./spend";
 import { assertModelSelectable, effectiveModel } from "./task-run-preparation";
+import { modelCheckFailure, modelCheckProblem, modelCheckTurn, readModelCheck, recordModelCheck } from "./model-check";
+import type { ModelCheckResult } from "../../shared/desktop-api";
 import {
   HEALTH_STATE_HEALTHY,
   WORKSPACE_SCOPES,
@@ -113,7 +115,6 @@ export class ProviderResponseRunner {
   }): Promise<AgentTurnResult & { requestId: string }> {
     const provider = loadConfiguredProvider(this.deps.db, input.orgId, input.providerId);
     const workspaceId = this.normalizeWorkspaceId(input.orgId, input.workspaceId ?? null);
-    this.enforcePolicy(input.orgId, provider, PROVIDER_EXECUTION_MODE.FOREGROUND, workspaceId);
     if (!supportsToolTurn(provider.provider_key)) {
       throw new Error("Agent execution is not available for this provider; assign the agent an OpenAI, Anthropic, xAI, Groq, Mistral, Cerebras, OpenRouter, GPT account, GitHub Copilot, Google Antigravity, or Claude Pro provider.");
     }
@@ -134,32 +135,67 @@ export class ProviderResponseRunner {
     if (!model) throw new Error("Provider model is required.");
     const owner: ProviderRow & { model: string } = { ...provider, model };
     const turn = this.deps.toolTurn ?? selectToolTurn(provider.provider_key);
-    const blocked = agent ? budgetBlock(this.deps.db, agent.id, isPriced(providerPrices(this.deps.db, provider.id)), new Date()) : null;
+    const caller = { orgId: input.orgId, owner, workspaceId, agent, taskRunId, turn };
+    const known = readModelCheck(this.deps.db, provider.id, model, requireCredentialId(provider));
+    const problem = known ? (known.passed ? null : known.detail) : await this.probeModel(caller);
+    if (problem) throw new Error(modelCheckFailure(provider.provider_key, model, problem));
+    return this.callTurn(caller, { instructions: input.instructions, messages: input.messages, tools: input.tools });
+  }
+
+  /** Probes a model for agent work and remembers the verdict; a request that fails records nothing. */
+  private async probeModel(caller: TurnCaller): Promise<string | null> {
+    const probe = modelCheckTurn();
+    const problem = modelCheckProblem(await this.callTurn(caller, probe), probe.code);
+    const { owner } = caller;
+    recordModelCheck(this.deps.db, { orgId: caller.orgId, providerId: owner.id, model: owner.model, credentialId: requireCredentialId(owner), problem, now: new Date() });
+    console.info(`[model-check] ${owner.provider_key} ${owner.model}: ${problem ?? "passed"}`);
+    return problem;
+  }
+
+  /** Runs the agent check of a provider's model now, replacing any earlier result. */
+  async checkModel(input: { orgId: string; providerId: string; model: string }): Promise<ModelCheckResult> {
+    const provider = loadConfiguredProvider(this.deps.db, input.orgId, input.providerId);
+    if (!supportsToolTurn(provider.provider_key)) throw new Error("Agent execution is not available for this provider.");
+    const model = input.model.trim();
+    if (!model) throw new Error("A model is required.");
+    assertModelSelectable(provider.provider_key, model, provider.model);
+    const owner = { ...provider, model };
+    const turn = this.deps.toolTurn ?? selectToolTurn(provider.provider_key);
+    const problem = await this.probeModel({ orgId: input.orgId, owner, workspaceId: null, agent: null, taskRunId: null, turn });
+    return problem === null ? { passed: true } : { passed: false, detail: problem };
+  }
+
+  private async callTurn(caller: TurnCaller, request: { instructions: string | null; messages: AgentTurnMessage[]; tools: ToolDefinition[] }): Promise<AgentTurnResult & { requestId: string }> {
+    const { orgId, owner, workspaceId, agent, taskRunId, turn } = caller;
+    const model = owner.model;
+    // Every request, a probe or the work, passes the provider's policy and quota and the agent's budget.
+    this.enforcePolicy(orgId, owner, PROVIDER_EXECUTION_MODE.FOREGROUND, workspaceId);
+    const blocked = agent ? budgetBlock(this.deps.db, agent.id, isPriced(providerPrices(this.deps.db, owner.id)), new Date()) : null;
     if (blocked) throw new Error(blocked);
     // Everything the model reads: instructions, the conversation, and the tool definitions.
-    const turnInputChars = (input.instructions?.length ?? 0) + JSON.stringify(input.messages).length + JSON.stringify(input.tools).length;
+    const turnInputChars = (request.instructions?.length ?? 0) + JSON.stringify(request.messages).length + JSON.stringify(request.tools).length;
     const requestId = this.deps.requestStore.create({
       agentId: agent?.id ?? null,
       taskRunId,
       department: agent?.department ?? null,
-      orgId: input.orgId,
+      orgId,
       provider: owner,
       model,
       workspaceId,
-      promptSha256: hashPrompt(JSON.stringify(input.messages)),
+      promptSha256: hashPrompt(JSON.stringify(request.messages)),
       inputCharCount: turnInputChars,
       executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND,
       createdAt: new Date().toISOString(),
     });
     const startedAt = Date.now();
     try {
-      const credential = await resolveTurnCredential({ authBroker: this.deps.authBroker, integrationKeys: this.deps.integrationKeys, orgId: input.orgId, provider });
-      const result = await turn({ credential, model, instructions: input.instructions, messages: input.messages, tools: input.tools });
-      recordObservedRateLimit(this.deps.rateLimitRegistry, input.orgId, owner, model, result.rateLimitState ?? null);
+      const credential = await resolveTurnCredential({ authBroker: this.deps.authBroker, integrationKeys: this.deps.integrationKeys, orgId, provider: owner });
+      const result = await turn({ credential, model, instructions: request.instructions, messages: request.messages, tools: request.tools });
+      recordObservedRateLimit(this.deps.rateLimitRegistry, orgId, owner, model, result.rateLimitState ?? null);
       // Tool calls are output the model wrote, as much as its text.
       const outputChars = result.text.length + (result.toolCalls.length > 0 ? JSON.stringify(result.toolCalls).length : 0);
       this.deps.requestStore.markSucceeded({
-        orgId: input.orgId,
+        orgId,
         requestId,
         responseText: result.text,
         rawResponseJson: result.rawResponseJson,
@@ -168,7 +204,7 @@ export class ProviderResponseRunner {
         completedAt: new Date().toISOString(),
       });
       this.deps.responseObserver.recordSuccess({
-        orgId: input.orgId,
+        orgId,
         provider: owner,
         requestId,
         executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND,
@@ -176,13 +212,13 @@ export class ProviderResponseRunner {
         action: "response_created",
       });
       this.deps.requestStore.settleUsage({
-        orgId: input.orgId, requestId, provider: owner, rawResponseJson: result.rawResponseJson,
+        orgId, requestId, provider: owner, rawResponseJson: result.rawResponseJson,
         inputCharCount: turnInputChars, outputCharCount: outputChars,
       });
       return { ...result, requestId };
     } catch (error) {
       this.deps.responseObserver.recordFailure({
-        orgId: input.orgId,
+        orgId,
         provider: owner,
         requestId,
         executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND,
@@ -391,3 +427,17 @@ interface PreparedProviderExecution {
   adapter: ProviderAdapterRegistry[string];
   credential: { secret: string };
 }
+
+interface TurnCaller {
+  orgId: string;
+  owner: ProviderRow & { model: string };
+  workspaceId: string | null;
+  agent: { id: string; department: string } | null;
+  taskRunId: string | null;
+  turn: ToolTurnFn;
+}
+
+function requireCredentialId(provider: ProviderRow): string {
+  if (!provider.credential_id) throw new Error("Provider credential is required.");
+  return provider.credential_id;
+}

src/main/services/provider-service.ts

@@ -18,6 +18,7 @@ import { ProviderResponseRunner, type ProviderResponseCreateInput } from "./prov
 import { launchProviderSignInWindow, type LaunchProviderSignInWindowInput } from "./provider-sign-in-window";
 import type { ToolTurnFn } from "./provider-tool-turn";
 import { stopRun } from "./run-control";
+import type { ModelCheckResult } from "../../shared/desktop-api";
 import {
   AUTH_STATE_CONFIGURED,
   AUTH_STATE_REVOKED,
@@ -194,6 +195,10 @@ export class ProviderService {
     return adapter.listModels({ authProfile, model: provider.model });
   }
 
+  checkModel(input: { orgId: string; providerId: string; model: string }): Promise<ModelCheckResult> {
+    return this.runner.checkModel(input);
+  }
+
   async checkHealth(orgId: string, providerId: string): Promise<void> {
     const provider = loadConfiguredProvider(this.db, orgId, providerId);
     const adapter = requireAdapter(this.adapters, provider.provider_key, provider.auth_type);

src/preload/index.ts

@@ -60,6 +60,7 @@ const api: DesktopApi = {
   cancelProviderOAuth: (input) => invoke(IPC_CHANNELS.cancelProviderOAuth, input),
   checkProviderHealth: (input) => invoke(IPC_CHANNELS.checkProviderHealth, input),
   listProviderModels: (input) => invoke(IPC_CHANNELS.listProviderModels, input),
+  checkProviderModel: (input) => invoke(IPC_CHANNELS.checkProviderModel, input),
   updateProviderPolicy: (input) => invoke(IPC_CHANNELS.updateProviderPolicy, input),
   setProviderPrices: (input) => invoke(IPC_CHANNELS.setProviderPrices, input),
   openExternalUrl: (url) => invoke(IPC_CHANNELS.openExternalUrl, url),

src/renderer/src/i18n/zh.ts

@@ -32,6 +32,10 @@ export const zh: Record<string, string> = {
   "superseded": "已被取代",
   "stalled": "已停滞",
   "Resume": "恢复",
+  "Checking...": "检查中...",
+  "Check for agent work": "检查能否胜任 Agent 工作",
+  "This model follows instructions and calls tools.": "该模型能遵循指令并调用工具。",
+  "This model cannot do agent work here: {detail}": "该模型无法在此完成 Agent 工作：{detail}",
   "{n} in the last 7 days": "近 7 天 {n}",
   "Success rate (7 days)": "成功率（近 7 天）",
   "{n} tokens in the last 7 days": "近 7 天 {n} token",

src/renderer/src/pages/settings/ProviderConnectionDialog.tsx

@@ -3,6 +3,7 @@ import type { ProviderCatalogEntry, ProviderRecord } from "../../../../shared/do
 import { ProviderOAuthConnect } from "../../shared/ProviderOAuthConnect";
 import { Field } from "../../shared/ui";
 import { t } from "../../i18n/i18n";
+import type { ModelCheckResult } from "../../../../shared/desktop-api";
 
 interface CloudFields {
   baseUrl: string;
@@ -97,8 +98,10 @@ function ModelPicker({ entry, provider }: { entry: ProviderCatalogEntry; provide
   const [fetched, setFetched] = useState<ProviderCatalogEntry["models"] | null>(null);
   const [loading, setLoading] = useState(false);
   const [error, setError] = useState<string | null>(null);
+  const [checks, setChecks] = useState<Record<string, ModelCheckResult | "running">>({});
   const [value, setValue] = useState(provider.model ?? entry.defaultModel ?? entry.models[0]?.id ?? "");
   const models = fetched ?? entry.models;
+  const check = checks[value];
   const connected = entry.supportsLiveModels && Boolean(provider.credentialId) && provider.enabled && provider.authState === "configured";
 
   const refresh = async () => {
@@ -114,6 +117,20 @@ function ModelPicker({ entry, provider }: { entry: ProviderCatalogEntry; provide
     if (result.data[0] && !result.data.some((model) => model.id === value)) setValue(result.data[0].id);
   };
 
+  const runCheck = async () => {
+    // A verdict belongs to the model it was asked about, whatever is selected when it arrives.
+    const model = value;
+    setChecks((current) => ({ ...current, [model]: "running" }));
+    const result = await window.agentCompany.checkProviderModel({ orgId: provider.orgId, providerId: provider.id, model });
+    setChecks((current) => {
+      const next = { ...current };
+      if (result.ok) next[model] = result.data;
+      else delete next[model];
+      return next;
+    });
+    if (!result.ok) setError(result.error.message);
+  };
+
   return (
     <Field label={t("Default model")}>
       {models.length ? (
@@ -130,6 +147,13 @@ function ModelPicker({ entry, provider }: { entry: ProviderCatalogEntry; provide
           {loading ? t("Loading models...") : t("Refresh models")}
         </button>
       )}
+      {provider.credentialId && provider.authState === "configured" && (
+        <button type="button" className="button ghost" style={{ marginTop: 6 }} disabled={check === "running" || !value} onClick={runCheck}>
+          {check === "running" ? t("Checking...") : t("Check for agent work")}
+        </button>
+      )}
+      {typeof check === "object" && check.passed && <div style={{ color: "var(--green)", fontSize: 12, marginTop: 4 }}>{t("This model follows instructions and calls tools.")}</div>}
+      {typeof check === "object" && !check.passed && <div style={{ color: "var(--red)", fontSize: 12, marginTop: 4 }}>{t("This model cannot do agent work here: {detail}", { detail: check.detail })}</div>}
       {error && <div style={{ color: "var(--red)", fontSize: 12, marginTop: 4 }}>{error}</div>}
     </Field>
   );

src/shared/desktop-api.ts

@@ -110,6 +110,9 @@ export type RendererEvent =
   | { type: "provider-response-stream"; event: ProviderStreamRendererEvent }
   | ProviderOAuthEvent;
 
+/** The verdict of a model's agent check. */
+export type ModelCheckResult = { passed: true } | { passed: false; detail: string };
+
 export interface DesktopApi {
   loadSnapshot(): Promise<DesktopResult<AppSnapshot>>;
   bootstrapOnboarding(input: BootstrapOnboardingInput): Promise<DesktopResult<BootstrapOnboardingSummary>>;
@@ -164,6 +167,8 @@ export interface DesktopApi {
   cancelProviderOAuth(input: CancelProviderOAuthInput): Promise<DesktopResult<null>>;
   checkProviderHealth(input: CheckProviderHealthInput): Promise<DesktopResult<AppSnapshot>>;
   listProviderModels(input: ListProviderModelsInput): Promise<DesktopResult<ProviderModelEntry[]>>;
+  /** Runs the agent check of a model: one turn that must follow system instructions and call a tool. */
+  checkProviderModel(input: { orgId: string; providerId: string; model: string }): Promise<DesktopResult<ModelCheckResult>>;
   updateProviderPolicy(input: UpdateProviderPolicyInput): Promise<DesktopResult<AppSnapshot>>;
   setProviderPrices(input: SetProviderPricesInput): Promise<DesktopResult<AppSnapshot>>;
   openExternalUrl(url: string): Promise<DesktopResult<null>>;

src/shared/ipc.ts

@@ -33,6 +33,7 @@ export const IPC_CHANNELS = {
   cancelProviderOAuth: "agentcompany:cancel-provider-oauth",
   checkProviderHealth: "agentcompany:check-provider-health",
   listProviderModels: "agentcompany:list-provider-models",
+  checkProviderModel: "agentcompany:check-provider-model",
   updateProviderPolicy: "agentcompany:update-provider-policy",
   setProviderPrices: "agentcompany:set-provider-prices",
   openExternalUrl: "agentcompany:open-external-url",

tests/helpers/model-check.ts

@@ -0,0 +1,22 @@
+import type { DatabaseSync } from "node:sqlite";
+import { MODEL_CHECK_TOOL } from "../../src/main/services/model-check";
+import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
+
+/** A scripted provider that also passes the agent check the way a capable model would. */
+export function answeringModelCheck<T extends (input: ProviderToolTurnInput) => Promise<AgentTurnResult>>(turn: T): T {
+  return (async (input: ProviderToolTurnInput) => {
+    if (!input.tools.some((tool) => tool.name === MODEL_CHECK_TOOL)) return turn(input);
+    const first = input.messages[0];
+    const code = first?.role === "user" ? /The code is (\d+)/.exec(first.text)?.[1] : undefined;
+    if (!code) throw new Error("The agent check sent no code.");
+    return { text: "", toolCalls: [{ id: "check", name: MODEL_CHECK_TOOL, arguments: { code } }], rawResponseJson: "{}" };
+  }) as T;
+}
+
+/** Records that a provider's model already passed the agent check, so a test sees only its own requests. */
+export function markModelChecked(db: DatabaseSync, providerId: string, model: string): void {
+  const provider = db.prepare("select org_id, credential_id from providers where id = ?").get(providerId) as { org_id: string; credential_id: string | null } | undefined;
+  if (!provider?.credential_id) throw new Error(`Provider ${providerId} has no credential to check a model against.`);
+  db.prepare("insert into model_checks(provider_id, model, credential_id, org_id, passed, detail, checked_at) values (?, ?, ?, ?, 1, null, ?)")
+    .run(providerId, model, provider.credential_id, provider.org_id, new Date().toISOString());
+}

tests/integration/agent-model.test.ts

@@ -9,6 +9,7 @@ import { providerAdapters } from "../../src/main/services/provider-adapters";
 import { TaskExecutionService } from "../../src/main/services/task-execution-service";
 import { RISK_LEVEL, TASK_PRIORITY, TASK_STATUS } from "../../src/shared/domain";
 import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -27,7 +28,7 @@ describe("agent model", () => {
     };
     const connection = new DatabaseConnection(join(tmpdir(), `ac-model-${randomUUID()}.db`));
     const organization = new OrganizationService(connection.db);
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";
     const tasks: Record<string, string> = {};
@@ -53,7 +54,7 @@ describe("agent model", () => {
     const turn = async (): Promise<AgentTurnResult> => ({ text: "done", toolCalls: [], rawResponseJson: "{}" });
     const connection = new DatabaseConnection(join(tmpdir(), `ac-model-az-${randomUUID()}.db`));
     const organization = new OrganizationService(connection.db);
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";
     let taskId = "";
@@ -83,7 +84,7 @@ describe("agent model", () => {
     };
     const connection = new DatabaseConnection(join(tmpdir(), `ac-model-pin-${randomUUID()}.db`));
     const organization = new OrganizationService(connection.db);
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";
     let taskId = "";

tests/integration/agent-reply.test.ts

@@ -11,6 +11,7 @@ import { SnapshotService } from "../../src/main/services/snapshot-service";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import { canonicalMentionHandle } from "../../src/shared/mention-handles";
 import type { AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -59,7 +60,7 @@ function scriptedTeam(): (input: ProviderToolTurnInput) => Promise<AgentTurnResu
 describe("inter-agent reply", () => {
   test("an @mention wakes the teammate, who replies in the same thread", async () => {
     const connection = new DatabaseConnection(join(tmpdir(), `ac-reply-${randomUUID()}.db`));
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, scriptedTeam());
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(scriptedTeam()));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const taskExecution = new TaskExecutionService(connection.db, providers);
     const snapshot = new SnapshotService(connection.db);

tests/integration/autonomous-run.test.ts

@@ -12,6 +12,7 @@ import { TaskExecutionService } from "../../src/main/services/task-execution-ser
 import { SnapshotService } from "../../src/main/services/snapshot-service";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import type { AgentTurnMessage, AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -89,7 +90,7 @@ describe("autonomous bootstrap run", () => {
     const workspacePath = join(tmpdir(), `ac-auto-ws-${randomUUID()}`);
     mkdirSync(workspacePath, { recursive: true });
 
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, scriptedCeo());
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(scriptedCeo()));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const snapshot = new SnapshotService(connection.db);
 
@@ -183,7 +184,7 @@ describe("autonomous bootstrap run", () => {
   test("a goal created after onboarding reaches the CEO and runs autonomously", async () => {
     const connection = new DatabaseConnection(join(tmpdir(), `ac-goal-${randomUUID()}.db`));
     // A minimal well-behaved agent: does its session, closes its task, then stops.
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, async (input: ProviderToolTurnInput) => {
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(async (input: ProviderToolTurnInput) => {
       const firstUser = input.messages.find((message) => message.role === "user");
       const taskId = firstUser && firstUser.role === "user" ? /Task id: (\S+)/.exec(firstUser.text)?.[1] : undefined;
       const alreadyClosed = input.messages.some(
@@ -201,7 +202,7 @@ describe("autonomous bootstrap run", () => {
         toolCalls: [],
         rawResponseJson: JSON.stringify({ id: "resp_goal" }),
       };
-    });
+    }));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const organization = new OrganizationService(connection.db);
     const taskExecution = new TaskExecutionService(connection.db, providers);

tests/integration/ceo-founder-chat.test.ts

@@ -11,6 +11,7 @@ import { ChatService } from "../../src/main/services/chat-service";
 import { ensureCeoDirectSpace } from "../../src/main/services/ceo-direct-space";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import type { AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -67,7 +68,7 @@ describe("founder⇄CEO chat", () => {
   test("a founder DM wakes the CEO, who sees who is asking and replies in the pinned conversation", async () => {
     const connection = new DatabaseConnection(join(tmpdir(), `ac-ceo-chat-${randomUUID()}.db`));
     const capturedWakePrompts: string[] = [];
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, scriptedCeo(capturedWakePrompts));
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(scriptedCeo(capturedWakePrompts)));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const taskExecution = new TaskExecutionService(connection.db, providers);
     const chat = new ChatService(connection.db);
@@ -143,7 +144,7 @@ describe("founder⇄CEO chat", () => {
 
   test("a CEO who answers in plain text still has the reply delivered into the founder's thread", async () => {
     const connection = new DatabaseConnection(join(tmpdir(), `ac-ceo-plain-${randomUUID()}.db`));
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, plainTextCeo());
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(plainTextCeo()));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const taskExecution = new TaskExecutionService(connection.db, providers);
     const chat = new ChatService(connection.db);

tests/integration/channel-anchor.test.ts

@@ -10,6 +10,7 @@ import { providerAdapters } from "../../src/main/services/provider-adapters";
 import { TaskExecutionService } from "../../src/main/services/task-execution-service";
 import { canonicalMentionHandle } from "../../src/shared/mention-handles";
 import type { AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -37,7 +38,7 @@ function scriptedCeoReply(): (input: ProviderToolTurnInput) => Promise<AgentTurn
 describe("channel anchor wake", () => {
   test("a plain user message in a channel wakes the most senior member, who replies", async () => {
     const connection = new DatabaseConnection(join(tmpdir(), `ac-anchor-${randomUUID()}.db`));
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, scriptedCeoReply());
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(scriptedCeoReply()));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const taskExecution = new TaskExecutionService(connection.db, providers);
     const chat = new ChatService(connection.db);

tests/integration/chat-context.test.ts

@@ -14,6 +14,7 @@ import { dispatchAgentTool, type AgentToolContext } from "../../src/main/service
 import { ensureCeoDirectSpace } from "../../src/main/services/ceo-direct-space";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -34,7 +35,7 @@ async function founderLine() {
     return { text: `Reply ${wakePrompts.length}.`, toolCalls: [], rawResponseJson: "{}" };
   };
   const connection = new DatabaseConnection(join(tmpdir(), `ac-chatctx-${randomUUID()}.db`));
-  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
   const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
   const execution = new TaskExecutionService(connection.db, providers);
   const chat = new ChatService(connection.db);
@@ -153,7 +154,7 @@ describe("chat wake context", () => {
       }
       return { text: "Switching to Postgres.", toolCalls: [], rawResponseJson: "{}" };
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const execution = new TaskExecutionService(connection.db, providers);
     let ceoId = "";

tests/integration/goal-changes.test.ts

@@ -9,6 +9,7 @@ import { providerAdapters } from "../../src/main/services/provider-adapters";
 import { TaskExecutionService } from "../../src/main/services/task-execution-service";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -17,7 +18,7 @@ const testVault = {
 
 function company(turn: (input: ProviderToolTurnInput, db: DatabaseConnection["db"]) => AgentTurnResult) {
   const connection = new DatabaseConnection(join(tmpdir(), `ac-goalchg-${randomUUID()}.db`));
-  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, async (input) => turn(input, connection.db));
+  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(async (input) => turn(input, connection.db)));
   const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
   const execution = new TaskExecutionService(connection.db, providers);
   let orgId = "";

tests/integration/interactive-lane.test.ts

@@ -12,6 +12,7 @@ import { TaskExecutionService } from "../../src/main/services/task-execution-ser
 import { dispatchAgentTool } from "../../src/main/services/agent-tools";
 import { claimNextQueuedRun, startTaskQueueWorker } from "../../src/main/app/task-queue-worker";
 import type { AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -75,7 +76,7 @@ describe("interactive execution lane", () => {
       }
       return turn("done");
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, script);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(script));
     providers.saveCredential({ orgId, providerId, secret: "sk-test-key", model: "gpt-5.4" });
     const taskExecution = new TaskExecutionService(connection.db, providers);
 
@@ -162,7 +163,7 @@ describe("interactive execution lane", () => {
       }
       return turn("done");
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, script);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(script));
     providers.saveCredential({ orgId, providerId, secret: "sk-test-key", model: "gpt-5.4" });
     const taskExecution = new TaskExecutionService(connection.db, providers);
 
@@ -227,7 +228,7 @@ describe("interactive execution lane", () => {
     const alwaysOverloaded = async (): Promise<AgentTurnResult> => {
       throw new Error("The engine is currently overloaded, please try again later");
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, alwaysOverloaded);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(alwaysOverloaded));
     providers.saveCredential({ orgId, providerId, secret: "sk-test-key", model: "gpt-5.4" });
     const taskExecution = new TaskExecutionService(connection.db, providers);
     const wakeDana = () => dispatchAgentTool({ db: connection.db, organization, chat }, {

tests/integration/run-checkpoint.test.ts

@@ -11,6 +11,7 @@ import { ChatService } from "../../src/main/services/chat-service";
 import { ensureCeoDirectSpace } from "../../src/main/services/ceo-direct-space";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import type { AgentTurnResult } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -24,7 +25,7 @@ describe("session checkpoints", () => {
       ? { text: "Audit done: all clean.", toolCalls: [], rawResponseJson: "{}" }
       : { text: "Looking into it.", toolCalls: [{ id: `c_${randomUUID()}`, name: "read_messages", arguments: {} }], rawResponseJson: "{}" };
     const connection = new DatabaseConnection(join(tmpdir(), `ac-checkpoint-${randomUUID()}.db`));
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";

tests/integration/run-journal.test.ts

@@ -9,6 +9,7 @@ import { providerAdapters } from "../../src/main/services/provider-adapters";
 import { TaskExecutionService } from "../../src/main/services/task-execution-service";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -41,7 +42,7 @@ describe("run journal", () => {
       throw new Error("Provider dropped the connection mid-run");
     };
     const connection = new DatabaseConnection(join(tmpdir(), `ac-journal-${randomUUID()}.db`));
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";

tests/integration/run-memory.test.ts

@@ -11,6 +11,7 @@ import { ChatService } from "../../src/main/services/chat-service";
 import { ensureCeoDirectSpace } from "../../src/main/services/ceo-direct-space";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -26,7 +27,7 @@ async function company(reply: (prompt: string) => AgentTurnResult) {
     return reply(prompt);
   };
   const connection = new DatabaseConnection(join(tmpdir(), `ac-memory-${randomUUID()}.db`));
-  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
   const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
   const execution = new TaskExecutionService(connection.db, providers);
   let orgId = "";

tests/integration/run-stop.test.ts

@@ -13,6 +13,7 @@ import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import { postJson } from "../../src/main/services/provider-http";
 import { RUN_STOPPED_MESSAGE, stopRun, withRunControl } from "../../src/main/services/run-control";
 import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -21,7 +22,7 @@ const testVault = {
 
 function company(turn: (input: ProviderToolTurnInput) => Promise<AgentTurnResult>) {
   const connection = new DatabaseConnection(join(tmpdir(), `ac-stop-${randomUUID()}.db`));
-  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
   const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
   const execution = new TaskExecutionService(connection.db, providers);
   let orgId = "";
@@ -44,7 +45,7 @@ describe("stopping runs", () => {
       return { text: "Announcing.", toolCalls: [{ id: "c1", name: "send_message", arguments: { channel: "general", body: "Should never be posted." } }], rawResponseJson: "{}" };
     };
     const connection = new DatabaseConnection(join(tmpdir(), `ac-stop-${randomUUID()}.db`));
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";
@@ -115,7 +116,7 @@ describe("stopping runs", () => {
       return { text: "done", toolCalls: [], rawResponseJson: "{}" };
     };
     const connection = new DatabaseConnection(join(tmpdir(), `ac-deadline-${randomUUID()}.db`));
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const execution = new TaskExecutionService(connection.db, providers, undefined, undefined, undefined, 200);
     let orgId = "";

tests/integration/spend-budget.test.ts

@@ -9,6 +9,7 @@ import { providerAdapters } from "../../src/main/services/provider-adapters";
 import { TaskExecutionService } from "../../src/main/services/task-execution-service";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import type { AgentTurnResult } from "../../src/main/services/provider-tool-turn";
+import { markModelChecked } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -39,6 +40,7 @@ describe("money budgets", () => {
       ceoId = summary.ceoAgentId;
     });
     const providerId = (connection.db.prepare("select model_provider_id as id from agents where id = ?").get(ceoId) as { id: string }).id;
+    markModelChecked(connection.db, providerId, "gpt-5.4");
     providers.setPrices({ orgId, providerId, inputDollarsPerMtok: 2.5, cachedInputDollarsPerMtok: null, outputDollarsPerMtok: 10 });
     connection.db.prepare("update agents set budget_cents = 2000 where id = ?").run(ceoId);
 

tests/integration/stale-queued-runs.test.ts

@@ -11,6 +11,7 @@ import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import { ChatService } from "../../src/main/services/chat-service";
 import { ensureCeoDirectSpace } from "../../src/main/services/ceo-direct-space";
 import type { AgentTurnResult } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -24,7 +25,7 @@ function company() {
     return { text: "done", toolCalls: [], rawResponseJson: "{}" };
   };
   const connection = new DatabaseConnection(join(tmpdir(), `ac-stale-${randomUUID()}.db`));
-  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
   const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
   const execution = new TaskExecutionService(connection.db, providers);
   let orgId = "";

tests/integration/task-context-scope.test.ts

@@ -13,6 +13,7 @@ import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import { canonicalMentionHandle } from "../../src/shared/mention-handles";
 import { RISK_LEVEL, TASK_PRIORITY, TASK_STATUS } from "../../src/shared/domain";
 import type { AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -49,7 +50,7 @@ describe("task prompt scope", () => {
       return { text: step.text, toolCalls: step.toolCalls, rawResponseJson: "{}" };
     };
     const connection = new DatabaseConnection(join(tmpdir(), `ac-scope-${randomUUID()}.db`));
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";
@@ -90,7 +91,7 @@ describe("task prompt scope", () => {
       calls += 1;
       return { text: "done", toolCalls: [], rawResponseJson: "{}" };
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const organization = new OrganizationService(connection.db);
     const execution = new TaskExecutionService(connection.db, providers);
@@ -125,7 +126,7 @@ describe("task prompt scope", () => {
   test("a manual write run waits while another run is changing the same workspace", async () => {
     const connection = new DatabaseConnection(join(tmpdir(), `ac-wsplock-${randomUUID()}.db`));
     const turn = async (): Promise<AgentTurnResult> => ({ text: "done", toolCalls: [], rawResponseJson: "{}" });
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
     const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
     const organization = new OrganizationService(connection.db);
     const execution = new TaskExecutionService(connection.db, providers);

tests/integration/task-dependencies.test.ts

@@ -13,6 +13,7 @@ import { dispatchAgentTool } from "../../src/main/services/agent-tools";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
 import { RISK_LEVEL, TASK_PRIORITY, TASK_STATUS } from "../../src/shared/domain";
 import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -28,7 +29,7 @@ function company() {
     prompts.push(first.text);
     return { text: "done", toolCalls: [], rawResponseJson: "{}" };
   };
-  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn));
   const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
   const execution = new TaskExecutionService(connection.db, providers);
   const organization = new OrganizationService(connection.db);

tests/unit/agent-instructions.test.ts

@@ -14,6 +14,7 @@ import { buildAgentInstructions, type TaskExecutionAgentRow } from "../../src/ma
 import { AGENT_TOOLS } from "../../src/main/services/agent-tools";
 import { goalKickoffDescription } from "../../src/main/services/goal-kickoff";
 import { CEO_FIRST_HIRES } from "../../src/main/services/org-building";
+import { answeringModelCheck } from "../helpers/model-check";
 
 function agentRow(overrides: Partial<TaskExecutionAgentRow>): TaskExecutionAgentRow {
   return {
@@ -134,7 +135,7 @@ describe("agent instruction wiring", () => {
       capturedInstructions = input.instructions ?? "";
       return { text: "done", toolCalls: [], rawResponseJson: JSON.stringify({ id: "resp-1" }) };
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn));
     const execution = new TaskExecutionService(connection.db, providers);
 
     let orgId = "";
@@ -189,7 +190,7 @@ describe("agent instruction wiring", () => {
       capturedPrompt = first.text;
       return { text: "done", toolCalls: [], rawResponseJson: JSON.stringify({ id: "resp-1" }) };
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn));
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";
     let taskId = "";

tests/unit/char-accounting.test.ts

@@ -7,6 +7,7 @@ import { OrganizationService } from "../../src/main/services/organization-servic
 import { ProviderService } from "../../src/main/services/provider-service";
 import { providerAdapters } from "../../src/main/services/provider-adapters";
 import type { AgentTurnResult, ToolDefinition } from "../../src/main/services/provider-tool-turn";
+import { markModelChecked } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -25,6 +26,7 @@ describe("character accounting", () => {
       orgId = new OrganizationService(connection.db).createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" });
       providerId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'openai'").get(orgId) as { id: string }).id;
       providers.saveCredential({ orgId, providerId, secret: "sk-test", model: "gpt-5.4" });
+      markModelChecked(connection.db, providerId, "gpt-5.4");
     });
     const instructions = "You run this company.";
     const messages = [{ role: "user" as const, text: "Kick off the launch." }];

tests/unit/domain-services.test.ts

@@ -24,7 +24,7 @@ describe("domain services", () => {
     const connection = createConnection();
 
     const rows = connection.db.prepare("select version from schema_migrations order by version asc").all() as Array<{ version: number }>;
-    expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76]);
+    expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77]);
     connection.close();
   });
 

tests/unit/model-check.test.ts

@@ -0,0 +1,84 @@
+import { randomUUID } from "node:crypto";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, test } from "vitest";
+import { DatabaseConnection } from "../../src/main/database/connection";
+import { OrganizationService } from "../../src/main/services/organization-service";
+import { ProviderService } from "../../src/main/services/provider-service";
+import { providerAdapters } from "../../src/main/services/provider-adapters";
+import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
+import { answeringModelCheck } from "../helpers/model-check";
+import { PROVIDER_DATA_POLICY } from "../../src/shared/domain";
+import { PROVIDER_MONTHLY_LIMIT_MESSAGE } from "../../src/shared/provider-runtime-domain";
+
+const testVault = {
+  encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
+  decryptSecret: (ciphertext: Buffer) => ciphertext.toString("utf8"),
+};
+
+const WORK = { workspaceId: null, instructions: "Do the work.", messages: [{ role: "user" as const, text: "go" }], tools: [] };
+const TEXT_ONLY = async (): Promise<AgentTurnResult> => ({ text: "Sure, the code is 123.", toolCalls: [], rawResponseJson: "{}" });
+
+function harness(turn: (input: ProviderToolTurnInput) => Promise<AgentTurnResult>) {
+  const connection = new DatabaseConnection(join(tmpdir(), `ac-model-check-${randomUUID()}.db`));
+  const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn);
+  let orgId = "";
+  let providerId = "";
+  connection.transaction(() => {
+    orgId = new OrganizationService(connection.db).createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" });
+    providerId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'openai'").get(orgId) as { id: string }).id;
+    providers.saveCredential({ orgId, providerId, secret: "sk-test", model: "gpt-5.4" });
+  });
+  const requests = () => (connection.db.prepare("select count(*) as n from provider_requests").get() as { n: number }).n;
+  return { connection, providers, orgId, providerId, requests };
+}
+
+describe("a model is checked for agent work before it does any", () => {
+  test("a model that answers in text instead of calling tools is refused, and the verdict is kept", async () => {
+    const h = harness(TEXT_ONLY);
+    await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK }))
+      .rejects.toThrow("gpt-5.4 on openai failed the agent check: it answered in text instead of calling the tool it was given");
+    expect(h.requests()).toBe(1);
+    await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("failed the agent check");
+    expect(h.requests()).toBe(1);
+    h.connection.close();
+  });
+
+  test("a capable model is probed once, then works; the probe is a request like any other and counts against the quota", async () => {
+    const seen: ProviderToolTurnInput[] = [];
+    const h = harness(answeringModelCheck(async (input) => { seen.push(input); return { text: "Done.", toolCalls: [], rawResponseJson: "{}" }; }));
+    expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
+    expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
+    expect([h.requests(), seen.length]).toEqual([3, 2]);
+
+    const tight = harness(answeringModelCheck(async () => ({ text: "Done.", toolCalls: [], rawResponseJson: "{}" })));
+    tight.providers.updatePolicy({ orgId: tight.orgId, providerId: tight.providerId, dataPolicy: PROVIDER_DATA_POLICY.STANDARD, backgroundEnabled: false, monthlyRequestLimit: 1, workspaceScope: "organization", contextWindowTokens: 128_000 });
+    await expect(tight.providers.runToolTurn({ orgId: tight.orgId, providerId: tight.providerId, ...WORK })).rejects.toThrow(PROVIDER_MONTHLY_LIMIT_MESSAGE);
+    expect(tight.requests()).toBe(1);
+    h.connection.close();
+    tight.connection.close();
+  });
+
+  test("a probe whose request fails records nothing; a new model gets its own probe; Settings can check again", async () => {
+    let mode: "down" | "text" | "capable" = "down";
+    const capable: (input: ProviderToolTurnInput) => Promise<AgentTurnResult> = answeringModelCheck(async () => ({ text: "ok", toolCalls: [], rawResponseJson: "{}" }));
+    const h = harness(async (input) => {
+      if (mode === "down") throw new Error("connect ECONNREFUSED");
+      return mode === "text" ? TEXT_ONLY() : capable(input);
+    });
+    await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("ECONNREFUSED");
+    expect(h.connection.db.prepare("select count(*) as n from model_checks").get()).toEqual({ n: 0 });
+
+    mode = "text";
+    expect(await h.providers.checkModel({ orgId: h.orgId, providerId: h.providerId, model: "gpt-5.4" }))
+      .toEqual({ passed: false, detail: "it answered in text instead of calling the tool it was given" });
+    mode = "capable";
+    expect(await h.providers.checkModel({ orgId: h.orgId, providerId: h.providerId, model: "gpt-5.4" })).toEqual({ passed: true });
+    expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("ok");
+
+    h.connection.db.prepare("update providers set model = 'gpt-5.5' where id = ?").run(h.providerId);
+    mode = "text";
+    await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("gpt-5.5 on openai failed the agent check");
+    h.connection.close();
+  });
+});

tests/unit/quota-reservation.test.ts

@@ -9,6 +9,7 @@ import { providerAdapters } from "../../src/main/services/provider-adapters";
 import { PROVIDER_MONTHLY_LIMIT_MESSAGE } from "../../src/shared/provider-runtime-domain";
 import { PROVIDER_DATA_POLICY } from "../../src/shared/domain";
 import type { AgentTurnResult } from "../../src/main/services/provider-tool-turn";
+import { markModelChecked } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -33,6 +34,7 @@ describe("monthly quota reservation", () => {
       orgId = new OrganizationService(connection.db).createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" });
       providerId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'openai'").get(orgId) as { id: string }).id;
       providers.saveCredential({ orgId, providerId, secret: "sk-test", model: "gpt-5.4" });
+      markModelChecked(connection.db, providerId, "gpt-5.4");
       providers.updatePolicy({ orgId, providerId, dataPolicy: PROVIDER_DATA_POLICY.STANDARD, backgroundEnabled: false, monthlyRequestLimit: 1, workspaceScope: "organization", contextWindowTokens: 128_000 });
     });
     const call = () => providers.runToolTurn({ orgId, providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] });

tests/unit/rate-limit-observation.test.ts

@@ -10,6 +10,7 @@ import { TaskExecutionService } from "../../src/main/services/task-execution-ser
 import { RISK_LEVEL, TASK_PRIORITY, TASK_STATUS } from "../../src/shared/domain";
 import type { AgentTurnResult } from "../../src/main/services/provider-tool-turn";
 import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
+import { markModelChecked } from "../helpers/model-check";
 
 const testVault = {
   encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
@@ -33,6 +34,7 @@ describe("rate limits seen on successful calls", () => {
       orgId = organization.createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" });
       providerId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'openai'").get(orgId) as { id: string }).id;
       providers.saveCredential({ orgId, providerId, secret: "sk-test", model: "gpt-5.4" });
+      markModelChecked(connection.db, providerId, "gpt-5.4");
       const agentId = organization.createAgent({ orgId, name: "Ada", role: "Engineer", department: "Engineering", modelProviderId: providerId, modelId: "gpt-5.4", budgetCents: 1000, managerAgentId: null, heartbeatEnabled: true });
       organization.createTask({ orgId, title: "Work", description: "d", status: TASK_STATUS.BACKLOG, assigneeAgentId: agentId, risk: RISK_LEVEL.LOW, projectId: null, goalId: null, workspaceId: null, priority: TASK_PRIORITY.NORMAL, requiresFilesystem: false });
     });

tests/unit/task-execution-service.test.ts

@@ -14,6 +14,7 @@ import { SnapshotService } from "../../src/main/services/snapshot-service";
 import { TaskExecutionService } from "../../src/main/services/task-execution-service";
 import { APPROVAL_STATE, PROVIDER_DATA_POLICY, RISK_LEVEL, TASK_PRIORITY, TASK_STATUS } from "../../src/shared/domain";
 import { currentProviderUsageWindow, PROVIDER_MONTHLY_LIMIT_MESSAGE } from "../../src/shared/provider-runtime-domain";
+import { answeringModelCheck } from "../helpers/model-check";
 
 function createConnection(): DatabaseConnection {
   return new DatabaseConnection(join(tmpdir(), `agentcompany-new-task-execution-${randomUUID()}.db`));
@@ -40,7 +41,7 @@ describe("task execution service", () => {
       if (first?.role === "user") capturedPrompt = first.text;
       return { text: "executed:gpt-5.4", toolCalls: [], rawResponseJson: JSON.stringify({ id: "resp-task-1" }) };
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn));
     const skills = new SkillService(connection.db, join(tmpdir(), `agentcompany-task-skills-${randomUUID()}`));
     const execution = new TaskExecutionService(connection.db, providers);
     const skillDir = join(tmpdir(), `agentcompany-task-skill-${randomUUID()}`);
@@ -152,7 +153,7 @@ describe("task execution service", () => {
       toolCalls: [],
       rawResponseJson: JSON.stringify({ id: "resp-task-approved" }),
     });
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn));
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";
     let employeeId = "";
@@ -291,7 +292,7 @@ describe("task execution service", () => {
     const fakeToolTurn = async (): Promise<AgentTurnResult> => {
       throw new Error("provider rejected task");
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn));
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";
     let taskId = "";
@@ -349,7 +350,7 @@ describe("task execution service", () => {
       turns += 1;
       return { text: "done", toolCalls: [], rawResponseJson: JSON.stringify({ id: `resp-${turns}` }) };
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn));
     const execution = new TaskExecutionService(connection.db, providers);
     let orgId = "";
     let taskId = "";
@@ -848,7 +849,7 @@ describe("task execution service", () => {
     const fakeToolTurn = async (): Promise<AgentTurnResult> => {
       throw new Error("The usage limit has been reached");
     };
-    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn));
     let nowMs = Date.now();
     const execution = new TaskExecutionService(connection.db, providers, undefined, undefined, () => nowMs);
     let orgId = "";