From 665bf3cae1fd1ce83d4099038b4c857dcb1442c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Thu, 8 Oct 2026 14:19:48 -0400 Subject: [PATCH] Q83: a model passes an agent check before it does agent work Co-Authored-By: Claude Opus 5.5 --- src/main/app/ipc-handlers.ts | 7 ++ .../database/migrations/077-model-checks.ts | 15 ++++ src/main/database/schema.ts | 2 + src/main/services/model-check.ts | 58 +++++++++++++ src/main/services/provider-response-runner.ts | 74 +++++++++++++--- src/main/services/provider-service.ts | 5 ++ src/preload/index.ts | 1 + src/renderer/src/i18n/zh.ts | 4 + .../settings/ProviderConnectionDialog.tsx | 24 ++++++ src/shared/desktop-api.ts | 5 ++ src/shared/ipc.ts | 1 + tests/helpers/model-check.ts | 22 +++++ tests/integration/agent-model.test.ts | 7 +- tests/integration/agent-reply.test.ts | 3 +- tests/integration/autonomous-run.test.ts | 7 +- tests/integration/ceo-founder-chat.test.ts | 5 +- tests/integration/channel-anchor.test.ts | 3 +- tests/integration/chat-context.test.ts | 5 +- tests/integration/goal-changes.test.ts | 3 +- tests/integration/interactive-lane.test.ts | 7 +- tests/integration/run-checkpoint.test.ts | 3 +- tests/integration/run-journal.test.ts | 3 +- tests/integration/run-memory.test.ts | 3 +- tests/integration/run-stop.test.ts | 7 +- tests/integration/spend-budget.test.ts | 2 + tests/integration/stale-queued-runs.test.ts | 3 +- tests/integration/task-context-scope.test.ts | 7 +- tests/integration/task-dependencies.test.ts | 3 +- tests/unit/agent-instructions.test.ts | 5 +- tests/unit/char-accounting.test.ts | 2 + tests/unit/domain-services.test.ts | 2 +- tests/unit/model-check.test.ts | 84 +++++++++++++++++++ tests/unit/quota-reservation.test.ts | 2 + tests/unit/rate-limit-observation.test.ts | 2 + tests/unit/task-execution-service.test.ts | 11 +-- 35 files changed, 350 insertions(+), 47 deletions(-) create mode 100644 src/main/database/migrations/077-model-checks.ts create mode 100644 src/main/services/model-check.ts create mode 100644 tests/helpers/model-check.ts create mode 100644 tests/unit/model-check.test.ts diff --git a/src/main/app/ipc-handlers.ts b/src/main/app/ipc-handlers.ts index 5f5647ff..883a08e5 100644 --- a/src/main/app/ipc-handlers.ts +++ b/src/main/app/ipc-handlers.ts @@ -228,6 +228,13 @@ export function registerIpcHandlers(deps: IpcDeps): void { return fail("provider_models_failed", error instanceof Error ? error.message : String(error)); } }); + ipcMain.handle(IPC_CHANNELS.checkProviderModel, async (_event, input) => { + try { + return ok(await providers.checkModel(input)); + } catch (error) { + return fail("provider_model_check_failed", error instanceof Error ? error.message : String(error)); + } + }); ipcMain.handle(IPC_CHANNELS.launchProviderSignIn, (_event, input) => { try { providers.launchSignInWindow(input); diff --git a/src/main/database/migrations/077-model-checks.ts b/src/main/database/migrations/077-model-checks.ts new file mode 100644 index 00000000..99de4d00 --- /dev/null +++ b/src/main/database/migrations/077-model-checks.ts @@ -0,0 +1,15 @@ +export const modelChecksSchema = ` + create table model_checks ( + provider_id text not null references providers(id) on delete cascade, + model text not null, + credential_id text not null, + org_id text not null, + passed integer not null, + detail text, + checked_at text not null, + primary key (provider_id, model, credential_id) + ); + + insert into schema_migrations(version, name, applied_at) + values (77, 'model_checks', datetime('now')); +`; diff --git a/src/main/database/schema.ts b/src/main/database/schema.ts index 30346d35..a844c3e3 100644 --- a/src/main/database/schema.ts +++ b/src/main/database/schema.ts @@ -75,6 +75,7 @@ import { usageWindowTokensSchema } from "./migrations/073-usage-window-tokens"; import { runFailureKindSchema } from "./migrations/074-run-failure-kind"; import { providerCooldownsSchema } from "./migrations/075-provider-cooldowns"; import { runPinsSchema } from "./migrations/076-run-pins"; +import { modelChecksSchema } from "./migrations/077-model-checks"; interface Migration { version: number; @@ -159,6 +160,7 @@ const MIGRATIONS: Migration[] = [ { version: 74, name: "run_failure_kind", sql: runFailureKindSchema }, { version: 75, name: "provider_cooldowns", sql: providerCooldownsSchema }, { version: 76, name: "run_pins", sql: runPinsSchema }, + { version: 77, name: "model_checks", sql: modelChecksSchema }, ]; const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version; diff --git a/src/main/services/model-check.ts b/src/main/services/model-check.ts new file mode 100644 index 00000000..2b12fba6 --- /dev/null +++ b/src/main/services/model-check.ts @@ -0,0 +1,58 @@ +import { randomInt } from "node:crypto"; +import type { DatabaseSync } from "node:sqlite"; +import type { AgentTurnMessage, AgentTurnResult, ToolDefinition } from "./provider-tool-turn"; + +export const MODEL_CHECK_TOOL = "confirm_ready"; + +export interface ModelCheckTurn { + code: string; + instructions: string; + messages: AgentTurnMessage[]; + tools: ToolDefinition[]; +} + +/** One turn that needs what agent work needs: system instructions, a tool, and arguments taken from the conversation. */ +export function modelCheckTurn(): ModelCheckTurn { + const code = String(randomInt(100_000, 1_000_000)); + return { + code, + instructions: `You are being checked for agent work. Answer only by calling ${MODEL_CHECK_TOOL} with the code from the user's message; never answer in text.`, + messages: [{ role: "user", text: `The code is ${code}.` }], + tools: [{ + name: MODEL_CHECK_TOOL, + description: "Confirms the agent is ready by echoing the code.", + parameters: { type: "object", properties: { code: { type: "string", description: "The code from the user's message." } }, required: ["code"] }, + }], + }; +} + +/** Why the answer shows the model cannot do agent work on this interface, or null when it passed. */ +export function modelCheckProblem(result: Pick, code: string): string | null { + const call = result.toolCalls.find((candidate) => candidate.name === MODEL_CHECK_TOOL); + if (!call) return result.toolCalls.length > 0 ? `it called ${result.toolCalls[0].name}, a tool it was not given` : "it answered in text instead of calling the tool it was given"; + if (call.argumentsError) return `its tool arguments were ${call.argumentsError}`; + if (String(call.arguments.code) !== code) return "it called the tool without the code its instructions asked for"; + return null; +} + +export type ModelCheck = { passed: true } | { passed: false; detail: string }; + +export function readModelCheck(db: DatabaseSync, providerId: string, model: string, credentialId: string): ModelCheck | null { + const row = db.prepare("select passed, detail from model_checks where provider_id = ? and model = ? and credential_id = ?") + .get(providerId, model, credentialId) as { passed: number; detail: string | null } | undefined; + if (!row) return null; + if (row.passed === 1) return { passed: true }; + if (!row.detail) throw new Error(`The failed agent check of ${model} has no reason recorded.`); + return { passed: false, detail: row.detail }; +} + +export function recordModelCheck(db: DatabaseSync, input: { orgId: string; providerId: string; model: string; credentialId: string; problem: string | null; now: Date }): void { + db.prepare(` + insert into model_checks(provider_id, model, credential_id, org_id, passed, detail, checked_at) values (?, ?, ?, ?, ?, ?, ?) + on conflict(provider_id, model, credential_id) do update set passed = excluded.passed, detail = excluded.detail, checked_at = excluded.checked_at + `).run(input.providerId, input.model, input.credentialId, input.orgId, input.problem ? 0 : 1, input.problem, input.now.toISOString()); +} + +export function modelCheckFailure(providerLabel: string, model: string, detail: string): string { + return `${model} on ${providerLabel} failed the agent check: ${detail}. Pick another model, or check it again in Settings.`; +} diff --git a/src/main/services/provider-response-runner.ts b/src/main/services/provider-response-runner.ts index bc76e94c..a3468a57 100644 --- a/src/main/services/provider-response-runner.ts +++ b/src/main/services/provider-response-runner.ts @@ -18,6 +18,8 @@ import { resolveTurnCredential } from "./provider-tool-turns/credential-resolver import { currentRunId } from "./run-control"; import { budgetBlock, isPriced, promptChars, providerPrices } from "./spend"; import { assertModelSelectable, effectiveModel } from "./task-run-preparation"; +import { modelCheckFailure, modelCheckProblem, modelCheckTurn, readModelCheck, recordModelCheck } from "./model-check"; +import type { ModelCheckResult } from "../../shared/desktop-api"; import { HEALTH_STATE_HEALTHY, WORKSPACE_SCOPES, @@ -113,7 +115,6 @@ export class ProviderResponseRunner { }): Promise { const provider = loadConfiguredProvider(this.deps.db, input.orgId, input.providerId); const workspaceId = this.normalizeWorkspaceId(input.orgId, input.workspaceId ?? null); - this.enforcePolicy(input.orgId, provider, PROVIDER_EXECUTION_MODE.FOREGROUND, workspaceId); if (!supportsToolTurn(provider.provider_key)) { throw new Error("Agent execution is not available for this provider; assign the agent an OpenAI, Anthropic, xAI, Groq, Mistral, Cerebras, OpenRouter, GPT account, GitHub Copilot, Google Antigravity, or Claude Pro provider."); } @@ -134,32 +135,67 @@ export class ProviderResponseRunner { if (!model) throw new Error("Provider model is required."); const owner: ProviderRow & { model: string } = { ...provider, model }; const turn = this.deps.toolTurn ?? selectToolTurn(provider.provider_key); - const blocked = agent ? budgetBlock(this.deps.db, agent.id, isPriced(providerPrices(this.deps.db, provider.id)), new Date()) : null; + const caller = { orgId: input.orgId, owner, workspaceId, agent, taskRunId, turn }; + const known = readModelCheck(this.deps.db, provider.id, model, requireCredentialId(provider)); + const problem = known ? (known.passed ? null : known.detail) : await this.probeModel(caller); + if (problem) throw new Error(modelCheckFailure(provider.provider_key, model, problem)); + return this.callTurn(caller, { instructions: input.instructions, messages: input.messages, tools: input.tools }); + } + + /** Probes a model for agent work and remembers the verdict; a request that fails records nothing. */ + private async probeModel(caller: TurnCaller): Promise { + const probe = modelCheckTurn(); + const problem = modelCheckProblem(await this.callTurn(caller, probe), probe.code); + const { owner } = caller; + recordModelCheck(this.deps.db, { orgId: caller.orgId, providerId: owner.id, model: owner.model, credentialId: requireCredentialId(owner), problem, now: new Date() }); + console.info(`[model-check] ${owner.provider_key} ${owner.model}: ${problem ?? "passed"}`); + return problem; + } + + /** Runs the agent check of a provider's model now, replacing any earlier result. */ + async checkModel(input: { orgId: string; providerId: string; model: string }): Promise { + const provider = loadConfiguredProvider(this.deps.db, input.orgId, input.providerId); + if (!supportsToolTurn(provider.provider_key)) throw new Error("Agent execution is not available for this provider."); + const model = input.model.trim(); + if (!model) throw new Error("A model is required."); + assertModelSelectable(provider.provider_key, model, provider.model); + const owner = { ...provider, model }; + const turn = this.deps.toolTurn ?? selectToolTurn(provider.provider_key); + const problem = await this.probeModel({ orgId: input.orgId, owner, workspaceId: null, agent: null, taskRunId: null, turn }); + return problem === null ? { passed: true } : { passed: false, detail: problem }; + } + + private async callTurn(caller: TurnCaller, request: { instructions: string | null; messages: AgentTurnMessage[]; tools: ToolDefinition[] }): Promise { + const { orgId, owner, workspaceId, agent, taskRunId, turn } = caller; + const model = owner.model; + // Every request, a probe or the work, passes the provider's policy and quota and the agent's budget. + this.enforcePolicy(orgId, owner, PROVIDER_EXECUTION_MODE.FOREGROUND, workspaceId); + const blocked = agent ? budgetBlock(this.deps.db, agent.id, isPriced(providerPrices(this.deps.db, owner.id)), new Date()) : null; if (blocked) throw new Error(blocked); // Everything the model reads: instructions, the conversation, and the tool definitions. - const turnInputChars = (input.instructions?.length ?? 0) + JSON.stringify(input.messages).length + JSON.stringify(input.tools).length; + const turnInputChars = (request.instructions?.length ?? 0) + JSON.stringify(request.messages).length + JSON.stringify(request.tools).length; const requestId = this.deps.requestStore.create({ agentId: agent?.id ?? null, taskRunId, department: agent?.department ?? null, - orgId: input.orgId, + orgId, provider: owner, model, workspaceId, - promptSha256: hashPrompt(JSON.stringify(input.messages)), + promptSha256: hashPrompt(JSON.stringify(request.messages)), inputCharCount: turnInputChars, executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND, createdAt: new Date().toISOString(), }); const startedAt = Date.now(); try { - const credential = await resolveTurnCredential({ authBroker: this.deps.authBroker, integrationKeys: this.deps.integrationKeys, orgId: input.orgId, provider }); - const result = await turn({ credential, model, instructions: input.instructions, messages: input.messages, tools: input.tools }); - recordObservedRateLimit(this.deps.rateLimitRegistry, input.orgId, owner, model, result.rateLimitState ?? null); + const credential = await resolveTurnCredential({ authBroker: this.deps.authBroker, integrationKeys: this.deps.integrationKeys, orgId, provider: owner }); + const result = await turn({ credential, model, instructions: request.instructions, messages: request.messages, tools: request.tools }); + recordObservedRateLimit(this.deps.rateLimitRegistry, orgId, owner, model, result.rateLimitState ?? null); // Tool calls are output the model wrote, as much as its text. const outputChars = result.text.length + (result.toolCalls.length > 0 ? JSON.stringify(result.toolCalls).length : 0); this.deps.requestStore.markSucceeded({ - orgId: input.orgId, + orgId, requestId, responseText: result.text, rawResponseJson: result.rawResponseJson, @@ -168,7 +204,7 @@ export class ProviderResponseRunner { completedAt: new Date().toISOString(), }); this.deps.responseObserver.recordSuccess({ - orgId: input.orgId, + orgId, provider: owner, requestId, executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND, @@ -176,13 +212,13 @@ export class ProviderResponseRunner { action: "response_created", }); this.deps.requestStore.settleUsage({ - orgId: input.orgId, requestId, provider: owner, rawResponseJson: result.rawResponseJson, + orgId, requestId, provider: owner, rawResponseJson: result.rawResponseJson, inputCharCount: turnInputChars, outputCharCount: outputChars, }); return { ...result, requestId }; } catch (error) { this.deps.responseObserver.recordFailure({ - orgId: input.orgId, + orgId, provider: owner, requestId, executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND, @@ -391,3 +427,17 @@ interface PreparedProviderExecution { adapter: ProviderAdapterRegistry[string]; credential: { secret: string }; } + +interface TurnCaller { + orgId: string; + owner: ProviderRow & { model: string }; + workspaceId: string | null; + agent: { id: string; department: string } | null; + taskRunId: string | null; + turn: ToolTurnFn; +} + +function requireCredentialId(provider: ProviderRow): string { + if (!provider.credential_id) throw new Error("Provider credential is required."); + return provider.credential_id; +} diff --git a/src/main/services/provider-service.ts b/src/main/services/provider-service.ts index 8eb402d8..3cc24c3c 100644 --- a/src/main/services/provider-service.ts +++ b/src/main/services/provider-service.ts @@ -18,6 +18,7 @@ import { ProviderResponseRunner, type ProviderResponseCreateInput } from "./prov import { launchProviderSignInWindow, type LaunchProviderSignInWindowInput } from "./provider-sign-in-window"; import type { ToolTurnFn } from "./provider-tool-turn"; import { stopRun } from "./run-control"; +import type { ModelCheckResult } from "../../shared/desktop-api"; import { AUTH_STATE_CONFIGURED, AUTH_STATE_REVOKED, @@ -194,6 +195,10 @@ export class ProviderService { return adapter.listModels({ authProfile, model: provider.model }); } + checkModel(input: { orgId: string; providerId: string; model: string }): Promise { + return this.runner.checkModel(input); + } + async checkHealth(orgId: string, providerId: string): Promise { const provider = loadConfiguredProvider(this.db, orgId, providerId); const adapter = requireAdapter(this.adapters, provider.provider_key, provider.auth_type); diff --git a/src/preload/index.ts b/src/preload/index.ts index 3d69502d..2b935f25 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -60,6 +60,7 @@ const api: DesktopApi = { cancelProviderOAuth: (input) => invoke(IPC_CHANNELS.cancelProviderOAuth, input), checkProviderHealth: (input) => invoke(IPC_CHANNELS.checkProviderHealth, input), listProviderModels: (input) => invoke(IPC_CHANNELS.listProviderModels, input), + checkProviderModel: (input) => invoke(IPC_CHANNELS.checkProviderModel, input), updateProviderPolicy: (input) => invoke(IPC_CHANNELS.updateProviderPolicy, input), setProviderPrices: (input) => invoke(IPC_CHANNELS.setProviderPrices, input), openExternalUrl: (url) => invoke(IPC_CHANNELS.openExternalUrl, url), diff --git a/src/renderer/src/i18n/zh.ts b/src/renderer/src/i18n/zh.ts index 4bc4d546..91a8682d 100644 --- a/src/renderer/src/i18n/zh.ts +++ b/src/renderer/src/i18n/zh.ts @@ -32,6 +32,10 @@ export const zh: Record = { "superseded": "已被取代", "stalled": "已停滞", "Resume": "恢复", + "Checking...": "检查中...", + "Check for agent work": "检查能否胜任 Agent 工作", + "This model follows instructions and calls tools.": "该模型能遵循指令并调用工具。", + "This model cannot do agent work here: {detail}": "该模型无法在此完成 Agent 工作：{detail}", "{n} in the last 7 days": "近 7 天 {n}", "Success rate (7 days)": "成功率（近 7 天）", "{n} tokens in the last 7 days": "近 7 天 {n} token", diff --git a/src/renderer/src/pages/settings/ProviderConnectionDialog.tsx b/src/renderer/src/pages/settings/ProviderConnectionDialog.tsx index 27fa1769..866aa0cc 100644 --- a/src/renderer/src/pages/settings/ProviderConnectionDialog.tsx +++ b/src/renderer/src/pages/settings/ProviderConnectionDialog.tsx @@ -3,6 +3,7 @@ import type { ProviderCatalogEntry, ProviderRecord } from "../../../../shared/do import { ProviderOAuthConnect } from "../../shared/ProviderOAuthConnect"; import { Field } from "../../shared/ui"; import { t } from "../../i18n/i18n"; +import type { ModelCheckResult } from "../../../../shared/desktop-api"; interface CloudFields { baseUrl: string; @@ -97,8 +98,10 @@ function ModelPicker({ entry, provider }: { entry: ProviderCatalogEntry; provide const [fetched, setFetched] = useState(null); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); + const [checks, setChecks] = useState>({}); const [value, setValue] = useState(provider.model ?? entry.defaultModel ?? entry.models[0]?.id ?? ""); const models = fetched ?? entry.models; + const check = checks[value]; const connected = entry.supportsLiveModels && Boolean(provider.credentialId) && provider.enabled && provider.authState === "configured"; const refresh = async () => { @@ -114,6 +117,20 @@ function ModelPicker({ entry, provider }: { entry: ProviderCatalogEntry; provide if (result.data[0] && !result.data.some((model) => model.id === value)) setValue(result.data[0].id); }; + const runCheck = async () => { + // A verdict belongs to the model it was asked about, whatever is selected when it arrives. + const model = value; + setChecks((current) => ({ ...current, [model]: "running" })); + const result = await window.agentCompany.checkProviderModel({ orgId: provider.orgId, providerId: provider.id, model }); + setChecks((current) => { + const next = { ...current }; + if (result.ok) next[model] = result.data; + else delete next[model]; + return next; + }); + if (!result.ok) setError(result.error.message); + }; + return ( {models.length ? ( @@ -130,6 +147,13 @@ function ModelPicker({ entry, provider }: { entry: ProviderCatalogEntry; provide {loading ? t("Loading models...") : t("Refresh models")} )} + {provider.credentialId && provider.authState === "configured" && ( + + )} + {typeof check === "object" && check.passed &&
{t("This model follows instructions and calls tools.")}
} + {typeof check === "object" && !check.passed &&
{t("This model cannot do agent work here: {detail}", { detail: check.detail })}
} {error &&
{error}
} ); diff --git a/src/shared/desktop-api.ts b/src/shared/desktop-api.ts index 777bbb23..42c7bbb3 100644 --- a/src/shared/desktop-api.ts +++ b/src/shared/desktop-api.ts @@ -110,6 +110,9 @@ export type RendererEvent = | { type: "provider-response-stream"; event: ProviderStreamRendererEvent } | ProviderOAuthEvent; +/** The verdict of a model's agent check. */ +export type ModelCheckResult = { passed: true } | { passed: false; detail: string }; + export interface DesktopApi { loadSnapshot(): Promise>; bootstrapOnboarding(input: BootstrapOnboardingInput): Promise>; @@ -164,6 +167,8 @@ export interface DesktopApi { cancelProviderOAuth(input: CancelProviderOAuthInput): Promise>; checkProviderHealth(input: CheckProviderHealthInput): Promise>; listProviderModels(input: ListProviderModelsInput): Promise>; + /** Runs the agent check of a model: one turn that must follow system instructions and call a tool. */ + checkProviderModel(input: { orgId: string; providerId: string; model: string }): Promise>; updateProviderPolicy(input: UpdateProviderPolicyInput): Promise>; setProviderPrices(input: SetProviderPricesInput): Promise>; openExternalUrl(url: string): Promise>; diff --git a/src/shared/ipc.ts b/src/shared/ipc.ts index b18415d3..3aec746d 100644 --- a/src/shared/ipc.ts +++ b/src/shared/ipc.ts @@ -33,6 +33,7 @@ export const IPC_CHANNELS = { cancelProviderOAuth: "agentcompany:cancel-provider-oauth", checkProviderHealth: "agentcompany:check-provider-health", listProviderModels: "agentcompany:list-provider-models", + checkProviderModel: "agentcompany:check-provider-model", updateProviderPolicy: "agentcompany:update-provider-policy", setProviderPrices: "agentcompany:set-provider-prices", openExternalUrl: "agentcompany:open-external-url", diff --git a/tests/helpers/model-check.ts b/tests/helpers/model-check.ts new file mode 100644 index 00000000..031b46be --- /dev/null +++ b/tests/helpers/model-check.ts @@ -0,0 +1,22 @@ +import type { DatabaseSync } from "node:sqlite"; +import { MODEL_CHECK_TOOL } from "../../src/main/services/model-check"; +import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn"; + +/** A scripted provider that also passes the agent check the way a capable model would. */ +export function answeringModelCheck Promise>(turn: T): T { + return (async (input: ProviderToolTurnInput) => { + if (!input.tools.some((tool) => tool.name === MODEL_CHECK_TOOL)) return turn(input); + const first = input.messages[0]; + const code = first?.role === "user" ? /The code is (\d+)/.exec(first.text)?.[1] : undefined; + if (!code) throw new Error("The agent check sent no code."); + return { text: "", toolCalls: [{ id: "check", name: MODEL_CHECK_TOOL, arguments: { code } }], rawResponseJson: "{}" }; + }) as T; +} + +/** Records that a provider's model already passed the agent check, so a test sees only its own requests. */ +export function markModelChecked(db: DatabaseSync, providerId: string, model: string): void { + const provider = db.prepare("select org_id, credential_id from providers where id = ?").get(providerId) as { org_id: string; credential_id: string | null } | undefined; + if (!provider?.credential_id) throw new Error(`Provider ${providerId} has no credential to check a model against.`); + db.prepare("insert into model_checks(provider_id, model, credential_id, org_id, passed, detail, checked_at) values (?, ?, ?, ?, 1, null, ?)") + .run(providerId, model, provider.credential_id, provider.org_id, new Date().toISOString()); +} diff --git a/tests/integration/agent-model.test.ts b/tests/integration/agent-model.test.ts index 318c72d8..88ed0943 100644 --- a/tests/integration/agent-model.test.ts +++ b/tests/integration/agent-model.test.ts @@ -9,6 +9,7 @@ import { providerAdapters } from "../../src/main/services/provider-adapters"; import { TaskExecutionService } from "../../src/main/services/task-execution-service"; import { RISK_LEVEL, TASK_PRIORITY, TASK_STATUS } from "../../src/shared/domain"; import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn"; +import { answeringModelCheck } from "../helpers/model-check"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -27,7 +28,7 @@ describe("agent model", () => { }; const connection = new DatabaseConnection(join(tmpdir(), `ac-model-${randomUUID()}.db`)); const organization = new OrganizationService(connection.db); - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn)); const execution = new TaskExecutionService(connection.db, providers); let orgId = ""; const tasks: Record = {}; @@ -53,7 +54,7 @@ describe("agent model", () => { const turn = async (): Promise => ({ text: "done", toolCalls: [], rawResponseJson: "{}" }); const connection = new DatabaseConnection(join(tmpdir(), `ac-model-az-${randomUUID()}.db`)); const organization = new OrganizationService(connection.db); - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn)); const execution = new TaskExecutionService(connection.db, providers); let orgId = ""; let taskId = ""; @@ -83,7 +84,7 @@ describe("agent model", () => { }; const connection = new DatabaseConnection(join(tmpdir(), `ac-model-pin-${randomUUID()}.db`)); const organization = new OrganizationService(connection.db); - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn)); const execution = new TaskExecutionService(connection.db, providers); let orgId = ""; let taskId = ""; diff --git a/tests/integration/agent-reply.test.ts b/tests/integration/agent-reply.test.ts index 210851b6..133ff795 100644 --- a/tests/integration/agent-reply.test.ts +++ b/tests/integration/agent-reply.test.ts @@ -11,6 +11,7 @@ import { SnapshotService } from "../../src/main/services/snapshot-service"; import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker"; import { canonicalMentionHandle } from "../../src/shared/mention-handles"; import type { AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn"; +import { answeringModelCheck } from "../helpers/model-check"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -59,7 +60,7 @@ function scriptedTeam(): (input: ProviderToolTurnInput) => Promise { test("an @mention wakes the teammate, who replies in the same thread", async () => { const connection = new DatabaseConnection(join(tmpdir(), `ac-reply-${randomUUID()}.db`)); - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, scriptedTeam()); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(scriptedTeam())); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const taskExecution = new TaskExecutionService(connection.db, providers); const snapshot = new SnapshotService(connection.db); diff --git a/tests/integration/autonomous-run.test.ts b/tests/integration/autonomous-run.test.ts index 411c6ba1..8ffdfdfd 100644 --- a/tests/integration/autonomous-run.test.ts +++ b/tests/integration/autonomous-run.test.ts @@ -12,6 +12,7 @@ import { TaskExecutionService } from "../../src/main/services/task-execution-ser import { SnapshotService } from "../../src/main/services/snapshot-service"; import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker"; import type { AgentTurnMessage, AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn"; +import { answeringModelCheck } from "../helpers/model-check"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -89,7 +90,7 @@ describe("autonomous bootstrap run", () => { const workspacePath = join(tmpdir(), `ac-auto-ws-${randomUUID()}`); mkdirSync(workspacePath, { recursive: true }); - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, scriptedCeo()); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(scriptedCeo())); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const snapshot = new SnapshotService(connection.db); @@ -183,7 +184,7 @@ describe("autonomous bootstrap run", () => { test("a goal created after onboarding reaches the CEO and runs autonomously", async () => { const connection = new DatabaseConnection(join(tmpdir(), `ac-goal-${randomUUID()}.db`)); // A minimal well-behaved agent: does its session, closes its task, then stops. - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, async (input: ProviderToolTurnInput) => { + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(async (input: ProviderToolTurnInput) => { const firstUser = input.messages.find((message) => message.role === "user"); const taskId = firstUser && firstUser.role === "user" ? /Task id: (\S+)/.exec(firstUser.text)?.[1] : undefined; const alreadyClosed = input.messages.some( @@ -201,7 +202,7 @@ describe("autonomous bootstrap run", () => { toolCalls: [], rawResponseJson: JSON.stringify({ id: "resp_goal" }), }; - }); + })); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const organization = new OrganizationService(connection.db); const taskExecution = new TaskExecutionService(connection.db, providers); diff --git a/tests/integration/ceo-founder-chat.test.ts b/tests/integration/ceo-founder-chat.test.ts index a0497408..f8109fa8 100644 --- a/tests/integration/ceo-founder-chat.test.ts +++ b/tests/integration/ceo-founder-chat.test.ts @@ -11,6 +11,7 @@ import { ChatService } from "../../src/main/services/chat-service"; import { ensureCeoDirectSpace } from "../../src/main/services/ceo-direct-space"; import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker"; import type { AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn"; +import { answeringModelCheck } from "../helpers/model-check"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -67,7 +68,7 @@ describe("founder⇄CEO chat", () => { test("a founder DM wakes the CEO, who sees who is asking and replies in the pinned conversation", async () => { const connection = new DatabaseConnection(join(tmpdir(), `ac-ceo-chat-${randomUUID()}.db`)); const capturedWakePrompts: string[] = []; - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, scriptedCeo(capturedWakePrompts)); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(scriptedCeo(capturedWakePrompts))); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const taskExecution = new TaskExecutionService(connection.db, providers); const chat = new ChatService(connection.db); @@ -143,7 +144,7 @@ describe("founder⇄CEO chat", () => { test("a CEO who answers in plain text still has the reply delivered into the founder's thread", async () => { const connection = new DatabaseConnection(join(tmpdir(), `ac-ceo-plain-${randomUUID()}.db`)); - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, plainTextCeo()); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(plainTextCeo())); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const taskExecution = new TaskExecutionService(connection.db, providers); const chat = new ChatService(connection.db); diff --git a/tests/integration/channel-anchor.test.ts b/tests/integration/channel-anchor.test.ts index a12c300d..e615041c 100644 --- a/tests/integration/channel-anchor.test.ts +++ b/tests/integration/channel-anchor.test.ts @@ -10,6 +10,7 @@ import { providerAdapters } from "../../src/main/services/provider-adapters"; import { TaskExecutionService } from "../../src/main/services/task-execution-service"; import { canonicalMentionHandle } from "../../src/shared/mention-handles"; import type { AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn"; +import { answeringModelCheck } from "../helpers/model-check"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -37,7 +38,7 @@ function scriptedCeoReply(): (input: ProviderToolTurnInput) => Promise { test("a plain user message in a channel wakes the most senior member, who replies", async () => { const connection = new DatabaseConnection(join(tmpdir(), `ac-anchor-${randomUUID()}.db`)); - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, scriptedCeoReply()); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(scriptedCeoReply())); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const taskExecution = new TaskExecutionService(connection.db, providers); const chat = new ChatService(connection.db); diff --git a/tests/integration/chat-context.test.ts b/tests/integration/chat-context.test.ts index cc26bbd7..db0eb702 100644 --- a/tests/integration/chat-context.test.ts +++ b/tests/integration/chat-context.test.ts @@ -14,6 +14,7 @@ import { dispatchAgentTool, type AgentToolContext } from "../../src/main/service import { ensureCeoDirectSpace } from "../../src/main/services/ceo-direct-space"; import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker"; import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn"; +import { answeringModelCheck } from "../helpers/model-check"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -34,7 +35,7 @@ async function founderLine() { return { text: `Reply ${wakePrompts.length}.`, toolCalls: [], rawResponseJson: "{}" }; }; const connection = new DatabaseConnection(join(tmpdir(), `ac-chatctx-${randomUUID()}.db`)); - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn)); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const execution = new TaskExecutionService(connection.db, providers); const chat = new ChatService(connection.db); @@ -153,7 +154,7 @@ describe("chat wake context", () => { } return { text: "Switching to Postgres.", toolCalls: [], rawResponseJson: "{}" }; }; - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn)); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const execution = new TaskExecutionService(connection.db, providers); let ceoId = ""; diff --git a/tests/integration/goal-changes.test.ts b/tests/integration/goal-changes.test.ts index 9e32cd22..aff5affc 100644 --- a/tests/integration/goal-changes.test.ts +++ b/tests/integration/goal-changes.test.ts @@ -9,6 +9,7 @@ import { providerAdapters } from "../../src/main/services/provider-adapters"; import { TaskExecutionService } from "../../src/main/services/task-execution-service"; import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker"; import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn"; +import { answeringModelCheck } from "../helpers/model-check"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -17,7 +18,7 @@ const testVault = { function company(turn: (input: ProviderToolTurnInput, db: DatabaseConnection["db"]) => AgentTurnResult) { const connection = new DatabaseConnection(join(tmpdir(), `ac-goalchg-${randomUUID()}.db`)); - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, async (input) => turn(input, connection.db)); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(async (input) => turn(input, connection.db))); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const execution = new TaskExecutionService(connection.db, providers); let orgId = ""; diff --git a/tests/integration/interactive-lane.test.ts b/tests/integration/interactive-lane.test.ts index e4e258f0..1f9a3cfd 100644 --- a/tests/integration/interactive-lane.test.ts +++ b/tests/integration/interactive-lane.test.ts @@ -12,6 +12,7 @@ import { TaskExecutionService } from "../../src/main/services/task-execution-ser import { dispatchAgentTool } from "../../src/main/services/agent-tools"; import { claimNextQueuedRun, startTaskQueueWorker } from "../../src/main/app/task-queue-worker"; import type { AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn"; +import { answeringModelCheck } from "../helpers/model-check"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -75,7 +76,7 @@ describe("interactive execution lane", () => { } return turn("done"); }; - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, script); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(script)); providers.saveCredential({ orgId, providerId, secret: "sk-test-key", model: "gpt-5.4" }); const taskExecution = new TaskExecutionService(connection.db, providers); @@ -162,7 +163,7 @@ describe("interactive execution lane", () => { } return turn("done"); }; - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, script); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(script)); providers.saveCredential({ orgId, providerId, secret: "sk-test-key", model: "gpt-5.4" }); const taskExecution = new TaskExecutionService(connection.db, providers); @@ -227,7 +228,7 @@ describe("interactive execution lane", () => { const alwaysOverloaded = async (): Promise => { throw new Error("The engine is currently overloaded, please try again later"); }; - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, alwaysOverloaded); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(alwaysOverloaded)); providers.saveCredential({ orgId, providerId, secret: "sk-test-key", model: "gpt-5.4" }); const taskExecution = new TaskExecutionService(connection.db, providers); const wakeDana = () => dispatchAgentTool({ db: connection.db, organization, chat }, { diff --git a/tests/integration/run-checkpoint.test.ts b/tests/integration/run-checkpoint.test.ts index 1191fabd..6736e025 100644 --- a/tests/integration/run-checkpoint.test.ts +++ b/tests/integration/run-checkpoint.test.ts @@ -11,6 +11,7 @@ import { ChatService } from "../../src/main/services/chat-service"; import { ensureCeoDirectSpace } from "../../src/main/services/ceo-direct-space"; import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker"; import type { AgentTurnResult } from "../../src/main/services/provider-tool-turn"; +import { answeringModelCheck } from "../helpers/model-check"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -24,7 +25,7 @@ describe("session checkpoints", () => { ? { text: "Audit done: all clean.", toolCalls: [], rawResponseJson: "{}" } : { text: "Looking into it.", toolCalls: [{ id: `c_${randomUUID()}`, name: "read_messages", arguments: {} }], rawResponseJson: "{}" }; const connection = new DatabaseConnection(join(tmpdir(), `ac-checkpoint-${randomUUID()}.db`)); - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn)); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const execution = new TaskExecutionService(connection.db, providers); let orgId = ""; diff --git a/tests/integration/run-journal.test.ts b/tests/integration/run-journal.test.ts index f0875a99..93d94a5c 100644 --- a/tests/integration/run-journal.test.ts +++ b/tests/integration/run-journal.test.ts @@ -9,6 +9,7 @@ import { providerAdapters } from "../../src/main/services/provider-adapters"; import { TaskExecutionService } from "../../src/main/services/task-execution-service"; import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker"; import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn"; +import { answeringModelCheck } from "../helpers/model-check"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -41,7 +42,7 @@ describe("run journal", () => { throw new Error("Provider dropped the connection mid-run"); }; const connection = new DatabaseConnection(join(tmpdir(), `ac-journal-${randomUUID()}.db`)); - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn)); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const execution = new TaskExecutionService(connection.db, providers); let orgId = ""; diff --git a/tests/integration/run-memory.test.ts b/tests/integration/run-memory.test.ts index 8fc4f821..b8240a4d 100644 --- a/tests/integration/run-memory.test.ts +++ b/tests/integration/run-memory.test.ts @@ -11,6 +11,7 @@ import { ChatService } from "../../src/main/services/chat-service"; import { ensureCeoDirectSpace } from "../../src/main/services/ceo-direct-space"; import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker"; import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn"; +import { answeringModelCheck } from "../helpers/model-check"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -26,7 +27,7 @@ async function company(reply: (prompt: string) => AgentTurnResult) { return reply(prompt); }; const connection = new DatabaseConnection(join(tmpdir(), `ac-memory-${randomUUID()}.db`)); - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn)); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const execution = new TaskExecutionService(connection.db, providers); let orgId = ""; diff --git a/tests/integration/run-stop.test.ts b/tests/integration/run-stop.test.ts index 92b38724..c016fe98 100644 --- a/tests/integration/run-stop.test.ts +++ b/tests/integration/run-stop.test.ts @@ -13,6 +13,7 @@ import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker"; import { postJson } from "../../src/main/services/provider-http"; import { RUN_STOPPED_MESSAGE, stopRun, withRunControl } from "../../src/main/services/run-control"; import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn"; +import { answeringModelCheck } from "../helpers/model-check"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -21,7 +22,7 @@ const testVault = { function company(turn: (input: ProviderToolTurnInput) => Promise) { const connection = new DatabaseConnection(join(tmpdir(), `ac-stop-${randomUUID()}.db`)); - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn)); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const execution = new TaskExecutionService(connection.db, providers); let orgId = ""; @@ -44,7 +45,7 @@ describe("stopping runs", () => { return { text: "Announcing.", toolCalls: [{ id: "c1", name: "send_message", arguments: { channel: "general", body: "Should never be posted." } }], rawResponseJson: "{}" }; }; const connection = new DatabaseConnection(join(tmpdir(), `ac-stop-${randomUUID()}.db`)); - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn)); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const execution = new TaskExecutionService(connection.db, providers); let orgId = ""; @@ -115,7 +116,7 @@ describe("stopping runs", () => { return { text: "done", toolCalls: [], rawResponseJson: "{}" }; }; const connection = new DatabaseConnection(join(tmpdir(), `ac-deadline-${randomUUID()}.db`)); - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn)); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const execution = new TaskExecutionService(connection.db, providers, undefined, undefined, undefined, 200); let orgId = ""; diff --git a/tests/integration/spend-budget.test.ts b/tests/integration/spend-budget.test.ts index 3fa46c30..38799af7 100644 --- a/tests/integration/spend-budget.test.ts +++ b/tests/integration/spend-budget.test.ts @@ -9,6 +9,7 @@ import { providerAdapters } from "../../src/main/services/provider-adapters"; import { TaskExecutionService } from "../../src/main/services/task-execution-service"; import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker"; import type { AgentTurnResult } from "../../src/main/services/provider-tool-turn"; +import { markModelChecked } from "../helpers/model-check"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -39,6 +40,7 @@ describe("money budgets", () => { ceoId = summary.ceoAgentId; }); const providerId = (connection.db.prepare("select model_provider_id as id from agents where id = ?").get(ceoId) as { id: string }).id; + markModelChecked(connection.db, providerId, "gpt-5.4"); providers.setPrices({ orgId, providerId, inputDollarsPerMtok: 2.5, cachedInputDollarsPerMtok: null, outputDollarsPerMtok: 10 }); connection.db.prepare("update agents set budget_cents = 2000 where id = ?").run(ceoId); diff --git a/tests/integration/stale-queued-runs.test.ts b/tests/integration/stale-queued-runs.test.ts index ddd7e230..24c8be46 100644 --- a/tests/integration/stale-queued-runs.test.ts +++ b/tests/integration/stale-queued-runs.test.ts @@ -11,6 +11,7 @@ import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker"; import { ChatService } from "../../src/main/services/chat-service"; import { ensureCeoDirectSpace } from "../../src/main/services/ceo-direct-space"; import type { AgentTurnResult } from "../../src/main/services/provider-tool-turn"; +import { answeringModelCheck } from "../helpers/model-check"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -24,7 +25,7 @@ function company() { return { text: "done", toolCalls: [], rawResponseJson: "{}" }; }; const connection = new DatabaseConnection(join(tmpdir(), `ac-stale-${randomUUID()}.db`)); - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn)); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const execution = new TaskExecutionService(connection.db, providers); let orgId = ""; diff --git a/tests/integration/task-context-scope.test.ts b/tests/integration/task-context-scope.test.ts index 267c1a6c..b3965ba0 100644 --- a/tests/integration/task-context-scope.test.ts +++ b/tests/integration/task-context-scope.test.ts @@ -13,6 +13,7 @@ import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker"; import { canonicalMentionHandle } from "../../src/shared/mention-handles"; import { RISK_LEVEL, TASK_PRIORITY, TASK_STATUS } from "../../src/shared/domain"; import type { AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn"; +import { answeringModelCheck } from "../helpers/model-check"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -49,7 +50,7 @@ describe("task prompt scope", () => { return { text: step.text, toolCalls: step.toolCalls, rawResponseJson: "{}" }; }; const connection = new DatabaseConnection(join(tmpdir(), `ac-scope-${randomUUID()}.db`)); - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn)); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const execution = new TaskExecutionService(connection.db, providers); let orgId = ""; @@ -90,7 +91,7 @@ describe("task prompt scope", () => { calls += 1; return { text: "done", toolCalls: [], rawResponseJson: "{}" }; }; - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn)); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const organization = new OrganizationService(connection.db); const execution = new TaskExecutionService(connection.db, providers); @@ -125,7 +126,7 @@ describe("task prompt scope", () => { test("a manual write run waits while another run is changing the same workspace", async () => { const connection = new DatabaseConnection(join(tmpdir(), `ac-wsplock-${randomUUID()}.db`)); const turn = async (): Promise => ({ text: "done", toolCalls: [], rawResponseJson: "{}" }); - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn)); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const organization = new OrganizationService(connection.db); const execution = new TaskExecutionService(connection.db, providers); diff --git a/tests/integration/task-dependencies.test.ts b/tests/integration/task-dependencies.test.ts index 54a958b5..3de159b4 100644 --- a/tests/integration/task-dependencies.test.ts +++ b/tests/integration/task-dependencies.test.ts @@ -13,6 +13,7 @@ import { dispatchAgentTool } from "../../src/main/services/agent-tools"; import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker"; import { RISK_LEVEL, TASK_PRIORITY, TASK_STATUS } from "../../src/shared/domain"; import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn"; +import { answeringModelCheck } from "../helpers/model-check"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -28,7 +29,7 @@ function company() { prompts.push(first.text); return { text: "done", toolCalls: [], rawResponseJson: "{}" }; }; - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(turn)); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const execution = new TaskExecutionService(connection.db, providers); const organization = new OrganizationService(connection.db); diff --git a/tests/unit/agent-instructions.test.ts b/tests/unit/agent-instructions.test.ts index d3749a26..d7141b43 100644 --- a/tests/unit/agent-instructions.test.ts +++ b/tests/unit/agent-instructions.test.ts @@ -14,6 +14,7 @@ import { buildAgentInstructions, type TaskExecutionAgentRow } from "../../src/ma import { AGENT_TOOLS } from "../../src/main/services/agent-tools"; import { goalKickoffDescription } from "../../src/main/services/goal-kickoff"; import { CEO_FIRST_HIRES } from "../../src/main/services/org-building"; +import { answeringModelCheck } from "../helpers/model-check"; function agentRow(overrides: Partial): TaskExecutionAgentRow { return { @@ -134,7 +135,7 @@ describe("agent instruction wiring", () => { capturedInstructions = input.instructions ?? ""; return { text: "done", toolCalls: [], rawResponseJson: JSON.stringify({ id: "resp-1" }) }; }; - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn)); const execution = new TaskExecutionService(connection.db, providers); let orgId = ""; @@ -189,7 +190,7 @@ describe("agent instruction wiring", () => { capturedPrompt = first.text; return { text: "done", toolCalls: [], rawResponseJson: JSON.stringify({ id: "resp-1" }) }; }; - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn)); const execution = new TaskExecutionService(connection.db, providers); let orgId = ""; let taskId = ""; diff --git a/tests/unit/char-accounting.test.ts b/tests/unit/char-accounting.test.ts index daaa84e8..eec661ee 100644 --- a/tests/unit/char-accounting.test.ts +++ b/tests/unit/char-accounting.test.ts @@ -7,6 +7,7 @@ import { OrganizationService } from "../../src/main/services/organization-servic import { ProviderService } from "../../src/main/services/provider-service"; import { providerAdapters } from "../../src/main/services/provider-adapters"; import type { AgentTurnResult, ToolDefinition } from "../../src/main/services/provider-tool-turn"; +import { markModelChecked } from "../helpers/model-check"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -25,6 +26,7 @@ describe("character accounting", () => { orgId = new OrganizationService(connection.db).createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" }); providerId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'openai'").get(orgId) as { id: string }).id; providers.saveCredential({ orgId, providerId, secret: "sk-test", model: "gpt-5.4" }); + markModelChecked(connection.db, providerId, "gpt-5.4"); }); const instructions = "You run this company."; const messages = [{ role: "user" as const, text: "Kick off the launch." }]; diff --git a/tests/unit/domain-services.test.ts b/tests/unit/domain-services.test.ts index 5975159f..49fd08ef 100644 --- a/tests/unit/domain-services.test.ts +++ b/tests/unit/domain-services.test.ts @@ -24,7 +24,7 @@ describe("domain services", () => { const connection = createConnection(); const rows = connection.db.prepare("select version from schema_migrations order by version asc").all() as Array<{ version: number }>; - expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76]); + expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77]); connection.close(); }); diff --git a/tests/unit/model-check.test.ts b/tests/unit/model-check.test.ts new file mode 100644 index 00000000..c4acbe8f --- /dev/null +++ b/tests/unit/model-check.test.ts @@ -0,0 +1,84 @@ +import { randomUUID } from "node:crypto"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, test } from "vitest"; +import { DatabaseConnection } from "../../src/main/database/connection"; +import { OrganizationService } from "../../src/main/services/organization-service"; +import { ProviderService } from "../../src/main/services/provider-service"; +import { providerAdapters } from "../../src/main/services/provider-adapters"; +import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn"; +import { answeringModelCheck } from "../helpers/model-check"; +import { PROVIDER_DATA_POLICY } from "../../src/shared/domain"; +import { PROVIDER_MONTHLY_LIMIT_MESSAGE } from "../../src/shared/provider-runtime-domain"; + +const testVault = { + encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), + decryptSecret: (ciphertext: Buffer) => ciphertext.toString("utf8"), +}; + +const WORK = { workspaceId: null, instructions: "Do the work.", messages: [{ role: "user" as const, text: "go" }], tools: [] }; +const TEXT_ONLY = async (): Promise => ({ text: "Sure, the code is 123.", toolCalls: [], rawResponseJson: "{}" }); + +function harness(turn: (input: ProviderToolTurnInput) => Promise) { + const connection = new DatabaseConnection(join(tmpdir(), `ac-model-check-${randomUUID()}.db`)); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn); + let orgId = ""; + let providerId = ""; + connection.transaction(() => { + orgId = new OrganizationService(connection.db).createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" }); + providerId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'openai'").get(orgId) as { id: string }).id; + providers.saveCredential({ orgId, providerId, secret: "sk-test", model: "gpt-5.4" }); + }); + const requests = () => (connection.db.prepare("select count(*) as n from provider_requests").get() as { n: number }).n; + return { connection, providers, orgId, providerId, requests }; +} + +describe("a model is checked for agent work before it does any", () => { + test("a model that answers in text instead of calling tools is refused, and the verdict is kept", async () => { + const h = harness(TEXT_ONLY); + await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })) + .rejects.toThrow("gpt-5.4 on openai failed the agent check: it answered in text instead of calling the tool it was given"); + expect(h.requests()).toBe(1); + await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("failed the agent check"); + expect(h.requests()).toBe(1); + h.connection.close(); + }); + + test("a capable model is probed once, then works; the probe is a request like any other and counts against the quota", async () => { + const seen: ProviderToolTurnInput[] = []; + const h = harness(answeringModelCheck(async (input) => { seen.push(input); return { text: "Done.", toolCalls: [], rawResponseJson: "{}" }; })); + expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done."); + expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done."); + expect([h.requests(), seen.length]).toEqual([3, 2]); + + const tight = harness(answeringModelCheck(async () => ({ text: "Done.", toolCalls: [], rawResponseJson: "{}" }))); + tight.providers.updatePolicy({ orgId: tight.orgId, providerId: tight.providerId, dataPolicy: PROVIDER_DATA_POLICY.STANDARD, backgroundEnabled: false, monthlyRequestLimit: 1, workspaceScope: "organization", contextWindowTokens: 128_000 }); + await expect(tight.providers.runToolTurn({ orgId: tight.orgId, providerId: tight.providerId, ...WORK })).rejects.toThrow(PROVIDER_MONTHLY_LIMIT_MESSAGE); + expect(tight.requests()).toBe(1); + h.connection.close(); + tight.connection.close(); + }); + + test("a probe whose request fails records nothing; a new model gets its own probe; Settings can check again", async () => { + let mode: "down" | "text" | "capable" = "down"; + const capable: (input: ProviderToolTurnInput) => Promise = answeringModelCheck(async () => ({ text: "ok", toolCalls: [], rawResponseJson: "{}" })); + const h = harness(async (input) => { + if (mode === "down") throw new Error("connect ECONNREFUSED"); + return mode === "text" ? TEXT_ONLY() : capable(input); + }); + await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("ECONNREFUSED"); + expect(h.connection.db.prepare("select count(*) as n from model_checks").get()).toEqual({ n: 0 }); + + mode = "text"; + expect(await h.providers.checkModel({ orgId: h.orgId, providerId: h.providerId, model: "gpt-5.4" })) + .toEqual({ passed: false, detail: "it answered in text instead of calling the tool it was given" }); + mode = "capable"; + expect(await h.providers.checkModel({ orgId: h.orgId, providerId: h.providerId, model: "gpt-5.4" })).toEqual({ passed: true }); + expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("ok"); + + h.connection.db.prepare("update providers set model = 'gpt-5.5' where id = ?").run(h.providerId); + mode = "text"; + await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("gpt-5.5 on openai failed the agent check"); + h.connection.close(); + }); +}); diff --git a/tests/unit/quota-reservation.test.ts b/tests/unit/quota-reservation.test.ts index 9963137b..e0c8c6d3 100644 --- a/tests/unit/quota-reservation.test.ts +++ b/tests/unit/quota-reservation.test.ts @@ -9,6 +9,7 @@ import { providerAdapters } from "../../src/main/services/provider-adapters"; import { PROVIDER_MONTHLY_LIMIT_MESSAGE } from "../../src/shared/provider-runtime-domain"; import { PROVIDER_DATA_POLICY } from "../../src/shared/domain"; import type { AgentTurnResult } from "../../src/main/services/provider-tool-turn"; +import { markModelChecked } from "../helpers/model-check"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -33,6 +34,7 @@ describe("monthly quota reservation", () => { orgId = new OrganizationService(connection.db).createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" }); providerId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'openai'").get(orgId) as { id: string }).id; providers.saveCredential({ orgId, providerId, secret: "sk-test", model: "gpt-5.4" }); + markModelChecked(connection.db, providerId, "gpt-5.4"); providers.updatePolicy({ orgId, providerId, dataPolicy: PROVIDER_DATA_POLICY.STANDARD, backgroundEnabled: false, monthlyRequestLimit: 1, workspaceScope: "organization", contextWindowTokens: 128_000 }); }); const call = () => providers.runToolTurn({ orgId, providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] }); diff --git a/tests/unit/rate-limit-observation.test.ts b/tests/unit/rate-limit-observation.test.ts index 51d31b26..abb70801 100644 --- a/tests/unit/rate-limit-observation.test.ts +++ b/tests/unit/rate-limit-observation.test.ts @@ -10,6 +10,7 @@ import { TaskExecutionService } from "../../src/main/services/task-execution-ser import { RISK_LEVEL, TASK_PRIORITY, TASK_STATUS } from "../../src/shared/domain"; import type { AgentTurnResult } from "../../src/main/services/provider-tool-turn"; import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker"; +import { markModelChecked } from "../helpers/model-check"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -33,6 +34,7 @@ describe("rate limits seen on successful calls", () => { orgId = organization.createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" }); providerId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'openai'").get(orgId) as { id: string }).id; providers.saveCredential({ orgId, providerId, secret: "sk-test", model: "gpt-5.4" }); + markModelChecked(connection.db, providerId, "gpt-5.4"); const agentId = organization.createAgent({ orgId, name: "Ada", role: "Engineer", department: "Engineering", modelProviderId: providerId, modelId: "gpt-5.4", budgetCents: 1000, managerAgentId: null, heartbeatEnabled: true }); organization.createTask({ orgId, title: "Work", description: "d", status: TASK_STATUS.BACKLOG, assigneeAgentId: agentId, risk: RISK_LEVEL.LOW, projectId: null, goalId: null, workspaceId: null, priority: TASK_PRIORITY.NORMAL, requiresFilesystem: false }); }); diff --git a/tests/unit/task-execution-service.test.ts b/tests/unit/task-execution-service.test.ts index fa7591f9..7c69157e 100644 --- a/tests/unit/task-execution-service.test.ts +++ b/tests/unit/task-execution-service.test.ts @@ -14,6 +14,7 @@ import { SnapshotService } from "../../src/main/services/snapshot-service"; import { TaskExecutionService } from "../../src/main/services/task-execution-service"; import { APPROVAL_STATE, PROVIDER_DATA_POLICY, RISK_LEVEL, TASK_PRIORITY, TASK_STATUS } from "../../src/shared/domain"; import { currentProviderUsageWindow, PROVIDER_MONTHLY_LIMIT_MESSAGE } from "../../src/shared/provider-runtime-domain"; +import { answeringModelCheck } from "../helpers/model-check"; function createConnection(): DatabaseConnection { return new DatabaseConnection(join(tmpdir(), `agentcompany-new-task-execution-${randomUUID()}.db`)); @@ -40,7 +41,7 @@ describe("task execution service", () => { if (first?.role === "user") capturedPrompt = first.text; return { text: "executed:gpt-5.4", toolCalls: [], rawResponseJson: JSON.stringify({ id: "resp-task-1" }) }; }; - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn)); const skills = new SkillService(connection.db, join(tmpdir(), `agentcompany-task-skills-${randomUUID()}`)); const execution = new TaskExecutionService(connection.db, providers); const skillDir = join(tmpdir(), `agentcompany-task-skill-${randomUUID()}`); @@ -152,7 +153,7 @@ describe("task execution service", () => { toolCalls: [], rawResponseJson: JSON.stringify({ id: "resp-task-approved" }), }); - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn)); const execution = new TaskExecutionService(connection.db, providers); let orgId = ""; let employeeId = ""; @@ -291,7 +292,7 @@ describe("task execution service", () => { const fakeToolTurn = async (): Promise => { throw new Error("provider rejected task"); }; - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn)); const execution = new TaskExecutionService(connection.db, providers); let orgId = ""; let taskId = ""; @@ -349,7 +350,7 @@ describe("task execution service", () => { turns += 1; return { text: "done", toolCalls: [], rawResponseJson: JSON.stringify({ id: `resp-${turns}` }) }; }; - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn)); const execution = new TaskExecutionService(connection.db, providers); let orgId = ""; let taskId = ""; @@ -848,7 +849,7 @@ describe("task execution service", () => { const fakeToolTurn = async (): Promise => { throw new Error("The usage limit has been reached"); }; - const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn)); let nowMs = Date.now(); const execution = new TaskExecutionService(connection.db, providers, undefined, undefined, () => nowMs); let orgId = "";