Q86: provider health is a real agent call to the default model

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/main/services/provider-adapter-contract.ts

@@ -32,16 +32,6 @@ export interface ProviderBackgroundRetrieveInput {
   responseId: string;
 }
 
-export interface ProviderHealthInput {
-  authProfile: ProviderAuthProfile;
-  model: string | null;
-}
-
-export interface ProviderHealthResult {
-  checkedAt: string;
-  rateLimitState: ProviderObservedRateLimitState | null;
-}
-
 export interface ProviderContext {
   authProfile: ProviderAuthProfile;
   model: string | null;
@@ -64,7 +54,6 @@ export interface ProviderAdapter {
   parseAuthProfile(secret: string): ProviderAuthProfile;
   listModels(ctx: ProviderContext): Promise<ProviderModelEntry[]>;
   getRateLimitState(ctx: ProviderContext): Promise<ProviderRateLimitState | null>;
-  checkHealth(input: ProviderHealthInput): Promise<ProviderHealthResult>;
   streamResponse(input: ProviderStreamInput): Promise<ProviderStreamResult>;
   cancelResponse(input: ProviderCancelInput): Promise<ProviderCancelResult>;
   createBackgroundResponse(input: ProviderBackgroundCreateInput): Promise<ProviderBackgroundCreateResult>;

src/main/services/provider-adapters.ts

@@ -1,6 +1,5 @@
 import { PROVIDER_AUTH_TYPE, type ProviderAuthType } from "../../shared/domain";
 import { listProviderModels, type ProviderModelEntry } from "../../shared/provider-models";
-import type { ProviderObservedRateLimitState } from "../../shared/provider-runtime-domain";
 import {
   parseAccessToken,
   parseAccountToken,
@@ -18,7 +17,6 @@ import {
   requireBedrockProfile,
   requireOpenAiCompatProfile,
   requireVertexProfile,
-  type ProviderAuthProfile,
 } from "./provider-auth-profiles";
 import { backgroundUnsupported, retrieveUnsupported } from "./provider-background";
 import { authorizeWithParser, refreshUnsupported, revokeLocalProfile } from "./provider-auth-lifecycle";
@@ -47,7 +45,6 @@ export * from "./provider-adapter-contract";
 import type {
   ProviderAdapter,
   ProviderAdapterRegistry,
-  ProviderHealthResult,
   ProviderRateLimitState,
 } from "./provider-adapter-contract";
 
@@ -63,7 +60,6 @@ export const providerAdapters: ProviderAdapterRegistry = {
       authorization: `Bearer ${requireApiKeyProfile(ctx.authProfile, "OpenAI").apiKey}`,
     }),
     getRateLimitState: providerRateLimitUnknown,
-    checkHealth: (input) => checkOpenAiHealth(input.authProfile, "https://api.openai.com/v1/models"),
     async streamResponse(input) {
       const profile = requireApiKeyProfile(input.authProfile, "OpenAI");
       const response = await postEventStream("https://api.openai.com/v1/responses", {
@@ -148,14 +144,6 @@ export const providerAdapters: ProviderAdapterRegistry = {
       "anthropic-version": ANTHROPIC_VERSION,
     }),
     getRateLimitState: providerRateLimitUnknown,
-    async checkHealth(input) {
-      const profile = requireApiKeyProfile(input.authProfile, "Anthropic");
-      const response = await getJson("https://api.anthropic.com/v1/models", {
-        "x-api-key": profile.apiKey,
-        "anthropic-version": "2023-06-01",
-      });
-      return healthy(response.rateLimitState);
-    },
     async streamResponse(input) {
       const profile = requireApiKeyProfile(input.authProfile, "Anthropic");
       const response = await postEventStream("https://api.anthropic.com/v1/messages", {
@@ -264,11 +252,6 @@ export const providerAdapters: ProviderAdapterRegistry = {
     cancelResponse: cancelUnsupported("Azure OpenAI"),
     createBackgroundResponse: backgroundUnsupported("Azure OpenAI"),
     retrieveResponse: retrieveUnsupported("Azure OpenAI"),
-    async checkHealth(input) {
-      const profile = requireAzureProfile(input.authProfile);
-      const response = await getJson(`${profile.baseUrl}/openai/deployments?api-version=2024-10-21`, { "api-key": profile.apiKey });
-      return healthy(response.rateLimitState);
-    },
   },
   "aws-bedrock": {
     providerKey: "aws-bedrock",
@@ -283,10 +266,6 @@ export const providerAdapters: ProviderAdapterRegistry = {
     cancelResponse: cancelUnsupported("AWS Bedrock"),
     createBackgroundResponse: backgroundUnsupported("AWS Bedrock"),
     retrieveResponse: retrieveUnsupported("AWS Bedrock"),
-    checkHealth: async (input) => {
-      requireBedrockProfile(input.authProfile);
-      return healthy(null);
-    },
   },
   "vertex-ai": {
     providerKey: "vertex-ai",
@@ -301,10 +280,6 @@ export const providerAdapters: ProviderAdapterRegistry = {
     cancelResponse: cancelUnsupported("Vertex AI"),
     createBackgroundResponse: backgroundUnsupported("Vertex AI"),
     retrieveResponse: retrieveUnsupported("Vertex AI"),
-    checkHealth: async (input) => {
-      requireVertexProfile(input.authProfile);
-      return healthy(null);
-    },
   },
   "custom-openai": {
     providerKey: "custom-openai",
@@ -319,10 +294,6 @@ export const providerAdapters: ProviderAdapterRegistry = {
     cancelResponse: cancelUnsupported("Custom OpenAI-compatible endpoint"),
     createBackgroundResponse: backgroundUnsupported("Custom OpenAI-compatible endpoint"),
     retrieveResponse: retrieveUnsupported("Custom OpenAI-compatible endpoint"),
-    checkHealth: async (input) => {
-      requireOpenAiCompatProfile(input.authProfile);
-      return healthy(null);
-    },
   },
   "custom-anthropic": {
     providerKey: "custom-anthropic",
@@ -337,10 +308,6 @@ export const providerAdapters: ProviderAdapterRegistry = {
     cancelResponse: cancelUnsupported("Custom Anthropic-compatible endpoint"),
     createBackgroundResponse: backgroundUnsupported("Custom Anthropic-compatible endpoint"),
     retrieveResponse: retrieveUnsupported("Custom Anthropic-compatible endpoint"),
-    checkHealth: async (input) => {
-      requireAnthropicCompatProfile(input.authProfile);
-      return healthy(null);
-    },
   },
 };
 
@@ -398,7 +365,6 @@ function openAiCompatibleAdapter({
       authorization: `Bearer ${requireApiKeyProfile(ctx.authProfile, label).apiKey}`,
     }),
     getRateLimitState: providerRateLimitUnknown,
-    checkHealth: (input) => checkOpenAiHealth(input.authProfile, healthUrl),
     async streamResponse(input) {
       const profile = requireApiKeyProfile(input.authProfile, label);
       const messages = responseMessages(input.instructions, input.prompt);
@@ -441,12 +407,6 @@ function responseMessages(instructions: string | null, prompt: string): Array<{
   ];
 }
 
-async function checkOpenAiHealth(authProfile: ProviderAuthProfile, healthUrl: string): Promise<ProviderHealthResult> {
-  const profile = requireApiKeyProfile(authProfile, "OpenAI-compatible provider");
-  const response = await getJson(healthUrl, { authorization: `Bearer ${profile.apiKey}` });
-  return healthy(response.rateLimitState);
-}
-
 function browserTokenAdapter(providerKey: string, authType: ProviderAuthType): ProviderAdapter {
   const parseAuthProfile = authType === PROVIDER_AUTH_TYPE.ACCOUNT_SUBSCRIPTION ? parseAccountToken : parseAccessToken;
   const assertProfile = authType === PROVIDER_AUTH_TYPE.ACCOUNT_SUBSCRIPTION ? requireAccountTokenProfile : requireAccessTokenProfile;
@@ -463,11 +423,6 @@ function browserTokenAdapter(providerKey: string, authType: ProviderAuthType): P
     cancelResponse: cancelUnsupported(providerKey),
     createBackgroundResponse: backgroundUnsupported(providerKey),
     retrieveResponse: retrieveUnsupported(providerKey),
-    checkHealth: async (input) => {
-      if (authType === PROVIDER_AUTH_TYPE.ACCOUNT_SUBSCRIPTION) requireAccountTokenProfile(input.authProfile);
-      else requireAccessTokenProfile(input.authProfile);
-      return healthy(null);
-    },
   };
 }
 
@@ -477,10 +432,6 @@ function requireResponseId(value: string): string {
   return responseId;
 }
 
-function healthy(rateLimitState: ProviderObservedRateLimitState | null): ProviderHealthResult {
-  return { checkedAt: new Date().toISOString(), rateLimitState };
-}
-
 function listStaticModels(providerKey: string): Promise<ProviderModelEntry[]> {
   return Promise.resolve(listProviderModels(providerKey));
 }

src/main/services/provider-service.ts

@@ -9,7 +9,7 @@ import { AuthBrokerService, type ProviderSecretVault } from "./auth-broker-servi
 import { OperationsService } from "./operations-service";
 import { IntegrationKeyStore } from "./integration-key-store";
 import { SecretVault } from "./secret-vault";
-import { providerAdapters, providerRateLimitStateFromError, type ProviderAdapterRegistry } from "./provider-adapters";
+import { providerAdapters, type ProviderAdapterRegistry } from "./provider-adapters";
 import { ProviderBackgroundService } from "./provider-background-service";
 import { ProviderRateLimitRegistry } from "./provider-rate-limit-registry";
 import { ProviderResponseObserver } from "./provider-response-observer";
@@ -28,7 +28,6 @@ import {
   WORKSPACE_SCOPES,
   loadConfiguredProvider,
   loadCredential,
-  recordObservedRateLimit,
   recordProviderHealth,
   requireAdapter,
   requireCatalogEntry,
@@ -199,14 +198,13 @@ export class ProviderService {
     return this.runner.checkModel(input);
   }
 
+  /** Health is a real agent call to the default model: account, endpoint and model all have to work. */
   async checkHealth(orgId: string, providerId: string): Promise<void> {
     const provider = loadConfiguredProvider(this.db, orgId, providerId);
-    const adapter = requireAdapter(this.adapters, provider.provider_key, provider.auth_type);
-    const credential = loadCredential(this.authBroker, orgId, provider);
     try {
-      const profile = adapter.parseAuthProfile(credential.secret);
+      if (!provider.model) throw new Error("Pick a default model; the health check is a real call to it.");
-      const result = await adapter.checkHealth({ authProfile: profile, model: provider.model });
+      const check = await this.runner.checkModel({ orgId, providerId, model: provider.model });
-      recordObservedRateLimit(this.rateLimitRegistry, orgId, provider, provider.model, result.rateLimitState);
+      if (!check.passed) throw new Error(`${provider.model} answered but cannot do agent work: ${check.detail}.`);
       recordProviderHealth(this.db, provider.id, orgId, HEALTH_STATE_HEALTHY, null);
       this.audit.record({
         orgId,
@@ -219,12 +217,12 @@ export class ProviderService {
       });
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
-      recordObservedRateLimit(this.rateLimitRegistry, orgId, provider, provider.model, providerRateLimitStateFromError(error));
       recordProviderHealth(this.db, provider.id, orgId, HEALTH_STATE_FAILED, message);
       throw error;
     }
   }
 
+
   launchSignInWindow(input: LaunchProviderSignInInput): void {
     this.assertEmployee(input.orgId, input.employeeId);
     const provider = this.db.prepare(`

tests/unit/operations-service.test.ts

@@ -64,7 +64,6 @@ describe("operations service", () => {
       openai: {
         ...providerAdapters.openai,
         parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
-        checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
         createResponse: async () => ({ text: "accepted", rawResponseJson: "{}", rateLimitState: null }),
       },
     };

tests/unit/provider-background-service.test.ts

@@ -243,7 +243,6 @@ function backgroundAdapters(status: string, text: string | null): ProviderAdapte
     openai: {
       ...providerAdapters.openai,
       parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
-      checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
       createBackgroundResponse: async () => ({
         responseId: "resp_bg_1",
         status: "queued",

tests/unit/provider-service-streaming.test.ts

@@ -43,7 +43,6 @@ describe("provider service streaming", () => {
       openai: {
         ...providerAdapters.openai,
         parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
-        checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
         streamResponse: async () => ({
           events: streamEvents([
             event("message_start", null, "created", "resp_live_1"),
@@ -152,7 +151,6 @@ describe("provider service streaming", () => {
       openai: {
         ...providerAdapters.openai,
         parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
-        checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
         cancelResponse: async ({ responseId }) => {
           cancelledResponseId = responseId;
           return { responseId, rawResponseJson: JSON.stringify({ id: responseId, status: "cancelled" }), rateLimitState: null };
@@ -227,7 +225,6 @@ describe("cancelled streams", () => {
       openai: {
         ...providerAdapters.openai,
         parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
-        checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
         streamResponse: async () => ({ events: liveEvents(), rateLimitState: null }),
         cancelResponse: async ({ responseId }) => ({ responseId, rawResponseJson: JSON.stringify({ id: responseId, status: "cancelled" }), rateLimitState: null }),
       },

tests/unit/provider-service.test.ts

@@ -5,10 +5,12 @@ import { describe, expect, test } from "vitest";
 import { DatabaseConnection } from "../../src/main/database/connection";
 import { OrganizationService } from "../../src/main/services/organization-service";
 import { ProviderService } from "../../src/main/services/provider-service";
-import { ProviderHttpError, providerAdapters, type ProviderAdapterRegistry, type ProviderAuthProfile } from "../../src/main/services/provider-adapters";
+import { ProviderHttpError, providerAdapters, type ProviderAdapterRegistry } from "../../src/main/services/provider-adapters";
 import { SnapshotService } from "../../src/main/services/snapshot-service";
 import { PROVIDER_DATA_POLICY, PROVIDER_EXECUTION_MODE } from "../../src/shared/domain";
 import { PROVIDER_CATALOG } from "../../src/shared/provider-catalog";
+import { answeringModelCheck } from "../helpers/model-check";
+import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
 
 function createConnection(): DatabaseConnection {
   return new DatabaseConnection(join(tmpdir(), `agentcompany-new-provider-${randomUUID()}.db`));
@@ -132,40 +134,42 @@ describe("provider service", () => {
     connection.close();
   });
 
-  test("checks provider health through its adapter", async () => {
+  test("health is a real agent call to the default model, not a look at the credential", async () => {
     const connection = createConnection();
     const organization = new OrganizationService(connection.db);
     const snapshot = new SnapshotService(connection.db);
-    let checkedProfile: ProviderAuthProfile | null = null;
+    let mode: "capable" | "text" | "down" = "capable";
-    const adapters: ProviderAdapterRegistry = {
+    const seen: string[] = [];
-      ...providerAdapters,
+    const capable: (input: ProviderToolTurnInput) => Promise<AgentTurnResult> = answeringModelCheck(async () => ({ text: "", toolCalls: [], rawResponseJson: "{}" }));
-      openai: {
+    const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, async (input) => {
-        ...providerAdapters.openai,
+      seen.push(`${input.credential.apiKey}:${input.model}`);
-        parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
+      if (mode === "down") throw new Error("401 invalid x-api-key");
-        checkHealth: async ({ authProfile }) => {
+      return mode === "text" ? { text: "I am healthy.", toolCalls: [], rawResponseJson: "{}" } : capable(input);
-          checkedProfile = authProfile;
+    });
-          return { checkedAt: new Date().toISOString(), rateLimitState: null };
-        },
-      },
-    };
-    const providers = new ProviderService(connection.db, adapters, testVault);
     let orgId = "";
     let openAiProviderId = "";
-
     connection.transaction(() => {
       orgId = createOrganization(organization);
       openAiProviderId = providerId(connection, orgId, "openai");
       providers.saveCredential({ orgId, providerId: openAiProviderId, secret: "sk-live-provider", model: "gpt-5.4" });
     });
-    await providers.checkHealth(orgId, openAiProviderId);
+    const health = () => snapshot.load().providers.find((item) => item.id === openAiProviderId);
 
-    const state = snapshot.load();
+    await providers.checkHealth(orgId, openAiProviderId);
-    const provider = state.providers.find((item) => item.id === openAiProviderId);
+    expect(seen).toEqual(["sk-live-provider:gpt-5.4"]);
-    if (!provider) throw new Error("Provider was not found in snapshot.");
+    expect(health()).toMatchObject({ healthState: "healthy", lastError: null });
-    expect(checkedProfile).toEqual({ type: "api_key", apiKey: "sk-live-provider" });
+    expect(snapshot.load().auditEvents.some((event) => event.entityType === "provider" && event.action === "health_checked")).toBe(true);
-    expect(provider.healthState).toBe("healthy");
+
-    expect(provider.lastCheckedAt).toHaveLength(24);
+    mode = "text";
-    expect(state.auditEvents.some((event) => event.entityType === "provider" && event.action === "health_checked")).toBe(true);
+    await expect(providers.checkHealth(orgId, openAiProviderId)).rejects.toThrow("gpt-5.4 answered but cannot do agent work: it answered in text instead of calling the tool it was given");
+    expect(health()).toMatchObject({ healthState: "failed", lastError: expect.stringContaining("cannot do agent work") });
+
+    mode = "down";
+    await expect(providers.checkHealth(orgId, openAiProviderId)).rejects.toThrow("401 invalid x-api-key");
+    expect(health()).toMatchObject({ healthState: "failed", lastError: "401 invalid x-api-key" });
+
+    connection.db.prepare("update providers set model = null where id = ?").run(openAiProviderId);
+    await expect(providers.checkHealth(orgId, openAiProviderId)).rejects.toThrow("Pick a default model");
     connection.close();
   });
 
@@ -274,7 +278,6 @@ describe("provider service", () => {
       openai: {
         ...providerAdapters.openai,
         parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
-        checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
         createResponse: async ({ authProfile, model, prompt }) => ({
           text: `${authProfile.type}:${model}:${prompt}`,
           rawResponseJson: JSON.stringify({ provider: "openai" }),
@@ -320,7 +323,6 @@ describe("provider service", () => {
       openai: {
         ...providerAdapters.openai,
         parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
-        checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
         createResponse: async () => ({
           text: "accepted",
           rawResponseJson: "{}",
@@ -373,7 +375,6 @@ describe("provider service", () => {
       openai: {
         ...providerAdapters.openai,
         parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
-        checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
         createResponse: async () => {
           throw new ProviderHttpError("rate limited", {
             requestLimit: 120,
@@ -430,7 +431,6 @@ describe("provider service", () => {
       openai: {
         ...providerAdapters.openai,
         parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
-        checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
         createResponse: async ({ prompt }) => ({ text: `accepted:${prompt}`, rawResponseJson: "{}", rateLimitState: null }),
       },
     };
@@ -499,7 +499,6 @@ describe("provider service", () => {
       openai: {
         ...providerAdapters.openai,
         parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
-        checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
         createResponse: async ({ prompt }) => ({ text: `accepted:${prompt}`, rawResponseJson: "{}", rateLimitState: null }),
       },
     };
@@ -561,7 +560,6 @@ describe("provider service", () => {
       openai: {
         ...providerAdapters.openai,
         parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
-        checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
         createResponse: async () => {
           adapterCalled = true;
           return { text: "blocked", rawResponseJson: "{}", rateLimitState: null };