diff --git a/src/main/services/provider-adapter-contract.ts b/src/main/services/provider-adapter-contract.ts index b7e6e7a7..f9612653 100644 --- a/src/main/services/provider-adapter-contract.ts +++ b/src/main/services/provider-adapter-contract.ts @@ -32,16 +32,6 @@ export interface ProviderBackgroundRetrieveInput { responseId: string; } -export interface ProviderHealthInput { - authProfile: ProviderAuthProfile; - model: string | null; -} - -export interface ProviderHealthResult { - checkedAt: string; - rateLimitState: ProviderObservedRateLimitState | null; -} - export interface ProviderContext { authProfile: ProviderAuthProfile; model: string | null; @@ -64,7 +54,6 @@ export interface ProviderAdapter { parseAuthProfile(secret: string): ProviderAuthProfile; listModels(ctx: ProviderContext): Promise; getRateLimitState(ctx: ProviderContext): Promise; - checkHealth(input: ProviderHealthInput): Promise; streamResponse(input: ProviderStreamInput): Promise; cancelResponse(input: ProviderCancelInput): Promise; createBackgroundResponse(input: ProviderBackgroundCreateInput): Promise; diff --git a/src/main/services/provider-adapters.ts b/src/main/services/provider-adapters.ts index 4ee0cb17..d8903d65 100644 --- a/src/main/services/provider-adapters.ts +++ b/src/main/services/provider-adapters.ts @@ -1,6 +1,5 @@ import { PROVIDER_AUTH_TYPE, type ProviderAuthType } from "../../shared/domain"; import { listProviderModels, type ProviderModelEntry } from "../../shared/provider-models"; -import type { ProviderObservedRateLimitState } from "../../shared/provider-runtime-domain"; import { parseAccessToken, parseAccountToken, @@ -18,7 +17,6 @@ import { requireBedrockProfile, requireOpenAiCompatProfile, requireVertexProfile, - type ProviderAuthProfile, } from "./provider-auth-profiles"; import { backgroundUnsupported, retrieveUnsupported } from "./provider-background"; import { authorizeWithParser, refreshUnsupported, revokeLocalProfile } from "./provider-auth-lifecycle"; @@ -47,7 +45,6 @@ export * from "./provider-adapter-contract"; import type { ProviderAdapter, ProviderAdapterRegistry, - ProviderHealthResult, ProviderRateLimitState, } from "./provider-adapter-contract"; @@ -63,7 +60,6 @@ export const providerAdapters: ProviderAdapterRegistry = { authorization: `Bearer ${requireApiKeyProfile(ctx.authProfile, "OpenAI").apiKey}`, }), getRateLimitState: providerRateLimitUnknown, - checkHealth: (input) => checkOpenAiHealth(input.authProfile, "https://api.openai.com/v1/models"), async streamResponse(input) { const profile = requireApiKeyProfile(input.authProfile, "OpenAI"); const response = await postEventStream("https://api.openai.com/v1/responses", { @@ -148,14 +144,6 @@ export const providerAdapters: ProviderAdapterRegistry = { "anthropic-version": ANTHROPIC_VERSION, }), getRateLimitState: providerRateLimitUnknown, - async checkHealth(input) { - const profile = requireApiKeyProfile(input.authProfile, "Anthropic"); - const response = await getJson("https://api.anthropic.com/v1/models", { - "x-api-key": profile.apiKey, - "anthropic-version": "2023-06-01", - }); - return healthy(response.rateLimitState); - }, async streamResponse(input) { const profile = requireApiKeyProfile(input.authProfile, "Anthropic"); const response = await postEventStream("https://api.anthropic.com/v1/messages", { @@ -264,11 +252,6 @@ export const providerAdapters: ProviderAdapterRegistry = { cancelResponse: cancelUnsupported("Azure OpenAI"), createBackgroundResponse: backgroundUnsupported("Azure OpenAI"), retrieveResponse: retrieveUnsupported("Azure OpenAI"), - async checkHealth(input) { - const profile = requireAzureProfile(input.authProfile); - const response = await getJson(`${profile.baseUrl}/openai/deployments?api-version=2024-10-21`, { "api-key": profile.apiKey }); - return healthy(response.rateLimitState); - }, }, "aws-bedrock": { providerKey: "aws-bedrock", @@ -283,10 +266,6 @@ export const providerAdapters: ProviderAdapterRegistry = { cancelResponse: cancelUnsupported("AWS Bedrock"), createBackgroundResponse: backgroundUnsupported("AWS Bedrock"), retrieveResponse: retrieveUnsupported("AWS Bedrock"), - checkHealth: async (input) => { - requireBedrockProfile(input.authProfile); - return healthy(null); - }, }, "vertex-ai": { providerKey: "vertex-ai", @@ -301,10 +280,6 @@ export const providerAdapters: ProviderAdapterRegistry = { cancelResponse: cancelUnsupported("Vertex AI"), createBackgroundResponse: backgroundUnsupported("Vertex AI"), retrieveResponse: retrieveUnsupported("Vertex AI"), - checkHealth: async (input) => { - requireVertexProfile(input.authProfile); - return healthy(null); - }, }, "custom-openai": { providerKey: "custom-openai", @@ -319,10 +294,6 @@ export const providerAdapters: ProviderAdapterRegistry = { cancelResponse: cancelUnsupported("Custom OpenAI-compatible endpoint"), createBackgroundResponse: backgroundUnsupported("Custom OpenAI-compatible endpoint"), retrieveResponse: retrieveUnsupported("Custom OpenAI-compatible endpoint"), - checkHealth: async (input) => { - requireOpenAiCompatProfile(input.authProfile); - return healthy(null); - }, }, "custom-anthropic": { providerKey: "custom-anthropic", @@ -337,10 +308,6 @@ export const providerAdapters: ProviderAdapterRegistry = { cancelResponse: cancelUnsupported("Custom Anthropic-compatible endpoint"), createBackgroundResponse: backgroundUnsupported("Custom Anthropic-compatible endpoint"), retrieveResponse: retrieveUnsupported("Custom Anthropic-compatible endpoint"), - checkHealth: async (input) => { - requireAnthropicCompatProfile(input.authProfile); - return healthy(null); - }, }, }; @@ -398,7 +365,6 @@ function openAiCompatibleAdapter({ authorization: `Bearer ${requireApiKeyProfile(ctx.authProfile, label).apiKey}`, }), getRateLimitState: providerRateLimitUnknown, - checkHealth: (input) => checkOpenAiHealth(input.authProfile, healthUrl), async streamResponse(input) { const profile = requireApiKeyProfile(input.authProfile, label); const messages = responseMessages(input.instructions, input.prompt); @@ -441,12 +407,6 @@ function responseMessages(instructions: string | null, prompt: string): Array<{ ]; } -async function checkOpenAiHealth(authProfile: ProviderAuthProfile, healthUrl: string): Promise { - const profile = requireApiKeyProfile(authProfile, "OpenAI-compatible provider"); - const response = await getJson(healthUrl, { authorization: `Bearer ${profile.apiKey}` }); - return healthy(response.rateLimitState); -} - function browserTokenAdapter(providerKey: string, authType: ProviderAuthType): ProviderAdapter { const parseAuthProfile = authType === PROVIDER_AUTH_TYPE.ACCOUNT_SUBSCRIPTION ? parseAccountToken : parseAccessToken; const assertProfile = authType === PROVIDER_AUTH_TYPE.ACCOUNT_SUBSCRIPTION ? requireAccountTokenProfile : requireAccessTokenProfile; @@ -463,11 +423,6 @@ function browserTokenAdapter(providerKey: string, authType: ProviderAuthType): P cancelResponse: cancelUnsupported(providerKey), createBackgroundResponse: backgroundUnsupported(providerKey), retrieveResponse: retrieveUnsupported(providerKey), - checkHealth: async (input) => { - if (authType === PROVIDER_AUTH_TYPE.ACCOUNT_SUBSCRIPTION) requireAccountTokenProfile(input.authProfile); - else requireAccessTokenProfile(input.authProfile); - return healthy(null); - }, }; } @@ -477,10 +432,6 @@ function requireResponseId(value: string): string { return responseId; } -function healthy(rateLimitState: ProviderObservedRateLimitState | null): ProviderHealthResult { - return { checkedAt: new Date().toISOString(), rateLimitState }; -} - function listStaticModels(providerKey: string): Promise { return Promise.resolve(listProviderModels(providerKey)); } diff --git a/src/main/services/provider-service.ts b/src/main/services/provider-service.ts index 3cc24c3c..3f341b55 100644 --- a/src/main/services/provider-service.ts +++ b/src/main/services/provider-service.ts @@ -9,7 +9,7 @@ import { AuthBrokerService, type ProviderSecretVault } from "./auth-broker-servi import { OperationsService } from "./operations-service"; import { IntegrationKeyStore } from "./integration-key-store"; import { SecretVault } from "./secret-vault"; -import { providerAdapters, providerRateLimitStateFromError, type ProviderAdapterRegistry } from "./provider-adapters"; +import { providerAdapters, type ProviderAdapterRegistry } from "./provider-adapters"; import { ProviderBackgroundService } from "./provider-background-service"; import { ProviderRateLimitRegistry } from "./provider-rate-limit-registry"; import { ProviderResponseObserver } from "./provider-response-observer"; @@ -28,7 +28,6 @@ import { WORKSPACE_SCOPES, loadConfiguredProvider, loadCredential, - recordObservedRateLimit, recordProviderHealth, requireAdapter, requireCatalogEntry, @@ -199,14 +198,13 @@ export class ProviderService { return this.runner.checkModel(input); } + /** Health is a real agent call to the default model: account, endpoint and model all have to work. */ async checkHealth(orgId: string, providerId: string): Promise { const provider = loadConfiguredProvider(this.db, orgId, providerId); - const adapter = requireAdapter(this.adapters, provider.provider_key, provider.auth_type); - const credential = loadCredential(this.authBroker, orgId, provider); try { - const profile = adapter.parseAuthProfile(credential.secret); - const result = await adapter.checkHealth({ authProfile: profile, model: provider.model }); - recordObservedRateLimit(this.rateLimitRegistry, orgId, provider, provider.model, result.rateLimitState); + if (!provider.model) throw new Error("Pick a default model; the health check is a real call to it."); + const check = await this.runner.checkModel({ orgId, providerId, model: provider.model }); + if (!check.passed) throw new Error(`${provider.model} answered but cannot do agent work: ${check.detail}.`); recordProviderHealth(this.db, provider.id, orgId, HEALTH_STATE_HEALTHY, null); this.audit.record({ orgId, @@ -219,12 +217,12 @@ export class ProviderService { }); } catch (error) { const message = error instanceof Error ? error.message : String(error); - recordObservedRateLimit(this.rateLimitRegistry, orgId, provider, provider.model, providerRateLimitStateFromError(error)); recordProviderHealth(this.db, provider.id, orgId, HEALTH_STATE_FAILED, message); throw error; } } + launchSignInWindow(input: LaunchProviderSignInInput): void { this.assertEmployee(input.orgId, input.employeeId); const provider = this.db.prepare(` diff --git a/tests/unit/operations-service.test.ts b/tests/unit/operations-service.test.ts index 9a7d4da1..8174398f 100644 --- a/tests/unit/operations-service.test.ts +++ b/tests/unit/operations-service.test.ts @@ -64,7 +64,6 @@ describe("operations service", () => { openai: { ...providerAdapters.openai, parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }), - checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }), createResponse: async () => ({ text: "accepted", rawResponseJson: "{}", rateLimitState: null }), }, }; diff --git a/tests/unit/provider-background-service.test.ts b/tests/unit/provider-background-service.test.ts index 97cd3c63..1e473d36 100644 --- a/tests/unit/provider-background-service.test.ts +++ b/tests/unit/provider-background-service.test.ts @@ -243,7 +243,6 @@ function backgroundAdapters(status: string, text: string | null): ProviderAdapte openai: { ...providerAdapters.openai, parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }), - checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }), createBackgroundResponse: async () => ({ responseId: "resp_bg_1", status: "queued", diff --git a/tests/unit/provider-service-streaming.test.ts b/tests/unit/provider-service-streaming.test.ts index 7f9493b3..b1db1d54 100644 --- a/tests/unit/provider-service-streaming.test.ts +++ b/tests/unit/provider-service-streaming.test.ts @@ -43,7 +43,6 @@ describe("provider service streaming", () => { openai: { ...providerAdapters.openai, parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }), - checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }), streamResponse: async () => ({ events: streamEvents([ event("message_start", null, "created", "resp_live_1"), @@ -152,7 +151,6 @@ describe("provider service streaming", () => { openai: { ...providerAdapters.openai, parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }), - checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }), cancelResponse: async ({ responseId }) => { cancelledResponseId = responseId; return { responseId, rawResponseJson: JSON.stringify({ id: responseId, status: "cancelled" }), rateLimitState: null }; @@ -227,7 +225,6 @@ describe("cancelled streams", () => { openai: { ...providerAdapters.openai, parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }), - checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }), streamResponse: async () => ({ events: liveEvents(), rateLimitState: null }), cancelResponse: async ({ responseId }) => ({ responseId, rawResponseJson: JSON.stringify({ id: responseId, status: "cancelled" }), rateLimitState: null }), }, diff --git a/tests/unit/provider-service.test.ts b/tests/unit/provider-service.test.ts index 0468cdb3..8649d975 100644 --- a/tests/unit/provider-service.test.ts +++ b/tests/unit/provider-service.test.ts @@ -5,10 +5,12 @@ import { describe, expect, test } from "vitest"; import { DatabaseConnection } from "../../src/main/database/connection"; import { OrganizationService } from "../../src/main/services/organization-service"; import { ProviderService } from "../../src/main/services/provider-service"; -import { ProviderHttpError, providerAdapters, type ProviderAdapterRegistry, type ProviderAuthProfile } from "../../src/main/services/provider-adapters"; +import { ProviderHttpError, providerAdapters, type ProviderAdapterRegistry } from "../../src/main/services/provider-adapters"; import { SnapshotService } from "../../src/main/services/snapshot-service"; import { PROVIDER_DATA_POLICY, PROVIDER_EXECUTION_MODE } from "../../src/shared/domain"; import { PROVIDER_CATALOG } from "../../src/shared/provider-catalog"; +import { answeringModelCheck } from "../helpers/model-check"; +import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn"; function createConnection(): DatabaseConnection { return new DatabaseConnection(join(tmpdir(), `agentcompany-new-provider-${randomUUID()}.db`)); @@ -132,40 +134,42 @@ describe("provider service", () => { connection.close(); }); - test("checks provider health through its adapter", async () => { + test("health is a real agent call to the default model, not a look at the credential", async () => { const connection = createConnection(); const organization = new OrganizationService(connection.db); const snapshot = new SnapshotService(connection.db); - let checkedProfile: ProviderAuthProfile | null = null; - const adapters: ProviderAdapterRegistry = { - ...providerAdapters, - openai: { - ...providerAdapters.openai, - parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }), - checkHealth: async ({ authProfile }) => { - checkedProfile = authProfile; - return { checkedAt: new Date().toISOString(), rateLimitState: null }; - }, - }, - }; - const providers = new ProviderService(connection.db, adapters, testVault); + let mode: "capable" | "text" | "down" = "capable"; + const seen: string[] = []; + const capable: (input: ProviderToolTurnInput) => Promise = answeringModelCheck(async () => ({ text: "", toolCalls: [], rawResponseJson: "{}" })); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, async (input) => { + seen.push(`${input.credential.apiKey}:${input.model}`); + if (mode === "down") throw new Error("401 invalid x-api-key"); + return mode === "text" ? { text: "I am healthy.", toolCalls: [], rawResponseJson: "{}" } : capable(input); + }); let orgId = ""; let openAiProviderId = ""; - connection.transaction(() => { orgId = createOrganization(organization); openAiProviderId = providerId(connection, orgId, "openai"); providers.saveCredential({ orgId, providerId: openAiProviderId, secret: "sk-live-provider", model: "gpt-5.4" }); }); - await providers.checkHealth(orgId, openAiProviderId); + const health = () => snapshot.load().providers.find((item) => item.id === openAiProviderId); - const state = snapshot.load(); - const provider = state.providers.find((item) => item.id === openAiProviderId); - if (!provider) throw new Error("Provider was not found in snapshot."); - expect(checkedProfile).toEqual({ type: "api_key", apiKey: "sk-live-provider" }); - expect(provider.healthState).toBe("healthy"); - expect(provider.lastCheckedAt).toHaveLength(24); - expect(state.auditEvents.some((event) => event.entityType === "provider" && event.action === "health_checked")).toBe(true); + await providers.checkHealth(orgId, openAiProviderId); + expect(seen).toEqual(["sk-live-provider:gpt-5.4"]); + expect(health()).toMatchObject({ healthState: "healthy", lastError: null }); + expect(snapshot.load().auditEvents.some((event) => event.entityType === "provider" && event.action === "health_checked")).toBe(true); + + mode = "text"; + await expect(providers.checkHealth(orgId, openAiProviderId)).rejects.toThrow("gpt-5.4 answered but cannot do agent work: it answered in text instead of calling the tool it was given"); + expect(health()).toMatchObject({ healthState: "failed", lastError: expect.stringContaining("cannot do agent work") }); + + mode = "down"; + await expect(providers.checkHealth(orgId, openAiProviderId)).rejects.toThrow("401 invalid x-api-key"); + expect(health()).toMatchObject({ healthState: "failed", lastError: "401 invalid x-api-key" }); + + connection.db.prepare("update providers set model = null where id = ?").run(openAiProviderId); + await expect(providers.checkHealth(orgId, openAiProviderId)).rejects.toThrow("Pick a default model"); connection.close(); }); @@ -274,7 +278,6 @@ describe("provider service", () => { openai: { ...providerAdapters.openai, parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }), - checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }), createResponse: async ({ authProfile, model, prompt }) => ({ text: `${authProfile.type}:${model}:${prompt}`, rawResponseJson: JSON.stringify({ provider: "openai" }), @@ -320,7 +323,6 @@ describe("provider service", () => { openai: { ...providerAdapters.openai, parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }), - checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }), createResponse: async () => ({ text: "accepted", rawResponseJson: "{}", @@ -373,7 +375,6 @@ describe("provider service", () => { openai: { ...providerAdapters.openai, parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }), - checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }), createResponse: async () => { throw new ProviderHttpError("rate limited", { requestLimit: 120, @@ -430,7 +431,6 @@ describe("provider service", () => { openai: { ...providerAdapters.openai, parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }), - checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }), createResponse: async ({ prompt }) => ({ text: `accepted:${prompt}`, rawResponseJson: "{}", rateLimitState: null }), }, }; @@ -499,7 +499,6 @@ describe("provider service", () => { openai: { ...providerAdapters.openai, parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }), - checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }), createResponse: async ({ prompt }) => ({ text: `accepted:${prompt}`, rawResponseJson: "{}", rateLimitState: null }), }, }; @@ -561,7 +560,6 @@ describe("provider service", () => { openai: { ...providerAdapters.openai, parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }), - checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }), createResponse: async () => { adapterCalled = true; return { text: "blocked", rawResponseJson: "{}", rateLimitState: null };