ZacharyZhang-NY/AgentCompany
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Q86: provider health is a real agent call to the default model

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ZacharyZhang-NYandClaude Opus 5.5 committed 2026-10-08 14:32:18 -04:00
1 parent da0b94ce59
commit 7ecffe84e3
7 files changed
+34 -103
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/unit/operations-service.test.ts
-1
View File
@@ -64,7 +64,6 @@ describe("operations service", () => {
openai: {
...providerAdapters.openai,
parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
createResponse: async () => ({ text: "accepted", rawResponseJson: "{}", rateLimitState: null }),
},
};
tests/unit/provider-background-service.test.ts
-1
View File
@@ -243,7 +243,6 @@ function backgroundAdapters(status: string, text: string | null): ProviderAdapte
openai: {
...providerAdapters.openai,
parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
createBackgroundResponse: async () => ({
responseId: "resp_bg_1",
status: "queued",
tests/unit/provider-service-streaming.test.ts
-3
View File
@@ -43,7 +43,6 @@ describe("provider service streaming", () => {
openai: {
...providerAdapters.openai,
parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
streamResponse: async () => ({
events: streamEvents([
event("message_start", null, "created", "resp_live_1"),
@@ -152,7 +151,6 @@ describe("provider service streaming", () => {
openai: {
...providerAdapters.openai,
parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
cancelResponse: async ({ responseId }) => {
cancelledResponseId = responseId;
return { responseId, rawResponseJson: JSON.stringify({ id: responseId, status: "cancelled" }), rateLimitState: null };
@@ -227,7 +225,6 @@ describe("cancelled streams", () => {
openai: {
...providerAdapters.openai,
parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
streamResponse: async () => ({ events: liveEvents(), rateLimitState: null }),
cancelResponse: async ({ responseId }) => ({ responseId, rawResponseJson: JSON.stringify({ id: responseId, status: "cancelled" }), rateLimitState: null }),
},
tests/unit/provider-service.test.ts
+28 -30
View File
@@ -5,10 +5,12 @@ import { describe, expect, test } from "vitest";
import { DatabaseConnection } from "../../src/main/database/connection";
import { OrganizationService } from "../../src/main/services/organization-service";
import { ProviderService } from "../../src/main/services/provider-service";
import { ProviderHttpError, providerAdapters, type ProviderAdapterRegistry, type ProviderAuthProfile } from "../../src/main/services/provider-adapters";
import { ProviderHttpError, providerAdapters, type ProviderAdapterRegistry } from "../../src/main/services/provider-adapters";
import { SnapshotService } from "../../src/main/services/snapshot-service";
import { PROVIDER_DATA_POLICY, PROVIDER_EXECUTION_MODE } from "../../src/shared/domain";
import { PROVIDER_CATALOG } from "../../src/shared/provider-catalog";
import { answeringModelCheck } from "../helpers/model-check";
import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
function createConnection(): DatabaseConnection {
return new DatabaseConnection(join(tmpdir(), `agentcompany-new-provider-${randomUUID()}.db`));
@@ -132,40 +134,42 @@ describe("provider service", () => {
connection.close();
});
test("checks provider health through its adapter", async () => {
test("health is a real agent call to the default model, not a look at the credential", async () => {
const connection = createConnection();
const organization = new OrganizationService(connection.db);
const snapshot = new SnapshotService(connection.db);
let checkedProfile: ProviderAuthProfile | null = null;
const adapters: ProviderAdapterRegistry = {
...providerAdapters,
openai: {
...providerAdapters.openai,
parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
checkHealth: async ({ authProfile }) => {
checkedProfile = authProfile;
return { checkedAt: new Date().toISOString(), rateLimitState: null };
},
},
};
const providers = new ProviderService(connection.db, adapters, testVault);
let mode: "capable" | "text" | "down" = "capable";
const seen: string[] = [];
const capable: (input: ProviderToolTurnInput) => Promise<AgentTurnResult> = answeringModelCheck(async () => ({ text: "", toolCalls: [], rawResponseJson: "{}" }));
const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, async (input) => {
seen.push(`${input.credential.apiKey}:${input.model}`);
if (mode === "down") throw new Error("401 invalid x-api-key");
return mode === "text" ? { text: "I am healthy.", toolCalls: [], rawResponseJson: "{}" } : capable(input);
});
let orgId = "";
let openAiProviderId = "";
connection.transaction(() => {
orgId = createOrganization(organization);
openAiProviderId = providerId(connection, orgId, "openai");
providers.saveCredential({ orgId, providerId: openAiProviderId, secret: "sk-live-provider", model: "gpt-5.4" });
});
await providers.checkHealth(orgId, openAiProviderId);
const health = () => snapshot.load().providers.find((item) => item.id === openAiProviderId);
const state = snapshot.load();
const provider = state.providers.find((item) => item.id === openAiProviderId);
if (!provider) throw new Error("Provider was not found in snapshot.");
expect(checkedProfile).toEqual({ type: "api_key", apiKey: "sk-live-provider" });
expect(provider.healthState).toBe("healthy");
expect(provider.lastCheckedAt).toHaveLength(24);
expect(state.auditEvents.some((event) => event.entityType === "provider" && event.action === "health_checked")).toBe(true);
await providers.checkHealth(orgId, openAiProviderId);
expect(seen).toEqual(["sk-live-provider:gpt-5.4"]);
expect(health()).toMatchObject({ healthState: "healthy", lastError: null });
expect(snapshot.load().auditEvents.some((event) => event.entityType === "provider" && event.action === "health_checked")).toBe(true);
mode = "text";
await expect(providers.checkHealth(orgId, openAiProviderId)).rejects.toThrow("gpt-5.4 answered but cannot do agent work: it answered in text instead of calling the tool it was given");
expect(health()).toMatchObject({ healthState: "failed", lastError: expect.stringContaining("cannot do agent work") });
mode = "down";
await expect(providers.checkHealth(orgId, openAiProviderId)).rejects.toThrow("401 invalid x-api-key");
expect(health()).toMatchObject({ healthState: "failed", lastError: "401 invalid x-api-key" });
connection.db.prepare("update providers set model = null where id = ?").run(openAiProviderId);
await expect(providers.checkHealth(orgId, openAiProviderId)).rejects.toThrow("Pick a default model");
connection.close();
});
@@ -274,7 +278,6 @@ describe("provider service", () => {
openai: {
...providerAdapters.openai,
parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
createResponse: async ({ authProfile, model, prompt }) => ({
text: `${authProfile.type}:${model}:${prompt}`,
rawResponseJson: JSON.stringify({ provider: "openai" }),
@@ -320,7 +323,6 @@ describe("provider service", () => {
openai: {
...providerAdapters.openai,
parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
createResponse: async () => ({
text: "accepted",
rawResponseJson: "{}",
@@ -373,7 +375,6 @@ describe("provider service", () => {
openai: {
...providerAdapters.openai,
parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
createResponse: async () => {
throw new ProviderHttpError("rate limited", {
requestLimit: 120,
@@ -430,7 +431,6 @@ describe("provider service", () => {
openai: {
...providerAdapters.openai,
parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
createResponse: async ({ prompt }) => ({ text: `accepted:${prompt}`, rawResponseJson: "{}", rateLimitState: null }),
},
};
@@ -499,7 +499,6 @@ describe("provider service", () => {
openai: {
...providerAdapters.openai,
parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
createResponse: async ({ prompt }) => ({ text: `accepted:${prompt}`, rawResponseJson: "{}", rateLimitState: null }),
},
};
@@ -561,7 +560,6 @@ describe("provider service", () => {
openai: {
...providerAdapters.openai,
parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }),
checkHealth: async () => ({ checkedAt: new Date().toISOString(), rateLimitState: null }),
createResponse: async () => {
adapterCalled = true;
return { text: "blocked", rawResponseJson: "{}", rateLimitState: null };