import { tableRows } from "../helpers/table-rows"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; import { describe, expect, test } from "vitest"; import { DatabaseConnection } from "../../src/main/database/connection"; import { OrganizationService } from "../../src/main/services/organization-service"; import { ProviderService } from "../../src/main/services/provider-service"; import { ProviderHttpError, providerAdapters, type ProviderAdapterRegistry } from "../../src/main/services/provider-adapters"; import { SnapshotService } from "../../src/main/services/snapshot-service"; import { PROVIDER_DATA_POLICY, PROVIDER_EXECUTION_MODE } from "../../src/shared/domain"; import { PROVIDER_CATALOG } from "../../src/shared/provider-catalog"; import { answeringModelCheck } from "../helpers/model-check"; import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn"; function createConnection(): DatabaseConnection { return new DatabaseConnection(join(tmpdir(), `agentcompany-new-provider-${randomUUID()}.db`)); } function createOrganization(organization: OrganizationService): string { return organization.createOrganization({ organizationName: "Acme Operations", employeeName: "Dana Lee", mission: "Coordinate agents with auditable approvals.", goal: "Ship a verified control plane.", }); } function providerId(connection: DatabaseConnection, orgId: string, key: string): string { return (connection.db.prepare("select id from providers where org_id = ? and provider_key = ?") .get(orgId, key) as { id: string }).id; } const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), decryptSecret: (ciphertext: Buffer) => ciphertext.toString("utf8"), }; describe("provider service", () => { test("registers an adapter for every catalog provider", () => { expect(Object.keys(providerAdapters).sort()).toEqual(PROVIDER_CATALOG.map((provider) => provider.key).sort()); for (const provider of PROVIDER_CATALOG) { expect(providerAdapters[provider.key]?.authType).toBe(provider.authType); } }); test("creates active auth broker profiles during credential rotation", () => { const connection = createConnection(); const organization = new OrganizationService(connection.db); const snapshot = new SnapshotService(connection.db); const providers = new ProviderService(connection.db, providerAdapters, testVault); let orgId = ""; let openAiProviderId = ""; connection.transaction(() => { orgId = createOrganization(organization); openAiProviderId = providerId(connection, orgId, "openai"); providers.saveCredential({ orgId, providerId: openAiProviderId, secret: "sk-live-provider-one", model: "gpt-5.4" }); providers.saveCredential({ orgId, providerId: openAiProviderId, secret: "sk-live-provider-two", model: "gpt-5.4" }); }); const rows = connection.db.prepare(` select credential_id, display_hint, source_type, capability_json, active from provider_auth_profiles where org_id = ? and provider_id = ? order by active desc, created_at desc `).all(orgId, openAiProviderId) as Array<{ credential_id: string; display_hint: string; source_type: string; capability_json: string; active: number; }>; const state = snapshot.load(); const activeProfile = state.providerAuthProfiles.find((profile) => profile.providerId === openAiProviderId && profile.active); const provider = state.providers.find((item) => item.id === openAiProviderId); expect(rows.map((row) => row.active)).toEqual([1, 0]); expect(rows[0]).toMatchObject({ display_hint: "sk-liv...-two", source_type: "manual_secret" }); expect(JSON.parse(rows[0].capability_json)).toMatchObject({ providerKey: "openai", capabilityTags: ["responses", "tools", "background"] }); expect(provider?.credentialId).toBe(rows[0].credential_id); expect(activeProfile?.credentialId).toBe(rows[0].credential_id); expect(state.auditEvents.some((event) => event.entityType === "provider_auth_profile" && event.action === "created")).toBe(true); connection.close(); }); test("revokes active auth broker profiles and disables provider credentials", async () => { const connection = createConnection(); const organization = new OrganizationService(connection.db); const snapshot = new SnapshotService(connection.db); const providers = new ProviderService(connection.db, providerAdapters, testVault); let orgId = ""; let openAiProviderId = ""; connection.transaction(() => { orgId = createOrganization(organization); openAiProviderId = providerId(connection, orgId, "openai"); providers.saveCredential({ orgId, providerId: openAiProviderId, secret: "sk-live-provider", model: "gpt-5.4" }); providers.revokeCredential({ orgId, providerId: openAiProviderId }); }); await expect(providers.createResponse({ orgId, providerId: openAiProviderId, prompt: "Draft board memo", instructions: null, })).rejects.toThrow("Provider credential is required."); const profileRows = connection.db.prepare(` select active from provider_auth_profiles where org_id = ? and provider_id = ? `).all(orgId, openAiProviderId) as Array<{ active: number }>; const credentialCount = connection.db.prepare(` select count(*) as count from credentials where org_id = ? and provider_key = 'openai' `).get(orgId) as { count: number }; const state = snapshot.load(); const provider = state.providers.find((item) => item.id === openAiProviderId); expect(profileRows).toEqual([{ active: 0 }]); expect(credentialCount.count).toBe(1); expect(provider).toMatchObject({ credentialId: null, enabled: false, authState: "revoked", healthState: "unchecked", lastError: null, lastCheckedAt: null, }); expect(state.providerAuthProfiles.find((profile) => profile.providerId === openAiProviderId)?.active).toBe(false); expect(state.auditEvents.some((event) => event.entityType === "provider_auth_profile" && event.action === "revoked")).toBe(true); expect(state.auditEvents.some((event) => event.entityType === "provider" && event.action === "credential_revoked")).toBe(true); connection.close(); }); test("health is a real agent call to the default model, not a look at the credential", async () => { const connection = createConnection(); const organization = new OrganizationService(connection.db); const snapshot = new SnapshotService(connection.db); let mode: "capable" | "text" | "down" = "capable"; const seen: string[] = []; const capable: (input: ProviderToolTurnInput) => Promise = answeringModelCheck(async () => ({ text: "", toolCalls: [], rawResponseJson: "{}" })); const providers = new ProviderService(connection.db, providerAdapters, testVault, async (input) => { seen.push(`${input.credential.apiKey}:${input.model}`); if (mode === "down") throw new Error("401 invalid x-api-key"); return mode === "text" ? { text: "I am healthy.", toolCalls: [], rawResponseJson: "{}" } : capable(input); }); let orgId = ""; let openAiProviderId = ""; connection.transaction(() => { orgId = createOrganization(organization); openAiProviderId = providerId(connection, orgId, "openai"); providers.saveCredential({ orgId, providerId: openAiProviderId, secret: "sk-live-provider", model: "gpt-5.4" }); }); const health = () => snapshot.load().providers.find((item) => item.id === openAiProviderId); await providers.checkHealth(orgId, openAiProviderId); expect(seen).toEqual(["sk-live-provider:gpt-5.4"]); expect(health()).toMatchObject({ healthState: "healthy", lastError: null }); expect(snapshot.load().auditEvents.some((event) => event.entityType === "provider" && event.action === "health_checked")).toBe(true); mode = "text"; await expect(providers.checkHealth(orgId, openAiProviderId)).rejects.toThrow("gpt-5.4 answered but cannot do agent work: it answered in text instead of calling the tool it was given"); expect(health()).toMatchObject({ healthState: "failed", lastError: expect.stringContaining("cannot do agent work") }); mode = "down"; await expect(providers.checkHealth(orgId, openAiProviderId)).rejects.toThrow("401 invalid x-api-key"); expect(health()).toMatchObject({ healthState: "failed", lastError: "401 invalid x-api-key" }); connection.db.prepare("update providers set model = null where id = ?").run(openAiProviderId); await expect(providers.checkHealth(orgId, openAiProviderId)).rejects.toThrow("Pick a default model"); connection.close(); }); test("a sign-in provider saves only a native sign-in, recorded as coming from the OAuth login", () => { const connection = createConnection(); const organization = new OrganizationService(connection.db); const snapshot = new SnapshotService(connection.db); const providers = new ProviderService(connection.db, providerAdapters, testVault); let orgId = ""; let copilotProviderId = ""; const signIn = JSON.stringify({ access: "ghu_live_provider", refresh: "ghr_refresh", expires: 1_900_000_000_000 }); connection.transaction(() => { orgId = createOrganization(organization); copilotProviderId = providerId(connection, orgId, "github-copilot"); }); expect(() => connection.transaction(() => providers.saveCredential({ orgId, providerId: copilotProviderId, secret: "ghu_live_provider", model: "claude-opus-4-5" }))) .toThrow("Sign in to GitHub Copilot; a pasted token cannot be renewed when it expires."); connection.transaction(() => providers.saveCredential({ orgId, providerId: copilotProviderId, secret: signIn, model: "claude-opus-4-5" })); const state = snapshot.load(); const profile = state.providerAuthProfiles.find((item) => item.providerId === copilotProviderId && item.active); expect(profile).toMatchObject({ sourceType: "provider_oauth_login", sourceRef: null }); expect(state.providers.find((item) => item.id === copilotProviderId)?.enabled).toBe(true); connection.close(); }); test("records provider request results through direct execution adapters", async () => { const connection = createConnection(); const organization = new OrganizationService(connection.db); const snapshot = new SnapshotService(connection.db); const adapters: ProviderAdapterRegistry = { ...providerAdapters, openai: { ...providerAdapters.openai, parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }), createResponse: async ({ authProfile, model, prompt }) => ({ text: `${authProfile.type}:${model}:${prompt}`, rawResponseJson: JSON.stringify({ provider: "openai" }), rateLimitState: null, }), }, }; const providers = new ProviderService(connection.db, adapters, testVault); let orgId = ""; let openAiProviderId = ""; connection.transaction(() => { orgId = createOrganization(organization); openAiProviderId = providerId(connection, orgId, "openai"); providers.saveCredential({ orgId, providerId: openAiProviderId, secret: "sk-live-provider", model: "gpt-5.4" }); }); await providers.createResponse({ orgId, providerId: openAiProviderId, prompt: "Draft board memo", instructions: "Return concise output.", }); const state = snapshot.load(); expect(state.providerRequests[0]).toMatchObject({ providerId: openAiProviderId, providerKey: "openai", model: "gpt-5.4", responseText: "api_key:gpt-5.4:Draft board memo", state: "succeeded", }); expect(tableRows(connection.db, "operational_events", "created_at desc", ["success"])[0]).toMatchObject({ signalKey: "provider_request_success", source: "provider_gateway", success: true }); expect(state.providers.find((provider) => provider.id === openAiProviderId)?.healthState).toBe("healthy"); connection.close(); }); test("records provider response rate limit observations", async () => { const connection = createConnection(); const organization = new OrganizationService(connection.db); const adapters: ProviderAdapterRegistry = { ...providerAdapters, openai: { ...providerAdapters.openai, parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }), createResponse: async () => ({ text: "accepted", rawResponseJson: "{}", rateLimitState: { requestLimit: 120, remainingRequests: 119, resetAt: "2026-05-23T12:00:00.000Z", retryAfterMs: null, rawStateJson: JSON.stringify({ "x-ratelimit-remaining-requests": "119" }), }, }), }, }; const providers = new ProviderService(connection.db, adapters, testVault); let orgId = ""; let openAiProviderId = ""; connection.transaction(() => { orgId = createOrganization(organization); openAiProviderId = providerId(connection, orgId, "openai"); providers.saveCredential({ orgId, providerId: openAiProviderId, secret: "sk-live-provider", model: "gpt-5.4" }); }); await providers.createResponse({ orgId, providerId: openAiProviderId, prompt: "Draft board memo", instructions: null, }); expect(tableRows(connection.db, "provider_rate_limit_observations", "observed_at desc")[0]).toMatchObject({ orgId, providerId: openAiProviderId, providerKey: "openai", model: "gpt-5.4", requestLimit: 120, remainingRequests: 119, resetAt: "2026-05-23T12:00:00.000Z", retryAfterMs: null, rawStateJson: JSON.stringify({ "x-ratelimit-remaining-requests": "119" }), }); connection.close(); }); test("records failed provider response rate limit observations", async () => { const connection = createConnection(); const organization = new OrganizationService(connection.db); const snapshot = new SnapshotService(connection.db); const adapters: ProviderAdapterRegistry = { ...providerAdapters, openai: { ...providerAdapters.openai, parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }), createResponse: async () => { throw new ProviderHttpError("rate limited", { requestLimit: 120, remainingRequests: 0, resetAt: "2026-05-23T12:00:00.000Z", retryAfterMs: 2000, rawStateJson: JSON.stringify({ "retry-after": "2" }), }); }, }, }; const providers = new ProviderService(connection.db, adapters, testVault); let orgId = ""; let openAiProviderId = ""; connection.transaction(() => { orgId = createOrganization(organization); openAiProviderId = providerId(connection, orgId, "openai"); providers.saveCredential({ orgId, providerId: openAiProviderId, secret: "sk-live-provider", model: "gpt-5.4" }); }); await expect(providers.createResponse({ orgId, providerId: openAiProviderId, prompt: "Draft board memo", instructions: null, })).rejects.toThrow("rate limited"); const state = snapshot.load(); expect(state.providerRequests[0]).toMatchObject({ providerId: openAiProviderId, state: "failed", errorMessage: "rate limited", }); expect(tableRows(connection.db, "provider_rate_limit_observations", "observed_at desc")[0]).toMatchObject({ orgId, providerId: openAiProviderId, providerKey: "openai", model: "gpt-5.4", requestLimit: 120, remainingRequests: 0, resetAt: "2026-05-23T12:00:00.000Z", retryAfterMs: 2000, rawStateJson: JSON.stringify({ "retry-after": "2" }), }); connection.close(); }); test("tracks provider usage windows and enforces monthly request limits", async () => { const connection = createConnection(); const organization = new OrganizationService(connection.db); const snapshot = new SnapshotService(connection.db); const adapters: ProviderAdapterRegistry = { ...providerAdapters, openai: { ...providerAdapters.openai, parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }), createResponse: async ({ prompt }) => ({ text: `accepted:${prompt}`, rawResponseJson: "{}", rateLimitState: null }), }, }; const providers = new ProviderService(connection.db, adapters, testVault); let orgId = ""; let openAiProviderId = ""; connection.transaction(() => { orgId = createOrganization(organization); openAiProviderId = providerId(connection, orgId, "openai"); providers.saveCredential({ orgId, providerId: openAiProviderId, secret: "sk-live-provider", model: "gpt-5.4" }); providers.updatePolicy({ orgId, providerId: openAiProviderId, dataPolicy: PROVIDER_DATA_POLICY.STANDARD, backgroundEnabled: false, monthlyRequestLimit: 1, workspaceScope: "organization", contextWindowTokens: 128_000, }); }); await providers.createResponse({ orgId, providerId: openAiProviderId, prompt: "Draft board memo", instructions: null, }); await expect(providers.createResponse({ orgId, providerId: openAiProviderId, prompt: "Second board memo", instructions: null, })).rejects.toThrow("Provider monthly request limit has been reached."); const usage = connection.db.prepare(` select request_count, input_char_count, output_char_count from provider_usage_windows where org_id = ? and provider_id = ? `).get(orgId, openAiProviderId) as { request_count: number; input_char_count: number; output_char_count: number }; expect(usage).toEqual({ request_count: 1, input_char_count: 16, output_char_count: 25 }); const state = snapshot.load(); expect(state.providerRequests[0]).toMatchObject({ providerId: openAiProviderId, inputCharCount: 16, outputCharCount: 25, executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND, policyDecision: "allowed:standard:organization", }); expect(state.providerRateLimits.find((item) => item.providerId === openAiProviderId)).toMatchObject({ providerId: openAiProviderId, monthlyRequestLimit: 1, requestCount: 1, inputCharCount: 16, outputCharCount: 25, remainingRequestCount: 0, exhausted: true, }); connection.close(); }); test("requires workspace context for workspace-scoped provider execution", async () => { const connection = createConnection(); const organization = new OrganizationService(connection.db); const snapshot = new SnapshotService(connection.db); const adapters: ProviderAdapterRegistry = { ...providerAdapters, openai: { ...providerAdapters.openai, parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }), createResponse: async ({ prompt }) => ({ text: `accepted:${prompt}`, rawResponseJson: "{}", rateLimitState: null }), }, }; const providers = new ProviderService(connection.db, adapters, testVault); let orgId = ""; let openAiProviderId = ""; let workspaceId = ""; connection.transaction(() => { orgId = createOrganization(organization); openAiProviderId = providerId(connection, orgId, "openai"); workspaceId = "wsp_release"; const now = new Date().toISOString(); connection.db.prepare(` insert into workspaces(id, org_id, name, local_path, repo_url, repo_ref, project_id, is_primary, created_at, updated_at) values (?, ?, ?, ?, ?, ?, null, ?, ?, ?) `).run(workspaceId, orgId, "Release", "C:/work/release", "https://example.com/release.git", "main", 1, now, now); providers.saveCredential({ orgId, providerId: openAiProviderId, secret: "sk-live-provider", model: "gpt-5.4" }); providers.updatePolicy({ orgId, providerId: openAiProviderId, dataPolicy: PROVIDER_DATA_POLICY.STANDARD, backgroundEnabled: false, monthlyRequestLimit: 10, workspaceScope: "workspace", contextWindowTokens: 128_000, }); }); await expect(providers.createResponse({ orgId, providerId: openAiProviderId, prompt: "Draft workspace memo", instructions: null, })).rejects.toThrow("Provider workspace scope requires a workspace context."); expect(connection.db.prepare("select count(*) as count from provider_requests").get()).toEqual({ count: 0 }); await providers.createResponse({ orgId, providerId: openAiProviderId, workspaceId, prompt: "Draft workspace memo", instructions: null, }); expect(snapshot.load().providerRequests[0]).toMatchObject({ providerId: openAiProviderId, workspaceId, policyDecision: `allowed:standard:workspace:${workspaceId}`, }); connection.close(); }); test("blocks background execution under zero data retention policy", async () => { const connection = createConnection(); const organization = new OrganizationService(connection.db); let adapterCalled = false; const adapters: ProviderAdapterRegistry = { ...providerAdapters, openai: { ...providerAdapters.openai, parseAuthProfile: (secret) => ({ type: "api_key", apiKey: secret.trim() }), createResponse: async () => { adapterCalled = true; return { text: "blocked", rawResponseJson: "{}", rateLimitState: null }; }, }, }; const providers = new ProviderService(connection.db, adapters, testVault); let orgId = ""; let openAiProviderId = ""; connection.transaction(() => { orgId = createOrganization(organization); openAiProviderId = providerId(connection, orgId, "openai"); providers.saveCredential({ orgId, providerId: openAiProviderId, secret: "sk-live-provider", model: "gpt-5.4" }); providers.updatePolicy({ orgId, providerId: openAiProviderId, dataPolicy: PROVIDER_DATA_POLICY.ZERO_DATA_RETENTION, backgroundEnabled: false, monthlyRequestLimit: 10, workspaceScope: "organization", contextWindowTokens: 128_000, }); }); await expect(providers.createResponse({ orgId, providerId: openAiProviderId, prompt: "Run background analysis", instructions: null, executionMode: PROVIDER_EXECUTION_MODE.BACKGROUND, })).rejects.toThrow("Background execution conflicts with zero data retention policy."); expect(adapterCalled).toBe(false); expect(connection.db.prepare("select count(*) as count from provider_requests").get()).toEqual({ count: 0 }); connection.close(); }); test("validates cloud auth profiles before saving credentials", () => { const connection = createConnection(); const organization = new OrganizationService(connection.db); const providers = new ProviderService(connection.db); let orgId = ""; let azureProviderId = ""; let customOpenAiProviderId = ""; connection.transaction(() => { orgId = createOrganization(organization); azureProviderId = providerId(connection, orgId, "azure-openai"); customOpenAiProviderId = providerId(connection, orgId, "custom-openai"); }); expect(() => connection.transaction(() => providers.saveCredential({ orgId, providerId: azureProviderId, secret: JSON.stringify({ baseUrl: "http://example.com", deployment: "gpt-5", secret: "key" }), model: "gpt-5", }))).toThrow("Azure base URL must use HTTPS."); expect(() => connection.transaction(() => providers.saveCredential({ orgId, providerId: customOpenAiProviderId, secret: JSON.stringify({ baseUrl: "http://api.example.com/v1", secret: "key" }), model: "gpt-5", }))).toThrow("Base URL must use HTTPS unless it is on this computer or the local network."); connection.close(); }); });