diff --git a/src/main/database/migrations/074-run-failure-kind.ts b/src/main/database/migrations/074-run-failure-kind.ts index a8d6a03c..6c4c7804 100644 --- a/src/main/database/migrations/074-run-failure-kind.ts +++ b/src/main/database/migrations/074-run-failure-kind.ts @@ -1,4 +1,5 @@ -import { TRANSIENT_PROVIDER_FAILURE_SIGNATURES } from "../../../shared/provider-runtime-domain"; +/** The message signatures earlier builds classified transient failures by; frozen here to heal their rows. */ +const TRANSIENT_PROVIDER_FAILURE_SIGNATURES = ["provider monthly request limit has been reached.", "usage limit", "rate limit", "too many requests", "overloaded"]; /** Runs record whether they failed for a passing provider reason; older rows are classified from their message once, here. */ export const runFailureKindSchema = ` diff --git a/src/main/database/migrations/078-request-error-codes.ts b/src/main/database/migrations/078-request-error-codes.ts new file mode 100644 index 00000000..ca07f214 --- /dev/null +++ b/src/main/database/migrations/078-request-error-codes.ts @@ -0,0 +1,7 @@ +export const requestErrorCodesSchema = ` + alter table provider_requests add column error_status integer; + alter table provider_requests add column error_code text; + + insert into schema_migrations(version, name, applied_at) + values (78, 'request_error_codes', datetime('now')); +`; diff --git a/src/main/database/schema.ts b/src/main/database/schema.ts index a844c3e3..1c4e4d74 100644 --- a/src/main/database/schema.ts +++ b/src/main/database/schema.ts @@ -76,6 +76,7 @@ import { runFailureKindSchema } from "./migrations/074-run-failure-kind"; import { providerCooldownsSchema } from "./migrations/075-provider-cooldowns"; import { runPinsSchema } from "./migrations/076-run-pins"; import { modelChecksSchema } from "./migrations/077-model-checks"; +import { requestErrorCodesSchema } from "./migrations/078-request-error-codes"; interface Migration { version: number; @@ -161,6 +162,7 @@ const MIGRATIONS: Migration[] = [ { version: 75, name: "provider_cooldowns", sql: providerCooldownsSchema }, { version: 76, name: "run_pins", sql: runPinsSchema }, { version: 77, name: "model_checks", sql: modelChecksSchema }, + { version: 78, name: "request_error_codes", sql: requestErrorCodesSchema }, ]; const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version; diff --git a/src/main/services/oauth/anthropic-claude-flow.ts b/src/main/services/oauth/anthropic-claude-flow.ts index 498be9f3..df8907b8 100644 --- a/src/main/services/oauth/anthropic-claude-flow.ts +++ b/src/main/services/oauth/anthropic-claude-flow.ts @@ -8,6 +8,7 @@ import { startOAuthCallbackServer } from "./oauth-callback-server"; import { generatePkce } from "./pkce"; import type { OAuthCredentials, OAuthFlow, OAuthLoginCallbacks } from "./oauth-types"; +import { httpFailure } from "../provider-http"; const CLIENT_ID = "9d1c250a-e61b-44d9-88ed-5944d1962f5e"; const AUTHORIZE_URL = "https://claude.ai/oauth/authorize"; @@ -61,7 +62,7 @@ export async function refreshClaudePro(refreshToken: string): Promise ""); - throw new Error(`${response.status} ${response.statusText}: ${text}`); + throw httpFailure(`${response.status} ${response.statusText}: ${text}`, response, text); } return (await response.json()) as Record; } diff --git a/src/main/services/oauth/google-antigravity-flow.ts b/src/main/services/oauth/google-antigravity-flow.ts index 2e2dcddd..f96e28a5 100644 --- a/src/main/services/oauth/google-antigravity-flow.ts +++ b/src/main/services/oauth/google-antigravity-flow.ts @@ -7,6 +7,7 @@ import { startOAuthCallbackServer } from "./oauth-callback-server"; import { generatePkce } from "./pkce"; import type { OAuthCredentials, OAuthFlow, OAuthLoginCallbacks } from "./oauth-types"; +import { httpFailure } from "../provider-http"; const CLIENT_ID = "1071006060591-tmhssin2h21lcre235vtolojh4g403ep.apps.googleusercontent.com"; const CALLBACK_PORT = 51121; @@ -115,7 +116,7 @@ export async function refreshAntigravity(refreshToken: string, projectId: string }), }); const text = await response.text(); - if (!response.ok) throw new Error(`Antigravity token refresh failed: ${response.status} ${text}`); + if (!response.ok) throw httpFailure(`Antigravity token refresh failed: ${response.status} ${text}`, response, text); const json = JSON.parse(text) as { access_token?: unknown; refresh_token?: unknown; expires_in?: unknown }; if (typeof json.access_token !== "string" || typeof json.expires_in !== "number") { throw new Error("Antigravity token refresh response was missing required fields."); diff --git a/src/main/services/oauth/kimi-code-flow.ts b/src/main/services/oauth/kimi-code-flow.ts index 9f4f6aad..d8d8cc53 100644 --- a/src/main/services/oauth/kimi-code-flow.ts +++ b/src/main/services/oauth/kimi-code-flow.ts @@ -8,6 +8,7 @@ import { createHash } from "node:crypto"; import { hostname, platform as osPlatform, release as osRelease, userInfo } from "node:os"; import type { OAuthCredentials, OAuthFlow, OAuthLoginCallbacks } from "./oauth-types"; +import { httpFailure } from "../provider-http"; // Published public client id of the official Kimi CLI; reused by third-party // coding agents per Kimi's device-flow integration guidance. @@ -47,7 +48,7 @@ function deviceHeaders(): Record { }; } -async function postForm(path: string, form: Record): Promise<{ status: number; data: Record }> { +async function postForm(path: string, form: Record): Promise<{ status: number; data: Record; response: Response; text: string }> { const response = await fetch(`${OAUTH_HOST}${path}`, { method: "POST", headers: { @@ -62,12 +63,12 @@ async function postForm(path: string, form: Record): Promise<{ s try { data = JSON.parse(text); } catch { - throw new Error(`Kimi OAuth endpoint returned a non-JSON response: ${response.status} ${text.slice(0, 200)}`); + throw httpFailure(`Kimi OAuth endpoint returned a non-JSON response: ${response.status} ${text.slice(0, 200)}`, response, text); } if (typeof data !== "object" || data === null || Array.isArray(data)) { - throw new Error(`Kimi OAuth endpoint returned an unexpected payload: ${response.status}`); + throw httpFailure(`Kimi OAuth endpoint returned an unexpected payload: ${response.status}`, response, text); } - return { status: response.status, data: data as Record }; + return { status: response.status, data: data as Record, response, text }; } interface DeviceAuthorization { @@ -167,12 +168,12 @@ async function pollForToken(device: DeviceAuthorization, signal?: AbortSignal): /** Refresh an expired Kimi Code access token (they live ~15 minutes). */ export async function refreshKimiCode(refreshToken: string): Promise { - const { status, data } = await postForm("/api/oauth/token", { + const { status, data, response, text } = await postForm("/api/oauth/token", { client_id: CLIENT_ID, grant_type: "refresh_token", refresh_token: refreshToken, }); - if (status !== 200) throw new Error(`Kimi token refresh failed: ${status} ${JSON.stringify(data)}`); + if (status !== 200) throw httpFailure(`Kimi token refresh failed: ${status} ${JSON.stringify(data)}`, response, text); return credentialsFromToken(data); } diff --git a/src/main/services/oauth/openai-codex-flow.ts b/src/main/services/oauth/openai-codex-flow.ts index b4870972..7335b6e5 100644 --- a/src/main/services/oauth/openai-codex-flow.ts +++ b/src/main/services/oauth/openai-codex-flow.ts @@ -7,6 +7,7 @@ import { startOAuthCallbackServer } from "./oauth-callback-server"; import { generatePkce } from "./pkce"; import type { OAuthCredentials, OAuthFlow, OAuthLoginCallbacks } from "./oauth-types"; +import { httpFailure } from "../provider-http"; const CLIENT_ID = "app_EMoamEEZ73f0CkXaXp7hrann"; const AUTHORIZE_URL = "https://auth.openai.com/oauth/authorize"; @@ -84,7 +85,7 @@ export async function refreshCodex(refreshToken: string): Promise 0 ? resetIn : null); - if (error.timedOut || (error.status !== null && (TRANSIENT_STATUSES.has(error.status) || (error.status >= 500 && error.status < 600)))) return { transient: true, retryAfterMs }; - if (error.status !== null) return { transient: isTransientProviderFailure(error.message), retryAfterMs }; + const transientStatus = error.status !== null && (TRANSIENT_STATUSES.has(error.status) || (error.status >= 500 && error.status < 600)); + return { transient: error.timedOut || transientStatus || TRANSIENT_PROVIDER_CODES.has(error.code ?? ""), retryAfterMs }; } - if (TRANSIENT_CODES.has(errorCode(error) ?? "")) return { transient: true, retryAfterMs: null }; - return { transient: isTransientProviderFailure(error instanceof Error ? error.message : String(error)), retryAfterMs: null }; + return { transient: TRANSIENT_CODES.has(errorCode(error) ?? ""), retryAfterMs: null }; } function errorCode(error: unknown): string | null { diff --git a/src/main/services/provider-http.ts b/src/main/services/provider-http.ts index 338e07ea..fbcfad21 100644 --- a/src/main/services/provider-http.ts +++ b/src/main/services/provider-http.ts @@ -40,6 +40,8 @@ export class ProviderHttpError extends Error { /** HTTP status of the failed response; null when no response arrived. */ readonly status: number | null = null, readonly timedOut = false, + /** The provider's machine-readable error code (OpenAI code/type, Anthropic type, Google status, AWS error type). */ + readonly code: string | null = null, ) { super(message); this.name = "ProviderHttpError"; @@ -92,7 +94,7 @@ export async function postEventStream( const raw = await withDeadline(controller, timeouts.bodyTimeoutMs ?? BODY_TIMEOUT_MS, () => response.text()); const data = parseProviderJson(raw, false); const message = readProviderError(data) ?? (raw.trim() || `Provider request failed with HTTP ${response.status}.`); - throw new ProviderHttpError(message, rateLimitState, response.status); + throw new ProviderHttpError(message, rateLimitState, response.status, false, readProviderErrorCode(data, response.headers)); } if (!response.body) throw new ProviderHttpError("Provider returned an empty event stream.", rateLimitState); return { messages: readServerSentEvents(response.body, timeouts.idleTimeoutMs ?? STREAM_IDLE_TIMEOUT_MS), rateLimitState }; @@ -106,7 +108,7 @@ async function requestJson(endpoint: string, init: RequestInit, timeouts: Provid const data = parseProviderJson(raw, response.ok); if (!response.ok) { const message = readProviderError(data) ?? (raw.trim() || `Provider request failed with HTTP ${response.status}.`); - throw new ProviderHttpError(message, rateLimitState, response.status); + throw new ProviderHttpError(message, rateLimitState, response.status, false, readProviderErrorCode(data, response.headers)); } return { data, rateLimitState }; } @@ -236,6 +238,33 @@ function readProviderError(data: JsonRecord): string | null { return null; } +/** A failed HTTP exchange outside the request helpers (OAuth refresh, token minting), with its status and provider code. */ +export function httpFailure(message: string, response: { status: number; headers: Headers }, text: string): ProviderHttpError { + let data: JsonRecord = {}; + try { + const parsed = JSON.parse(text) as unknown; + if (isRecord(parsed)) data = parsed; + } catch { + data = {}; + } + return new ProviderHttpError(message, parseProviderRateLimitHeaders(response.headers), response.status, false, readProviderErrorCode(data, response.headers)); +} + +/** The provider's error code: an error object's code, type or status, OAuth's error token, or AWS's error type. */ +export function readProviderErrorCode(data: JsonRecord, headers?: Headers): string | null { + const error = data.error; + // OAuth (RFC 6749 5.2) puts the code itself in `error`; a sentence there is a message, not a code. + if (typeof error === "string" && /^[A-Za-z0-9_.-]+$/.test(error)) return error; + if (isRecord(error)) { + for (const key of ["code", "type", "status"]) { + const value = error[key]; + if (typeof value === "string" && value) return value; + } + } + const awsType = headers?.get("x-amzn-errortype") ?? (typeof data.__type === "string" ? data.__type : null); + return awsType ? awsType.split(":")[0].split("#").pop() ?? null : null; +} + export function isRecord(value: unknown): value is JsonRecord { return typeof value === "object" && value !== null && !Array.isArray(value); } diff --git a/src/main/services/provider-request-store.ts b/src/main/services/provider-request-store.ts index b68fd5da..350550e7 100644 --- a/src/main/services/provider-request-store.ts +++ b/src/main/services/provider-request-store.ts @@ -6,6 +6,7 @@ import { providerPrices, readTokenUsage, requestCostCents, type TokenUsage } fro /** Providers already logged as not reporting token usage. */ const providersWithoutUsage = new Set(); import { currentProviderUsageWindow, PROVIDER_MONTHLY_LIMIT_MESSAGE } from "../../shared/provider-runtime-domain"; +import { ProviderLimitError } from "./provider-failure"; export interface ProviderRequestOwner { id: string; @@ -69,6 +70,8 @@ export interface ProviderRequestFailureInput { orgId: string; requestId: string; errorMessage: string; + errorStatus: number | null; + errorCode: string | null; completedAt: string; rawResponseJson?: string | null; } @@ -216,9 +219,9 @@ export class ProviderRequestStore { markFailed(input: ProviderRequestFailureInput): boolean { return this.db.prepare(` update provider_requests - set state = ?, error_message = ?, raw_response_json = coalesce(?, raw_response_json), completed_at = ? + set state = ?, error_message = ?, error_status = ?, error_code = ?, raw_response_json = coalesce(?, raw_response_json), completed_at = ? where id = ? and org_id = ? and state not in ('succeeded', 'failed', 'cancelled') - `).run(PROVIDER_REQUEST_STATE.FAILED, input.errorMessage, input.rawResponseJson ?? null, input.completedAt, input.requestId, input.orgId).changes === 1; + `).run(PROVIDER_REQUEST_STATE.FAILED, input.errorMessage, input.errorStatus, input.errorCode, input.rawResponseJson ?? null, input.completedAt, input.requestId, input.orgId).changes === 1; } markCancelled(input: ProviderRequestCancellationInput): void { @@ -282,7 +285,7 @@ export class ProviderRequestStore { select count(*) as n from provider_requests where org_id = ? and provider_id = ? and state in (?, ?) and created_at >= ? `).get(orgId, provider.id, PROVIDER_REQUEST_STATE.PENDING, PROVIDER_REQUEST_STATE.STREAMING, window.start) as { n: number }; - if ((finished?.request_count ?? 0) + inFlight.n >= provider.monthly_request_limit) throw new Error(PROVIDER_MONTHLY_LIMIT_MESSAGE); + if ((finished?.request_count ?? 0) + inFlight.n >= provider.monthly_request_limit) throw new ProviderLimitError(PROVIDER_MONTHLY_LIMIT_MESSAGE); } /** Settles a successful call: reads the provider's token usage, prices it, and adds it to the request and the month's totals. */ diff --git a/src/main/services/provider-response-observer.ts b/src/main/services/provider-response-observer.ts index c677596a..dfb4df6d 100644 --- a/src/main/services/provider-response-observer.ts +++ b/src/main/services/provider-response-observer.ts @@ -6,6 +6,7 @@ import { OperationsService } from "./operations-service"; import { providerRateLimitStateFromError } from "./provider-adapters"; import { ProviderRateLimitRegistry } from "./provider-rate-limit-registry"; import { ProviderRequestStore } from "./provider-request-store"; +import { ProviderHttpError } from "./provider-http"; const HEALTH_STATE_HEALTHY = "healthy"; const HEALTH_STATE_FAILED = "failed"; @@ -73,6 +74,8 @@ export class ProviderResponseObserver { orgId: input.orgId, requestId: input.requestId, errorMessage: message, + errorStatus: input.error instanceof ProviderHttpError ? input.error.status : null, + errorCode: input.error instanceof ProviderHttpError ? input.error.code : null, completedAt: new Date().toISOString(), rawResponseJson: input.rawResponseJson, }); diff --git a/src/main/services/provider-response-runner.ts b/src/main/services/provider-response-runner.ts index a3468a57..9cca6671 100644 --- a/src/main/services/provider-response-runner.ts +++ b/src/main/services/provider-response-runner.ts @@ -20,6 +20,7 @@ import { budgetBlock, isPriced, promptChars, providerPrices } from "./spend"; import { assertModelSelectable, effectiveModel } from "./task-run-preparation"; import { modelCheckFailure, modelCheckProblem, modelCheckTurn, readModelCheck, recordModelCheck } from "./model-check"; import type { ModelCheckResult } from "../../shared/desktop-api"; +import { isRecord, ProviderHttpError, readProviderErrorCode } from "./provider-http"; import { HEALTH_STATE_HEALTHY, WORKSPACE_SCOPES, @@ -248,7 +249,10 @@ export class ProviderResponseRunner { }); recordObservedRateLimit(this.deps.rateLimitRegistry, input.orgId, execution.provider, execution.provider.model, result.rateLimitState); for await (const event of result.events) { - if (event.type === "error") throw new Error(event.textDelta ?? "Provider stream failed."); + if (event.type === "error") { + const failure = streamErrorFields(event.rawEventJson); + throw new ProviderHttpError(event.textDelta ?? "Provider stream failed.", null, failure.status, false, failure.code); + } rawEvents.push(event.rawEventJson); if (event.type === "usage") continue; if (event.type === "message_stop") finished = true; @@ -441,3 +445,12 @@ function requireCredentialId(provider: ProviderRow): string { if (!provider.credential_id) throw new Error("Provider credential is required."); return provider.credential_id; } + +/** A stream error event's status (a numeric error code, as OpenRouter and Google send) and provider code. */ +function streamErrorFields(rawEventJson: string): { status: number | null; code: string | null } { + const data = JSON.parse(rawEventJson) as unknown; + if (!isRecord(data)) return { status: null, code: null }; + const error = data.error ?? (isRecord(data.response) ? data.response.error : undefined); + const status = isRecord(error) && typeof error.code === "number" ? error.code : null; + return { status, code: readProviderErrorCode({ error }) ?? (typeof data.code === "string" ? data.code : null) }; +} diff --git a/src/main/services/provider-streaming.ts b/src/main/services/provider-streaming.ts index 6c295914..c35a629e 100644 --- a/src/main/services/provider-streaming.ts +++ b/src/main/services/provider-streaming.ts @@ -65,6 +65,11 @@ export async function* mapOpenAiResponseStream(messages: AsyncIterable): AsyncIterable { let started = false; for await (const message of messages) { + // A mid-stream failure (OpenRouter) arrives as an error object; it ends the stream with the provider's own words. + if (message.data.error !== undefined) { + yield streamEvent("error", readErrorText(message.data) ?? "Provider stream failed.", message); + return; + } const delta = readChatDeltaContent(message.data); if (delta) { if (!started) { diff --git a/src/main/services/provider-tool-turns/chat-format.ts b/src/main/services/provider-tool-turns/chat-format.ts index 2b1c98c4..7ad285e0 100644 --- a/src/main/services/provider-tool-turns/chat-format.ts +++ b/src/main/services/provider-tool-turns/chat-format.ts @@ -1,5 +1,5 @@ /** OpenAI Chat Completions encoding/decoding (xAI, Groq, Mistral, Cerebras, OpenRouter, Kimi family). */ -import { isRecord, ProviderHttpError, type JsonRecord, type ProviderEventStreamMessage } from "../provider-http"; +import { isRecord, ProviderHttpError, readProviderErrorCode, type JsonRecord, type ProviderEventStreamMessage } from "../provider-http"; import { FINISHED_STOP_REASONS, replayIn, requireFinished, type AgentTurnMessage, type AgentTurnResult, type AssistantReplay, toolCallFromJson, type ToolCall, type ToolDefinition } from "../provider-tool-turn"; /** How each Chat Completions provider names the output cap: OpenAI's current field, or the older one the others document. */ @@ -94,7 +94,7 @@ export async function accumulateChatStream(messages: AsyncIterable 0 && isRecord(choices[0]) ? choices[0] : null; if (first && typeof first.finish_reason === "string") finishReason = first.finish_reason; diff --git a/src/main/services/provider-tool-turns/gcp-auth.ts b/src/main/services/provider-tool-turns/gcp-auth.ts index 5c999371..a4f4cc79 100644 --- a/src/main/services/provider-tool-turns/gcp-auth.ts +++ b/src/main/services/provider-tool-turns/gcp-auth.ts @@ -4,6 +4,7 @@ * per service account until shortly before expiry. */ import { createSign } from "node:crypto"; +import { httpFailure } from "../provider-http"; interface ServiceAccount { client_email: string; @@ -48,7 +49,7 @@ export async function getGcpAccessToken(serviceAccountJson: string): Promise = { server_error: 500, rate_limit_exceeded: 429 }; - /** The error of an `error` event, or of a `response.failed` event's response. */ function streamError(data: JsonRecord): unknown { return data.error ?? (isRecord(data.response) ? data.response.error : undefined); @@ -100,10 +97,10 @@ function errorText(data: JsonRecord): string { return "Provider stream failed."; } -function errorStatus(data: JsonRecord): number | null { +/** The error's code: the error object's code or type, or the event's own code. */ +function errorCode(data: JsonRecord): string | null { const error = streamError(data); - const code = isRecord(error) ? error.code : data.code; - return typeof code === "string" ? RESPONSES_ERROR_STATUS[code] ?? null : null; + return isRecord(error) ? readProviderErrorCode({ error }) : typeof data.code === "string" ? data.code : null; } /** @@ -123,7 +120,7 @@ export async function accumulateResponsesStream(messages: AsyncIterable signature.toLowerCase()); - -export function isTransientProviderFailure(message: string | null | undefined): boolean { - if (!message) return false; - const normalized = message.toLowerCase(); - return TRANSIENT_PROVIDER_FAILURE_SIGNATURES.some((signature) => normalized.includes(signature)); -} - export function currentProviderUsageWindow(now = new Date()): { start: string; end: string } { const start = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1)); const end = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + 1, 1)); diff --git a/tests/integration/interactive-lane.test.ts b/tests/integration/interactive-lane.test.ts index 1f9a3cfd..f332da2e 100644 --- a/tests/integration/interactive-lane.test.ts +++ b/tests/integration/interactive-lane.test.ts @@ -13,6 +13,7 @@ import { dispatchAgentTool } from "../../src/main/services/agent-tools"; import { claimNextQueuedRun, startTaskQueueWorker } from "../../src/main/app/task-queue-worker"; import type { AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn"; import { answeringModelCheck } from "../helpers/model-check"; +import { ProviderHttpError } from "../../src/main/services/provider-http"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -154,7 +155,7 @@ describe("interactive execution lane", () => { const script = async (input: ProviderToolTurnInput): Promise => { const prompt = input.messages.find((message) => message.role === "user")?.text ?? ""; if (prompt.includes("Grind through the migration backlog")) { - throw new Error("The engine is currently overloaded, please try again later"); + throw new ProviderHttpError("The engine is currently overloaded, please try again later", null, 529, false, "overloaded_error"); } const threadId = /thread_id="([^"]+)"/.exec(prompt)?.[1]; if (prompt.includes("Are we blocked") && threadId && !ceoReplied) { @@ -226,7 +227,7 @@ describe("interactive execution lane", () => { }); const chat = new ChatService(connection.db); const alwaysOverloaded = async (): Promise => { - throw new Error("The engine is currently overloaded, please try again later"); + throw new ProviderHttpError("The engine is currently overloaded, please try again later", null, 529, false, "overloaded_error"); }; const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(alwaysOverloaded)); providers.saveCredential({ orgId, providerId, secret: "sk-test-key", model: "gpt-5.4" }); diff --git a/tests/integration/run-memory.test.ts b/tests/integration/run-memory.test.ts index b8240a4d..421a7091 100644 --- a/tests/integration/run-memory.test.ts +++ b/tests/integration/run-memory.test.ts @@ -12,6 +12,7 @@ import { ensureCeoDirectSpace } from "../../src/main/services/ceo-direct-space"; import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker"; import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn"; import { answeringModelCheck } from "../helpers/model-check"; +import { ProviderHttpError } from "../../src/main/services/provider-http"; const testVault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), @@ -64,16 +65,16 @@ describe("run memory", () => { }); test("a real failure is remembered; a transient provider failure is not", async () => { - let failure = "Workspace build failed: missing package.json"; - const h = await company(() => { throw new Error(failure); }); + let failure: Error = new Error("Workspace build failed: missing package.json"); + const h = await company(() => { throw failure; }); const first = claimNextQueuedRun(h.connection)!; - await expect(h.execution.executeRun(h.orgId, first.id)).rejects.toThrow(failure); - expect(h.memories().map((entry) => [entry.category, entry.content])).toEqual([["failure_cause", failure]]); + await expect(h.execution.executeRun(h.orgId, first.id)).rejects.toThrow(failure.message); + expect(h.memories().map((entry) => [entry.category, entry.content])).toEqual([["failure_cause", failure.message]]); - failure = "429 rate limit exceeded"; + failure = new ProviderHttpError("429 rate limit exceeded", null, 429, false, "rate_limit_exceeded"); h.connection.db.prepare("update task_runs set state = 'queued' where id = ?").run(first.id); const second = claimNextQueuedRun(h.connection)!; - await expect(h.execution.executeRun(h.orgId, second.id)).rejects.toThrow(failure); + await expect(h.execution.executeRun(h.orgId, second.id)).rejects.toThrow(failure.message); expect(h.memories()).toHaveLength(1); h.connection.close(); }); diff --git a/tests/unit/domain-services.test.ts b/tests/unit/domain-services.test.ts index 49fd08ef..69ea8937 100644 --- a/tests/unit/domain-services.test.ts +++ b/tests/unit/domain-services.test.ts @@ -24,7 +24,7 @@ describe("domain services", () => { const connection = createConnection(); const rows = connection.db.prepare("select version from schema_migrations order by version asc").all() as Array<{ version: number }>; - expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77]); + expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78]); connection.close(); }); diff --git a/tests/unit/provider-background-service.test.ts b/tests/unit/provider-background-service.test.ts index 1e473d36..40c8756d 100644 --- a/tests/unit/provider-background-service.test.ts +++ b/tests/unit/provider-background-service.test.ts @@ -8,6 +8,7 @@ import { providerAdapters, type ProviderAdapterRegistry } from "../../src/main/s import { ProviderBackgroundService } from "../../src/main/services/provider-background-service"; import { ProviderService } from "../../src/main/services/provider-service"; import { PROVIDER_DATA_POLICY, PROVIDER_EXECUTION_MODE, PROVIDER_REQUEST_STATE } from "../../src/shared/domain"; +import { ProviderHttpError } from "../../src/main/services/provider-http"; function createConnection(): DatabaseConnection { return new DatabaseConnection(join(tmpdir(), `agentcompany-new-provider-background-${randomUUID()}.db`)); @@ -166,13 +167,14 @@ describe("provider background service", () => { await background.poll({ orgId, requestId }); const request = connection.db.prepare(` - select state, error_message, raw_response_json + select state, error_message, error_code, raw_response_json from provider_requests where id = ? - `).get(requestId) as { state: string; error_message: string; raw_response_json: string }; + `).get(requestId) as { state: string; error_message: string; error_code: string | null; raw_response_json: string }; expect(request.state).toBe(PROVIDER_REQUEST_STATE.FAILED); expect(request.error_message).toBe("provider worker failed"); + expect(request.error_code).toBe("server_error"); expect(JSON.parse(request.raw_response_json)).toMatchObject({ id: "resp_bg_1", status: "failed" }); connection.close(); }); @@ -192,7 +194,7 @@ describe("background polling failures", () => { retrieveResponse: async (input) => { if (pollFailures > 0) { pollFailures -= 1; - throw new Error("fetch failed: ECONNRESET"); + throw new ProviderHttpError("fetch failed: ECONNRESET", null, 503, false, "UNAVAILABLE"); } const result = await base.openai.retrieveResponse(input); return { ...result, status }; @@ -224,8 +226,8 @@ describe("background polling failures", () => { pollFailures = 100; const doomed = await background.create({ orgId, providerId: openAiProviderId, prompt: "Research again", instructions: null, executionMode: PROVIDER_EXECUTION_MODE.BACKGROUND }); for (let attempt = 0; attempt < 10; attempt += 1) await background.poll({ orgId, requestId: doomed }); - expect(connection.db.prepare("select state, error_message from provider_requests where id = ?").get(doomed)) - .toEqual({ state: PROVIDER_REQUEST_STATE.FAILED, error_message: "Gave up after 10 failed status checks: fetch failed: ECONNRESET" }); + expect(connection.db.prepare("select state, error_message, error_status, error_code from provider_requests where id = ?").get(doomed)) + .toEqual({ state: PROVIDER_REQUEST_STATE.FAILED, error_message: "Gave up after 10 failed status checks: fetch failed: ECONNRESET", error_status: 503, error_code: "UNAVAILABLE" }); // An answer that is not a valid result (unknown status) also counts, so it cannot stay pending forever. pollFailures = 0; @@ -256,7 +258,7 @@ function backgroundAdapters(status: string, text: string | null): ProviderAdapte rawResponseJson: JSON.stringify({ id: "resp_bg_1", status, - error: status === "failed" ? { message: "provider worker failed" } : null, + error: status === "failed" ? { message: "provider worker failed", code: "server_error" } : null, output_text: text, }), rateLimitState: null, diff --git a/tests/unit/provider-error-codes.test.ts b/tests/unit/provider-error-codes.test.ts new file mode 100644 index 00000000..52895cf5 --- /dev/null +++ b/tests/unit/provider-error-codes.test.ts @@ -0,0 +1,132 @@ +import { randomUUID } from "node:crypto"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import { classifyProviderFailure } from "../../src/main/services/provider-failure"; +import { refreshCodex } from "../../src/main/services/oauth/openai-codex-flow"; +import { refreshKimiCode } from "../../src/main/services/oauth/kimi-code-flow"; +import { postJson, ProviderHttpError } from "../../src/main/services/provider-http"; +import { accumulateResponsesStream } from "../../src/main/services/provider-tool-turns/responses-format"; +import { accumulateGeminiStream } from "../../src/main/services/provider-tool-turns/gemini-format"; +import { DatabaseConnection } from "../../src/main/database/connection"; +import { OrganizationService } from "../../src/main/services/organization-service"; +import { ProviderService } from "../../src/main/services/provider-service"; +import { providerAdapters } from "../../src/main/services/provider-adapters"; +import { markModelChecked } from "../helpers/model-check"; +import { PROVIDER_DATA_POLICY, PROVIDER_EXECUTION_MODE } from "../../src/shared/domain"; +import { mapAnthropicMessageStream, mapOpenAiChatCompletionStream, mapOpenAiResponseStream } from "../../src/main/services/provider-streaming"; +import { PROVIDER_MONTHLY_LIMIT_MESSAGE } from "../../src/shared/provider-runtime-domain"; + +const NOW = Date.parse("2026-10-08T12:00:00.000Z"); + +async function httpFailure(status: number, body: unknown, headers: Record = {}): Promise { + vi.stubGlobal("fetch", async () => new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json", ...headers } })); + const error = await postJson("https://provider.test/v1", {}, {}).catch((caught: unknown) => caught); + if (!(error instanceof ProviderHttpError)) throw new Error("expected a provider HTTP error"); + return error; +} + +afterEach(() => vi.unstubAllGlobals()); + +describe("provider errors keep their status and machine code", () => { + test("each provider's error code is read from where it puts it", async () => { + expect(await httpFailure(400, { error: { message: "bad", type: "invalid_request_error", code: "context_length_exceeded" } })).toMatchObject({ status: 400, code: "context_length_exceeded" }); + expect(await httpFailure(529, { type: "error", error: { type: "overloaded_error", message: "Overloaded" } })).toMatchObject({ status: 529, code: "overloaded_error" }); + expect(await httpFailure(429, { error: { code: 429, message: "Quota exceeded", status: "RESOURCE_EXHAUSTED" } })).toMatchObject({ status: 429, code: "RESOURCE_EXHAUSTED" }); + expect(await httpFailure(400, { message: "Rate exceeded" }, { "x-amzn-ErrorType": "ThrottlingException:http://internal.amazon.com/coral/com.amazon.bedrock/" })).toMatchObject({ status: 400, code: "ThrottlingException" }); + expect(await httpFailure(500, { oops: true })).toMatchObject({ status: 500, code: null }); + }); + + test("a token refresh the provider rate-limits is a structured, transient failure", async () => { + vi.stubGlobal("fetch", async () => new Response(JSON.stringify({ error: { code: "rate_limit_exceeded", message: "slow down" } }), { status: 429, headers: { "content-type": "application/json", "retry-after": "30" } })); + const failure = await refreshCodex("refresh-token").catch((caught: unknown) => caught); + expect(failure).toMatchObject({ status: 429, code: "rate_limit_exceeded" }); + expect(classifyProviderFailure(failure, NOW)).toEqual({ transient: true, retryAfterMs: 30_000 }); + }); + + test("OAuth error tokens are codes, sentences are not; Kimi's refresh keeps status, wait and code even without JSON", async () => { + expect(await httpFailure(400, { error: "invalid_grant", error_description: "expired" })).toMatchObject({ status: 400, code: "invalid_grant" }); + expect(await httpFailure(400, { error: "Rate limit exceeded, slow down" })).toMatchObject({ code: null }); + vi.stubGlobal("fetch", async () => new Response("busy", { status: 503, headers: { "retry-after": "300" } })); + const busy = await refreshKimiCode("refresh-token").catch((caught: unknown) => caught); + expect(busy).toMatchObject({ status: 503 }); + expect(classifyProviderFailure(busy, NOW)).toEqual({ transient: true, retryAfterMs: 300_000 }); + vi.stubGlobal("fetch", async () => new Response(JSON.stringify({ error: "rate_limited" }), { status: 429, headers: { "content-type": "application/json", "retry-after": "30" } })); + const limited = await refreshKimiCode("refresh-token").catch((caught: unknown) => caught); + expect(limited).toMatchObject({ status: 429, code: "rate_limited" }); + expect(classifyProviderFailure(limited, NOW).retryAfterMs).toBe(30_000); + }); + + test("a provider's transient code is transient even under a status that is not", async () => { + expect(classifyProviderFailure(await httpFailure(400, { message: "Rate exceeded" }, { "x-amzn-ErrorType": "ThrottlingException:x" }), NOW).transient).toBe(true); + expect(classifyProviderFailure(await httpFailure(400, { error: { type: "invalid_request_error", message: "rate limit in prompt" } }), NOW).transient).toBe(false); + }); + + test("stream errors carry the provider's code, not a status made up for it", async () => { + async function* events(data: Record) { yield { eventName: null, data, rawData: JSON.stringify(data) }; } + const codex = await accumulateResponsesStream(events({ type: "error", error: { type: "usage_limit_reached", message: "The usage limit has been reached" } })).catch((caught: unknown) => caught); + expect(codex).toMatchObject({ status: null, code: "usage_limit_reached" }); + expect(classifyProviderFailure(codex, NOW).transient).toBe(true); + const gemini = await accumulateGeminiStream(events({ error: { code: 503, message: "busy", status: "UNAVAILABLE" } })).catch((caught: unknown) => caught); + expect(gemini).toMatchObject({ status: 503, code: "UNAVAILABLE" }); + }); + + test("a streamed reply's error event keeps the provider's code on the error and the request", async () => { + const connection = new DatabaseConnection(join(tmpdir(), `ac-codes-stream-${randomUUID()}.db`)); + const vault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), decryptSecret: (ciphertext: Buffer) => ciphertext.toString("utf8") }; + async function* overloaded() { const data = { type: "error", error: { type: "overloaded_error", message: "Overloaded" } }; yield { eventName: null, data, rawData: JSON.stringify(data) }; } + const adapters = { ...providerAdapters, anthropic: { ...providerAdapters.anthropic, streamResponse: async () => ({ events: mapAnthropicMessageStream(overloaded()), rateLimitState: null }) } }; + const providers = new ProviderService(connection.db, adapters, vault); + let orgId = ""; + let providerId = ""; + connection.transaction(() => { + orgId = new OrganizationService(connection.db).createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" }); + providerId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'anthropic'").get(orgId) as { id: string }).id; + providers.saveCredential({ orgId, providerId, secret: "sk-ant-test", model: "claude-sonnet-4-5" }); + }); + const failure = await providers.streamResponse({ orgId, providerId, prompt: "p", instructions: null, executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND }, () => {}).catch((caught: unknown) => caught); + expect(failure).toMatchObject({ message: "Overloaded", code: "overloaded_error" }); + async function* failedResponse() { const data = { type: "response.failed", response: { error: { code: "server_error", message: "The server had an error." } } }; yield { eventName: null, data, rawData: JSON.stringify(data) }; } + const openAi = new ProviderService(connection.db, { ...adapters, openai: { ...providerAdapters.openai, streamResponse: async () => ({ events: mapOpenAiResponseStream(failedResponse()), rateLimitState: null }) } }, vault); + const openAiId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'openai'").get(orgId) as { id: string }).id; + connection.transaction(() => openAi.saveCredential({ orgId, providerId: openAiId, secret: "sk-test", model: "gpt-5.4" })); + const serverError = await openAi.streamResponse({ orgId, providerId: openAiId, prompt: "p", instructions: null, executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND }, () => {}).catch((caught: unknown) => caught); + expect(serverError).toMatchObject({ code: "server_error" }); + expect(classifyProviderFailure(serverError, NOW).transient).toBe(true); + + async function* midStream() { const data = { id: "gen-1", error: { code: 503, message: "Provider unavailable" }, choices: [{ delta: { content: "" }, finish_reason: "error" }] }; yield { eventName: null, data, rawData: JSON.stringify(data) }; } + const router = new ProviderService(connection.db, { ...adapters, openrouter: { ...providerAdapters.openrouter, streamResponse: async () => ({ events: mapOpenAiChatCompletionStream(midStream()), rateLimitState: null }) } }, vault); + const routerId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'openrouter'").get(orgId) as { id: string }).id; + connection.transaction(() => router.saveCredential({ orgId, providerId: routerId, secret: "sk-or-test", model: "openai/gpt-5.4" })); + const unavailable = await router.streamResponse({ orgId, providerId: routerId, prompt: "p", instructions: null, executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND }, () => {}).catch((caught: unknown) => caught); + expect(unavailable).toMatchObject({ message: "Provider unavailable", status: 503 }); + expect(classifyProviderFailure(unavailable, NOW).transient).toBe(true); + expect(classifyProviderFailure(failure, NOW).transient).toBe(true); + expect(connection.db.prepare("select error_code from provider_requests order by rowid").all()).toEqual([{ error_code: "overloaded_error" }, { error_code: "server_error" }, { error_code: null }]); + connection.close(); + }); + + test("our own monthly limit is a typed error, and a failed request keeps its status and code", async () => { + const connection = new DatabaseConnection(join(tmpdir(), `ac-codes-${randomUUID()}.db`)); + const vault = { encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), decryptSecret: (ciphertext: Buffer) => ciphertext.toString("utf8") }; + const providers = new ProviderService(connection.db, providerAdapters, vault, undefined, async () => { throw new ProviderHttpError("Overloaded", null, 529, false, "overloaded_error"); }); + let orgId = ""; + let providerId = ""; + connection.transaction(() => { + orgId = new OrganizationService(connection.db).createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" }); + providerId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'openai'").get(orgId) as { id: string }).id; + providers.saveCredential({ orgId, providerId, secret: "sk-test", model: "gpt-5.4" }); + markModelChecked(connection.db, providerId, "gpt-5.4"); + }); + const call = () => providers.runToolTurn({ orgId, providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] }); + await expect(call()).rejects.toThrow("Overloaded"); + expect(connection.db.prepare("select error_status, error_code from provider_requests").get()).toEqual({ error_status: 529, error_code: "overloaded_error" }); + + providers.updatePolicy({ orgId, providerId, dataPolicy: PROVIDER_DATA_POLICY.STANDARD, backgroundEnabled: false, monthlyRequestLimit: 1, workspaceScope: "organization", contextWindowTokens: 128_000 }); + const limited = await call().catch((caught: unknown) => caught); + expect((limited as Error).message).toBe(PROVIDER_MONTHLY_LIMIT_MESSAGE); + expect(classifyProviderFailure(limited, NOW).transient).toBe(true); + expect(classifyProviderFailure(new Error((limited as Error).message), NOW).transient).toBe(false); + connection.close(); + }); +}); diff --git a/tests/unit/provider-failure.test.ts b/tests/unit/provider-failure.test.ts index 5aa5f179..3e3b98ac 100644 --- a/tests/unit/provider-failure.test.ts +++ b/tests/unit/provider-failure.test.ts @@ -14,7 +14,9 @@ describe("provider failure classification", () => { expect(classifyProviderFailure(new ProviderHttpError("{\"type\":\"overloaded_error\"}", null, 529), NOW)).toEqual({ transient: true, retryAfterMs: null }); expect(classifyProviderFailure(new ProviderHttpError("Provider request timed out.", null, null, true), NOW)).toEqual({ transient: true, retryAfterMs: null }); expect(classifyProviderFailure(new ProviderHttpError("invalid api key", null, 401), NOW)).toEqual({ transient: false, retryAfterMs: null }); - expect(classifyProviderFailure(new ProviderHttpError("The usage limit has been reached", null, 400), NOW)).toEqual({ transient: true, retryAfterMs: null }); + expect(classifyProviderFailure(new ProviderHttpError("The usage limit has been reached", null, 400, false, "usage_limit_reached"), NOW)).toEqual({ transient: true, retryAfterMs: null }); + expect(classifyProviderFailure(new ProviderHttpError("Please do not rate limit me", null, 400, false, "invalid_request_error"), NOW)).toEqual({ transient: false, retryAfterMs: null }); + expect(classifyProviderFailure(new Error("rate limit reached, too many requests"), NOW)).toEqual({ transient: false, retryAfterMs: null }); const reset = Object.assign(new Error("read ECONNRESET"), { code: "ECONNRESET" }); expect(classifyProviderFailure(new TypeError("fetch failed", { cause: reset }), NOW)).toEqual({ transient: true, retryAfterMs: null }); expect(classifyProviderFailure(new ProviderHttpError("", null, 522), NOW)).toEqual({ transient: true, retryAfterMs: null }); diff --git a/tests/unit/provider-runtime-domain.test.ts b/tests/unit/provider-runtime-domain.test.ts deleted file mode 100644 index fbc7ec31..00000000 --- a/tests/unit/provider-runtime-domain.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { isTransientProviderFailure, PROVIDER_MONTHLY_LIMIT_MESSAGE } from "../../src/shared/provider-runtime-domain"; - -describe("isTransientProviderFailure", () => { - test("classifies provider-side limit and overload verdicts as transient", () => { - expect(isTransientProviderFailure(PROVIDER_MONTHLY_LIMIT_MESSAGE)).toBe(true); - expect(isTransientProviderFailure("The usage limit has been reached")).toBe(true); - expect(isTransientProviderFailure("Our servers are currently overloaded. Please try again later.")).toBe(true); - expect(isTransientProviderFailure("Rate limit exceeded, retry after 20s")).toBe(true); - expect(isTransientProviderFailure("429 Too Many Requests")).toBe(true); - }); - - test("keeps genuine task failures counting toward the retry budget", () => { - expect(isTransientProviderFailure(null)).toBe(false); - expect(isTransientProviderFailure(undefined)).toBe(false); - expect(isTransientProviderFailure("")).toBe(false); - expect(isTransientProviderFailure("provider rejected task")).toBe(false); - expect(isTransientProviderFailure("Agent run did not converge within 12 iterations.")).toBe(false); - expect(isTransientProviderFailure("Provider returned an empty turn: no text and no tool calls.")).toBe(false); - }); -}); diff --git a/tests/unit/task-execution-service.test.ts b/tests/unit/task-execution-service.test.ts index 7c69157e..cd2dd9b5 100644 --- a/tests/unit/task-execution-service.test.ts +++ b/tests/unit/task-execution-service.test.ts @@ -15,6 +15,7 @@ import { TaskExecutionService } from "../../src/main/services/task-execution-ser import { APPROVAL_STATE, PROVIDER_DATA_POLICY, RISK_LEVEL, TASK_PRIORITY, TASK_STATUS } from "../../src/shared/domain"; import { currentProviderUsageWindow, PROVIDER_MONTHLY_LIMIT_MESSAGE } from "../../src/shared/provider-runtime-domain"; import { answeringModelCheck } from "../helpers/model-check"; +import { ProviderHttpError } from "../../src/main/services/provider-http"; function createConnection(): DatabaseConnection { return new DatabaseConnection(join(tmpdir(), `agentcompany-new-task-execution-${randomUUID()}.db`)); @@ -847,7 +848,7 @@ describe("task execution service", () => { const connection = createConnection(); const organization = new OrganizationService(connection.db); const fakeToolTurn = async (): Promise => { - throw new Error("The usage limit has been reached"); + throw new ProviderHttpError("The usage limit has been reached", null, 429, false, "usage_limit_reached"); }; const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, answeringModelCheck(fakeToolTurn)); let nowMs = Date.now();