Q69: provider failures classify by status and code; cooldowns follow Retry-After or back off

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/unit/domain-services.test.ts

@@ -24,7 +24,7 @@ describe("domain services", () => {
     const connection = createConnection();
 
     const rows = connection.db.prepare("select version from schema_migrations order by version asc").all() as Array<{ version: number }>;
-    expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73]);
+    expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74]);
     connection.close();
   });
 

tests/unit/provider-cooldown-backoff.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, test, vi } from "vitest";
+import { ProviderCooldowns } from "../../src/main/services/provider-cooldowns";
+
+describe("provider cooldown sizing", () => {
+  test("a provider-given wait is honored; otherwise parks back off and reset once a request is served", () => {
+    vi.spyOn(console, "warn").mockImplementation(() => {});
+    vi.spyOn(console, "info").mockImplementation(() => {});
+    let now = 0;
+    const cooldowns = new ProviderCooldowns(() => now);
+    const provider = { id: "prv_1", label: "OpenAI" };
+    const parkedFor = (): number => {
+      let elapsed = 0;
+      while (cooldowns.cooledProviderIds().includes(provider.id)) {
+        now += 1_000;
+        elapsed += 1_000;
+      }
+      return elapsed;
+    };
+    cooldowns.park(provider, "429", 45_000);
+    expect(parkedFor()).toBe(45_000);
+    cooldowns.park(provider, "503", null);
+    expect(parkedFor()).toBe(4 * 60_000);
+    cooldowns.park(provider, "503", null);
+    expect(parkedFor()).toBe(8 * 60_000);
+    cooldowns.clear(provider);
+    cooldowns.park(provider, "503", null);
+    expect(parkedFor()).toBe(2 * 60_000);
+    cooldowns.park(provider, "429", 5 * 3_600_000);
+    expect(parkedFor()).toBe(60 * 60_000);
+    vi.restoreAllMocks();
+  });
+});

tests/unit/provider-failure.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, test } from "vitest";
+import { classifyProviderFailure } from "../../src/main/services/provider-failure";
+import { ProviderHttpError } from "../../src/main/services/provider-http";
+import { accumulateChatStream } from "../../src/main/services/provider-tool-turns/chat-format";
+import { accumulateResponsesStream } from "../../src/main/services/provider-tool-turns/responses-format";
+
+const NOW = Date.parse("2026-10-08T12:00:00.000Z");
+const limits = (retryAfterMs: number | null, resetAt: string | null) => ({ requestLimit: null, remainingRequests: 0, resetAt, retryAfterMs, rawStateJson: "{}" });
+
+describe("provider failure classification", () => {
+  test("status, timeouts and network codes decide, whatever the message says", () => {
+    expect(classifyProviderFailure(new ProviderHttpError("slow down", limits(45_000, null), 429), NOW)).toEqual({ transient: true, retryAfterMs: 45_000 });
+    expect(classifyProviderFailure(new ProviderHttpError("", limits(null, "2026-10-08T12:05:00.000Z"), 503), NOW)).toEqual({ transient: true, retryAfterMs: 300_000 });
+    expect(classifyProviderFailure(new ProviderHttpError("{\"type\":\"overloaded_error\"}", null, 529), NOW)).toEqual({ transient: true, retryAfterMs: null });
+    expect(classifyProviderFailure(new ProviderHttpError("Provider request timed out.", null, null, true), NOW)).toEqual({ transient: true, retryAfterMs: null });
+    expect(classifyProviderFailure(new ProviderHttpError("invalid api key", null, 401), NOW)).toEqual({ transient: false, retryAfterMs: null });
+    expect(classifyProviderFailure(new ProviderHttpError("The usage limit has been reached", null, 400), NOW)).toEqual({ transient: true, retryAfterMs: null });
+    const reset = Object.assign(new Error("read ECONNRESET"), { code: "ECONNRESET" });
+    expect(classifyProviderFailure(new TypeError("fetch failed", { cause: reset }), NOW)).toEqual({ transient: true, retryAfterMs: null });
+    expect(classifyProviderFailure(new ProviderHttpError("", null, 522), NOW)).toEqual({ transient: true, retryAfterMs: null });
+    expect(classifyProviderFailure(new Error("Agent has no enabled skills."), NOW)).toEqual({ transient: false, retryAfterMs: null });
+  });
+
+  test("an error event inside a chat stream keeps its status code", async () => {
+    async function* stream() {
+      yield { eventName: null, data: { error: { code: 500, message: "Internal server error" } }, rawData: "{}" };
+    }
+    const failure = await accumulateChatStream(stream()).catch((error: unknown) => error);
+    expect(classifyProviderFailure(failure, NOW)).toEqual({ transient: true, retryAfterMs: null });
+  });
+
+  test("a failed Responses stream keeps its server_error and rate-limit codes", async () => {
+    async function* failed(code: string) {
+      yield { eventName: null, data: { type: "response.failed", response: { error: { code, message: "The server had an error." } } }, rawData: "{}" };
+    }
+    const serverError = await accumulateResponsesStream(failed("server_error")).catch((error: unknown) => error);
+    expect((serverError as Error).message).toBe("The server had an error.");
+    expect(classifyProviderFailure(serverError, NOW)).toEqual({ transient: true, retryAfterMs: null });
+    expect(classifyProviderFailure(await accumulateResponsesStream(failed("rate_limit_exceeded")).catch((error: unknown) => error), NOW).transient).toBe(true);
+    expect(classifyProviderFailure(await accumulateResponsesStream(failed("invalid_prompt")).catch((error: unknown) => error), NOW).transient).toBe(false);
+  });
+});

tests/unit/task-execution-service.test.ts

@@ -553,8 +553,8 @@ describe("task execution service", () => {
         insert into task_runs(
           id, org_id, task_id, agent_id, provider_id, provider_request_id,
           execution_context_id, service_identity_id, workspace_mount_json, secret_scope_json,
-          trigger, state, created_at, started_at, completed_at, error_message
-        ) values (?, ?, ?, ?, ?, null, ?, ?, '[]', '{}', 'heartbeat', 'failed', ?, ?, ?, ?)
+          trigger, state, created_at, started_at, completed_at, error_message, failure_kind
+        ) values (?, ?, ?, ?, ?, null, ?, ?, '[]', '{}', 'heartbeat', 'failed', ?, ?, ?, ?, 'transient')
       `);
       const now = new Date().toISOString();
       for (const index of [1, 2, 3]) {
@@ -638,8 +638,8 @@ describe("task execution service", () => {
         insert into task_runs(
           id, org_id, task_id, agent_id, provider_id, provider_request_id,
           execution_context_id, service_identity_id, workspace_mount_json, secret_scope_json,
-          trigger, state, created_at, started_at, completed_at, error_message
-        ) values (?, ?, ?, ?, ?, null, ?, ?, '[]', '{}', 'heartbeat', 'failed', ?, ?, ?, ?)
+          trigger, state, created_at, started_at, completed_at, error_message, failure_kind
+        ) values (?, ?, ?, ?, ?, null, ?, ?, '[]', '{}', 'heartbeat', 'failed', ?, ?, ?, ?, 'transient')
       `);
       const now = new Date().toISOString();
       for (const index of [1, 2, 3]) {