Q61: a failed status check no longer ends recovery of a background response

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/main/database/migrations/071-background-poll-failures.ts

@@ -0,0 +1,7 @@
+export const backgroundPollFailuresSchema = `
+  alter table provider_requests add column poll_failures integer not null default 0;
+  alter table provider_requests add column last_poll_error text;
+
+  insert into schema_migrations(version, name, applied_at)
+  values (71, 'background_poll_failures', datetime('now'));
+`;

src/main/database/schema.ts

@@ -69,6 +69,7 @@ import { taskRunBudgetSchema } from "./migrations/067-task-run-budget";
 import { taskReviewLinkSchema } from "./migrations/068-task-review-link";
 import { goalStallSchema } from "./migrations/069-goal-stall";
 import { runActionsSchema } from "./migrations/070-run-actions";
+import { backgroundPollFailuresSchema } from "./migrations/071-background-poll-failures";
 
 interface Migration {
   version: number;
@@ -147,6 +148,7 @@ const MIGRATIONS: Migration[] = [
   { version: 68, name: "task_review_link", sql: taskReviewLinkSchema },
   { version: 69, name: "goal_stall", sql: goalStallSchema },
   { version: 70, name: "run_actions", sql: runActionsSchema },
+  { version: 71, name: "background_poll_failures", sql: backgroundPollFailuresSchema },
 ];
 
 const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version;

src/main/services/provider-background-service.ts

@@ -24,6 +24,7 @@ const AUTH_STATE_CONFIGURED = "configured";
 const HEALTH_STATE_HEALTHY = "healthy";
 const ACTIVE_BACKGROUND_STATUSES = new Set(["queued", "in_progress"]);
 const FAILED_BACKGROUND_STATUSES = new Set(["failed", "incomplete"]);
+const MAX_POLL_FAILURES = 10;
 const CANCELLED_BACKGROUND_STATUSES = new Set(["cancelled"]);
 
 type BackgroundCreateInput = CreateProviderResponseInput & {
@@ -137,6 +138,7 @@ export class ProviderBackgroundService {
       });
       this.recordObservedRateLimit(request.org_id, providerRow, providerRow.model, result.rateLimitState);
       if (ACTIVE_BACKGROUND_STATUSES.has(result.status)) {
+        this.requestStore.clearPollFailures(request.id);
         this.requestStore.markBackgroundProgress({
           orgId: request.org_id,
           requestId: request.id,
@@ -149,6 +151,7 @@ export class ProviderBackgroundService {
       }
       if (result.status === "completed") {
         const responseText = requireBackgroundText(result.text, provider.provider_key);
+        this.requestStore.clearPollFailures(request.id);
         this.requestStore.markSucceeded({
           orgId: request.org_id,
           requestId: request.id,
@@ -170,6 +173,7 @@ export class ProviderBackgroundService {
         return;
       }
       if (CANCELLED_BACKGROUND_STATUSES.has(result.status)) {
+        this.requestStore.clearPollFailures(request.id);
         this.requestStore.markCancelled({
           orgId: request.org_id,
           requestId: request.id,
@@ -185,6 +189,15 @@ export class ProviderBackgroundService {
       }
       throw new Error(`Provider background response returned unsupported status ${result.status}.`);
     } catch (error) {
+      // The provider reporting failure is final; a failed status check is not, since the remote job may still finish.
+      let finalError = error;
+      if (!(error instanceof ProviderBackgroundStatusError)) {
+        const message = error instanceof Error ? error.message : String(error);
+        const failures = this.requestStore.recordPollFailure(request.id, message);
+        console.warn(`[provider-background] status check ${failures}/${MAX_POLL_FAILURES} for request ${request.id} failed: ${message}`);
+        if (failures < MAX_POLL_FAILURES) return;
+        finalError = new Error(`Gave up after ${MAX_POLL_FAILURES} failed status checks: ${message}`);
+      }
       this.responseObserver.recordFailure({
         orgId: request.org_id,
         provider,
@@ -192,7 +205,7 @@ export class ProviderBackgroundService {
         executionMode: PROVIDER_EXECUTION_MODE.BACKGROUND,
         startedAt,
         inputCharCount: request.input_char_count,
-        error,
+        error: finalError,
         rawResponseJson: error instanceof ProviderBackgroundStatusError ? error.rawResponseJson : null,
       });
     }

src/main/services/provider-request-store.ts

@@ -156,6 +156,16 @@ export class ProviderRequestStore {
     );
   }
 
+  /** Counts a failed status check on a background request and returns how many happened in a row. */
+  recordPollFailure(requestId: string, message: string): number {
+    this.db.prepare("update provider_requests set poll_failures = poll_failures + 1, last_poll_error = ? where id = ?").run(message, requestId);
+    return (this.db.prepare("select poll_failures from provider_requests where id = ?").get(requestId) as { poll_failures: number }).poll_failures;
+  }
+
+  clearPollFailures(requestId: string): void {
+    this.db.prepare("update provider_requests set poll_failures = 0, last_poll_error = null where id = ?").run(requestId);
+  }
+
   markBackgroundProgress(input: ProviderBackgroundProgressInput): void {
     this.db.prepare(`
       update provider_requests

tests/unit/domain-services.test.ts

@@ -24,7 +24,7 @@ describe("domain services", () => {
     const connection = createConnection();
 
     const rows = connection.db.prepare("select version from schema_migrations order by version asc").all() as Array<{ version: number }>;
-    expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70]);
+    expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71]);
     connection.close();
   });
 

tests/unit/provider-background-service.test.ts

@@ -178,6 +178,65 @@ describe("provider background service", () => {
   });
 });
 
+describe("background polling failures", () => {
+  test("a failed status check keeps the request recoverable; only repeated failures give up", async () => {
+    const connection = createConnection();
+    const organization = new OrganizationService(connection.db);
+    let pollFailures = 2;
+    let status = "completed";
+    const base = backgroundAdapters("completed", "Late result.");
+    const adapters: ProviderAdapterRegistry = {
+      ...base,
+      openai: {
+        ...base.openai,
+        retrieveResponse: async (input) => {
+          if (pollFailures > 0) {
+            pollFailures -= 1;
+            throw new Error("fetch failed: ECONNRESET");
+          }
+          const result = await base.openai.retrieveResponse(input);
+          return { ...result, status };
+        },
+      },
+    };
+    const providers = new ProviderService(connection.db, adapters, testVault);
+    const background = new ProviderBackgroundService(connection.db, adapters, testVault);
+    let orgId = "";
+    let openAiProviderId = "";
+    connection.transaction(() => {
+      orgId = createOrganization(organization);
+      openAiProviderId = providerId(connection, orgId, "openai");
+      providers.saveCredential({ orgId, providerId: openAiProviderId, secret: "sk-live-provider", model: "gpt-5.4" });
+      providers.updatePolicy({
+        orgId, providerId: openAiProviderId, dataPolicy: PROVIDER_DATA_POLICY.STANDARD, backgroundEnabled: true,
+        monthlyRequestLimit: 10, workspaceScope: "organization", contextWindowTokens: 128_000,
+      });
+    });
+    const requestId = await background.create({ orgId, providerId: openAiProviderId, prompt: "Research", instructions: null, executionMode: PROVIDER_EXECUTION_MODE.BACKGROUND });
+    const row = () => connection.db.prepare("select state, poll_failures, last_poll_error, response_text from provider_requests where id = ?").get(requestId);
+
+    expect(await background.recoverBackgroundResponses()).toBe(1);
+    expect(row()).toMatchObject({ state: PROVIDER_REQUEST_STATE.PENDING, poll_failures: 1, last_poll_error: "fetch failed: ECONNRESET" });
+    await background.recoverBackgroundResponses();
+    await background.recoverBackgroundResponses();
+    expect(row()).toMatchObject({ state: PROVIDER_REQUEST_STATE.SUCCEEDED, poll_failures: 0, response_text: "Late result." });
+
+    pollFailures = 100;
+    const doomed = await background.create({ orgId, providerId: openAiProviderId, prompt: "Research again", instructions: null, executionMode: PROVIDER_EXECUTION_MODE.BACKGROUND });
+    for (let attempt = 0; attempt < 10; attempt += 1) await background.poll({ orgId, requestId: doomed });
+    expect(connection.db.prepare("select state, error_message from provider_requests where id = ?").get(doomed))
+      .toEqual({ state: PROVIDER_REQUEST_STATE.FAILED, error_message: "Gave up after 10 failed status checks: fetch failed: ECONNRESET" });
+
+    // An answer that is not a valid result (unknown status) also counts, so it cannot stay pending forever.
+    pollFailures = 0;
+    status = "mystery";
+    const odd = await background.create({ orgId, providerId: openAiProviderId, prompt: "Odd", instructions: null, executionMode: PROVIDER_EXECUTION_MODE.BACKGROUND });
+    for (let attempt = 0; attempt < 10; attempt += 1) await background.poll({ orgId, requestId: odd });
+    expect(connection.db.prepare("select state from provider_requests where id = ?").get(odd)).toEqual({ state: PROVIDER_REQUEST_STATE.FAILED });
+    connection.close();
+  });
+});
+
 function backgroundAdapters(status: string, text: string | null): ProviderAdapterRegistry {
   return {
     ...providerAdapters,