From 46cd167c2c09d420dbbe858e92c5ef9bee5ff227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Thu, 8 Oct 2026 10:42:57 -0400 Subject: [PATCH] Q61: a failed status check no longer ends recovery of a background response Co-Authored-By: Claude Opus 5.5 --- .../071-background-poll-failures.ts | 7 +++ src/main/database/schema.ts | 2 + .../services/provider-background-service.ts | 15 ++++- src/main/services/provider-request-store.ts | 10 ++++ tests/unit/domain-services.test.ts | 2 +- .../unit/provider-background-service.test.ts | 59 +++++++++++++++++++ 6 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 src/main/database/migrations/071-background-poll-failures.ts diff --git a/src/main/database/migrations/071-background-poll-failures.ts b/src/main/database/migrations/071-background-poll-failures.ts new file mode 100644 index 00000000..ec61e5ef --- /dev/null +++ b/src/main/database/migrations/071-background-poll-failures.ts @@ -0,0 +1,7 @@ +export const backgroundPollFailuresSchema = ` + alter table provider_requests add column poll_failures integer not null default 0; + alter table provider_requests add column last_poll_error text; + + insert into schema_migrations(version, name, applied_at) + values (71, 'background_poll_failures', datetime('now')); +`; diff --git a/src/main/database/schema.ts b/src/main/database/schema.ts index dd9ea6f7..0b61c513 100644 --- a/src/main/database/schema.ts +++ b/src/main/database/schema.ts @@ -69,6 +69,7 @@ import { taskRunBudgetSchema } from "./migrations/067-task-run-budget"; import { taskReviewLinkSchema } from "./migrations/068-task-review-link"; import { goalStallSchema } from "./migrations/069-goal-stall"; import { runActionsSchema } from "./migrations/070-run-actions"; +import { backgroundPollFailuresSchema } from "./migrations/071-background-poll-failures"; interface Migration { version: number; @@ -147,6 +148,7 @@ const MIGRATIONS: Migration[] = [ { version: 68, name: "task_review_link", sql: taskReviewLinkSchema }, { version: 69, name: "goal_stall", sql: goalStallSchema }, { version: 70, name: "run_actions", sql: runActionsSchema }, + { version: 71, name: "background_poll_failures", sql: backgroundPollFailuresSchema }, ]; const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version; diff --git a/src/main/services/provider-background-service.ts b/src/main/services/provider-background-service.ts index 174171c4..56c5b7a3 100644 --- a/src/main/services/provider-background-service.ts +++ b/src/main/services/provider-background-service.ts @@ -24,6 +24,7 @@ const AUTH_STATE_CONFIGURED = "configured"; const HEALTH_STATE_HEALTHY = "healthy"; const ACTIVE_BACKGROUND_STATUSES = new Set(["queued", "in_progress"]); const FAILED_BACKGROUND_STATUSES = new Set(["failed", "incomplete"]); +const MAX_POLL_FAILURES = 10; const CANCELLED_BACKGROUND_STATUSES = new Set(["cancelled"]); type BackgroundCreateInput = CreateProviderResponseInput & { @@ -137,6 +138,7 @@ export class ProviderBackgroundService { }); this.recordObservedRateLimit(request.org_id, providerRow, providerRow.model, result.rateLimitState); if (ACTIVE_BACKGROUND_STATUSES.has(result.status)) { + this.requestStore.clearPollFailures(request.id); this.requestStore.markBackgroundProgress({ orgId: request.org_id, requestId: request.id, @@ -149,6 +151,7 @@ export class ProviderBackgroundService { } if (result.status === "completed") { const responseText = requireBackgroundText(result.text, provider.provider_key); + this.requestStore.clearPollFailures(request.id); this.requestStore.markSucceeded({ orgId: request.org_id, requestId: request.id, @@ -170,6 +173,7 @@ export class ProviderBackgroundService { return; } if (CANCELLED_BACKGROUND_STATUSES.has(result.status)) { + this.requestStore.clearPollFailures(request.id); this.requestStore.markCancelled({ orgId: request.org_id, requestId: request.id, @@ -185,6 +189,15 @@ export class ProviderBackgroundService { } throw new Error(`Provider background response returned unsupported status ${result.status}.`); } catch (error) { + // The provider reporting failure is final; a failed status check is not, since the remote job may still finish. + let finalError = error; + if (!(error instanceof ProviderBackgroundStatusError)) { + const message = error instanceof Error ? error.message : String(error); + const failures = this.requestStore.recordPollFailure(request.id, message); + console.warn(`[provider-background] status check ${failures}/${MAX_POLL_FAILURES} for request ${request.id} failed: ${message}`); + if (failures < MAX_POLL_FAILURES) return; + finalError = new Error(`Gave up after ${MAX_POLL_FAILURES} failed status checks: ${message}`); + } this.responseObserver.recordFailure({ orgId: request.org_id, provider, @@ -192,7 +205,7 @@ export class ProviderBackgroundService { executionMode: PROVIDER_EXECUTION_MODE.BACKGROUND, startedAt, inputCharCount: request.input_char_count, - error, + error: finalError, rawResponseJson: error instanceof ProviderBackgroundStatusError ? error.rawResponseJson : null, }); } diff --git a/src/main/services/provider-request-store.ts b/src/main/services/provider-request-store.ts index b80f9fa0..e91b1ef7 100644 --- a/src/main/services/provider-request-store.ts +++ b/src/main/services/provider-request-store.ts @@ -156,6 +156,16 @@ export class ProviderRequestStore { ); } + /** Counts a failed status check on a background request and returns how many happened in a row. */ + recordPollFailure(requestId: string, message: string): number { + this.db.prepare("update provider_requests set poll_failures = poll_failures + 1, last_poll_error = ? where id = ?").run(message, requestId); + return (this.db.prepare("select poll_failures from provider_requests where id = ?").get(requestId) as { poll_failures: number }).poll_failures; + } + + clearPollFailures(requestId: string): void { + this.db.prepare("update provider_requests set poll_failures = 0, last_poll_error = null where id = ?").run(requestId); + } + markBackgroundProgress(input: ProviderBackgroundProgressInput): void { this.db.prepare(` update provider_requests diff --git a/tests/unit/domain-services.test.ts b/tests/unit/domain-services.test.ts index 15556e03..e9d458b0 100644 --- a/tests/unit/domain-services.test.ts +++ b/tests/unit/domain-services.test.ts @@ -24,7 +24,7 @@ describe("domain services", () => { const connection = createConnection(); const rows = connection.db.prepare("select version from schema_migrations order by version asc").all() as Array<{ version: number }>; - expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70]); + expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71]); connection.close(); }); diff --git a/tests/unit/provider-background-service.test.ts b/tests/unit/provider-background-service.test.ts index 0a49bf6d..4ac978b0 100644 --- a/tests/unit/provider-background-service.test.ts +++ b/tests/unit/provider-background-service.test.ts @@ -178,6 +178,65 @@ describe("provider background service", () => { }); }); +describe("background polling failures", () => { + test("a failed status check keeps the request recoverable; only repeated failures give up", async () => { + const connection = createConnection(); + const organization = new OrganizationService(connection.db); + let pollFailures = 2; + let status = "completed"; + const base = backgroundAdapters("completed", "Late result."); + const adapters: ProviderAdapterRegistry = { + ...base, + openai: { + ...base.openai, + retrieveResponse: async (input) => { + if (pollFailures > 0) { + pollFailures -= 1; + throw new Error("fetch failed: ECONNRESET"); + } + const result = await base.openai.retrieveResponse(input); + return { ...result, status }; + }, + }, + }; + const providers = new ProviderService(connection.db, adapters, testVault); + const background = new ProviderBackgroundService(connection.db, adapters, testVault); + let orgId = ""; + let openAiProviderId = ""; + connection.transaction(() => { + orgId = createOrganization(organization); + openAiProviderId = providerId(connection, orgId, "openai"); + providers.saveCredential({ orgId, providerId: openAiProviderId, secret: "sk-live-provider", model: "gpt-5.4" }); + providers.updatePolicy({ + orgId, providerId: openAiProviderId, dataPolicy: PROVIDER_DATA_POLICY.STANDARD, backgroundEnabled: true, + monthlyRequestLimit: 10, workspaceScope: "organization", contextWindowTokens: 128_000, + }); + }); + const requestId = await background.create({ orgId, providerId: openAiProviderId, prompt: "Research", instructions: null, executionMode: PROVIDER_EXECUTION_MODE.BACKGROUND }); + const row = () => connection.db.prepare("select state, poll_failures, last_poll_error, response_text from provider_requests where id = ?").get(requestId); + + expect(await background.recoverBackgroundResponses()).toBe(1); + expect(row()).toMatchObject({ state: PROVIDER_REQUEST_STATE.PENDING, poll_failures: 1, last_poll_error: "fetch failed: ECONNRESET" }); + await background.recoverBackgroundResponses(); + await background.recoverBackgroundResponses(); + expect(row()).toMatchObject({ state: PROVIDER_REQUEST_STATE.SUCCEEDED, poll_failures: 0, response_text: "Late result." }); + + pollFailures = 100; + const doomed = await background.create({ orgId, providerId: openAiProviderId, prompt: "Research again", instructions: null, executionMode: PROVIDER_EXECUTION_MODE.BACKGROUND }); + for (let attempt = 0; attempt < 10; attempt += 1) await background.poll({ orgId, requestId: doomed }); + expect(connection.db.prepare("select state, error_message from provider_requests where id = ?").get(doomed)) + .toEqual({ state: PROVIDER_REQUEST_STATE.FAILED, error_message: "Gave up after 10 failed status checks: fetch failed: ECONNRESET" }); + + // An answer that is not a valid result (unknown status) also counts, so it cannot stay pending forever. + pollFailures = 0; + status = "mystery"; + const odd = await background.create({ orgId, providerId: openAiProviderId, prompt: "Odd", instructions: null, executionMode: PROVIDER_EXECUTION_MODE.BACKGROUND }); + for (let attempt = 0; attempt < 10; attempt += 1) await background.poll({ orgId, requestId: odd }); + expect(connection.db.prepare("select state from provider_requests where id = ?").get(odd)).toEqual({ state: PROVIDER_REQUEST_STATE.FAILED }); + connection.close(); + }); +}); + function backgroundAdapters(status: string, text: string | null): ProviderAdapterRegistry { return { ...providerAdapters,