Recover interrupted runs and requests at boot

A task_run still marked running at startup has no executor - the
single-instance app that owned it is gone - yet nothing failed it, the
queue worker only claims queued rows, and the heartbeat excludes tasks
with a running run, so one interrupted shutdown wedged the company
forever. Boot now fails orphaned running runs and their in-flight
foreground provider requests (auditably), which hands the task back to
the heartbeat for an automatic retry. Background provider requests are
untouched; the background worker resumes those by external id.

AgentCompany-New/tests/unit/startup-recovery.test.ts

@@ -0,0 +1,90 @@
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { randomUUID } from "node:crypto";
+import { describe, expect, test } from "vitest";
+import { DatabaseConnection } from "../../src/main/database/connection";
+import { OnboardingService } from "../../src/main/services/onboarding-service";
+import { TaskExecutionService } from "../../src/main/services/task-execution-service";
+import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
+import { recoverInterruptedWork } from "../../src/main/app/startup-recovery";
+import { createId } from "../../src/shared/ids";
+
+describe("startup recovery", () => {
+  test("orphaned running runs fail at boot and the heartbeat re-enqueues the task", () => {
+    const connection = new DatabaseConnection(join(tmpdir(), `ac-recover-${randomUUID()}.db`));
+    const onboarding = new OnboardingService({
+      db: connection.db,
+      saveProviderCredential: ({ orgId, providerId, model }) => {
+        connection.db.prepare("update providers set credential_id = ?, model = ?, enabled = 1 where id = ? and org_id = ?")
+          .run("cred_test", model, providerId, orgId);
+      },
+    });
+
+    let orgId = "";
+    let providerId = "";
+    connection.transaction(() => {
+      const summary = onboarding.bootstrap({
+        companyName: "Acme Operations",
+        companyDescription: "Coordinate agents with auditable approvals.",
+        employeeName: "Dana Lee",
+        workspacePath: "",
+        executionMode: "ai_ceo",
+        autoApproveHires: true,
+        reviewDeliverables: false,
+        goalTitle: "Ship a verified control plane",
+        goalDescription: "Bring the desktop operating loop online.",
+        providerKey: "openai",
+        providerSecret: "test-secret",
+        providerModel: "gpt-5.1",
+      });
+      orgId = summary.orgId;
+      providerId = summary.providerId;
+    });
+
+    // The previous session claimed the run and left in-flight provider requests.
+    const claimed = claimNextQueuedRun(connection);
+    expect(claimed).toBeTruthy();
+    const insertRequest = (id: string, executionMode: string, state: string) => {
+      connection.db.prepare(`
+        insert into provider_requests(
+          id, org_id, provider_id, provider_key, workspace_id, model, prompt_sha256,
+          response_text, raw_response_json, state, error_message, input_char_count,
+          output_char_count, execution_mode, policy_decision, external_response_id,
+          stream_event_count, cancelled_at, created_at, completed_at
+        )
+        values (?, ?, ?, 'openai', null, 'gpt-5.1', 'sha', null, null, ?, null, 10, 0, ?, 'allowed', ?, 0, null, ?, null)
+      `).run(id, orgId, providerId, state, executionMode, executionMode === "background" ? "resp_ext" : null, new Date().toISOString());
+    };
+    const foregroundId = createId("preq");
+    const backgroundId = createId("preq");
+    insertRequest(foregroundId, "foreground", "pending");
+    insertRequest(backgroundId, "background", "pending");
+
+    connection.transaction(() => recoverInterruptedWork(connection.db));
+
+    const run = connection.db.prepare("select state, error_message, completed_at from task_runs where id = ?")
+      .get(claimed!.id) as { state: string; error_message: string; completed_at: string };
+    expect(run.state).toBe("failed");
+    expect(run.error_message).toContain("Interrupted");
+    expect(run.completed_at).toBeTruthy();
+
+    // Foreground orphans fail; background ones stay for the background worker.
+    expect(connection.db.prepare("select state from provider_requests where id = ?").get(foregroundId)).toMatchObject({ state: "failed" });
+    expect(connection.db.prepare("select state from provider_requests where id = ?").get(backgroundId)).toMatchObject({ state: "pending" });
+
+    // The interruption is auditable and the heartbeat immediately retries the task.
+    const audit = connection.db.prepare(
+      "select count(*) as n from audit_events where org_id = ? and entity_type = 'task_run' and action = 'failed'",
+    ).get(orgId) as { n: number };
+    expect(audit.n).toBe(1);
+    const taskExecution = new TaskExecutionService(connection.db);
+    const enqueued = connection.transaction(() => taskExecution.enqueueHeartbeatRuns());
+    expect(enqueued).toHaveLength(1);
+
+    // Recovery is idempotent: a second boot changes nothing.
+    connection.transaction(() => recoverInterruptedWork(connection.db));
+    const failedRuns = connection.db.prepare("select count(*) as n from task_runs where state = 'failed'").get() as { n: number };
+    expect(failedRuns.n).toBe(1);
+    connection.close();
+  });
+});