fix(worker): recover tasks after worker crash

tests/unit/worker-crash-recovery.test.ts

@@ -0,0 +1,201 @@
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { mkdtemp, rm } from "node:fs/promises";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("electron", () => ({
+  safeStorage: {
+    isEncryptionAvailable: () => false,
+    getSelectedStorageBackend: () => "basic_text",
+    encryptString: (value: string) => Buffer.from(value, "utf8"),
+    decryptString: (value: Buffer) => value.toString("utf8"),
+  },
+}));
+
+import { AppDatabase } from "../../src/main/database";
+import { handleWorkerExit, requeueQueuedRunsAfterWorkerRestart } from "../../src/main/worker/entry/crash-recovery";
+
+function createLogger() {
+  return {
+    debug: vi.fn(),
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+  };
+}
+
+function createCompany(db: AppDatabase): string {
+  return db.core.company.saveCompany({
+    name: "Crash Recovery Co",
+    description: "Worker crash recovery contract test",
+    status: "active",
+  });
+}
+
+function createConnector(db: AppDatabase): void {
+  db.core.connectors.saveConnector({
+    id: "openai-codex",
+    model: "gpt-5.4",
+    envBindingText: "",
+    notes: "Test connector",
+  });
+}
+
+function createWorkspace(db: AppDatabase, companyId: string, localPath: string): string {
+  return db.core.workspaces.saveWorkspace({
+    companyId,
+    name: "Primary",
+    localPath,
+    repoUrl: "",
+    repoRef: "",
+    isPrimary: true,
+  });
+}
+
+function createAgent(db: AppDatabase, companyId: string, workspaceId: string): string {
+  return db.core.agents.saveAgent({
+    companyId,
+    name: "Recovery Agent",
+    role: "Engineer",
+    title: "Engineer",
+    status: "active",
+    reportsTo: null,
+    connectorId: "openai-codex",
+    workspaceId,
+    model: "gpt-5.4",
+    capabilities: "Continues work after runtime crashes",
+    budgetMonthlyUsd: 100,
+  });
+}
+
+describe("worker crash recovery", () => {
+  let tempDir: string;
+  let db: AppDatabase;
+
+  beforeEach(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), "agentcompany-worker-crash-"));
+    db = new AppDatabase(tempDir);
+    db.init();
+  });
+
+  afterEach(async () => {
+    db.close();
+    await rm(tempDir, { recursive: true, force: true });
+  });
+
+  it("returns interrupted running tasks to checkoutable work and wakes the original agent", () => {
+    const companyId = createCompany(db);
+    createConnector(db);
+    const workspaceId = createWorkspace(db, companyId, tempDir);
+    const agentId = createAgent(db, companyId, workspaceId);
+    const taskId = db.core.tasks.saveTask({
+      companyId,
+      title: "Continue Phase 2",
+      description: "This task must remain recoverable after the worker process exits.",
+      assigneeAgentId: agentId,
+      workspaceId,
+      priority: "high",
+      status: "todo",
+    });
+    const run = db.core.runs.createTaskRun({
+      companyId,
+      taskId,
+      agentId,
+      workspaceId,
+      connectorId: "openai-codex",
+    });
+    expect(run).not.toBeNull();
+    db.core.runs.markRunStatus(run!.id, "running", "Started");
+    expect(db.core.entity.getTask(taskId).status).toBe("in_progress");
+    expect(db.core.runs.hasActiveRun({ companyId, agentId })).toBe(true);
+
+    const wakeAgentIfPossible = vi.fn(() => null);
+    handleWorkerExit(
+      1,
+      {
+        db,
+        logger: createLogger(),
+        isQuitting: () => false,
+        publishDomainChanged: vi.fn(),
+        emitEvent: vi.fn(),
+        wakeAgentIfPossible,
+      },
+    );
+
+    expect(db.core.runs.getRun(run!.id).status).toBe("interrupted");
+    expect(db.core.entity.getTask(taskId).status).toBe("todo");
+    expect(db.core.runs.hasActiveRun({ companyId, agentId })).toBe(false);
+    expect(wakeAgentIfPossible).toHaveBeenCalledWith(agentId, companyId, "recovery");
+
+    const retryRun = db.core.runs.createTaskRun({
+      companyId,
+      taskId,
+      agentId,
+      workspaceId,
+      connectorId: "openai-codex",
+    });
+    expect(retryRun).not.toBeNull();
+  });
+
+  it("requeues queued runs after the replacement worker is ready", () => {
+    const companyId = createCompany(db);
+    createConnector(db);
+    const workspaceId = createWorkspace(db, companyId, tempDir);
+    const agentId = createAgent(db, companyId, workspaceId);
+    const taskId = db.core.tasks.saveTask({
+      companyId,
+      title: "Queued recovery task",
+      description: "This run was queued when the worker process exited.",
+      assigneeAgentId: agentId,
+      workspaceId,
+      priority: "high",
+      status: "todo",
+    });
+    const run = db.core.runs.createTaskRun({
+      companyId,
+      taskId,
+      agentId,
+      workspaceId,
+      connectorId: "openai-codex",
+    });
+    expect(run).not.toBeNull();
+    expect(db.core.runs.getRun(run!.id).status).toBe("queued");
+    expect(db.core.runs.hasActiveRun({ companyId, agentId })).toBe(true);
+
+    const dispatchRunToWorker = vi.fn();
+    const requeueCandidates = handleWorkerExit(
+      1,
+      {
+        db,
+        logger: createLogger(),
+        isQuitting: () => false,
+        publishDomainChanged: vi.fn(),
+        emitEvent: vi.fn(),
+        wakeAgentIfPossible: vi.fn(() => null),
+      },
+    );
+
+    expect(dispatchRunToWorker).not.toHaveBeenCalled();
+    expect(requeueCandidates).toHaveLength(1);
+    expect(db.core.runs.getRun(run!.id).status).toBe("queued");
+
+    requeueQueuedRunsAfterWorkerRestart(
+      requeueCandidates,
+      {
+        db,
+        logger: createLogger(),
+        dispatchRunToWorker,
+      },
+    );
+
+    expect(dispatchRunToWorker).toHaveBeenCalledOnce();
+    expect(dispatchRunToWorker).toHaveBeenCalledWith(expect.objectContaining({
+      runId: run!.id,
+      companyId,
+      task: expect.objectContaining({ id: taskId }),
+      agent: expect.objectContaining({ id: agentId }),
+      trigger: "recovery",
+    }));
+    expect(db.core.runs.hasActiveRun({ companyId, agentId })).toBe(true);
+  });
+});

tests/unit/worker-service-shutdown.test.ts

@@ -1,5 +1,5 @@
 import { EventEmitter } from "node:events";
-import { describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 
 const mocks = vi.hoisted(() => ({
   forkMock: vi.fn(),
@@ -25,6 +25,15 @@ function createWorker() {
 }
 
 describe("worker service shutdown", () => {
+  beforeEach(() => {
+    mocks.forkMock.mockReset();
+  });
+
+  afterEach(() => {
+    vi.clearAllTimers();
+    vi.useRealTimers();
+  });
+
   it("keeps completed run outcomes when cleaning shutdown runs", async () => {
     const worker = createWorker();
     mocks.forkMock.mockReturnValueOnce(worker);
@@ -83,4 +92,128 @@ describe("worker service shutdown", () => {
       status: "interrupted",
     }));
   });
+
+  it("clears the crashed worker reference before recovery runs", () => {
+    vi.useFakeTimers();
+    const worker = createWorker();
+    mocks.forkMock.mockReturnValueOnce(worker);
+    const publishDomainChanged = vi.fn();
+    const service = createWorkerService({
+      db: {
+        projection: {
+          snapshot: {
+            listSnapshot: vi.fn(() => ({ runs: [] })),
+          },
+        },
+      },
+      logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+      workerPath: "/tmp/worker.js",
+      isQuitting: () => false,
+      publishDomainChanged,
+      emitEvent: vi.fn(),
+      wakeAgentIfPossible: vi.fn(() => null),
+      dispatchRunToWorker: vi.fn(),
+      processRunCompletion: vi.fn(),
+      onRunStarted: vi.fn(),
+      onRunCompleted: vi.fn(),
+      notify: vi.fn(),
+      budgetService: undefined,
+      sessionService: undefined,
+      getRuntimeBridge: vi.fn(() => null),
+      getRuntimeMessageInjector: vi.fn(() => null),
+    } as never);
+
+    service.createWorker();
+    expect(service.getWorker()).toBe(worker);
+    worker.emit("exit", 1);
+
+    expect(service.getWorker()).toBeNull();
+    expect(publishDomainChanged).toHaveBeenCalledOnce();
+  });
+
+  it("requeues queued crash runs after the replacement worker is created", () => {
+    vi.useFakeTimers();
+    const crashedWorker = createWorker();
+    const replacementWorker = createWorker();
+    mocks.forkMock
+      .mockReturnValueOnce(crashedWorker)
+      .mockReturnValueOnce(replacementWorker);
+    const queuedRun = {
+      id: "run-queued",
+      status: "queued",
+      taskId: "task-1",
+      agentId: "agent-1",
+      companyId: "company-1",
+      workspaceId: null,
+      connectorId: "openai-codex",
+    };
+    const dispatchRunToWorker = vi.fn();
+    const service = createWorkerService({
+      db: {
+        projection: {
+          snapshot: {
+            listSnapshot: vi.fn(() => ({
+              runs: [queuedRun],
+              companies: [{ id: "company-1", name: "Company", description: "Recovered company" }],
+            })),
+          },
+        },
+        core: {
+          runs: {
+            getRun: vi.fn(() => queuedRun),
+          },
+          entity: {
+            getTask: vi.fn(() => ({
+              id: "task-1",
+              companyId: "company-1",
+              workspaceId: null,
+              projectId: null,
+              requiresFilesystem: false,
+            })),
+            getAgent: vi.fn(() => ({
+              id: "agent-1",
+              companyId: "company-1",
+              connectorId: "openai-codex",
+              workspaceId: null,
+            })),
+          },
+          connectors: {
+            getConnector: vi.fn(() => ({ id: "openai-codex" })),
+          },
+          workspaces: {
+            releaseWorkspaceLock: vi.fn(),
+            getCompanyObservationWorkspace: vi.fn(() => null),
+          },
+        },
+      },
+      logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+      workerPath: "/tmp/worker.js",
+      isQuitting: () => false,
+      publishDomainChanged: vi.fn(),
+      emitEvent: vi.fn(),
+      wakeAgentIfPossible: vi.fn(() => null),
+      dispatchRunToWorker,
+      processRunCompletion: vi.fn(),
+      onRunStarted: vi.fn(),
+      onRunCompleted: vi.fn(),
+      notify: vi.fn(),
+      budgetService: undefined,
+      sessionService: undefined,
+      getRuntimeBridge: vi.fn(() => null),
+      getRuntimeMessageInjector: vi.fn(() => null),
+    } as never);
+
+    service.createWorker();
+    crashedWorker.emit("exit", 1);
+    expect(dispatchRunToWorker).not.toHaveBeenCalled();
+
+    vi.advanceTimersByTime(1_000);
+
+    expect(service.getWorker()).toBe(replacementWorker);
+    expect(dispatchRunToWorker).toHaveBeenCalledWith(expect.objectContaining({
+      runId: "run-queued",
+      companyId: "company-1",
+      trigger: "recovery",
+    }));
+  });
 });