fix(worker): recover tasks after worker crash

This commit is contained in:
ZacharyZhang-NY committed 2026-04-28 23:37:24 -04:00
1 parent d3e108ec18
commit 7dbaaf0022
6 files changed
+437 -101

No files matched your search

+134 -1
View File
@@ -1,5 +1,5 @@
import { EventEmitter } from "node:events";
import { describe, expect, it, vi } from "vitest";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
forkMock: vi.fn(),
@@ -25,6 +25,15 @@ function createWorker() {
}
describe("worker service shutdown", () => {
beforeEach(() => {
mocks.forkMock.mockReset();
});
afterEach(() => {
vi.clearAllTimers();
vi.useRealTimers();
});
it("keeps completed run outcomes when cleaning shutdown runs", async () => {
const worker = createWorker();
mocks.forkMock.mockReturnValueOnce(worker);
@@ -83,4 +92,128 @@ describe("worker service shutdown", () => {
status: "interrupted",
}));
});
it("clears the crashed worker reference before recovery runs", () => {
vi.useFakeTimers();
const worker = createWorker();
mocks.forkMock.mockReturnValueOnce(worker);
const publishDomainChanged = vi.fn();
const service = createWorkerService({
db: {
projection: {
snapshot: {
listSnapshot: vi.fn(() => ({ runs: [] })),
},
},
},
logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
workerPath: "/tmp/worker.js",
isQuitting: () => false,
publishDomainChanged,
emitEvent: vi.fn(),
wakeAgentIfPossible: vi.fn(() => null),
dispatchRunToWorker: vi.fn(),
processRunCompletion: vi.fn(),
onRunStarted: vi.fn(),
onRunCompleted: vi.fn(),
notify: vi.fn(),
budgetService: undefined,
sessionService: undefined,
getRuntimeBridge: vi.fn(() => null),
getRuntimeMessageInjector: vi.fn(() => null),
} as never);
service.createWorker();
expect(service.getWorker()).toBe(worker);
worker.emit("exit", 1);
expect(service.getWorker()).toBeNull();
expect(publishDomainChanged).toHaveBeenCalledOnce();
});
it("requeues queued crash runs after the replacement worker is created", () => {
vi.useFakeTimers();
const crashedWorker = createWorker();
const replacementWorker = createWorker();
mocks.forkMock
.mockReturnValueOnce(crashedWorker)
.mockReturnValueOnce(replacementWorker);
const queuedRun = {
id: "run-queued",
status: "queued",
taskId: "task-1",
agentId: "agent-1",
companyId: "company-1",
workspaceId: null,
connectorId: "openai-codex",
};
const dispatchRunToWorker = vi.fn();
const service = createWorkerService({
db: {
projection: {
snapshot: {
listSnapshot: vi.fn(() => ({
runs: [queuedRun],
companies: [{ id: "company-1", name: "Company", description: "Recovered company" }],
})),
},
},
core: {
runs: {
getRun: vi.fn(() => queuedRun),
},
entity: {
getTask: vi.fn(() => ({
id: "task-1",
companyId: "company-1",
workspaceId: null,
projectId: null,
requiresFilesystem: false,
})),
getAgent: vi.fn(() => ({
id: "agent-1",
companyId: "company-1",
connectorId: "openai-codex",
workspaceId: null,
})),
},
connectors: {
getConnector: vi.fn(() => ({ id: "openai-codex" })),
},
workspaces: {
releaseWorkspaceLock: vi.fn(),
getCompanyObservationWorkspace: vi.fn(() => null),
},
},
},
logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
workerPath: "/tmp/worker.js",
isQuitting: () => false,
publishDomainChanged: vi.fn(),
emitEvent: vi.fn(),
wakeAgentIfPossible: vi.fn(() => null),
dispatchRunToWorker,
processRunCompletion: vi.fn(),
onRunStarted: vi.fn(),
onRunCompleted: vi.fn(),
notify: vi.fn(),
budgetService: undefined,
sessionService: undefined,
getRuntimeBridge: vi.fn(() => null),
getRuntimeMessageInjector: vi.fn(() => null),
} as never);
service.createWorker();
crashedWorker.emit("exit", 1);
expect(dispatchRunToWorker).not.toHaveBeenCalled();
vi.advanceTimersByTime(1_000);
expect(service.getWorker()).toBe(replacementWorker);
expect(dispatchRunToWorker).toHaveBeenCalledWith(expect.objectContaining({
runId: "run-queued",
companyId: "company-1",
trigger: "recovery",
}));
});
});