fix(worker): recover tasks after worker crash
This commit is contained in:
1 parent
d3e108ec18
commit
7dbaaf0022
6 files changed
+437
-101
No files matched your search
@@ -1,5 +1,5 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
forkMock: vi.fn(),
|
||||
@@ -25,6 +25,15 @@ function createWorker() {
|
||||
}
|
||||
|
||||
describe("worker service shutdown", () => {
|
||||
beforeEach(() => {
|
||||
mocks.forkMock.mockReset();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("keeps completed run outcomes when cleaning shutdown runs", async () => {
|
||||
const worker = createWorker();
|
||||
mocks.forkMock.mockReturnValueOnce(worker);
|
||||
@@ -83,4 +92,128 @@ describe("worker service shutdown", () => {
|
||||
status: "interrupted",
|
||||
}));
|
||||
});
|
||||
|
||||
it("clears the crashed worker reference before recovery runs", () => {
|
||||
vi.useFakeTimers();
|
||||
const worker = createWorker();
|
||||
mocks.forkMock.mockReturnValueOnce(worker);
|
||||
const publishDomainChanged = vi.fn();
|
||||
const service = createWorkerService({
|
||||
db: {
|
||||
projection: {
|
||||
snapshot: {
|
||||
listSnapshot: vi.fn(() => ({ runs: [] })),
|
||||
},
|
||||
},
|
||||
},
|
||||
logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
workerPath: "/tmp/worker.js",
|
||||
isQuitting: () => false,
|
||||
publishDomainChanged,
|
||||
emitEvent: vi.fn(),
|
||||
wakeAgentIfPossible: vi.fn(() => null),
|
||||
dispatchRunToWorker: vi.fn(),
|
||||
processRunCompletion: vi.fn(),
|
||||
onRunStarted: vi.fn(),
|
||||
onRunCompleted: vi.fn(),
|
||||
notify: vi.fn(),
|
||||
budgetService: undefined,
|
||||
sessionService: undefined,
|
||||
getRuntimeBridge: vi.fn(() => null),
|
||||
getRuntimeMessageInjector: vi.fn(() => null),
|
||||
} as never);
|
||||
|
||||
service.createWorker();
|
||||
expect(service.getWorker()).toBe(worker);
|
||||
worker.emit("exit", 1);
|
||||
|
||||
expect(service.getWorker()).toBeNull();
|
||||
expect(publishDomainChanged).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("requeues queued crash runs after the replacement worker is created", () => {
|
||||
vi.useFakeTimers();
|
||||
const crashedWorker = createWorker();
|
||||
const replacementWorker = createWorker();
|
||||
mocks.forkMock
|
||||
.mockReturnValueOnce(crashedWorker)
|
||||
.mockReturnValueOnce(replacementWorker);
|
||||
const queuedRun = {
|
||||
id: "run-queued",
|
||||
status: "queued",
|
||||
taskId: "task-1",
|
||||
agentId: "agent-1",
|
||||
companyId: "company-1",
|
||||
workspaceId: null,
|
||||
connectorId: "openai-codex",
|
||||
};
|
||||
const dispatchRunToWorker = vi.fn();
|
||||
const service = createWorkerService({
|
||||
db: {
|
||||
projection: {
|
||||
snapshot: {
|
||||
listSnapshot: vi.fn(() => ({
|
||||
runs: [queuedRun],
|
||||
companies: [{ id: "company-1", name: "Company", description: "Recovered company" }],
|
||||
})),
|
||||
},
|
||||
},
|
||||
core: {
|
||||
runs: {
|
||||
getRun: vi.fn(() => queuedRun),
|
||||
},
|
||||
entity: {
|
||||
getTask: vi.fn(() => ({
|
||||
id: "task-1",
|
||||
companyId: "company-1",
|
||||
workspaceId: null,
|
||||
projectId: null,
|
||||
requiresFilesystem: false,
|
||||
})),
|
||||
getAgent: vi.fn(() => ({
|
||||
id: "agent-1",
|
||||
companyId: "company-1",
|
||||
connectorId: "openai-codex",
|
||||
workspaceId: null,
|
||||
})),
|
||||
},
|
||||
connectors: {
|
||||
getConnector: vi.fn(() => ({ id: "openai-codex" })),
|
||||
},
|
||||
workspaces: {
|
||||
releaseWorkspaceLock: vi.fn(),
|
||||
getCompanyObservationWorkspace: vi.fn(() => null),
|
||||
},
|
||||
},
|
||||
},
|
||||
logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
workerPath: "/tmp/worker.js",
|
||||
isQuitting: () => false,
|
||||
publishDomainChanged: vi.fn(),
|
||||
emitEvent: vi.fn(),
|
||||
wakeAgentIfPossible: vi.fn(() => null),
|
||||
dispatchRunToWorker,
|
||||
processRunCompletion: vi.fn(),
|
||||
onRunStarted: vi.fn(),
|
||||
onRunCompleted: vi.fn(),
|
||||
notify: vi.fn(),
|
||||
budgetService: undefined,
|
||||
sessionService: undefined,
|
||||
getRuntimeBridge: vi.fn(() => null),
|
||||
getRuntimeMessageInjector: vi.fn(() => null),
|
||||
} as never);
|
||||
|
||||
service.createWorker();
|
||||
crashedWorker.emit("exit", 1);
|
||||
expect(dispatchRunToWorker).not.toHaveBeenCalled();
|
||||
|
||||
vi.advanceTimersByTime(1_000);
|
||||
|
||||
expect(service.getWorker()).toBe(replacementWorker);
|
||||
expect(dispatchRunToWorker).toHaveBeenCalledWith(expect.objectContaining({
|
||||
runId: "run-queued",
|
||||
companyId: "company-1",
|
||||
trigger: "recovery",
|
||||
}));
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user