Package release 0.4.182 from the current workspace snapshot
Constraint: Windows and Linux artifacts are built on native GitHub runners from a temporary snapshot branch Rejected: Build Windows/Linux locally on Apple Silicon | docker/wine packaging is not stable on this host Confidence: medium Scope-risk: moderate Directive: This snapshot branch is transport-only and should be deleted after the release completes Tested: local verify/build stages before snapshot creation Not-tested: remote packaging before workflow execution
This commit is contained in:
1 parent
49ea508ee1
commit
6d2b685e3a
66 files changed
+2300
-323
No files matched your search
@@ -107,12 +107,14 @@ describe("pi-mono migrations 27 + 28", () => {
|
||||
db.init();
|
||||
|
||||
const rows = db.projection.snapshot.listSnapshot().connectors.map((entry) => entry.id);
|
||||
expect(rows.length).toBe(26);
|
||||
expect(rows.length).toBe(28);
|
||||
expect(rows).toContain("claude-pro");
|
||||
expect(rows).toContain("amazon-bedrock");
|
||||
expect(rows).toContain("google-vertex");
|
||||
expect(rows).toContain("fireworks");
|
||||
expect(rows).toContain("fireworks-firepass");
|
||||
expect(rows).toContain("ollama");
|
||||
expect(rows).toContain("lm-studio");
|
||||
expect(rows.find((id) => (id as string) === "claude_local")).toBeUndefined();
|
||||
expect(rows.find((id) => (id as string) === "codex_local")).toBeUndefined();
|
||||
expect(rows.find((id) => (id as string) === "claude_compatible")).toBeUndefined();
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { AppDatabase } from "../../src/main/database";
|
||||
|
||||
vi.mock("electron", () => ({
|
||||
safeStorage: {
|
||||
isEncryptionAvailable: () => false,
|
||||
getSelectedStorageBackend: () => "basic_text",
|
||||
encryptString: (value: string) => Buffer.from(value, "utf8"),
|
||||
decryptString: (value: Buffer) => value.toString("utf8"),
|
||||
},
|
||||
}));
|
||||
|
||||
type RawDb = {
|
||||
db: {
|
||||
prepare: (sql: string) => {
|
||||
run: (...args: unknown[]) => void;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
function createCompany(db: AppDatabase): string {
|
||||
return db.core.company.saveCompany({
|
||||
name: "Projection Co",
|
||||
description: "Active run projection contract",
|
||||
status: "active",
|
||||
});
|
||||
}
|
||||
|
||||
function createWorkspace(db: AppDatabase, companyId: string, localPath: string): string {
|
||||
return db.core.workspaces.saveWorkspace({
|
||||
companyId,
|
||||
name: "Primary",
|
||||
localPath,
|
||||
repoUrl: "",
|
||||
repoRef: "",
|
||||
isPrimary: true,
|
||||
});
|
||||
}
|
||||
|
||||
function createAgent(db: AppDatabase, companyId: string, workspaceId: string): string {
|
||||
return db.core.agents.saveAgent({
|
||||
companyId,
|
||||
name: "Projection Agent",
|
||||
role: "Engineer",
|
||||
title: "Engineer",
|
||||
status: "idle",
|
||||
reportsTo: null,
|
||||
connectorId: "openai-codex",
|
||||
workspaceId,
|
||||
model: "gpt-5.4",
|
||||
capabilities: "Tests active run projections",
|
||||
budgetMonthlyUsd: 100,
|
||||
});
|
||||
}
|
||||
|
||||
describe("active run projection", () => {
|
||||
let tempDir: string;
|
||||
let db: AppDatabase;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), "agentcompany-active-run-projection-"));
|
||||
db = new AppDatabase(tempDir);
|
||||
db.init();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
db.close();
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("ignores terminal run leases when projecting activeRunId", () => {
|
||||
const companyId = createCompany(db);
|
||||
const workspaceId = createWorkspace(db, companyId, tempDir);
|
||||
const agentId = createAgent(db, companyId, workspaceId);
|
||||
const taskId = db.core.tasks.saveTask({
|
||||
companyId,
|
||||
title: "Recoverable task",
|
||||
description: "A stale terminal lease must not hide this task from recovery.",
|
||||
assigneeAgentId: agentId,
|
||||
workspaceId,
|
||||
priority: "high",
|
||||
status: "todo",
|
||||
});
|
||||
const run = db.core.runs.createRun({
|
||||
companyId,
|
||||
taskId,
|
||||
agentId,
|
||||
workspaceId: null,
|
||||
connectorId: "openai-codex",
|
||||
});
|
||||
db.core.runs.finishRun({
|
||||
runId: run.id,
|
||||
status: "succeeded",
|
||||
summary: "Done",
|
||||
errorMessage: null,
|
||||
exitCode: 0,
|
||||
signal: null,
|
||||
model: null,
|
||||
unattributedCost: true,
|
||||
});
|
||||
const now = new Date().toISOString();
|
||||
(db as unknown as RawDb).db
|
||||
.prepare(
|
||||
`insert into task_leases (
|
||||
task_id, company_id, run_id, agent_id, acquired_at, renewed_at, expires_at, created_at, updated_at
|
||||
) values (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
)
|
||||
.run(taskId, companyId, run.id, agentId, now, now, now, now, now);
|
||||
|
||||
expect(db.core.entity.getTask(taskId).activeRunId).toBeNull();
|
||||
expect(db.core.tasks.listAgentTasks(agentId, companyId, ["todo"])[0]?.activeRunId).toBeNull();
|
||||
expect(db.projection.snapshot.listSnapshot({ companyId }).tasks.find((task) => task.id === taskId)?.activeRunId).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -4,6 +4,7 @@ import { tmpdir } from "node:os";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { AppDatabase } from "../../src/main/database";
|
||||
import { handleApprovalResolved } from "../../src/main/automation/automation-approval-handler";
|
||||
import { createHeartbeat } from "../../src/main/orchestration/heartbeat";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
@@ -18,7 +19,9 @@ afterEach(async () => {
|
||||
});
|
||||
|
||||
describe("deliverable review feedback writeback", () => {
|
||||
it("persists high-importance knowledge when the board requests revisions on a deliverable", async () => {
|
||||
it.each(["rejected", "revision_requested"] as const)(
|
||||
"persists high-importance knowledge when the board returns %s on a deliverable",
|
||||
async (decision) => {
|
||||
const profileDir = await createTempDir("agentcompany-review-feedback-");
|
||||
const db = new AppDatabase(profileDir);
|
||||
db.init();
|
||||
@@ -87,7 +90,7 @@ describe("deliverable review feedback writeback", () => {
|
||||
|
||||
db.core.approvals.decideApproval({
|
||||
approvalId,
|
||||
state: "revision_requested",
|
||||
state: decision,
|
||||
decisionNote: "Add stronger competitor evidence and a clearer recommendation.",
|
||||
});
|
||||
|
||||
@@ -105,7 +108,7 @@ describe("deliverable review feedback writeback", () => {
|
||||
handleTaskStatusChange: vi.fn(),
|
||||
runWorkflowPipeline: async () => {},
|
||||
trackTimer: vi.fn(),
|
||||
}, approvalId, companyId, "revision_requested");
|
||||
}, approvalId, companyId, decision);
|
||||
|
||||
const knowledgeEntries = db.projection.snapshot.listSnapshot().knowledgeBase.filter((entry) => entry.companyId === companyId);
|
||||
expect(knowledgeEntries).toHaveLength(1);
|
||||
@@ -113,13 +116,147 @@ describe("deliverable review feedback writeback", () => {
|
||||
expect(knowledgeEntries[0]?.importance).toBe("high");
|
||||
expect(knowledgeEntries[0]?.referencedEntityType).toBe("task");
|
||||
expect(knowledgeEntries[0]?.referencedEntityId).toBe(taskId);
|
||||
expect(knowledgeEntries[0]?.content).toContain("revision_requested");
|
||||
expect(knowledgeEntries[0]?.content).toContain(decision);
|
||||
expect(knowledgeEntries[0]?.content).toContain("stronger competitor evidence");
|
||||
expect(wakeAgentIfPossible).toHaveBeenCalledWith(agentId, companyId, "revision_needed");
|
||||
expect(db.core.entity.getTask(taskId).status).toBe("in_progress");
|
||||
expect(wakeAgentIfPossible).toHaveBeenCalledTimes(1);
|
||||
expect(db.core.entity.getTask(taskId).status).toBe("todo");
|
||||
const document = db.projection.snapshot.listSnapshot().documents.find((entry) => entry.id === documentId);
|
||||
expect(document?.status).toBe("draft");
|
||||
|
||||
db.close();
|
||||
},
|
||||
);
|
||||
|
||||
it("makes a revision request immediately checkoutable by the assignee heartbeat", async () => {
|
||||
const profileDir = await createTempDir("agentcompany-review-checkout-");
|
||||
const db = new AppDatabase(profileDir);
|
||||
db.init();
|
||||
|
||||
db.core.connectors.saveConnector({
|
||||
id: "openai-codex",
|
||||
model: "gpt-5.4",
|
||||
envBindingText: "",
|
||||
notes: "Test connector",
|
||||
});
|
||||
|
||||
const companyId = db.core.company.saveCompany({
|
||||
name: "Review Checkout Co",
|
||||
description: "Tests review revision checkout",
|
||||
status: "active",
|
||||
});
|
||||
const workspaceId = db.core.workspaces.saveWorkspace({
|
||||
companyId,
|
||||
name: "Primary",
|
||||
localPath: profileDir,
|
||||
repoUrl: "",
|
||||
repoRef: "",
|
||||
isPrimary: true,
|
||||
});
|
||||
const agentId = db.core.agents.saveAgent({
|
||||
companyId,
|
||||
name: "Research Lead",
|
||||
role: "Researcher",
|
||||
title: "Research Lead",
|
||||
status: "active",
|
||||
reportsTo: null,
|
||||
connectorId: "openai-codex",
|
||||
workspaceId,
|
||||
model: "gpt-5.4",
|
||||
capabilities: "Research and synthesis",
|
||||
budgetMonthlyUsd: 180,
|
||||
department: "research",
|
||||
});
|
||||
const taskId = db.core.tasks.saveTask({
|
||||
companyId,
|
||||
title: "Draft market report",
|
||||
description: "Produce a market report with concrete evidence and business insight.",
|
||||
assigneeAgentId: agentId,
|
||||
workspaceId,
|
||||
priority: "high",
|
||||
status: "in_review",
|
||||
});
|
||||
const documentId = db.core.documents.saveDocument({
|
||||
companyId,
|
||||
type: "knowledge_article",
|
||||
title: "Market report",
|
||||
content: "Draft report body",
|
||||
authorAgentId: agentId,
|
||||
status: "in_review",
|
||||
});
|
||||
const approvalId = db.core.approvals.requestApproval({
|
||||
companyId,
|
||||
relatedTaskId: taskId,
|
||||
requestedByAgentId: agentId,
|
||||
type: "deliverable_review",
|
||||
payloadSummary: "Review market report deliverable",
|
||||
impactSummary: "Board review required before the report is finalized.",
|
||||
payloadJson: JSON.stringify({ documentIds: [documentId] }),
|
||||
});
|
||||
|
||||
db.core.approvals.decideApproval({
|
||||
approvalId,
|
||||
state: "revision_requested",
|
||||
decisionNote: "Add stronger competitor evidence and a clearer recommendation.",
|
||||
});
|
||||
|
||||
const dispatchRunToWorker = vi.fn();
|
||||
const heartbeat = createHeartbeat(
|
||||
{
|
||||
db,
|
||||
logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
publishDomainChanged: vi.fn(),
|
||||
emitEvent: vi.fn(),
|
||||
notify: vi.fn(),
|
||||
createRunToken: () => "test-token",
|
||||
getApiPort: () => 3210,
|
||||
dispatchAutomation: vi.fn(),
|
||||
hasWorkerReady: () => true,
|
||||
enqueueWorkerRun: vi.fn(),
|
||||
isQuitting: () => false,
|
||||
listRuntimeSkillEntries: () => [],
|
||||
budgetService: {
|
||||
autoCloseExpiredIncidents: vi.fn(() => 0),
|
||||
evaluateBudgetGate: vi.fn(() => ({ allowed: true, blockedBy: [] })),
|
||||
} as never,
|
||||
checkpointEngine: { shouldPauseHeartbeat: () => false },
|
||||
},
|
||||
() => ({
|
||||
workspace: null,
|
||||
lockMode: "none",
|
||||
source: "none",
|
||||
observationPath: profileDir,
|
||||
}),
|
||||
() => null,
|
||||
dispatchRunToWorker,
|
||||
(filter) => db.core.runs.hasActiveRun(filter),
|
||||
);
|
||||
|
||||
handleApprovalResolved({
|
||||
db,
|
||||
logger: { info: () => {}, warn: () => {}, error: () => {} },
|
||||
publishDomainChanged: () => {},
|
||||
notify: vi.fn(),
|
||||
wakeAgentIfPossible: heartbeat.wakeAgentIfPossible,
|
||||
queueBrowserActionExecution: () => {},
|
||||
isQuitting: () => false,
|
||||
}, {
|
||||
dispatchAutomation: vi.fn(),
|
||||
handleTaskStatusChange: vi.fn(),
|
||||
runWorkflowPipeline: async () => {},
|
||||
trackTimer: vi.fn(),
|
||||
}, approvalId, companyId, "revision_requested");
|
||||
|
||||
const task = db.core.entity.getTask(taskId);
|
||||
expect(task.status).toBe("in_progress");
|
||||
expect(db.core.runs.hasActiveRun({ companyId, taskId, agentId })).toBe(true);
|
||||
expect(dispatchRunToWorker).toHaveBeenCalledTimes(1);
|
||||
expect(dispatchRunToWorker).toHaveBeenCalledWith(expect.objectContaining({
|
||||
companyId,
|
||||
task: expect.objectContaining({ id: taskId }),
|
||||
trigger: "revision_needed",
|
||||
}));
|
||||
|
||||
db.close();
|
||||
});
|
||||
});
|
||||
@@ -7,14 +7,16 @@ import { findConnectorDefinition, listFeaturedOnboardingConnectors } from "../..
|
||||
* Settings panel) depends on. */
|
||||
|
||||
describe("CONNECTORS catalog", () => {
|
||||
it("contains exactly 26 connectors (5 OAuth + 18 API key + 3 Cloud)", () => {
|
||||
expect(CONNECTORS.length).toBe(26);
|
||||
it("contains exactly 28 connectors (5 OAuth + 18 API key + 3 Cloud + 2 Local)", () => {
|
||||
expect(CONNECTORS.length).toBe(28);
|
||||
const oauth = CONNECTORS.filter((c) => c.auth?.type === "oauth");
|
||||
const apiKey = CONNECTORS.filter((c) => c.auth?.type === "api_key");
|
||||
const cloud = CONNECTORS.filter((c) => c.auth ? ["aws", "azure", "gcp_adc"].includes(c.auth.type) : false);
|
||||
const local = CONNECTORS.filter((c) => c.auth?.type === "local");
|
||||
expect(oauth.length).toBe(5);
|
||||
expect(apiKey.length).toBe(18);
|
||||
expect(cloud.length).toBe(3);
|
||||
expect(local.length).toBe(2);
|
||||
});
|
||||
|
||||
it("uses unique connector ids", () => {
|
||||
@@ -39,7 +41,7 @@ describe("CONNECTORS catalog", () => {
|
||||
"minimax", "minimax-cn", "huggingface", "opencode", "opencode-go", "kimi-coding",
|
||||
]);
|
||||
// Providers wired through src/main/model-resolver.ts instead of pi's catalog.
|
||||
const localProviders = new Set(["fireworks", "fireworks-firepass"]);
|
||||
const localProviders = new Set(["fireworks", "fireworks-firepass", "ollama", "lm-studio"]);
|
||||
for (const connector of CONNECTORS) {
|
||||
const provider = connector.provider ?? "";
|
||||
const recognised = knownPiProviders.has(provider) || localProviders.has(provider);
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { AppDatabase } from "../../src/main/database";
|
||||
import { createBudgetEnforcementService } from "../../src/main/budget-enforcement-service";
|
||||
import { createRunDispatch } from "../../src/main/orchestration/run-dispatch";
|
||||
import { createTaskRunner } from "../../src/main/orchestration/task-runner";
|
||||
|
||||
vi.mock("electron", () => ({
|
||||
safeStorage: {
|
||||
isEncryptionAvailable: () => false,
|
||||
getSelectedStorageBackend: () => "basic_text",
|
||||
encryptString: (value: string) => Buffer.from(value, "utf8"),
|
||||
decryptString: (value: Buffer) => value.toString("utf8"),
|
||||
},
|
||||
}));
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
async function createTempDir(prefix: string) {
|
||||
const dir = await mkdtemp(join(tmpdir(), prefix));
|
||||
tempDirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
function createLogger() {
|
||||
return {
|
||||
debug: vi.fn(),
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
function createHarness(db: AppDatabase) {
|
||||
const logger = createLogger();
|
||||
const enqueueWorkerRun = vi.fn();
|
||||
const budgetService = createBudgetEnforcementService({
|
||||
db,
|
||||
logger,
|
||||
notify: vi.fn(),
|
||||
publishDomainChanged: vi.fn(),
|
||||
});
|
||||
const deps = {
|
||||
db,
|
||||
logger,
|
||||
publishDomainChanged: vi.fn(),
|
||||
emitEvent: vi.fn(),
|
||||
notify: vi.fn(),
|
||||
createRunToken: () => "token",
|
||||
getApiPort: () => 3210,
|
||||
dispatchAutomation: vi.fn(),
|
||||
hasWorkerReady: () => true,
|
||||
enqueueWorkerRun,
|
||||
isQuitting: () => false,
|
||||
listRuntimeSkillEntries: () => [],
|
||||
budgetService,
|
||||
checkpointEngine: { shouldPauseHeartbeat: () => false },
|
||||
};
|
||||
const runDispatch = createRunDispatch(deps);
|
||||
const taskRunner = createTaskRunner(
|
||||
deps,
|
||||
runDispatch.resolveWorkspace,
|
||||
runDispatch.getRunEligibilityFailure,
|
||||
runDispatch.dispatchRunToWorker,
|
||||
(filter) => db.core.runs.hasActiveRun(filter),
|
||||
);
|
||||
return { enqueueWorkerRun, taskRunner };
|
||||
}
|
||||
|
||||
function createCompanyGraph(db: AppDatabase, profileDir: string) {
|
||||
db.core.connectors.updateConnectorHealth({
|
||||
id: "openai-codex",
|
||||
status: "ready",
|
||||
authState: "authenticated",
|
||||
version: "test",
|
||||
lastError: null,
|
||||
});
|
||||
const companyId = db.core.company.saveCompany({
|
||||
name: "Queued Recovery Co",
|
||||
description: "Recovers queued runs deterministically",
|
||||
status: "active",
|
||||
});
|
||||
const workspaceId = db.core.workspaces.saveWorkspace({
|
||||
companyId,
|
||||
name: "Primary",
|
||||
localPath: profileDir,
|
||||
repoUrl: "",
|
||||
repoRef: "",
|
||||
isPrimary: true,
|
||||
});
|
||||
const agentId = db.core.agents.saveAgent({
|
||||
companyId,
|
||||
name: "Engineer",
|
||||
role: "Engineer",
|
||||
title: "Engineer",
|
||||
status: "idle",
|
||||
reportsTo: null,
|
||||
connectorId: "openai-codex",
|
||||
workspaceId,
|
||||
model: "gpt-5.4",
|
||||
capabilities: "Runs queued recovery tasks",
|
||||
budgetMonthlyUsd: 100,
|
||||
});
|
||||
return { companyId, workspaceId, agentId };
|
||||
}
|
||||
|
||||
function createQueuedTaskRun(db: AppDatabase, input: {
|
||||
companyId: string;
|
||||
workspaceId: string;
|
||||
agentId: string;
|
||||
title: string;
|
||||
}) {
|
||||
const taskId = db.core.tasks.saveTask({
|
||||
companyId: input.companyId,
|
||||
title: input.title,
|
||||
description: "Queued before worker startup recovery.",
|
||||
assigneeAgentId: input.agentId,
|
||||
workspaceId: input.workspaceId,
|
||||
priority: "high",
|
||||
status: "todo",
|
||||
});
|
||||
const run = db.core.runs.createTaskRun({
|
||||
companyId: input.companyId,
|
||||
taskId,
|
||||
agentId: input.agentId,
|
||||
workspaceId: input.workspaceId,
|
||||
connectorId: "openai-codex",
|
||||
});
|
||||
if (!run) throw new Error("Expected queued run to be created.");
|
||||
return { taskId, runId: run.id };
|
||||
}
|
||||
|
||||
describe("queued run recovery", () => {
|
||||
it("dispatches the first queued run and releases same-agent queued siblings", async () => {
|
||||
const profileDir = await createTempDir("agentcompany-queued-recovery-");
|
||||
const db = new AppDatabase(profileDir);
|
||||
db.init();
|
||||
const { companyId, workspaceId, agentId } = createCompanyGraph(db, profileDir);
|
||||
const first = createQueuedTaskRun(db, { companyId, workspaceId, agentId, title: "First queued task" });
|
||||
const second = createQueuedTaskRun(db, { companyId, workspaceId, agentId, title: "Second queued task" });
|
||||
const { enqueueWorkerRun, taskRunner } = createHarness(db);
|
||||
|
||||
taskRunner.resumeQueuedRuns();
|
||||
|
||||
expect(enqueueWorkerRun).toHaveBeenCalledOnce();
|
||||
expect(enqueueWorkerRun.mock.calls[0]?.[0]).toEqual(expect.objectContaining({
|
||||
runId: first.runId,
|
||||
taskId: first.taskId,
|
||||
agentId,
|
||||
trigger: "recovery",
|
||||
}));
|
||||
expect(db.core.runs.getRun(second.runId).status).toBe("cancelled");
|
||||
expect(db.core.entity.getTask(second.taskId).status).toBe("todo");
|
||||
db.close();
|
||||
});
|
||||
|
||||
it("releases queued runs when recovery is blocked by connector readiness", async () => {
|
||||
const profileDir = await createTempDir("agentcompany-queued-connector-block-");
|
||||
const db = new AppDatabase(profileDir);
|
||||
db.init();
|
||||
const { companyId, workspaceId, agentId } = createCompanyGraph(db, profileDir);
|
||||
const queued = createQueuedTaskRun(db, { companyId, workspaceId, agentId, title: "Connector blocked task" });
|
||||
db.core.connectors.updateConnectorHealth({
|
||||
id: "openai-codex",
|
||||
status: "auth_required",
|
||||
authState: "missing",
|
||||
version: "test",
|
||||
lastError: "Login required",
|
||||
});
|
||||
const { enqueueWorkerRun, taskRunner } = createHarness(db);
|
||||
|
||||
taskRunner.resumeQueuedRuns();
|
||||
|
||||
expect(enqueueWorkerRun).not.toHaveBeenCalled();
|
||||
expect(db.core.runs.getRun(queued.runId).status).toBe("cancelled");
|
||||
expect(db.core.entity.getTask(queued.taskId).status).toBe("todo");
|
||||
db.close();
|
||||
});
|
||||
});
|
||||
@@ -199,6 +199,38 @@ describe("unified scheduler heartbeat cadence", () => {
|
||||
expect(wakeAgentIfPossible).toHaveBeenCalledWith(agentId, companyId, "timer");
|
||||
});
|
||||
|
||||
it("does not wake assigned work for an agent with heartbeat disabled", () => {
|
||||
const companyId = createCompany(db);
|
||||
createConnector(db);
|
||||
const workspaceId = createWorkspace(db, companyId, tempDir);
|
||||
const agentId = createAgent(db, companyId, workspaceId);
|
||||
db.core.agents.setHeartbeat(agentId, false, 1);
|
||||
db.core.tasks.saveTask({
|
||||
companyId,
|
||||
title: "Paused operating work",
|
||||
description: "Manual heartbeat disable should pause periodic recovery.",
|
||||
assigneeAgentId: agentId,
|
||||
workspaceId,
|
||||
priority: "high",
|
||||
status: "todo",
|
||||
});
|
||||
const wakeAgentIfPossible = vi.fn();
|
||||
const scheduler = createScheduler(
|
||||
createDeps(db),
|
||||
() => false,
|
||||
wakeAgentIfPossible,
|
||||
);
|
||||
|
||||
try {
|
||||
scheduler.startAllSchedulers();
|
||||
vi.advanceTimersByTime(60_000);
|
||||
} finally {
|
||||
scheduler.stopAllSchedulers();
|
||||
}
|
||||
|
||||
expect(wakeAgentIfPossible).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("wakes an agent with a queued wakeup request on heartbeat cadence", () => {
|
||||
const companyId = createCompany(db);
|
||||
createConnector(db);
|
||||
|
||||
@@ -24,19 +24,45 @@ function runFinishedMessage(runId: string) {
|
||||
};
|
||||
}
|
||||
|
||||
function createHandlerDeps(calls: string[]) {
|
||||
const releaseWorkspaceLock = vi.fn(() => calls.push("releaseWorkspaceLock"));
|
||||
function createHandlerDeps(calls: string[], options: {
|
||||
taskId?: string | null;
|
||||
taskStatus?: string;
|
||||
sessionAccumulateThrows?: boolean;
|
||||
processRunCompletionThrows?: boolean;
|
||||
activityThrows?: boolean;
|
||||
emitEventThrows?: boolean;
|
||||
wakeupThrows?: boolean;
|
||||
releaseWorkspaceThrows?: boolean;
|
||||
markRunCompletionFailureThrows?: boolean;
|
||||
} = {}) {
|
||||
const releaseWorkspaceLock = vi.fn(() => {
|
||||
calls.push("releaseWorkspaceLock");
|
||||
if (options.releaseWorkspaceThrows) {
|
||||
throw new Error("workspace release failed");
|
||||
}
|
||||
});
|
||||
const finishRun = vi.fn(() => {
|
||||
calls.push("finishRun");
|
||||
return { budgetHardStopped: false };
|
||||
});
|
||||
const rotateSession = vi.fn(() => calls.push("rotateSession"));
|
||||
const processRunCompletion = vi.fn(() => calls.push("processRunCompletion"));
|
||||
const processRunCompletion = vi.fn(() => {
|
||||
calls.push("processRunCompletion");
|
||||
if (options.processRunCompletionThrows) {
|
||||
throw new Error("processRunCompletion failed");
|
||||
}
|
||||
});
|
||||
const releaseRunResources = vi.fn((): RunResourceReleaseResult => {
|
||||
calls.push("releaseRunResources");
|
||||
return { ok: true as const };
|
||||
});
|
||||
const markRunCompletionFailure = vi.fn(() => calls.push("markRunCompletionFailure"));
|
||||
const markRunCompletionFailure = vi.fn(() => {
|
||||
calls.push("markRunCompletionFailure");
|
||||
if (options.markRunCompletionFailureThrows) {
|
||||
throw new Error("mark completion failed");
|
||||
}
|
||||
});
|
||||
const advanceTaskStatus = vi.fn(() => calls.push("advanceTaskStatus"));
|
||||
|
||||
const db = {
|
||||
core: {
|
||||
@@ -45,14 +71,21 @@ function createHandlerDeps(calls: string[]) {
|
||||
id: RUN_ID,
|
||||
companyId: "company-1",
|
||||
agentId: "agent-1",
|
||||
taskId: null,
|
||||
taskId: options.taskId ?? null,
|
||||
workspaceId: "workspace-1",
|
||||
})),
|
||||
finishRun,
|
||||
markRunCompletionFailure,
|
||||
},
|
||||
workspaces: { releaseWorkspaceLock },
|
||||
activity: { addActivity: vi.fn(() => calls.push("addActivity")) },
|
||||
activity: {
|
||||
addActivity: vi.fn(() => {
|
||||
calls.push("addActivity");
|
||||
if (options.activityThrows) {
|
||||
throw new Error("activity write failed");
|
||||
}
|
||||
}),
|
||||
},
|
||||
entity: {
|
||||
getAgent: vi.fn(() => ({
|
||||
id: "agent-1",
|
||||
@@ -60,11 +93,19 @@ function createHandlerDeps(calls: string[]) {
|
||||
reportsTo: "manager-1",
|
||||
sessionId: "session-1",
|
||||
})),
|
||||
getTask: vi.fn(() => ({
|
||||
id: options.taskId ?? "task-1",
|
||||
companyId: "company-1",
|
||||
agentId: "agent-1",
|
||||
title: "Completion task",
|
||||
status: options.taskStatus ?? "in_progress",
|
||||
})),
|
||||
countActiveGoalsForCompany: vi.fn(() => 0),
|
||||
},
|
||||
tasks: {
|
||||
getSessionById: vi.fn(() => ({ id: "session-1" })),
|
||||
listAgentTasks: vi.fn(() => []),
|
||||
advanceTaskStatus,
|
||||
},
|
||||
agents: {
|
||||
getDirectReports: vi.fn(() => []),
|
||||
@@ -76,14 +117,24 @@ function createHandlerDeps(calls: string[]) {
|
||||
},
|
||||
automation: {
|
||||
wakeup: {
|
||||
promoteWakeupRequests: vi.fn(() => 0),
|
||||
promoteWakeupRequests: vi.fn(() => {
|
||||
if (options.wakeupThrows) {
|
||||
throw new Error("wakeup promotion failed");
|
||||
}
|
||||
return 0;
|
||||
}),
|
||||
getPreferredWakeTrigger: vi.fn(() => null),
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const sessionService = {
|
||||
accumulateTokenUsage: vi.fn(() => calls.push("accumulateTokenUsage")),
|
||||
accumulateTokenUsage: vi.fn(() => {
|
||||
calls.push("accumulateTokenUsage");
|
||||
if (options.sessionAccumulateThrows) {
|
||||
throw new Error("session accumulation failed");
|
||||
}
|
||||
}),
|
||||
evaluateRotation: vi.fn(() => ({ shouldRotate: true, reason: "unit-test" })),
|
||||
rotateSession,
|
||||
};
|
||||
@@ -98,7 +149,12 @@ function createHandlerDeps(calls: string[]) {
|
||||
error: vi.fn(),
|
||||
},
|
||||
isQuitting: vi.fn(() => false),
|
||||
emitEvent: vi.fn(() => calls.push("emitEvent")),
|
||||
emitEvent: vi.fn(() => {
|
||||
calls.push("emitEvent");
|
||||
if (options.emitEventThrows) {
|
||||
throw new Error("emit failed");
|
||||
}
|
||||
}),
|
||||
wakeAgentIfPossible: vi.fn(() => null),
|
||||
processRunCompletion,
|
||||
notify: vi.fn(),
|
||||
@@ -111,6 +167,7 @@ function createHandlerDeps(calls: string[]) {
|
||||
processRunCompletion,
|
||||
releaseRunResources,
|
||||
markRunCompletionFailure,
|
||||
advanceTaskStatus,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -227,6 +284,265 @@ describe("worker completion boundary", () => {
|
||||
expect(worker.postMessage).toHaveBeenCalledWith({ type: "run-completion-ack", runId: RUN_ID });
|
||||
});
|
||||
|
||||
it("continues completion when artifact failure marking throws", async () => {
|
||||
const calls: string[] = [];
|
||||
const {
|
||||
deps,
|
||||
releaseWorkspaceLock,
|
||||
processRunCompletion,
|
||||
releaseRunResources,
|
||||
markRunCompletionFailure,
|
||||
} = createHandlerDeps(calls, { markRunCompletionFailureThrows: true });
|
||||
releaseRunResources.mockImplementationOnce(() => {
|
||||
calls.push("releaseRunResources");
|
||||
return { ok: false as const, errorMessage: "completion_reconcile failed" };
|
||||
});
|
||||
const worker = { postMessage: vi.fn(() => calls.push("completionAck")) };
|
||||
|
||||
await handleWorkerMessage(
|
||||
runFinishedMessage(RUN_ID),
|
||||
worker as never,
|
||||
deps as never,
|
||||
workerMessageSchema,
|
||||
false,
|
||||
2,
|
||||
30_000,
|
||||
new Map(),
|
||||
3,
|
||||
new Set(),
|
||||
new Map(),
|
||||
);
|
||||
|
||||
expect(markRunCompletionFailure).toHaveBeenCalledWith(RUN_ID, "completion_reconcile failed");
|
||||
expect(processRunCompletion).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: RUN_ID }),
|
||||
expect.objectContaining({
|
||||
status: "failed",
|
||||
errorMessage: "completion_reconcile failed",
|
||||
usageStatus: "error_stop",
|
||||
}),
|
||||
);
|
||||
expect(releaseWorkspaceLock).toHaveBeenCalledWith("workspace-1", RUN_ID);
|
||||
expect(worker.postMessage).toHaveBeenCalledWith({ type: "run-completion-ack", runId: RUN_ID });
|
||||
});
|
||||
|
||||
it("continues completion automation when session accumulation fails", async () => {
|
||||
const calls: string[] = [];
|
||||
const {
|
||||
deps,
|
||||
releaseWorkspaceLock,
|
||||
processRunCompletion,
|
||||
} = createHandlerDeps(calls, { sessionAccumulateThrows: true });
|
||||
const worker = { postMessage: vi.fn(() => calls.push("completionAck")) };
|
||||
|
||||
await handleWorkerMessage(
|
||||
runFinishedMessage(RUN_ID),
|
||||
worker as never,
|
||||
deps as never,
|
||||
workerMessageSchema,
|
||||
false,
|
||||
2,
|
||||
30_000,
|
||||
new Map(),
|
||||
3,
|
||||
new Set(),
|
||||
new Map(),
|
||||
);
|
||||
|
||||
expect(processRunCompletion).toHaveBeenCalled();
|
||||
expect(releaseWorkspaceLock).toHaveBeenCalledWith("workspace-1", RUN_ID);
|
||||
expect(worker.postMessage).toHaveBeenCalledWith({ type: "run-completion-ack", runId: RUN_ID });
|
||||
});
|
||||
|
||||
it("marks completion failed and blocks the task when completion automation throws", async () => {
|
||||
const calls: string[] = [];
|
||||
const {
|
||||
deps,
|
||||
releaseWorkspaceLock,
|
||||
markRunCompletionFailure,
|
||||
advanceTaskStatus,
|
||||
} = createHandlerDeps(calls, {
|
||||
taskId: "task-1",
|
||||
taskStatus: "in_progress",
|
||||
processRunCompletionThrows: true,
|
||||
});
|
||||
const worker = { postMessage: vi.fn(() => calls.push("completionAck")) };
|
||||
|
||||
await handleWorkerMessage(
|
||||
runFinishedMessage(RUN_ID),
|
||||
worker as never,
|
||||
deps as never,
|
||||
workerMessageSchema,
|
||||
false,
|
||||
2,
|
||||
30_000,
|
||||
new Map(),
|
||||
3,
|
||||
new Set(),
|
||||
new Map(),
|
||||
);
|
||||
|
||||
expect(markRunCompletionFailure).toHaveBeenCalledWith(RUN_ID, "processRunCompletion failed");
|
||||
expect(advanceTaskStatus).toHaveBeenCalledWith(
|
||||
"task-1",
|
||||
"blocked",
|
||||
expect.stringContaining("Run completion automation failed: processRunCompletion failed"),
|
||||
);
|
||||
expect(releaseWorkspaceLock).toHaveBeenCalledWith("workspace-1", RUN_ID);
|
||||
expect(worker.postMessage).toHaveBeenCalledWith({ type: "run-completion-ack", runId: RUN_ID });
|
||||
});
|
||||
|
||||
it("releases workspace and acknowledges completion when activity logging fails", async () => {
|
||||
const calls: string[] = [];
|
||||
const {
|
||||
deps,
|
||||
releaseWorkspaceLock,
|
||||
processRunCompletion,
|
||||
} = createHandlerDeps(calls, { activityThrows: true });
|
||||
const worker = { postMessage: vi.fn(() => calls.push("completionAck")) };
|
||||
|
||||
await handleWorkerMessage(
|
||||
runFinishedMessage(RUN_ID),
|
||||
worker as never,
|
||||
deps as never,
|
||||
workerMessageSchema,
|
||||
false,
|
||||
2,
|
||||
30_000,
|
||||
new Map(),
|
||||
3,
|
||||
new Set(),
|
||||
new Map(),
|
||||
);
|
||||
|
||||
expect(processRunCompletion).toHaveBeenCalled();
|
||||
expect(releaseWorkspaceLock).toHaveBeenCalledWith("workspace-1", RUN_ID);
|
||||
expect(worker.postMessage).toHaveBeenCalledWith({ type: "run-completion-ack", runId: RUN_ID });
|
||||
});
|
||||
|
||||
it("releases workspace and acknowledges completion when renderer event emission fails", async () => {
|
||||
const calls: string[] = [];
|
||||
const {
|
||||
deps,
|
||||
releaseWorkspaceLock,
|
||||
processRunCompletion,
|
||||
} = createHandlerDeps(calls, { emitEventThrows: true });
|
||||
const worker = { postMessage: vi.fn(() => calls.push("completionAck")) };
|
||||
|
||||
await handleWorkerMessage(
|
||||
runFinishedMessage(RUN_ID),
|
||||
worker as never,
|
||||
deps as never,
|
||||
workerMessageSchema,
|
||||
false,
|
||||
2,
|
||||
30_000,
|
||||
new Map(),
|
||||
3,
|
||||
new Set(),
|
||||
new Map(),
|
||||
);
|
||||
|
||||
expect(processRunCompletion).toHaveBeenCalled();
|
||||
expect(releaseWorkspaceLock).toHaveBeenCalledWith("workspace-1", RUN_ID);
|
||||
expect(worker.postMessage).toHaveBeenCalledWith({ type: "run-completion-ack", runId: RUN_ID });
|
||||
});
|
||||
|
||||
it("releases workspace and acknowledges completion when deferred wake promotion fails", async () => {
|
||||
const calls: string[] = [];
|
||||
const {
|
||||
deps,
|
||||
releaseWorkspaceLock,
|
||||
processRunCompletion,
|
||||
} = createHandlerDeps(calls, { wakeupThrows: true });
|
||||
const worker = { postMessage: vi.fn(() => calls.push("completionAck")) };
|
||||
|
||||
await handleWorkerMessage(
|
||||
runFinishedMessage(RUN_ID),
|
||||
worker as never,
|
||||
deps as never,
|
||||
workerMessageSchema,
|
||||
false,
|
||||
2,
|
||||
30_000,
|
||||
new Map(),
|
||||
3,
|
||||
new Set(),
|
||||
new Map(),
|
||||
);
|
||||
|
||||
expect(processRunCompletion).toHaveBeenCalled();
|
||||
expect(releaseWorkspaceLock).toHaveBeenCalledWith("workspace-1", RUN_ID);
|
||||
expect(worker.postMessage).toHaveBeenCalledWith({ type: "run-completion-ack", runId: RUN_ID });
|
||||
});
|
||||
|
||||
it("acknowledges completion when workspace release fails", async () => {
|
||||
const calls: string[] = [];
|
||||
const {
|
||||
deps,
|
||||
releaseWorkspaceLock,
|
||||
processRunCompletion,
|
||||
} = createHandlerDeps(calls, { releaseWorkspaceThrows: true });
|
||||
const worker = { postMessage: vi.fn(() => calls.push("completionAck")) };
|
||||
|
||||
await handleWorkerMessage(
|
||||
runFinishedMessage(RUN_ID),
|
||||
worker as never,
|
||||
deps as never,
|
||||
workerMessageSchema,
|
||||
false,
|
||||
2,
|
||||
30_000,
|
||||
new Map(),
|
||||
3,
|
||||
new Set(),
|
||||
new Map(),
|
||||
);
|
||||
|
||||
expect(processRunCompletion).toHaveBeenCalled();
|
||||
expect(releaseWorkspaceLock).toHaveBeenCalledWith("workspace-1", RUN_ID);
|
||||
expect(worker.postMessage).toHaveBeenCalledWith({ type: "run-completion-ack", runId: RUN_ID });
|
||||
});
|
||||
|
||||
it("blocks the task and acknowledges completion when marking completion failure throws", async () => {
|
||||
const calls: string[] = [];
|
||||
const {
|
||||
deps,
|
||||
releaseWorkspaceLock,
|
||||
markRunCompletionFailure,
|
||||
advanceTaskStatus,
|
||||
} = createHandlerDeps(calls, {
|
||||
taskId: "task-1",
|
||||
taskStatus: "in_progress",
|
||||
processRunCompletionThrows: true,
|
||||
markRunCompletionFailureThrows: true,
|
||||
});
|
||||
const worker = { postMessage: vi.fn(() => calls.push("completionAck")) };
|
||||
|
||||
await handleWorkerMessage(
|
||||
runFinishedMessage(RUN_ID),
|
||||
worker as never,
|
||||
deps as never,
|
||||
workerMessageSchema,
|
||||
false,
|
||||
2,
|
||||
30_000,
|
||||
new Map(),
|
||||
3,
|
||||
new Set(),
|
||||
new Map(),
|
||||
);
|
||||
|
||||
expect(markRunCompletionFailure).toHaveBeenCalledWith(RUN_ID, "processRunCompletion failed");
|
||||
expect(advanceTaskStatus).toHaveBeenCalledWith(
|
||||
"task-1",
|
||||
"blocked",
|
||||
expect.stringContaining("Run completion automation failed: processRunCompletion failed"),
|
||||
);
|
||||
expect(releaseWorkspaceLock).toHaveBeenCalledWith("workspace-1", RUN_ID);
|
||||
expect(worker.postMessage).toHaveBeenCalledWith({ type: "run-completion-ack", runId: RUN_ID });
|
||||
});
|
||||
|
||||
it("keeps same-workspace runs queued until parent acknowledges completion", async () => {
|
||||
const parentPort = new EventEmitter() as EventEmitter & {
|
||||
postMessage: ReturnType<typeof vi.fn>;
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("electron", () => ({
|
||||
safeStorage: {
|
||||
isEncryptionAvailable: () => false,
|
||||
getSelectedStorageBackend: () => "basic_text",
|
||||
encryptString: (value: string) => Buffer.from(value, "utf8"),
|
||||
decryptString: (value: Buffer) => value.toString("utf8"),
|
||||
},
|
||||
}));
|
||||
|
||||
import { AppDatabase } from "../../src/main/database";
|
||||
import { handleWorkerExit, requeueQueuedRunsAfterWorkerRestart } from "../../src/main/worker/entry/crash-recovery";
|
||||
|
||||
function createLogger() {
|
||||
return {
|
||||
debug: vi.fn(),
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
function createCompany(db: AppDatabase): string {
|
||||
return db.core.company.saveCompany({
|
||||
name: "Crash Recovery Co",
|
||||
description: "Worker crash recovery contract test",
|
||||
status: "active",
|
||||
});
|
||||
}
|
||||
|
||||
function createConnector(db: AppDatabase): void {
|
||||
db.core.connectors.saveConnector({
|
||||
id: "openai-codex",
|
||||
model: "gpt-5.4",
|
||||
envBindingText: "",
|
||||
notes: "Test connector",
|
||||
});
|
||||
}
|
||||
|
||||
function createWorkspace(db: AppDatabase, companyId: string, localPath: string): string {
|
||||
return db.core.workspaces.saveWorkspace({
|
||||
companyId,
|
||||
name: "Primary",
|
||||
localPath,
|
||||
repoUrl: "",
|
||||
repoRef: "",
|
||||
isPrimary: true,
|
||||
});
|
||||
}
|
||||
|
||||
function createAgent(db: AppDatabase, companyId: string, workspaceId: string): string {
|
||||
return db.core.agents.saveAgent({
|
||||
companyId,
|
||||
name: "Recovery Agent",
|
||||
role: "Engineer",
|
||||
title: "Engineer",
|
||||
status: "active",
|
||||
reportsTo: null,
|
||||
connectorId: "openai-codex",
|
||||
workspaceId,
|
||||
model: "gpt-5.4",
|
||||
capabilities: "Continues work after runtime crashes",
|
||||
budgetMonthlyUsd: 100,
|
||||
});
|
||||
}
|
||||
|
||||
describe("worker crash recovery", () => {
|
||||
let tempDir: string;
|
||||
let db: AppDatabase;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), "agentcompany-worker-crash-"));
|
||||
db = new AppDatabase(tempDir);
|
||||
db.init();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
db.close();
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("returns interrupted running tasks to checkoutable work and wakes the original agent", () => {
|
||||
const companyId = createCompany(db);
|
||||
createConnector(db);
|
||||
const workspaceId = createWorkspace(db, companyId, tempDir);
|
||||
const agentId = createAgent(db, companyId, workspaceId);
|
||||
const taskId = db.core.tasks.saveTask({
|
||||
companyId,
|
||||
title: "Continue Phase 2",
|
||||
description: "This task must remain recoverable after the worker process exits.",
|
||||
assigneeAgentId: agentId,
|
||||
workspaceId,
|
||||
priority: "high",
|
||||
status: "todo",
|
||||
});
|
||||
const run = db.core.runs.createTaskRun({
|
||||
companyId,
|
||||
taskId,
|
||||
agentId,
|
||||
workspaceId,
|
||||
connectorId: "openai-codex",
|
||||
});
|
||||
expect(run).not.toBeNull();
|
||||
db.core.runs.markRunStatus(run!.id, "running", "Started");
|
||||
expect(db.core.entity.getTask(taskId).status).toBe("in_progress");
|
||||
expect(db.core.runs.hasActiveRun({ companyId, agentId })).toBe(true);
|
||||
|
||||
const wakeAgentIfPossible = vi.fn(() => null);
|
||||
handleWorkerExit(
|
||||
1,
|
||||
{
|
||||
db,
|
||||
logger: createLogger(),
|
||||
isQuitting: () => false,
|
||||
publishDomainChanged: vi.fn(),
|
||||
emitEvent: vi.fn(),
|
||||
wakeAgentIfPossible,
|
||||
},
|
||||
);
|
||||
|
||||
expect(db.core.runs.getRun(run!.id).status).toBe("interrupted");
|
||||
expect(db.core.entity.getTask(taskId).status).toBe("todo");
|
||||
expect(db.core.runs.hasActiveRun({ companyId, agentId })).toBe(false);
|
||||
expect(wakeAgentIfPossible).toHaveBeenCalledWith(agentId, companyId, "recovery");
|
||||
|
||||
const retryRun = db.core.runs.createTaskRun({
|
||||
companyId,
|
||||
taskId,
|
||||
agentId,
|
||||
workspaceId,
|
||||
connectorId: "openai-codex",
|
||||
});
|
||||
expect(retryRun).not.toBeNull();
|
||||
});
|
||||
|
||||
it("requeues queued runs after the replacement worker is ready", () => {
|
||||
const companyId = createCompany(db);
|
||||
createConnector(db);
|
||||
const workspaceId = createWorkspace(db, companyId, tempDir);
|
||||
const agentId = createAgent(db, companyId, workspaceId);
|
||||
const taskId = db.core.tasks.saveTask({
|
||||
companyId,
|
||||
title: "Queued recovery task",
|
||||
description: "This run was queued when the worker process exited.",
|
||||
assigneeAgentId: agentId,
|
||||
workspaceId,
|
||||
priority: "high",
|
||||
status: "todo",
|
||||
});
|
||||
const run = db.core.runs.createTaskRun({
|
||||
companyId,
|
||||
taskId,
|
||||
agentId,
|
||||
workspaceId,
|
||||
connectorId: "openai-codex",
|
||||
});
|
||||
expect(run).not.toBeNull();
|
||||
expect(db.core.runs.getRun(run!.id).status).toBe("queued");
|
||||
expect(db.core.runs.hasActiveRun({ companyId, agentId })).toBe(true);
|
||||
|
||||
const dispatchRunToWorker = vi.fn();
|
||||
const requeueCandidates = handleWorkerExit(
|
||||
1,
|
||||
{
|
||||
db,
|
||||
logger: createLogger(),
|
||||
isQuitting: () => false,
|
||||
publishDomainChanged: vi.fn(),
|
||||
emitEvent: vi.fn(),
|
||||
wakeAgentIfPossible: vi.fn(() => null),
|
||||
},
|
||||
);
|
||||
|
||||
expect(dispatchRunToWorker).not.toHaveBeenCalled();
|
||||
expect(requeueCandidates).toHaveLength(1);
|
||||
expect(db.core.runs.getRun(run!.id).status).toBe("queued");
|
||||
|
||||
requeueQueuedRunsAfterWorkerRestart(
|
||||
requeueCandidates,
|
||||
{
|
||||
db,
|
||||
logger: createLogger(),
|
||||
dispatchRunToWorker,
|
||||
},
|
||||
);
|
||||
|
||||
expect(dispatchRunToWorker).toHaveBeenCalledOnce();
|
||||
expect(dispatchRunToWorker).toHaveBeenCalledWith(expect.objectContaining({
|
||||
runId: run!.id,
|
||||
companyId,
|
||||
task: expect.objectContaining({ id: taskId }),
|
||||
agent: expect.objectContaining({ id: agentId }),
|
||||
trigger: "recovery",
|
||||
}));
|
||||
expect(db.core.runs.hasActiveRun({ companyId, agentId })).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
forkMock: vi.fn(),
|
||||
@@ -25,6 +25,15 @@ function createWorker() {
|
||||
}
|
||||
|
||||
describe("worker service shutdown", () => {
|
||||
beforeEach(() => {
|
||||
mocks.forkMock.mockReset();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("keeps completed run outcomes when cleaning shutdown runs", async () => {
|
||||
const worker = createWorker();
|
||||
mocks.forkMock.mockReturnValueOnce(worker);
|
||||
@@ -83,4 +92,128 @@ describe("worker service shutdown", () => {
|
||||
status: "interrupted",
|
||||
}));
|
||||
});
|
||||
|
||||
it("clears the crashed worker reference before recovery runs", () => {
|
||||
vi.useFakeTimers();
|
||||
const worker = createWorker();
|
||||
mocks.forkMock.mockReturnValueOnce(worker);
|
||||
const publishDomainChanged = vi.fn();
|
||||
const service = createWorkerService({
|
||||
db: {
|
||||
projection: {
|
||||
snapshot: {
|
||||
listSnapshot: vi.fn(() => ({ runs: [] })),
|
||||
},
|
||||
},
|
||||
},
|
||||
logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
workerPath: "/tmp/worker.js",
|
||||
isQuitting: () => false,
|
||||
publishDomainChanged,
|
||||
emitEvent: vi.fn(),
|
||||
wakeAgentIfPossible: vi.fn(() => null),
|
||||
dispatchRunToWorker: vi.fn(),
|
||||
processRunCompletion: vi.fn(),
|
||||
onRunStarted: vi.fn(),
|
||||
onRunCompleted: vi.fn(),
|
||||
notify: vi.fn(),
|
||||
budgetService: undefined,
|
||||
sessionService: undefined,
|
||||
getRuntimeBridge: vi.fn(() => null),
|
||||
getRuntimeMessageInjector: vi.fn(() => null),
|
||||
} as never);
|
||||
|
||||
service.createWorker();
|
||||
expect(service.getWorker()).toBe(worker);
|
||||
worker.emit("exit", 1);
|
||||
|
||||
expect(service.getWorker()).toBeNull();
|
||||
expect(publishDomainChanged).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("requeues queued crash runs after the replacement worker is created", () => {
|
||||
vi.useFakeTimers();
|
||||
const crashedWorker = createWorker();
|
||||
const replacementWorker = createWorker();
|
||||
mocks.forkMock
|
||||
.mockReturnValueOnce(crashedWorker)
|
||||
.mockReturnValueOnce(replacementWorker);
|
||||
const queuedRun = {
|
||||
id: "run-queued",
|
||||
status: "queued",
|
||||
taskId: "task-1",
|
||||
agentId: "agent-1",
|
||||
companyId: "company-1",
|
||||
workspaceId: null,
|
||||
connectorId: "openai-codex",
|
||||
};
|
||||
const dispatchRunToWorker = vi.fn();
|
||||
const service = createWorkerService({
|
||||
db: {
|
||||
projection: {
|
||||
snapshot: {
|
||||
listSnapshot: vi.fn(() => ({
|
||||
runs: [queuedRun],
|
||||
companies: [{ id: "company-1", name: "Company", description: "Recovered company" }],
|
||||
})),
|
||||
},
|
||||
},
|
||||
core: {
|
||||
runs: {
|
||||
getRun: vi.fn(() => queuedRun),
|
||||
},
|
||||
entity: {
|
||||
getTask: vi.fn(() => ({
|
||||
id: "task-1",
|
||||
companyId: "company-1",
|
||||
workspaceId: null,
|
||||
projectId: null,
|
||||
requiresFilesystem: false,
|
||||
})),
|
||||
getAgent: vi.fn(() => ({
|
||||
id: "agent-1",
|
||||
companyId: "company-1",
|
||||
connectorId: "openai-codex",
|
||||
workspaceId: null,
|
||||
})),
|
||||
},
|
||||
connectors: {
|
||||
getConnector: vi.fn(() => ({ id: "openai-codex" })),
|
||||
},
|
||||
workspaces: {
|
||||
releaseWorkspaceLock: vi.fn(),
|
||||
getCompanyObservationWorkspace: vi.fn(() => null),
|
||||
},
|
||||
},
|
||||
},
|
||||
logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
workerPath: "/tmp/worker.js",
|
||||
isQuitting: () => false,
|
||||
publishDomainChanged: vi.fn(),
|
||||
emitEvent: vi.fn(),
|
||||
wakeAgentIfPossible: vi.fn(() => null),
|
||||
dispatchRunToWorker,
|
||||
processRunCompletion: vi.fn(),
|
||||
onRunStarted: vi.fn(),
|
||||
onRunCompleted: vi.fn(),
|
||||
notify: vi.fn(),
|
||||
budgetService: undefined,
|
||||
sessionService: undefined,
|
||||
getRuntimeBridge: vi.fn(() => null),
|
||||
getRuntimeMessageInjector: vi.fn(() => null),
|
||||
} as never);
|
||||
|
||||
service.createWorker();
|
||||
crashedWorker.emit("exit", 1);
|
||||
expect(dispatchRunToWorker).not.toHaveBeenCalled();
|
||||
|
||||
vi.advanceTimersByTime(1_000);
|
||||
|
||||
expect(service.getWorker()).toBe(replacementWorker);
|
||||
expect(dispatchRunToWorker).toHaveBeenCalledWith(expect.objectContaining({
|
||||
runId: "run-queued",
|
||||
companyId: "company-1",
|
||||
trigger: "recovery",
|
||||
}));
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user