Package release 0.4.182 from the current workspace snapshot
Constraint: Windows and Linux artifacts are built on native GitHub runners from a temporary snapshot branch Rejected: Build Windows/Linux locally on Apple Silicon | docker/wine packaging is not stable on this host Confidence: medium Scope-risk: moderate Directive: This snapshot branch is transport-only and should be deleted after the release completes Tested: local verify/build stages before snapshot creation Not-tested: remote packaging before workflow execution
This commit is contained in:
1 parent
49ea508ee1
commit
6d2b685e3a
66 files changed
+2300
-323
No files matched your search
@@ -1,6 +1,15 @@
|
||||
import type { WorkerServiceDependencies } from "./types";
|
||||
import { resolveWorkspaceWithLockMode } from "../../orchestration/resolve-workspace";
|
||||
|
||||
export interface QueuedRunRecoveryCandidate {
|
||||
id: string;
|
||||
taskId: string | null;
|
||||
agentId: string;
|
||||
companyId: string;
|
||||
workspaceId: string | null;
|
||||
connectorId: string;
|
||||
}
|
||||
|
||||
export function handleWorkerExit(
|
||||
code: number | null,
|
||||
{
|
||||
@@ -10,26 +19,16 @@ export function handleWorkerExit(
|
||||
publishDomainChanged,
|
||||
emitEvent: _emitEvent,
|
||||
wakeAgentIfPossible,
|
||||
dispatchRunToWorker,
|
||||
}: Pick<WorkerServiceDependencies, "db" | "logger" | "isQuitting" | "publishDomainChanged" | "emitEvent" | "wakeAgentIfPossible" | "dispatchRunToWorker">,
|
||||
createWorker: () => void,
|
||||
getWorker: () => ReturnType<typeof import("electron").utilityProcess.fork> | null,
|
||||
) {
|
||||
if (isQuitting()) return;
|
||||
}: Pick<WorkerServiceDependencies, "db" | "logger" | "isQuitting" | "publishDomainChanged" | "emitEvent" | "wakeAgentIfPossible">,
|
||||
): QueuedRunRecoveryCandidate[] {
|
||||
if (isQuitting()) return [];
|
||||
|
||||
logger.warn(`[worker] Worker exited with code ${code}, recovering...`);
|
||||
|
||||
const snapshot = db.projection.snapshot.listSnapshot();
|
||||
const affectedAgentIds = new Set<string>();
|
||||
const affectedCompanyIds = new Map<string, string>();
|
||||
const requeueCandidates: Array<{
|
||||
id: string;
|
||||
taskId: string | null;
|
||||
agentId: string;
|
||||
companyId: string;
|
||||
workspaceId: string | null;
|
||||
connectorId: string;
|
||||
}> = [];
|
||||
const requeueCandidates: QueuedRunRecoveryCandidate[] = [];
|
||||
|
||||
for (const run of snapshot.runs) {
|
||||
if (run.status === "running") {
|
||||
@@ -42,14 +41,11 @@ export function handleWorkerExit(
|
||||
signal: null,
|
||||
model: null,
|
||||
unattributedCost: true,
|
||||
recoverTaskForRetry: run.taskId !== null,
|
||||
});
|
||||
if (run.workspaceId) {
|
||||
db.core.workspaces.releaseWorkspaceLock(run.workspaceId, run.id);
|
||||
}
|
||||
// EXEC-02: Release execution lock on crash recovery
|
||||
if (run.taskId) {
|
||||
db.core.tasks.releaseTaskLock(run.taskId, run.id);
|
||||
}
|
||||
affectedAgentIds.add(run.agentId);
|
||||
affectedCompanyIds.set(run.agentId, run.companyId);
|
||||
|
||||
@@ -77,61 +73,70 @@ export function handleWorkerExit(
|
||||
logger.info(`[worker] Promoted ${promoted} deferred wake(s) for agent ${agentId} after crash recovery`);
|
||||
}
|
||||
const pendingWakeTrigger = db.automation.wakeup.getPreferredWakeTrigger(agentId, cid);
|
||||
if (pendingWakeTrigger) {
|
||||
wakeAgentIfPossible(agentId, cid, pendingWakeTrigger);
|
||||
}
|
||||
wakeAgentIfPossible(agentId, cid, pendingWakeTrigger ?? "recovery");
|
||||
}
|
||||
|
||||
publishDomainChanged();
|
||||
|
||||
createWorker();
|
||||
return requeueCandidates;
|
||||
}
|
||||
|
||||
const worker = getWorker();
|
||||
if (requeueCandidates.length > 0 && worker) {
|
||||
logger.info(`[worker] Re-enqueuing ${requeueCandidates.length} queued run(s) after worker restart.`);
|
||||
const recoverySnapshot = db.projection.snapshot.listSnapshot();
|
||||
for (const candidate of requeueCandidates) {
|
||||
try {
|
||||
// The crashed run may still own a workspace lock acquired during the
|
||||
// original dispatch; release it before re-dispatch so the replay
|
||||
// does not self-collide with "Workspace is locked by another run".
|
||||
if (candidate.workspaceId) {
|
||||
db.core.workspaces.releaseWorkspaceLock(candidate.workspaceId, candidate.id);
|
||||
}
|
||||
const task = candidate.taskId ? db.core.entity.getTask(candidate.taskId) : null;
|
||||
const agent = db.core.entity.getAgent(candidate.agentId);
|
||||
const connector = db.core.connectors.getConnector(agent.connectorId);
|
||||
const { workspace, lockMode } = resolveWorkspaceWithLockMode(db, task, agent, candidate.companyId);
|
||||
const recoveryCompany = recoverySnapshot.companies.find((company) => company.id === candidate.companyId);
|
||||
export function requeueQueuedRunsAfterWorkerRestart(
|
||||
requeueCandidates: QueuedRunRecoveryCandidate[],
|
||||
{
|
||||
db,
|
||||
logger,
|
||||
dispatchRunToWorker,
|
||||
}: Pick<WorkerServiceDependencies, "db" | "logger" | "dispatchRunToWorker">,
|
||||
): void {
|
||||
if (requeueCandidates.length === 0) return;
|
||||
logger.info(`[worker] Re-enqueuing ${requeueCandidates.length} queued run(s) after worker restart.`);
|
||||
const recoverySnapshot = db.projection.snapshot.listSnapshot();
|
||||
for (const candidate of requeueCandidates) {
|
||||
try {
|
||||
const currentRun = db.core.runs.getRun(candidate.id);
|
||||
if (currentRun.status !== "queued") continue;
|
||||
// The crashed run may still own a workspace lock acquired during the
|
||||
// original dispatch; release it before re-dispatch so the replay
|
||||
// does not self-collide with "Workspace is locked by another run".
|
||||
if (candidate.workspaceId) {
|
||||
db.core.workspaces.releaseWorkspaceLock(candidate.workspaceId, candidate.id);
|
||||
}
|
||||
const task = candidate.taskId ? db.core.entity.getTask(candidate.taskId) : null;
|
||||
const agent = db.core.entity.getAgent(candidate.agentId);
|
||||
const connector = db.core.connectors.getConnector(agent.connectorId);
|
||||
const { workspace, lockMode, observationPath } = resolveWorkspaceWithLockMode(db, task, agent, candidate.companyId);
|
||||
const recoveryCompany = recoverySnapshot.companies.find((company) => company.id === candidate.companyId);
|
||||
|
||||
dispatchRunToWorker({
|
||||
runId: candidate.id,
|
||||
companyId: candidate.companyId,
|
||||
agent,
|
||||
task,
|
||||
connector,
|
||||
workspace,
|
||||
workspaceMode: lockMode,
|
||||
companyName: recoveryCompany?.name ?? "Company",
|
||||
companyDescription: recoveryCompany?.description ?? "",
|
||||
snapshot: recoverySnapshot,
|
||||
trigger: "recovery",
|
||||
});
|
||||
} catch (error) {
|
||||
logger.warn(`[worker] Failed to re-enqueue run ${candidate.id}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
db.core.runs.finishRun({
|
||||
runId: candidate.id,
|
||||
status: "interrupted",
|
||||
summary: "Could not re-enqueue after worker restart.",
|
||||
errorMessage: error instanceof Error ? error.message : String(error),
|
||||
exitCode: null,
|
||||
signal: null,
|
||||
model: null,
|
||||
unattributedCost: true,
|
||||
});
|
||||
if (candidate.workspaceId) {
|
||||
db.core.workspaces.releaseWorkspaceLock(candidate.workspaceId, candidate.id);
|
||||
}
|
||||
dispatchRunToWorker({
|
||||
runId: candidate.id,
|
||||
companyId: candidate.companyId,
|
||||
agent,
|
||||
task,
|
||||
connector,
|
||||
workspace,
|
||||
workspaceMode: lockMode,
|
||||
observationPath,
|
||||
companyName: recoveryCompany?.name ?? "Company",
|
||||
companyDescription: recoveryCompany?.description ?? "",
|
||||
snapshot: recoverySnapshot,
|
||||
trigger: "recovery",
|
||||
});
|
||||
} catch (error) {
|
||||
logger.warn(`[worker] Failed to re-enqueue run ${candidate.id}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
db.core.runs.finishRun({
|
||||
runId: candidate.id,
|
||||
status: "interrupted",
|
||||
summary: "Could not re-enqueue after worker restart.",
|
||||
errorMessage: error instanceof Error ? error.message : String(error),
|
||||
exitCode: null,
|
||||
signal: null,
|
||||
model: null,
|
||||
unattributedCost: true,
|
||||
recoverTaskForRetry: candidate.taskId !== null,
|
||||
});
|
||||
if (candidate.workspaceId) {
|
||||
db.core.workspaces.releaseWorkspaceLock(candidate.workspaceId, candidate.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,14 +60,26 @@ export async function handleWorkerMessage(
|
||||
deps.getRuntimeBridge?.()?.releaseRunResources(event.runId) ?? { ok: true };
|
||||
return runtimeReleaseResult;
|
||||
};
|
||||
// The worker's per-run state machine releases `active[runId]` only
|
||||
// after receiving run-completion-ack. Any uncaught throw between
|
||||
// run-finished and posting the ack permanently leaks the agent —
|
||||
// every subsequent wake then enqueues behind a queue whose
|
||||
// `isAgentBusy` check refuses to start it. The ack must therefore
|
||||
// be posted unconditionally.
|
||||
try {
|
||||
completion = handleRunFinished(event, deps, context, {
|
||||
beforePostCompletionAutomation: releaseRuntimeResources,
|
||||
});
|
||||
} finally {
|
||||
releaseRuntimeResources();
|
||||
try {
|
||||
completion = handleRunFinished(event, deps, context, {
|
||||
beforePostCompletionAutomation: releaseRuntimeResources,
|
||||
});
|
||||
} finally {
|
||||
releaseRuntimeResources();
|
||||
}
|
||||
releaseCompletedRunWorkspaceLock(completion, deps);
|
||||
} catch (error) {
|
||||
deps.logger.error(
|
||||
`[worker] run-finished post-handling failed for run ${event.runId}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
releaseCompletedRunWorkspaceLock(completion, deps);
|
||||
if (!worker) {
|
||||
throw new Error(`run-completion-ack worker missing for run ${event.runId}`);
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { NotificationOptions } from "../../notification-service";
|
||||
import type { createBudgetEnforcementService } from "../../budget-enforcement-service";
|
||||
import type { createSessionManagementService } from "../../session-management-service";
|
||||
import type { RuntimeBridge, RuntimeMessageInjector } from "../../agent-runtime";
|
||||
import type { RunDispatchParams } from "../../orchestration/turn-context";
|
||||
|
||||
const runStatusSchema = z.enum(["queued", "running", "succeeded", "failed", "cancelled", "timed_out", "interrupted"]);
|
||||
const runUsageStatusSchema = z.enum(RUN_USAGE_STATUSES);
|
||||
@@ -110,19 +111,7 @@ export interface WorkerServiceDependencies {
|
||||
publishDomainChanged: () => void;
|
||||
emitEvent: (event: DesktopEvent) => void;
|
||||
wakeAgentIfPossible: (agentId: string | null | undefined, companyId: string, trigger: string) => string | null;
|
||||
dispatchRunToWorker: (params: {
|
||||
runId: string;
|
||||
companyId: string;
|
||||
agent: ReturnType<AppDatabase["core"]["entity"]["getAgent"]>;
|
||||
task: ReturnType<AppDatabase["core"]["entity"]["getTask"]> | null;
|
||||
connector: ReturnType<AppDatabase["core"]["connectors"]["getConnector"]>;
|
||||
workspace: ReturnType<AppDatabase["core"]["workspaces"]["getWorkspace"]>;
|
||||
workspaceMode: "exclusive" | "shared" | "none";
|
||||
companyName: string;
|
||||
companyDescription: string;
|
||||
snapshot: ReturnType<AppDatabase["projection"]["snapshot"]["listSnapshot"]>;
|
||||
trigger: string;
|
||||
}) => void;
|
||||
dispatchRunToWorker: (params: RunDispatchParams) => void;
|
||||
processRunCompletion: (run: RunRecord, event: z.infer<typeof workerMessageSchema> & { type: "run-finished" }) => void;
|
||||
onRunStarted?: (runId: string, taskId: string | null, agentId: string, companyId: string) => void;
|
||||
onRunCompleted?: (runId: string, taskId: string | null, agentId: string, companyId: string, status: string, summary: string | null, errorMessage: string | null) => void;
|
||||
|
||||
Reference in new issue
Block a user