fix(runtime): fail runs on context input errors
This commit is contained in:
1 parent
43583ef30b
commit
33c6ede381
5 files changed
+317
-100
No files matched your search
@@ -4,6 +4,7 @@ import type { OrchestrationDependencies } from "./types";
|
||||
import { buildRunPayload, type RunDispatchParams } from "./turn-context";
|
||||
import { validateConnectorModelSelection } from "../connector-model-policy";
|
||||
import { resolveWorkspaceWithLockMode } from "./resolve-workspace";
|
||||
import { AppError } from "../errors/index.js";
|
||||
|
||||
type ResolveWorkspaceTaskInput =
|
||||
| (Pick<TaskRecord, "workspaceId" | "projectId"> & { requiresFilesystem?: boolean })
|
||||
@@ -24,6 +25,7 @@ export function createRunDispatch(deps: OrchestrationDependencies) {
|
||||
const {
|
||||
db,
|
||||
logger,
|
||||
emitEvent,
|
||||
isQuitting,
|
||||
hasWorkerReady,
|
||||
getApiPort,
|
||||
@@ -38,6 +40,47 @@ export function createRunDispatch(deps: OrchestrationDependencies) {
|
||||
const resolveWorkspace: ResolveWorkspaceFn = (task, agent, companyId) =>
|
||||
resolveWorkspaceWithLockMode(db, task, agent, companyId);
|
||||
|
||||
function formatDispatchError(error: unknown): string {
|
||||
if (error instanceof AppError) {
|
||||
return `[${error.code}] ${error.message}`;
|
||||
}
|
||||
if (error instanceof Error) {
|
||||
return error.message;
|
||||
}
|
||||
return String(error);
|
||||
}
|
||||
|
||||
function failRunBeforeWorkerEnqueue(
|
||||
params: RunDispatchParams,
|
||||
error: unknown,
|
||||
workspaceLockClaimed: boolean,
|
||||
): void {
|
||||
const errorMessage = formatDispatchError(error);
|
||||
logger.error(`[run] Dispatch failed before worker enqueue for ${params.runId}: ${errorMessage}`);
|
||||
db.runs.finishRun({
|
||||
runId: params.runId,
|
||||
status: "failed",
|
||||
summary: "Run failed before worker enqueue.",
|
||||
errorMessage,
|
||||
exitCode: null,
|
||||
signal: null,
|
||||
model: params.resolvedModel ?? null,
|
||||
unattributedCost: true,
|
||||
});
|
||||
if (workspaceLockClaimed && params.workspaceMode === "exclusive" && params.workspace) {
|
||||
db.workspacesMod.releaseWorkspaceLock(params.workspace.id, params.runId);
|
||||
}
|
||||
db.activityMod.addActivity({
|
||||
companyId: params.companyId,
|
||||
actor: params.agent.id,
|
||||
action: "run.failed",
|
||||
entityType: "run",
|
||||
entityId: params.runId,
|
||||
detail: errorMessage,
|
||||
});
|
||||
emitEvent({ type: "run-updated", runId: params.runId, action: "completed" });
|
||||
}
|
||||
|
||||
function getRunEligibilityFailure(
|
||||
agent: { status: string; budgetMonthlyUsd: number; spentMonthlyUsd: number; name: string; connectorId: string; id: string; companyId: string; model?: string | null },
|
||||
connector: Pick<ConnectorRecord, "id" | "label" | "status" | "capabilityMatrix">,
|
||||
@@ -74,57 +117,62 @@ export function createRunDispatch(deps: OrchestrationDependencies) {
|
||||
}
|
||||
|
||||
function dispatchRunToWorker(params: RunDispatchParams) {
|
||||
if (isQuitting()) {
|
||||
throw new Error("Application is shutting down — refusing to dispatch new run.");
|
||||
}
|
||||
if (!hasWorkerReady()) {
|
||||
throw new Error("Orchestration worker is not ready.");
|
||||
}
|
||||
const connectorIssue = getConnectorExecutionReadinessIssue(params.connector);
|
||||
if (connectorIssue) {
|
||||
throw new Error(connectorIssue);
|
||||
}
|
||||
let workspaceLockClaimed = false;
|
||||
try {
|
||||
if (isQuitting()) {
|
||||
throw new Error("Application is shutting down — refusing to dispatch new run.");
|
||||
}
|
||||
if (!hasWorkerReady()) {
|
||||
throw new Error("Orchestration worker is not ready.");
|
||||
}
|
||||
const connectorIssue = getConnectorExecutionReadinessIssue(params.connector);
|
||||
if (connectorIssue) {
|
||||
throw new Error(connectorIssue);
|
||||
}
|
||||
|
||||
// D-02: Claim the workspace lock synchronously before enqueuing the run.
|
||||
// Previously the worker claimed the lock only after it emitted `running`,
|
||||
// so two runs could race between enqueue and startup and both read the
|
||||
// same workspace mid-write. Main is the authoritative coordinator.
|
||||
//
|
||||
// On contention we mark the run `cancelled`, not `failed`. The Dashboard
|
||||
// "Needs attention" card only surfaces failed/timed_out/interrupted runs
|
||||
// (see src/renderer/components/dashboard/Dashboard.tsx), so the user is
|
||||
// not alarmed by a transient backoff. The task row is rolled back to
|
||||
// `todo` via the `finishRun` never-started branch, so heartbeat picks
|
||||
// it up on the next cycle with no manual intervention.
|
||||
if (
|
||||
params.workspaceMode === "exclusive" &&
|
||||
params.workspace &&
|
||||
!db.workspacesMod.claimWorkspaceLock(params.workspace.id, params.runId)
|
||||
) {
|
||||
db.runs.finishRun({
|
||||
runId: params.runId,
|
||||
status: "cancelled",
|
||||
summary: null,
|
||||
errorMessage: `Workspace ${params.workspace.id} is in use by another run; will retry on the next heartbeat.`,
|
||||
exitCode: null,
|
||||
signal: null,
|
||||
model: null,
|
||||
unattributedCost: true,
|
||||
// D-02: Claim the workspace lock synchronously before enqueuing the run.
|
||||
// Previously the worker claimed the lock only after it emitted `running`,
|
||||
// so two runs could race between enqueue and startup and both read the
|
||||
// same workspace mid-write. Main is the authoritative coordinator.
|
||||
//
|
||||
// On contention we mark the run `cancelled`, not `failed`. The Dashboard
|
||||
// "Needs attention" card only surfaces failed/timed_out/interrupted runs
|
||||
// (see src/renderer/components/dashboard/Dashboard.tsx), so the user is
|
||||
// not alarmed by a transient backoff. The task row is rolled back to
|
||||
// `todo` via the `finishRun` never-started branch, so heartbeat picks
|
||||
// it up on the next cycle with no manual intervention.
|
||||
if (params.workspaceMode === "exclusive" && params.workspace) {
|
||||
if (!db.workspacesMod.claimWorkspaceLock(params.workspace.id, params.runId)) {
|
||||
db.runs.finishRun({
|
||||
runId: params.runId,
|
||||
status: "cancelled",
|
||||
summary: null,
|
||||
errorMessage: `Workspace ${params.workspace.id} is in use by another run; will retry on the next heartbeat.`,
|
||||
exitCode: null,
|
||||
signal: null,
|
||||
model: null,
|
||||
unattributedCost: true,
|
||||
});
|
||||
emitEvent({ type: "run-updated", runId: params.runId, action: "completed" });
|
||||
return;
|
||||
}
|
||||
workspaceLockClaimed = true;
|
||||
}
|
||||
|
||||
const payload = buildRunPayload(params, {
|
||||
db,
|
||||
logger,
|
||||
getApiPort,
|
||||
createRunToken,
|
||||
listRuntimeSkillEntries,
|
||||
getSessionHandoff,
|
||||
evaluatePreRunSessionRotation,
|
||||
});
|
||||
return;
|
||||
|
||||
enqueueWorkerRun(payload);
|
||||
} catch (error) {
|
||||
failRunBeforeWorkerEnqueue(params, error, workspaceLockClaimed);
|
||||
}
|
||||
|
||||
const payload = buildRunPayload(params, {
|
||||
db,
|
||||
logger,
|
||||
getApiPort,
|
||||
createRunToken,
|
||||
listRuntimeSkillEntries,
|
||||
getSessionHandoff,
|
||||
evaluatePreRunSessionRotation,
|
||||
});
|
||||
|
||||
enqueueWorkerRun(payload);
|
||||
}
|
||||
|
||||
return { resolveWorkspace, getRunEligibilityFailure, dispatchRunToWorker };
|
||||
|
||||
Reference in new issue
Block a user