update

src/main/worker/failure-escalation.ts

@@ -76,8 +76,10 @@ export function handleFailureEscalation(
         `[escalation] Escalated task ${run.taskId} to manager ${chain[0].name} (${chain[0].id})`,
       );
     }
-  } catch {
-    // Chain lookup failed (agent deleted)
+  } catch (error) {
+    logger.warn(
+      `[escalation] Chain-of-command lookup failed for agent ${run.agentId}: ${error instanceof Error ? error.message : String(error)}`,
+    );
   }
 
   if (!escalated) {
@@ -90,7 +92,11 @@ export function handleFailureEscalation(
           `[escalation] Escalated task ${run.taskId} to CEO ${ceo.name} (${ceo.id})`,
         );
       }
-    } catch { /* CEO lookup failed */ }
+    } catch (error) {
+      logger.warn(
+        `[escalation] CEO lookup failed for company ${run.companyId}: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
   }
 
   if (!escalated) {
@@ -111,8 +117,13 @@ export function handleFailureEscalation(
 export function handleSuccessReset(
   db: EscalationDeps["db"],
   taskId: string,
+  logger?: EscalationDeps["logger"],
 ): void {
   try {
     db.resetTaskRetryCount(taskId);
-  } catch { /* best effort */ }
+  } catch (error) {
+    logger?.warn(
+      `[escalation] resetTaskRetryCount failed for task ${taskId}: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
 }

src/main/worker/handlers/run-finished-helpers.ts

@@ -26,7 +26,10 @@ export function finalizeRun(
   let run: RunRecord;
   try {
     run = db.getRun(event.runId);
-  } catch {
+  } catch (error) {
+    logger.warn(
+      `[finalizeRun] getRun(${event.runId}) failed; run may have been deleted: ${error instanceof Error ? error.message : String(error)}`,
+    );
     return null;
   }
 
@@ -55,7 +58,10 @@ export function finalizeRun(
   let task: TaskRecord | null = null;
   try {
     task = db.getTask(run.taskId);
-  } catch {
+  } catch (error) {
+    logger.warn(
+      `[finalizeRun] getTask(${run.taskId}) failed; task may have been deleted: ${error instanceof Error ? error.message : String(error)}`,
+    );
     task = null;
   }
 
@@ -120,7 +126,7 @@ export function emitAndNotifyCompletion(
   deps: HandlerDeps,
   rfCtx: RunFinishedContext,
 ): void {
-  const { db, emitEvent, notify } = deps;
+  const { db, emitEvent, notify, logger } = deps;
   const { run, task, runOutcome } = rfCtx;
 
   emitEvent({ type: "run-updated", runId: run.id, action: "completed" });
@@ -143,7 +149,10 @@ export function emitAndNotifyCompletion(
   let updatedAgent: ReturnType<HandlerDeps["db"]["getAgent"]> | null = null;
   try {
     updatedAgent = db.getAgent(run.agentId);
-  } catch {
+  } catch (error) {
+    logger.warn(
+      `[emitAndNotifyCompletion] getAgent(${run.agentId}) failed; skipping budget-paused notification: ${error instanceof Error ? error.message : String(error)}`,
+    );
     updatedAgent = null;
   }
 
@@ -177,14 +186,11 @@ export function runPostCompletionAutomation(
   const { destroyed, MAX_RETRY_ATTEMPTS, RETRY_COOLDOWN_MS, pendingRetryTimers } = context;
   const { run, task } = rfCtx;
 
-  // Policy-based incident check after run completion (Phase 5)
+  // Policy-based incident check after run completion (Phase 5). Reuse the
+  // `task` already resolved in finalizeRun instead of double-fetching with
+  // its own catch-and-swallow; if task is null the project-scope is null.
   if (budgetService && typeof event.costUsd === "number" && Number.isFinite(event.costUsd)) {
-    try {
-      const taskForBudget = db.getTask(run.taskId);
-      budgetService.checkAndCreateIncidents(run.companyId, run.agentId, taskForBudget?.projectId ?? null);
-    } catch (err) {
-      logger.error(`[budget] Incident check failed: ${err instanceof Error ? err.message : String(err)}`);
-    }
+    budgetService.checkAndCreateIncidents(run.companyId, run.agentId, task?.projectId ?? null);
   }
 
   if ((event.status === "failed" || event.status === "timed_out") && !isQuitting() && !destroyed) {
@@ -252,7 +258,7 @@ export function runPostCompletionAutomation(
 
   // Reset retry count on successful completion
   if (event.status === "succeeded") {
-    handleSuccessReset(db, run.taskId);
+    handleSuccessReset(db, run.taskId, logger);
   }
 
   return { hadDeferredWakes };
@@ -294,13 +300,17 @@ export function triggerSelfWake(
         if (chain.length > 0) {
           wakeAgentIfPossible(chain[0].id, run.companyId, "report_stuck");
         }
-      } catch {
-        /* best effort */
+      } catch (error) {
+        logger.warn(
+          `[self-wake] chain-of-command lookup failed for agent ${run.agentId}: ${error instanceof Error ? error.message : String(error)}`,
+        );
       }
     } else {
       wakeAgentIfPossible(run.agentId, run.companyId, "task_continuation");
     }
-  } catch {
-    // Best-effort self-wake; agent will pick up work on the next heartbeat cycle.
+  } catch (error) {
+    logger.warn(
+      `[self-wake] self-wake evaluation failed for agent ${run.agentId}: ${error instanceof Error ? error.message : String(error)}`,
+    );
   }
 }

src/main/worker/worker-crash-recovery.ts

@@ -92,6 +92,12 @@ export function handleWorkerExit(
     const recoverySnapshot = db.listSnapshot();
     for (const candidate of requeueCandidates) {
       try {
+        // The crashed run may still own a workspace lock acquired during the
+        // original dispatch; release it before re-dispatch so the replay
+        // does not self-collide with "Workspace is locked by another run".
+        if (candidate.workspaceId) {
+          db.releaseWorkspaceLock(candidate.workspaceId, candidate.id);
+        }
         const task = db.getTask(candidate.taskId);
         const agent = db.getAgent(candidate.agentId);
         const connector = db.getConnector(agent.connectorId);

src/main/worker/worker-entry.ts

@@ -37,33 +37,39 @@ function enqueue(job: RunJob) {
     status: "queued",
     message: "Run queued in orchestration worker.",
   });
-  void maybeStartQueuedRuns();
+  maybeStartQueuedRuns();
 }
 
-async function maybeStartQueuedRuns() {
-  const readyJobs = [...queue];
-  for (const job of readyJobs) {
-    if (activeByAgent.has(job.agentId)) continue;
+/**
+ * Walks the queue in-place and dispatches every job whose agent + workspace
+ * are not currently busy. Claims (activeByAgent / activeByWorkspace) are
+ * written BEFORE the job is removed from the queue and BEFORE `startRun`
+ * fires — this is the atomic point of commit for orchestrator ordering.
+ * Hardens against a future refactor that adds an `await` before the claim
+ * inside `startRun`, which would open a re-entrancy window.
+ */
+function maybeStartQueuedRuns() {
+  let i = 0;
+  while (i < queue.length) {
+    const job = queue[i];
+    if (activeByAgent.has(job.agentId)) { i += 1; continue; }
     const lockKey = workspaceKey(job);
-    if (lockKey && activeByWorkspace.has(lockKey)) continue;
-    const index = queue.findIndex((entry) => entry.runId === job.runId);
-    if (index >= 0) queue.splice(index, 1);
-    void startRun(job);
+    if (lockKey && activeByWorkspace.has(lockKey)) { i += 1; continue; }
+    activeByAgent.set(job.agentId, job.runId);
+    if (lockKey) activeByWorkspace.set(lockKey, job.runId);
+    queue.splice(i, 1);
+    void startRun(job, lockKey);
   }
 }
 
-async function startRun(job: RunJob) {
-  const lockKey = workspaceKey(job);
-  activeByAgent.set(job.agentId, job.runId);
-  if (lockKey) activeByWorkspace.set(lockKey, job.runId);
-
+async function startRun(job: RunJob, lockKey: string | null) {
   try {
     await runner.run(job);
   } finally {
     cancelledRuns.delete(job.runId);
     activeByAgent.delete(job.agentId);
     if (lockKey) activeByWorkspace.delete(lockKey);
-    void maybeStartQueuedRuns();
+    maybeStartQueuedRuns();
   }
 }
 
@@ -109,8 +115,17 @@ parentPort.on("message", (event) => {
     return;
   }
   if (message.type === "shutdown") {
-    runner.cancelAll();
-    rpc.cancelAll("Worker shutting down.");
-    process.exit(0);
+    void (async () => {
+      // Graceful drain: abort every active run and await its promise
+      // settlement so that final `run-finished` messages and
+      // `persistMessage` RPCs reach the parent before we exit. Bounded by
+      // a 5s budget so a stuck run does not wedge the shutdown handshake
+      // (the parent has its own kill-timer as a safety net).
+      const drainBudgetMs = 5_000;
+      const timeout = new Promise<void>((resolve) => setTimeout(resolve, drainBudgetMs));
+      await Promise.race([runner.cancelAll(), timeout]);
+      rpc.cancelAll("Worker shutting down.");
+      process.exit(0);
+    })();
   }
 });

src/main/worker/worker-helpers.ts

@@ -19,7 +19,6 @@ export interface RunJob {
   sessionParams: Record<string, unknown> | null;
   systemPrompt: string | null;
   trigger: string | null;
-  managedSkills: Array<{ runtimeName: string; sourcePath: string }>;
   taskSessionId: string;
   executionMode: "ai_ceo" | "ai_cofounder";
 }
@@ -80,7 +79,16 @@ export function buildRedactor(env: Record<string, string>): (text: string) => st
   };
 }
 
-/** Pushes extra env vars onto process.env and returns a restore function. */
+/**
+ * Pushes extra env vars onto `process.env` and returns a restore function
+ * that is **infallible** — each individual restore is guarded so a single
+ * malformed key cannot leave a partial restore that keeps a provider
+ * secret in the worker's env for the next run. Intended for short-lived
+ * turn-scope env (`AGENT_COMPANY_*`, connector secrets). The worker runs
+ * one agent at a time, so this is safe for the current scheduler; do not
+ * convert it to concurrent run isolation — pass per-spawn env at the
+ * child-process level instead.
+ */
 export function applyEnvOverrides(overrides: Record<string, string>): () => void {
   const saved: Record<string, string | undefined> = {};
   for (const [key, value] of Object.entries(overrides)) {
@@ -89,8 +97,15 @@ export function applyEnvOverrides(overrides: Record<string, string>): () => void
   }
   return () => {
     for (const [key, previous] of Object.entries(saved)) {
-      if (previous === undefined) delete process.env[key];
-      else process.env[key] = previous;
+      try {
+        if (previous === undefined) delete process.env[key];
+        else process.env[key] = previous;
+      } catch {
+        // Individual restore failure is never fatal — move on so the
+        // remaining keys still revert. Should never happen in practice
+        // (the Node env-var assignment is synchronous and infallible for
+        // string values), but the guard lets the outer finally complete.
+      }
     }
   };
 }

src/main/worker/worker-message-handler.ts

@@ -42,9 +42,14 @@ export async function handleWorkerMessage(
     case "run-status":
       handleRunStatus(event, deps, context);
       return;
-    case "run-finished":
-      handleRunFinished(event, deps, context);
+    case "run-finished": {
+      try {
+        handleRunFinished(event, deps, context);
+      } finally {
+        deps.getRuntimeBridge?.()?.releaseRunResources(event.runId);
+      }
       return;
+    }
     case "agent-event": {
       const bridge = deps.getRuntimeBridge?.();
       if (bridge) {

src/main/worker/worker-rpc.ts

@@ -3,17 +3,33 @@ import type { ChildMessage } from "./worker-helpers";
 
 interface PendingRpc {
   resolve: (value: { ok: true; result: unknown } | { ok: false; error: string }) => void;
+  timer: NodeJS.Timeout;
 }
 
+const DEFAULT_RPC_TIMEOUT_MS = 30_000;
+
 export class WorkerRpcClient {
   private readonly pending = new Map<string, PendingRpc>();
 
-  constructor(private readonly post: (message: ChildMessage) => void) {}
+  constructor(
+    private readonly post: (message: ChildMessage) => void,
+    private readonly timeoutMs: number = DEFAULT_RPC_TIMEOUT_MS,
+  ) {}
 
-  invoke(runId: string, method: string, payload: unknown): Promise<{ ok: true; result: unknown } | { ok: false; error: string }> {
+  invoke(
+    runId: string,
+    method: string,
+    payload: unknown,
+  ): Promise<{ ok: true; result: unknown } | { ok: false; error: string }> {
     const rpcId = randomUUID();
     return new Promise((resolve) => {
-      this.pending.set(rpcId, { resolve });
+      const timer = setTimeout(() => {
+        const pending = this.pending.get(rpcId);
+        if (!pending) return;
+        this.pending.delete(rpcId);
+        pending.resolve({ ok: false, error: `RPC ${method} timed out after ${this.timeoutMs}ms` });
+      }, this.timeoutMs);
+      this.pending.set(rpcId, { resolve, timer });
       this.post({ type: "rpc-request", rpcId, runId, method, payload });
     });
   }
@@ -21,12 +37,14 @@ export class WorkerRpcClient {
   resolve(rpcId: string, response: { ok: true; result: unknown } | { ok: false; error: string }): void {
     const pending = this.pending.get(rpcId);
     if (!pending) return;
+    clearTimeout(pending.timer);
     this.pending.delete(rpcId);
     pending.resolve(response);
   }
 
   cancelAll(reason: string): void {
     for (const [id, pending] of this.pending) {
+      clearTimeout(pending.timer);
       pending.resolve({ ok: false, error: reason });
       this.pending.delete(id);
     }

src/main/worker/worker-runner.ts

@@ -23,6 +23,7 @@ interface RunnerDeps {
 
 export class WorkerRunner {
   private readonly abortControllers = new Map<string, AbortController>();
+  private readonly runPromises = new Map<string, Promise<void>>();
 
   constructor(private readonly deps: RunnerDeps) {}
 
@@ -31,21 +32,37 @@ export class WorkerRunner {
     if (controller) controller.abort();
   }
 
-  cancelAll(): void {
+  /**
+   * Aborts every in-flight run and returns a promise that resolves once
+   * each `run()` has settled (success, failure, or abort). Callers (the
+   * worker shutdown handler) must `await` this before `process.exit` so
+   * pending `persistMessage` RPCs and the final `run-finished` post reach
+   * the parent before the worker dies.
+   */
+  async cancelAll(): Promise<void> {
     for (const controller of this.abortControllers.values()) {
       controller.abort();
     }
+    const pending = [...this.runPromises.values()];
     this.abortControllers.clear();
+    await Promise.allSettled(pending);
   }
 
-  async run(job: RunJob): Promise<void> {
+  run(job: RunJob): Promise<void> {
+    const promise = this.runImpl(job).finally(() => {
+      this.runPromises.delete(job.runId);
+    });
+    this.runPromises.set(job.runId, promise);
+    return promise;
+  }
+
+  private async runImpl(job: RunJob): Promise<void> {
     const { post, rpc, isCancelled } = this.deps;
     const redact = buildRedactor(job.env);
     const abortController = new AbortController();
     this.abortControllers.set(job.runId, abortController);
 
     post({ type: "run-status", runId: job.runId, status: "running", message: `Running agent ${job.agentName}.` });
-    post({ type: "run-log", runId: job.runId, stream: "stdout", chunk: `[runner] starting run for ${job.agentName} (${job.connectorId}/${job.model ?? "?"})\n` });
 
     let resolvedModelId: string | null = job.model;
     let finalUsage: { inputTokens: number | null; outputTokens: number | null; costUsd: number | null } = {
@@ -55,6 +72,14 @@ export class WorkerRunner {
     };
     let errorMessage: string | null = null;
 
+    // Inject the AgentCompany job env (AGENT_COMPANY_API_URL / API_KEY /
+    // AGENT_ID / COMPANY_ID / RUN_ID / TASK_ID / WAKE_REASON + provider
+    // secret bindings) into the worker's process.env BEFORE Agent
+    // construction, so the agent's bash-tool children can call back into
+    // our HTTP API via fetch(process.env.AGENT_COMPANY_API_URL). Restored
+    // in the outer finally regardless of success / abort / throw.
+    const restoreJobEnv = applyEnvOverrides(job.env);
+
     try {
       if (!job.model) {
         throw new Error(`Run ${job.runId} has no model selected.`);
@@ -73,8 +98,7 @@ export class WorkerRunner {
       }
       const prepared = prepareResponse.result as PrepareRunResult;
 
-      post({ type: "run-log", runId: job.runId, stream: "stdout", chunk: `[runner] prepareRun ok, resolving model\n` });
-      const restoreEnv = applyEnvOverrides(prepared.extraEnv);
+      const restoreProviderEnv = applyEnvOverrides(prepared.extraEnv);
       try {
         const model = resolveModel(prepared.provider, job.model);
         if (!model) {
@@ -82,7 +106,6 @@ export class WorkerRunner {
         }
         resolvedModelId = model.id;
         const extraHeaders = resolveExtraStreamHeaders(prepared.provider, prepared.apiKey);
-        post({ type: "run-log", runId: job.runId, stream: "stdout", chunk: `[runner] model=${model.id} provider=${prepared.provider} — constructing Agent\n` });
 
         const cwd = job.workspacePath ?? process.cwd();
         const tools = createCodingTools(cwd) as never;
@@ -100,36 +123,10 @@ export class WorkerRunner {
               apiKey: prepared.apiKey,
               headers: { ...(options?.headers ?? {}), ...(extraHeaders ?? {}) },
             }),
-          beforeToolCall: async (context) => {
-            const verdict = await rpc.invoke(job.runId, "beforeToolCall", {
-              agentId: job.agentId,
-              companyId: job.companyId,
-              taskId: job.taskId,
-              toolCallId: context.toolCall.id,
-              toolName: context.toolCall.name,
-              args: context.args,
-            });
-            if (verdict.ok) {
-              return verdict.result as { block?: boolean; reason?: string } | undefined;
-            }
-            return { block: true, reason: verdict.error };
-          },
-          afterToolCall: async (context) => {
-            await rpc.invoke(job.runId, "afterToolCall", {
-              toolCallId: context.toolCall.id,
-              toolName: context.toolCall.name,
-              isError: context.isError,
-            });
-            return undefined;
-          },
         });
 
         const unsubscribe = agent.subscribe(async (event) => {
           const timestamp = new Date().toISOString();
-          // Surface every pi AgentEvent type as a raw Console probe so the
-          // operator can see the live event stream even when projection
-          // misses a type or the agent is stuck inside a particular phase.
-          post({ type: "run-log", runId: job.runId, stream: "stdout", chunk: `[pi-event] ${event.type}\n` });
           post({
             type: "agent-event",
             runId: job.runId,
@@ -166,22 +163,21 @@ export class WorkerRunner {
             content: [{ type: "text", text: promptContent }],
             timestamp: Date.now(),
           };
-          post({ type: "run-log", runId: job.runId, stream: "stdout", chunk: `[runner] calling agent.prompt (${promptContent.length} chars)\n` });
           const runPromise = agent.prompt(promptMessage);
           abortController.signal.addEventListener("abort", () => agent.abort(), { once: true });
           await runPromise;
-          post({ type: "run-log", runId: job.runId, stream: "stdout", chunk: `[runner] agent.prompt resolved\n` });
         } finally {
           unsubscribe();
         }
       } finally {
-        restoreEnv();
+        restoreProviderEnv();
       }
     } catch (error) {
       errorMessage = error instanceof Error ? error.message : String(error);
       post({ type: "run-log", runId: job.runId, stream: "stderr", chunk: redact(errorMessage) });
     } finally {
       this.abortControllers.delete(job.runId);
+      restoreJobEnv();
     }
 
     const status: RunStatus = isCancelled(job.runId)

src/main/worker/worker-types.ts

@@ -50,7 +50,7 @@ export const workerMessageSchema = z.union([
     type: z.literal("rpc-request"),
     rpcId: z.string(),
     runId: z.string().uuid(),
-    method: z.enum(["beforeToolCall", "afterToolCall", "persistMessage", "prepareRun"]),
+    method: z.enum(["persistMessage", "prepareRun"]),
     payload: z.unknown(),
   }),
 ]);