Enable lock-free parallel agent runs

This commit is contained in:
ZacharyZhang-NY committed 2026-04-22 20:23:44 -04:00
1 parent d13c9d83e8
commit 1a7eb8f1b3
18 files changed
+327 -194

No files matched your search

@@ -81,6 +81,9 @@ export function reconcileWorkspaceArtifacts(
db: AppDatabase,
ctx: ArtifactProjectorRunContext,
): ProjectionOutcome[] {
if (ctx.workspaceMode !== "exclusive") {
return [{ kind: "skipped", reason: "workspace_not_exclusive" }];
}
return projectCollectedChanges(db, ctx, collectChangedPaths(ctx));
}
@@ -88,6 +91,9 @@ function projectBashArtifacts(
db: AppDatabase,
ctx: ArtifactProjectorRunContext,
): ProjectionOutcome[] {
if (ctx.workspaceMode !== "exclusive") {
return [{ kind: "skipped", reason: "workspace_not_exclusive" }];
}
return projectCollectedChanges(db, ctx, collectChangedPaths(ctx));
}
+1
View File
@@ -14,6 +14,7 @@ export type ArtifactProjectorRunContext = {
taskId: string | null;
workspaceId: string | null;
workspacePath: string | null;
workspaceMode: "exclusive" | "shared" | "none";
logger?: Logger;
/**
* mtime snapshot captured at run start for workspaces that are NOT
+2
View File
@@ -56,6 +56,7 @@ export interface ActiveRunContext {
taskSessionId: string;
workspaceId: string | null;
workspacePath: string | null;
workspaceMode: "exclusive" | "shared" | "none";
workspaceBaseline: import("./artifact-projector").WorkspaceBaseline | null;
state: ActiveRunLifecycleState;
}
@@ -162,6 +163,7 @@ export function handleAfterToolCall(
taskId: ctx.taskId,
workspaceId: ctx.workspaceId,
workspacePath: ctx.workspacePath,
workspaceMode: ctx.workspaceMode,
logger: deps.logger,
workspaceBaseline: ctx.workspaceBaseline,
};
+8 -7
View File
@@ -152,6 +152,7 @@ export class RuntimeBridge {
taskId: ctx.taskId,
workspaceId: ctx.workspaceId,
workspacePath: ctx.workspacePath,
workspaceMode: ctx.workspaceMode,
logger: this.deps.logger,
workspaceBaseline: ctx.workspaceBaseline,
});
@@ -394,14 +395,13 @@ export class RuntimeBridge {
modelId: string;
workspaceId: string | null;
workspacePath: string | null;
workspaceMode: "exclusive" | "shared" | "none";
};
// Capture an mtime baseline only for non-git workspaces so `bash`
// and the end-of-run reconciliation can still diff changed files
// against a known-start reference. Git workspaces skip the walk —
// `git status` is the source of truth there, and paying for an
// O(tree) scan on every run just to throw the result away is a
// hot-path tax the scheduler should not be paying.
const baseline = captureBaselineIfRequired(payload.workspacePath, this.deps.logger);
// Baseline scans are safe only while this run owns the workspace lock.
const baseline =
payload.workspaceMode === "exclusive"
? captureBaselineIfRequired(payload.workspacePath, this.deps.logger)
: null;
this.registerActiveRun({
runId,
agentId: payload.agentId,
@@ -412,6 +412,7 @@ export class RuntimeBridge {
taskSessionId: payload.taskSessionId,
workspaceId: payload.workspaceId,
workspacePath: payload.workspacePath,
workspaceMode: payload.workspaceMode,
workspaceBaseline: baseline,
state: "preparing",
});
+31 -22
View File
@@ -5,7 +5,7 @@ import type { WorkspaceLockMode } from "./types";
/**
* Per-task workspace lock inference (v52+).
*
* The core invariant is: `lockMode='exclusive'` is an *opt-in*,
* The core invariant is: `lockMode='exclusive'` is an opt-in,
* `lockMode='none'` is the default. Previously every resolved
* workspace was held exclusively, which folded four runnable
* agents onto a single company workspace lock and serialised the
@@ -103,16 +103,14 @@ type AgentInput = Pick<AgentRecord, "workspaceId"> & {
*
* Decision table:
*
* 1. `task.workspaceId` set -> exclusive on that workspace
* (explicit opt-in by the task author, e.g. PR branch tasks).
* 1. Cognitive task (`requiresFilesystem !== true`) -> none.
* Workspace binding remains available as an observation root.
*
* 2. Task signals filesystem need (`requiresFilesystem === true`)
* -> exclusive on agent.workspaceId / project primary. Throws if
* neither exists.
* 2. Filesystem task with `task.workspaceId` set -> exclusive on
* that workspace.
*
* 3. Otherwise -> none / null. Pure cognitive work: strategy,
* planning, document drafting, financial analysis, PM decisions
* runs with no workspace and no lock.
* 3. Filesystem task without `task.workspaceId` -> exclusive on
* agent.workspaceId / project primary. Throws if neither exists.
*/
function inferLockMode(
db: AppDatabase,
@@ -120,18 +118,15 @@ function inferLockMode(
agent: AgentInput,
companyId: string,
): { lockMode: WorkspaceLockMode; workspaceId: string | null; source: WorkspaceResolutionSource } {
// Rule 1: task-level explicit workspace binding is the strongest
// signal available: the task row literally names the workspace.
if (task?.workspaceId) {
return { lockMode: "exclusive", workspaceId: task.workspaceId, source: "task" };
}
if (task?.requiresFilesystem !== true) {
return { lockMode: "none", workspaceId: null, source: "none" };
}
// Rule 2: filesystem-touching task -> agent workspace first, then
// project primary.
if (task.workspaceId) {
return { lockMode: "exclusive", workspaceId: task.workspaceId, source: "task" };
}
// Filesystem-touching task -> agent workspace first, then project primary.
if (agent.workspaceId) {
return { lockMode: "exclusive", workspaceId: agent.workspaceId, source: "agent" };
}
@@ -175,11 +170,25 @@ export function resolveWorkspaceWithLockMode(
if (decision.lockMode === "none") {
// No lock claim, but the cognitive run can still touch the
// filesystem via `bash`. Pick a stable observation root so the
// artifact projector can baseline + diff the run's footprint.
// `getCompanyObservationWorkspace` returns the primary or oldest
// workspace; null only when the company has no workspace row at
// all, in which case there is genuinely nothing to observe.
const observation = db.core.workspaces.getCompanyObservationWorkspace(companyId);
// artifact projector can baseline + diff the run's footprint. A
// task-bound workspace is the strongest observation signal; the
// company primary/oldest workspace remains the company-level root.
let observation: WorkspaceRecord | null = null;
if (task?.workspaceId) {
observation = db.core.workspaces.getWorkspace(task.workspaceId);
if (!observation) {
throw new WorkspaceConfigurationError(
`Task observation workspace ${task.workspaceId} does not exist (companyId=${companyId}).`,
);
}
if (observation.companyId !== companyId) {
throw new WorkspaceConfigurationError(
`Task observation workspace ${observation.id} belongs to company ${observation.companyId}, not ${companyId}.`,
);
}
} else {
observation = db.core.workspaces.getCompanyObservationWorkspace(companyId);
}
return {
workspace: null,
lockMode: "none",
+6 -5
View File
@@ -15,11 +15,12 @@ interface ActiveRun {
}
/**
* Single source of truth for scheduling invariants: "one agent at a time"
* and "one exclusive-workspace lock at a time". Prior layout kept three
* parallel structures (`activeByAgent`, `activeByWorkspace`, queue) that
* had to be updated in lockstep. The Map-of-ActiveRun keyed by runId is
* the only authoritative record; agent/workspace busy checks iterate it.
* Single source of truth for scheduling invariants: one active run per
* agent and one active run per exclusive workspace lock. Prior layout
* kept three parallel structures (`activeByAgent`, `activeByWorkspace`,
* queue) that had to be updated in lockstep. The Map-of-ActiveRun keyed
* by runId is the only authoritative record; agent/workspace busy checks
* iterate it.
*/
const active = new Map<string, ActiveRun>();
const queue: RunJob[] = [];
-31
View File
@@ -109,34 +109,3 @@ export function buildRedactor(env: Record<string, string>): (text: string) => st
return result;
};
}
/**
* Pushes extra env vars onto `process.env` and returns a restore function
* that is **infallible** — each individual restore is guarded so a single
* malformed key cannot leave a partial restore that keeps a provider
* secret in the worker's env for the next run. Intended for short-lived
* turn-scope env (`AGENT_COMPANY_*`, connector secrets). The worker runs
* one agent at a time, so this is safe for the current scheduler; do not
* convert it to concurrent run isolation — pass per-spawn env at the
* child-process level instead.
*/
export function applyEnvOverrides(overrides: Record<string, string>): () => void {
const saved: Record<string, string | undefined> = {};
for (const [key, value] of Object.entries(overrides)) {
saved[key] = process.env[key];
process.env[key] = value;
}
return () => {
for (const [key, previous] of Object.entries(saved)) {
try {
if (previous === undefined) delete process.env[key];
else process.env[key] = previous;
} catch {
// Individual restore failure is never fatal — move on so the
// remaining keys still revert. Should never happen in practice
// (the Node env-var assignment is synchronous and infallible for
// string values), but the guard lets the outer finally complete.
}
}
};
}
+104 -115
View File
@@ -2,12 +2,12 @@ import type { Agent, AgentMessage } from "@mariozechner/pi-agent-core";
import { resolveModel, resolveExtraStreamHeaders } from "../../model-resolver";
import type { RunStatus } from "@shared/types";
import type { ChildMessage, RunJob } from "./helpers";
import { applyEnvOverrides, buildRedactor } from "./helpers";
import { createCodingTools } from "@mariozechner/pi-coding-agent";
import { buildRedactor } from "./helpers";
import { createPiAgent, resolveThinkingLevel, type ErasedAgentTool } from "./pi-agent-factory";
import type { WorkerRpcClient } from "../entry/rpc";
import { createHireAgentTool, HIRE_AGENT_TOOL_NAME } from "./internal-tools/hire-agent-tool";
import { createFileApprovalTool, FILE_APPROVAL_TOOL_NAME } from "./internal-tools/file-approval-tool";
import { createRunScopedCodingTools, mergeRunEnvs } from "./run-tools";
import {
classifyRunUsage,
hasAttributableAssistantUsage,
@@ -95,7 +95,7 @@ export class WorkerRunner {
private async runImpl(job: RunJob): Promise<void> {
const { post, rpc, isCancelled } = this.deps;
const redact = buildRedactor(job.env);
let redact = buildRedactor(job.env);
const abortController = new AbortController();
this.abortControllers.set(job.runId, abortController);
@@ -110,14 +110,6 @@ export class WorkerRunner {
let terminalStopReason: TerminalAssistantStopReason | null = null;
let errorMessage: string | null = null;
// Inject the AgentCompany job env (AGENT_COMPANY_API_URL / API_KEY /
// AGENT_ID / COMPANY_ID / RUN_ID / TASK_ID / WAKE_REASON + provider
// secret bindings) into the worker's process.env BEFORE Agent
// construction, so the agent's bash-tool children can call back into
// our HTTP API via fetch(process.env.AGENT_COMPANY_API_URL). Restored
// in the outer finally regardless of success / abort / throw.
const restoreJobEnv = applyEnvOverrides(job.env);
try {
if (!job.model) {
throw new Error(`Run ${job.runId} has no model selected.`);
@@ -132,134 +124,131 @@ export class WorkerRunner {
modelId: job.model,
workspaceId: job.workspaceId ?? null,
workspacePath: job.workspacePath ?? null,
workspaceMode: job.workspaceMode,
});
if (!prepareResponse.ok) {
throw new Error(prepareResponse.error);
}
const prepared = prepareResponse.result as PrepareRunResult;
const runEnv = mergeRunEnvs(job.env, prepared.extraEnv);
redact = buildRedactor(runEnv);
const restoreProviderEnv = applyEnvOverrides(prepared.extraEnv);
try {
const model = resolveModel(prepared.provider, job.model);
if (!model) {
throw new Error(`Unknown model ${job.model} for provider ${prepared.provider}.`);
}
resolvedModelId = model.id;
const extraHeaders = resolveExtraStreamHeaders(prepared.provider, prepared.apiKey);
const model = resolveModel(prepared.provider, job.model);
if (!model) {
throw new Error(`Unknown model ${job.model} for provider ${prepared.provider}.`);
}
resolvedModelId = model.id;
const extraHeaders = resolveExtraStreamHeaders(prepared.provider, prepared.apiKey);
const cwd = job.workspacePath ?? process.cwd();
const codingTools = createCodingTools(cwd);
const internalTools: ErasedAgentTool[] = [];
for (const toolName of job.internalTools ?? []) {
switch (toolName) {
case HIRE_AGENT_TOOL_NAME:
internalTools.push(createHireAgentTool(rpc, job.runId));
break;
case FILE_APPROVAL_TOOL_NAME:
internalTools.push(createFileApprovalTool(rpc, job.runId));
break;
default: {
// Exhaustiveness guard: a new entry in the `internalTools`
// array without a matching case should be a compile-time
// regression.
const _exhaustive: never = toolName;
throw new Error(`Unknown internal tool requested: ${String(_exhaustive)}`);
}
const cwd = job.workspacePath ?? process.cwd();
const codingTools = createRunScopedCodingTools(cwd, runEnv);
const internalTools: ErasedAgentTool[] = [];
for (const toolName of job.internalTools ?? []) {
switch (toolName) {
case HIRE_AGENT_TOOL_NAME:
internalTools.push(createHireAgentTool(rpc, job.runId));
break;
case FILE_APPROVAL_TOOL_NAME:
internalTools.push(createFileApprovalTool(rpc, job.runId));
break;
default: {
// Exhaustiveness guard: a new entry in the `internalTools`
// array without a matching case should be a compile-time
// regression.
const _exhaustive: never = toolName;
throw new Error(`Unknown internal tool requested: ${String(_exhaustive)}`);
}
}
const tools: ErasedAgentTool[] = [...codingTools, ...internalTools];
}
const tools: ErasedAgentTool[] = [...codingTools, ...internalTools];
const agent = createPiAgent({
runId: job.runId,
taskSessionId: job.taskSessionId,
systemPrompt: job.systemPrompt ?? "",
model,
tools,
priorMessages: prepared.priorMessages,
thinkingLevel: resolveThinkingLevel(job.thinkingEffort),
apiKey: prepared.apiKey,
extraHeaders,
rpc,
onContextCompaction: (telemetry) => {
post({
type: "context-compaction",
runId: telemetry.runId,
emittedAt: telemetry.emittedAt,
estimatedTokens: telemetry.estimatedTokens,
originalMessageCount: telemetry.originalMessageCount,
compactedMessageCount: telemetry.compactedMessageCount,
compactedOlderMessageCount: telemetry.compactedOlderMessageCount,
retainedRecentMessageCount: telemetry.retainedRecentMessageCount,
});
},
});
this.activeAgents.set(job.runId, agent);
post({ type: "run-agent-state", runId: job.runId, state: "agent_active" });
const unsubscribe = agent.subscribe(async (event) => {
const timestamp = new Date().toISOString();
const agent = createPiAgent({
runId: job.runId,
taskSessionId: job.taskSessionId,
systemPrompt: job.systemPrompt ?? "",
model,
tools,
priorMessages: prepared.priorMessages,
thinkingLevel: resolveThinkingLevel(job.thinkingEffort),
apiKey: prepared.apiKey,
extraHeaders,
rpc,
onContextCompaction: (telemetry) => {
post({
type: "agent-event",
runId: job.runId,
agentId: job.agentId,
companyId: job.companyId,
taskId: job.taskId,
emittedAt: timestamp,
event: event as { type: string } & Record<string, unknown>,
type: "context-compaction",
runId: telemetry.runId,
emittedAt: telemetry.emittedAt,
estimatedTokens: telemetry.estimatedTokens,
originalMessageCount: telemetry.originalMessageCount,
compactedMessageCount: telemetry.compactedMessageCount,
compactedOlderMessageCount: telemetry.compactedOlderMessageCount,
retainedRecentMessageCount: telemetry.retainedRecentMessageCount,
});
},
});
this.activeAgents.set(job.runId, agent);
post({ type: "run-agent-state", runId: job.runId, state: "agent_active" });
if (event.type === "message_end" && event.message) {
await rpc.invoke(job.runId, "persistMessage", {
taskSessionId: job.taskSessionId,
message: event.message,
});
const maybeAssistant = event.message as {
role?: string;
stopReason?: string;
errorMessage?: string;
usage?: { input?: number; output?: number; cost?: { total?: number } };
};
if (maybeAssistant.role === "assistant" && isTerminalAssistantStopReason(maybeAssistant.stopReason)) {
terminalStopReason = maybeAssistant.stopReason;
errorMessage = maybeAssistant.errorMessage ?? `Agent stopped with ${maybeAssistant.stopReason}.`;
}
if (hasAttributableAssistantUsage(maybeAssistant)) {
sawAttributableUsage = true;
finalUsage = {
inputTokens: typeof maybeAssistant.usage.input === "number" ? maybeAssistant.usage.input : finalUsage.inputTokens,
outputTokens: typeof maybeAssistant.usage.output === "number" ? maybeAssistant.usage.output : finalUsage.outputTokens,
};
}
}
const unsubscribe = agent.subscribe(async (event) => {
const timestamp = new Date().toISOString();
post({
type: "agent-event",
runId: job.runId,
agentId: job.agentId,
companyId: job.companyId,
taskId: job.taskId,
emittedAt: timestamp,
event: event as { type: string } & Record<string, unknown>,
});
try {
const promptContent = job.initialUserTurn.trim();
if (promptContent.length === 0) {
throw new Error(`Run ${job.runId} has an empty initial user turn.`);
}
const promptMessage: AgentMessage = {
role: "user",
content: [{ type: "text", text: promptContent }],
timestamp: Date.now(),
if (event.type === "message_end" && event.message) {
await rpc.invoke(job.runId, "persistMessage", {
taskSessionId: job.taskSessionId,
message: event.message,
});
const maybeAssistant = event.message as {
role?: string;
stopReason?: string;
errorMessage?: string;
usage?: { input?: number; output?: number; cost?: { total?: number } };
};
const runPromise = agent.prompt(promptMessage);
abortController.signal.addEventListener("abort", () => agent.abort(), { once: true });
await runPromise;
} finally {
post({ type: "run-agent-state", runId: job.runId, state: "closing" });
unsubscribe();
this.activeAgents.delete(job.runId);
if (maybeAssistant.role === "assistant" && isTerminalAssistantStopReason(maybeAssistant.stopReason)) {
terminalStopReason = maybeAssistant.stopReason;
errorMessage = maybeAssistant.errorMessage ?? `Agent stopped with ${maybeAssistant.stopReason}.`;
}
if (hasAttributableAssistantUsage(maybeAssistant)) {
sawAttributableUsage = true;
finalUsage = {
inputTokens: typeof maybeAssistant.usage.input === "number" ? maybeAssistant.usage.input : finalUsage.inputTokens,
outputTokens: typeof maybeAssistant.usage.output === "number" ? maybeAssistant.usage.output : finalUsage.outputTokens,
};
}
}
});
try {
const promptContent = job.initialUserTurn.trim();
if (promptContent.length === 0) {
throw new Error(`Run ${job.runId} has an empty initial user turn.`);
}
const promptMessage: AgentMessage = {
role: "user",
content: [{ type: "text", text: promptContent }],
timestamp: Date.now(),
};
const runPromise = agent.prompt(promptMessage);
abortController.signal.addEventListener("abort", () => agent.abort(), { once: true });
await runPromise;
} finally {
restoreProviderEnv();
post({ type: "run-agent-state", runId: job.runId, state: "closing" });
unsubscribe();
this.activeAgents.delete(job.runId);
}
} catch (error) {
errorMessage = error instanceof Error ? error.message : String(error);
post({ type: "run-log", runId: job.runId, stream: "stderr", chunk: redact(errorMessage) });
} finally {
this.abortControllers.delete(job.runId);
restoreJobEnv();
}
const status: RunStatus = isCancelled(job.runId)
+35
View File
@@ -0,0 +1,35 @@
import { createCodingTools, type BashToolOptions } from "@mariozechner/pi-coding-agent";
import type { AgentState } from "@mariozechner/pi-agent-core";
type ErasedAgentTool = AgentState["tools"][number];
export function mergeRunEnvs(
jobEnv: Record<string, string>,
preparedEnv: Record<string, string>,
): Record<string, string> {
return { ...jobEnv, ...preparedEnv };
}
export function createRunScopedBashOptions(
runEnv: Record<string, string>,
): BashToolOptions {
const envSnapshot = { ...runEnv };
return {
spawnHook: (context) => ({
...context,
env: {
...context.env,
...envSnapshot,
},
}),
};
}
export function createRunScopedCodingTools(
cwd: string,
runEnv: Record<string, string>,
): ErasedAgentTool[] {
return createCodingTools(cwd, {
bash: createRunScopedBashOptions(runEnv),
}) as ErasedAgentTool[];
}