Q55: the operator can stop in-flight runs; pause stops them too

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ZacharyZhang-NYandClaude Opus 5.5 committed 2026-10-08 09:59:03 -04:00
1 parent e809cfefea
commit 9e1f8da11c
18 files changed
+259 -17

No files matched your search

+2
View File
@@ -6,6 +6,7 @@ import type { AgentToolContext, AgentToolDeps } from "./agent-tools";
import type { JsonRecord } from "./provider-http";
import type { ProviderService } from "./provider-service";
import type { AgentTurnMessage, ToolCall, ToolDefinition } from "./provider-tool-turn";
import { throwIfRunStopped } from "./run-control";
const AUDIT_MAX_ITERATIONS = 6;
const STATUS_TASK_LINES = 8;
@@ -83,6 +84,7 @@ export async function completeGoalTool(args: GoalCompletionArgs): Promise<string
if (!verdict.approved) {
return `The independent audit rejected closing the goal: ${verdict.reason} The goal stays active — close that gap or create the tasks that will.`;
}
throwIfRunStopped();
deps.db.prepare("update goals set status = 'achieved', stalled_reason = null, updated_at = ? where id = ? and org_id = ?")
.run(new Date().toISOString(), goal.id, ctx.orgId);
new AuditService(deps.db).record({
+13 -6
View File
@@ -8,6 +8,7 @@ import type { AgentTurnMessage, ToolCall, ToolDefinition } from "./provider-tool
import { isComposioTool, executeComposioTool } from "./composio/composio-tool-bridge";
import { capToolOutput, compactToolHistory } from "./run-context-compaction";
import { fitContextBudget } from "./context-budget";
import { throwIfRunStopped, untilStopped } from "./run-control";
const MAX_ITERATIONS = 12;
@@ -67,14 +68,19 @@ export async function runAgentLoop(deps: AgentRunLoopDeps, input: AgentRunLoopIn
console.info(`[run-loop] iteration ${iteration}: elided ${compaction.elided} earlier tool result(s)`);
}
fitContextBudget(input.instructions, messages, allTools, input.inputBudgetTokens, iteration);
const turn = await deps.providers.runToolTurn({
throwIfRunStopped();
const turn = await untilStopped(deps.providers.runToolTurn({
orgId: input.orgId,
providerId: input.providerId,
workspaceId: input.workspaceId,
instructions: input.instructions,
messages,
tools: allTools,
})).catch((error: unknown) => {
throwIfRunStopped();
throw error;
});
throwIfRunStopped();
lastRequestId = turn.requestId;
input.onModelTurn?.();
@@ -96,18 +102,19 @@ export async function runAgentLoop(deps: AgentRunLoopDeps, input: AgentRunLoopIn
messages.push({ role: "assistant", text: turn.text, toolCalls: turn.toolCalls });
const results: Array<{ toolCallId: string; toolName: string; output: string }> = [];
for (const call of turn.toolCalls) {
throwIfRunStopped();
toolCallCount += 1;
let output: string;
if (isComposioTool(call.name)) {
output = await executeComposioTool(deps.tools.db, deps.providers.integrationKeys, input.orgId, input.toolContext.agent.id, call);
output = await untilStopped(executeComposioTool(deps.tools.db, deps.providers.integrationKeys, input.orgId, input.toolContext.agent.id, call));
} else if (isWorkspaceTool(call.name)) {
output = await executeWorkspaceTool(input.toolContext, call);
output = await untilStopped(executeWorkspaceTool(input.toolContext, call));
} else if (isWebTool(call.name)) {
output = await executeWebTool(call);
output = await untilStopped(executeWebTool(call));
} else if (call.name === "complete_goal") {
// Runs an independent audit turn against the provider, so it needs
// the loop's provider access rather than the sync dispatcher.
output = await completeGoalTool({
output = await untilStopped(completeGoalTool({
deps: deps.tools,
ctx: input.toolContext,
providers: deps.providers,
@@ -115,7 +122,7 @@ export async function runAgentLoop(deps: AgentRunLoopDeps, input: AgentRunLoopIn
workspaceId: input.workspaceId,
inputBudgetTokens: input.inputBudgetTokens,
call,
});
}));
} else {
output = dispatchAgentTool(deps.tools, input.toolContext, call, {
announceTaskAssignments: shouldAnnounceTaskAssignment(call, turn.toolCalls),
@@ -1,4 +1,5 @@
import { net } from "electron";
import { currentRunSignal, RUN_STOPPED_MESSAGE } from "../run-control";
const BASE_URL = "https://backend.composio.dev/api/v3";
@@ -153,6 +154,15 @@ export class ComposioClient {
): Promise<Record<string, unknown>> {
return new Promise((resolve, reject) => {
const request = net.request({ method, url });
const runSignal = currentRunSignal();
if (runSignal?.aborted) {
reject(new Error(RUN_STOPPED_MESSAGE));
return;
}
runSignal?.addEventListener("abort", () => {
request.abort();
reject(new Error(RUN_STOPPED_MESSAGE));
}, { once: true });
request.setHeader("x-api-key", this.apiKey);
request.setHeader("Accept", "application/json");
if (body) request.setHeader("Content-Type", "application/json");
+2 -2
View File
@@ -55,7 +55,7 @@ export class HeartbeatScheduler {
and not exists (select 1 from tasks c where c.parent_task_id = t.id and c.status <> 'done')
and (t.origin is not null or t.goal_id is null or exists (select 1 from goals g where g.id = t.goal_id and g.status = 'active'))
and (a.heartbeat_interval_sec <= 0 or not exists (
select 1 from task_runs hb where hb.agent_id = a.id and hb.trigger = 'heartbeat' and hb.state <> 'superseded'
select 1 from task_runs hb where hb.agent_id = a.id and hb.trigger = 'heartbeat' and hb.state not in ('superseded', 'stopped')
and hb.created_at > strftime('%Y-%m-%dT%H:%M:%fZ', ?, '-' || a.heartbeat_interval_sec || ' seconds')
))
order by case t.priority when 'high' then 0 when 'normal' then 1 else 2 end, t.created_at
@@ -64,7 +64,7 @@ export class HeartbeatScheduler {
const cooling = new Set(this.cooldowns.cooledProviderIds());
const attemptMessages = this.db.prepare(`
select r.error_message from task_runs r join tasks t on t.id = r.task_id
where r.task_id = ? and r.state <> 'superseded' and r.rowid > t.run_budget_floor
where r.task_id = ? and r.state not in ('superseded', 'stopped') and r.rowid > t.run_budget_floor
`);
const created: string[] = [];
const pacedAgents = new Set<string>();
+13 -2
View File
@@ -1,5 +1,6 @@
import type { ProviderObservedRateLimitState } from "../../shared/provider-runtime-domain";
import { parseProviderRateLimitHeaders } from "./provider-rate-limit-registry";
import { currentRunSignal, throwIfRunStopped } from "./run-control";
export type JsonRecord = Record<string, unknown>;
@@ -69,7 +70,7 @@ export async function postEventStream(
body: JsonRecord,
timeouts: ProviderRequestTimeouts = {},
): Promise<ProviderEventStreamResponse> {
const controller = new AbortController();
const controller = linkedController();
const response = await withDeadline(controller, timeouts.headersTimeoutMs ?? HEADERS_TIMEOUT_MS, () => fetch(endpoint, {
method: "POST",
headers: {
@@ -92,7 +93,7 @@ export async function postEventStream(
}
async function requestJson(endpoint: string, init: RequestInit, timeouts: ProviderRequestTimeouts = {}): Promise<ProviderJsonResponse> {
const controller = new AbortController();
const controller = linkedController();
const response = await withDeadline(controller, timeouts.headersTimeoutMs ?? HEADERS_TIMEOUT_MS, () => fetch(endpoint, { ...init, signal: controller.signal }));
const rateLimitState = parseProviderRateLimitHeaders(response.headers);
const raw = await withDeadline(controller, timeouts.bodyTimeoutMs ?? BODY_TIMEOUT_MS, () => response.text());
@@ -110,11 +111,21 @@ async function requestJson(endpoint: string, init: RequestInit, timeouts: Provid
* then stalls the body must not hang the caller, so the body read gets its
* own (longer) deadline after the headers arrive.
*/
/** A request controller that also aborts when the operator stops the run it belongs to. */
function linkedController(): AbortController {
const controller = new AbortController();
const runSignal = currentRunSignal();
if (runSignal?.aborted) controller.abort();
else runSignal?.addEventListener("abort", () => controller.abort(), { once: true });
return controller;
}
async function withDeadline<T>(controller: AbortController, timeoutMs: number, operation: () => Promise<T>): Promise<T> {
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
return await operation();
} catch (error) {
throwIfRunStopped();
if (controller.signal.aborted) throw new ProviderHttpError("Provider request timed out.", null);
throw error;
} finally {
+54
View File
@@ -0,0 +1,54 @@
import { AsyncLocalStorage } from "node:async_hooks";
/** The error message a run ends with when the operator stops it. */
export const RUN_STOPPED_MESSAGE = "Stopped by the operator.";
const runSignals = new AsyncLocalStorage<AbortSignal>();
const controllers = new Map<string, AbortController>();
/** Runs a session under a stop signal that every provider request and tool inside it observes. */
export async function withRunControl<T>(runId: string, operation: () => Promise<T>): Promise<T> {
const controller = new AbortController();
controllers.set(runId, controller);
try {
return await runSignals.run(controller.signal, operation);
} finally {
controllers.delete(runId);
}
}
/** Stops one in-flight run; false when it is not running in this process. */
export function stopRun(runId: string): boolean {
const controller = controllers.get(runId);
if (!controller) return false;
console.info(`[run-control] stopping run ${runId}`);
controller.abort();
return true;
}
export function stopAllRuns(): number {
const ids = [...controllers.keys()];
for (const id of ids) stopRun(id);
return ids.length;
}
/** The stop signal of the run this code executes in, if any. */
export function currentRunSignal(): AbortSignal | undefined {
return runSignals.getStore();
}
export function throwIfRunStopped(): void {
if (currentRunSignal()?.aborted) throw new Error(RUN_STOPPED_MESSAGE);
}
/** Settles with the operation, or rejects as soon as the run is stopped; work without its own abort is abandoned. */
export function untilStopped<T>(operation: Promise<T>): Promise<T> {
const signal = currentRunSignal();
if (!signal) return operation;
if (signal.aborted) return Promise.reject(new Error(RUN_STOPPED_MESSAGE));
return new Promise<T>((resolve, reject) => {
const onAbort = () => reject(new Error(RUN_STOPPED_MESSAGE));
signal.addEventListener("abort", onAbort, { once: true });
operation.then(resolve, reject).finally(() => signal.removeEventListener("abort", onAbort));
});
}
+19 -2
View File
@@ -13,6 +13,7 @@ import { runAgentLoop, runTools } from "./agent-run-loop";
import type { RunModel } from "./skill-runtime";
import { combineMidRunPolls, createGoalChangePoller } from "./goal-change-poller";
import { recordRunAction, runJournalLines } from "./run-journal";
import { currentRunSignal, RUN_STOPPED_MESSAGE, withRunControl } from "./run-control";
import type { AgentToolContext } from "./agent-tools";
import {
buildAgentInstructions,
@@ -46,6 +47,7 @@ const TASK_RUN_STATE = {
COMPLETED: "completed",
FAILED: "failed",
SUPERSEDED: "superseded",
STOPPED: "stopped",
} as const;
@@ -97,7 +99,7 @@ export class TaskExecutionService {
const workspace = loadWorkspace(this.db, task.workspace_id);
const isolation = createIsolationContext(this.db, input.orgId, task, agent, provider, workspace);
const runId = this.createRun(input.orgId, task.id, agent.id, provider.id, isolation);
return this.executeAgentRun({ orgId: input.orgId, runId, task, agent, provider, workspace, isolation });
return withRunControl(runId, () => this.executeAgentRun({ orgId: input.orgId, runId, task, agent, provider, workspace, isolation }));
}
/** Executes a claimed run after re-checking that the task still matches it; a mismatch supersedes the run. */
@@ -155,7 +157,7 @@ export class TaskExecutionService {
if (task && task.origin === null) recordRunMemory(this.db, { orgId, runId, projectId: task.project_id, taskTitle: task.title, outcome: "failed", text: message });
throw error;
}
return this.executeAgentRun(prepared);
return withRunControl(runId, () => this.executeAgentRun(prepared));
}
private async executeAgentRun(args: {
@@ -280,6 +282,11 @@ export class TaskExecutionService {
return runId;
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
// A stop wins over whatever error the interrupted step raised.
if (currentRunSignal()?.aborted) {
this.markStopped(orgId, runId, providerRequestId, { taskId: task.id, agentId: agent.id, providerId: provider.id });
throw error;
}
this.failRun(orgId, runId, providerRequestId, message);
if (isTransientProviderFailure(message)) this.cooldowns.park(provider, message);
else if (task.origin === null) recordRunMemory(this.db, { orgId, runId, projectId: task.project_id, taskTitle: task.title, outcome: "failed", text: message });
@@ -415,6 +422,16 @@ export class TaskExecutionService {
this.organization.updateTaskStatus(orgId, run.task_id, TASK_STATUS.DONE);
}
/** The operator stopped the run: not a failure, no retry budget spent, no memory recorded. */
private markStopped(orgId: string, runId: string, providerRequestId: string | null, payload: { taskId: string; agentId: string; providerId: string }): void {
this.db.prepare(`
update task_runs set state = ?, provider_request_id = coalesce(?, provider_request_id), completed_at = ?, error_message = ?
where id = ? and org_id = ?
`).run(TASK_RUN_STATE.STOPPED, providerRequestId, new Date().toISOString(), RUN_STOPPED_MESSAGE, runId, orgId);
this.closeChatWakeTask(orgId, runId);
this.audit.record({ orgId, entityType: "task_run", entityId: runId, actorType: "employee", actorId: null, action: "stopped", payload });
}
/** Closes a queued run that no longer matches its task; it is not a failure and spends no retry budget. */
private supersedeRun(orgId: string, runId: string, run: RunRow, reason: string): string {
const message = `Superseded: ${reason}`;
+2
View File
@@ -1,4 +1,5 @@
import { spawn } from "node:child_process";
import { currentRunSignal } from "./run-control";
const COMMAND_TIMEOUT_MS = 120_000;
const KEEP_BYTES = 8_000;
@@ -51,6 +52,7 @@ export function runCommand(workspaceRoot: string, command: string): Promise<stri
killedByTimeout = true;
child.kill();
}, COMMAND_TIMEOUT_MS);
currentRunSignal()?.addEventListener("abort", () => child.kill(), { once: true });
const collect = (chunk: Buffer): void => output.push(chunk);
child.stdout.on("data", collect);
child.stderr.on("data", collect);