This commit is contained in:
ZacharyZhang-NY committed 2026-04-05 20:14:48 -04:00
1 parent 09971cabc2
commit 9145572422
163 files changed
+3348 -2874

No files matched your search

+49
View File
@@ -0,0 +1,49 @@
/**
* Self-wake circuit breaker.
*
* Prevents infinite self-wake loops when an agent is stuck on a task.
* After MAX_CONSECUTIVE_WAKES consecutive wakes without progress (i.e.,
* without a successful run), self-wake is suppressed so the manager can
* be notified instead.
*/
export const MAX_CONSECUTIVE_WAKES = 5;
export interface CircuitBreaker {
/** Attempt a self-wake. Returns true if allowed, false if circuit breaker trips. */
tryWake(agentId: string, companyId: string): boolean;
/** Reset the counter for an agent (called on task progress / successful run). */
reset(agentId: string, companyId: string): void;
/** Get the current consecutive-wake count for an agent. */
getCount(agentId: string, companyId: string): number;
}
export function createCircuitBreaker(
consecutiveWakeMap: Map<string, number>,
maxConsecutive: number = MAX_CONSECUTIVE_WAKES,
): CircuitBreaker {
function wakeKey(agentId: string, companyId: string): string {
return `${agentId}:${companyId}`;
}
return {
tryWake(agentId: string, companyId: string): boolean {
const key = wakeKey(agentId, companyId);
const count = consecutiveWakeMap.get(key) ?? 0;
if (count >= maxConsecutive) {
consecutiveWakeMap.delete(key);
return false;
}
consecutiveWakeMap.set(key, count + 1);
return true;
},
reset(agentId: string, companyId: string): void {
consecutiveWakeMap.delete(wakeKey(agentId, companyId));
},
getCount(agentId: string, companyId: string): number {
return consecutiveWakeMap.get(wakeKey(agentId, companyId)) ?? 0;
},
};
}
+118
View File
@@ -0,0 +1,118 @@
/**
* Retry-aware failure escalation pipeline.
*
* State machine:
* 1. Task fails with retry_count < MAX_RETRY_ATTEMPTS -> increment count, re-queue
* 2. Task fails with retry_count >= MAX_RETRY_ATTEMPTS -> escalate to manager chain
* 3. No manager in chain -> fall through to CEO
* 4. No CEO available -> notify user
* 5. Task succeeds -> reset retry count
*/
export interface EscalationDeps {
db: {
getTaskRetryCount: (taskId: string) => number;
incrementTaskRetryCount: (taskId: string) => void;
resetTaskRetryCount: (taskId: string) => void;
getChainOfCommand: (agentId: string) => Array<{ id: string; name: string }>;
getCeoForCompany: (companyId: string) => { id: string; name: string } | null;
advanceTaskStatus: (taskId: string, status: string, note: string) => unknown;
};
wakeAgentIfPossible: (agentId: string, companyId: string, trigger: string) => unknown;
notify: (title: string, body: string) => void;
logger: {
info: (...args: unknown[]) => void;
warn: (...args: unknown[]) => void;
error: (...args: unknown[]) => void;
};
}
export interface EscalationResult {
action: "retry" | "escalated" | "user_notified" | "none";
retryScheduled?: boolean;
}
export function handleFailureEscalation(
deps: EscalationDeps,
run: { taskId: string; agentId: string; companyId: string },
task: { title: string } | null,
eventStatus: "failed" | "timed_out",
maxRetryAttempts: number,
retryCooldownMs: number,
scheduleRetry: (fn: () => void, delayMs: number) => void,
): EscalationResult {
const { db, wakeAgentIfPossible, notify, logger } = deps;
const retryCount = db.getTaskRetryCount(run.taskId);
if (retryCount < maxRetryAttempts) {
db.incrementTaskRetryCount(run.taskId);
db.advanceTaskStatus(
run.taskId,
"todo",
`Run ${eventStatus} (attempt ${retryCount + 1}/${maxRetryAttempts}). Auto-retrying after ${retryCooldownMs / 1000}s cooldown.`,
);
logger.info(
`[escalation] Retrying task ${run.taskId} (attempt ${retryCount + 1}/${maxRetryAttempts}) for agent ${run.agentId}`,
);
scheduleRetry(() => {
wakeAgentIfPossible(run.agentId, run.companyId, "retry_failed");
}, retryCooldownMs);
return { action: "retry", retryScheduled: true };
}
// Retry limit reached — escalate up the chain
db.resetTaskRetryCount(run.taskId);
logger.warn(
`[escalation] Task ${run.taskId} failed ${maxRetryAttempts} times. Escalating to manager.`,
);
let escalated = false;
try {
const chain = db.getChainOfCommand(run.agentId);
if (chain.length > 0) {
wakeAgentIfPossible(chain[0].id, run.companyId, "report_failed");
escalated = true;
logger.info(
`[escalation] Escalated task ${run.taskId} to manager ${chain[0].name} (${chain[0].id})`,
);
}
} catch {
// Chain lookup failed (agent deleted)
}
if (!escalated) {
try {
const ceo = db.getCeoForCompany(run.companyId);
if (ceo) {
wakeAgentIfPossible(ceo.id, run.companyId, "report_failed");
escalated = true;
logger.info(
`[escalation] Escalated task ${run.taskId} to CEO ${ceo.name} (${ceo.id})`,
);
}
} catch { /* CEO lookup failed */ }
}
if (!escalated) {
const taskTitle = task?.title ?? run.taskId;
notify(
"Task failed -- manual intervention needed",
`"${taskTitle}" failed ${maxRetryAttempts} times and no manager is available to handle it.`,
);
logger.error(
`[escalation] Task ${run.taskId} failed with no escalation target. User notified.`,
);
return { action: "user_notified" };
}
return { action: "escalated" };
}
export function handleSuccessReset(
db: EscalationDeps["db"],
taskId: string,
): void {
try {
db.resetTaskRetryCount(taskId);
} catch { /* best effort */ }
}
+8 -3
View File
@@ -80,7 +80,12 @@ const activeByAgent = new Map<string, string>();
const activeByWorkspace = new Map<string, string>();
const queue: RunJob[] = [];
const cancelledRuns = new Set<string>();
const allowUnsafeConnectorBypass = process.env.AGENTCOMPANY_ALLOW_UNSAFE_CONNECTOR_BYPASS !== "false";
const allowUnsafeConnectorBypass =
process.env.AGENTCOMPANY_ALLOW_UNSAFE_CONNECTOR_BYPASS === "true" &&
process.env.NODE_ENV === "development";
if (allowUnsafeConnectorBypass) {
process.stderr.write("[SECURITY WARNING] AGENTCOMPANY_ALLOW_UNSAFE_CONNECTOR_BYPASS is enabled (development only) — sandbox, approvals, and permissions checks are disabled.\n");
}
function post(message: ChildMessage) {
parentPort.postMessage(message);
@@ -133,8 +138,8 @@ function buildConnectorExecutionConfig(job: RunJob, sanitizedEnv: Record<string,
case "gemini_local":
return {
...baseConfig,
yolo: true,
approvalMode: "yolo",
yolo: allowUnsafeConnectorBypass,
approvalMode: allowUnsafeConnectorBypass ? "yolo" : "suggest",
sandbox: true,
};
default:
+30 -65
View File
@@ -1,6 +1,8 @@
import { z } from "zod";
import { getConnectorDefinition } from "../connectors";
import type { WorkerServiceDependencies, workerMessageSchema } from "./worker-types";
import { createCircuitBreaker } from "./circuit-breaker";
import { handleFailureEscalation, handleSuccessReset } from "./failure-escalation";
const LEASE_RENEWAL_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
@@ -261,61 +263,27 @@ export async function handleWorkerMessage(
}
if ((event.status === "failed" || event.status === "timed_out") && !isQuitting() && !destroyed) {
const retryCount = db.getTaskRetryCount(run.taskId);
if (retryCount < MAX_RETRY_ATTEMPTS) {
// Retry same agent after cooldown
db.incrementTaskRetryCount(run.taskId);
db.advanceTaskStatus(run.taskId, "todo",
`Run ${event.status} (attempt ${retryCount + 1}/${MAX_RETRY_ATTEMPTS}). Auto-retrying after ${RETRY_COOLDOWN_MS / 1000}s cooldown.`);
logger.info(`[escalation] Retrying task ${run.taskId} (attempt ${retryCount + 1}/${MAX_RETRY_ATTEMPTS}) for agent ${run.agentId}`);
const retryTimer = setTimeout(() => {
pendingRetryTimers.delete(retryTimer);
if (isQuitting() || destroyed) return;
wakeAgentIfPossible(run.agentId, run.companyId, "retry_failed");
}, RETRY_COOLDOWN_MS);
pendingRetryTimers.add(retryTimer);
} else {
// Retry limit reached -- escalate up the chain
db.resetTaskRetryCount(run.taskId);
logger.warn(`[escalation] Task ${run.taskId} failed ${MAX_RETRY_ATTEMPTS} times. Escalating to manager.`);
let escalated = false;
try {
const chain = db.getChainOfCommand(run.agentId);
if (chain.length > 0) {
wakeAgentIfPossible(chain[0].id, run.companyId, "report_failed");
escalated = true;
logger.info(`[escalation] Escalated task ${run.taskId} to manager ${chain[0].name} (${chain[0].id})`);
}
} catch {
// Chain lookup failed (agent deleted)
}
if (!escalated) {
// No manager found -- try CEO
try {
const ceo = db.getCeoForCompany(run.companyId);
if (ceo) {
wakeAgentIfPossible(ceo.id, run.companyId, "report_failed");
escalated = true;
logger.info(`[escalation] Escalated task ${run.taskId} to CEO ${ceo.name} (${ceo.id})`);
}
} catch { /* CEO lookup failed */ }
}
if (!escalated) {
// No one to escalate to -- notify user
const taskTitle = task?.title ?? run.taskId;
notify({
title: "Attention Required",
body: `${taskTitle} needs manual intervention`,
urgency: "critical",
navigation: { section: "tasks", entityId: run.taskId },
});
logger.error(`[escalation] Task ${run.taskId} failed with no escalation target. User notified.`);
}
}
handleFailureEscalation(
{
db,
wakeAgentIfPossible,
notify: (title, body) => notify({ title, body, urgency: "critical", navigation: { section: "tasks", entityId: run.taskId } }),
logger,
},
{ taskId: run.taskId, agentId: run.agentId, companyId: run.companyId },
task,
event.status,
MAX_RETRY_ATTEMPTS,
RETRY_COOLDOWN_MS,
(fn, delayMs) => {
const retryTimer = setTimeout(() => {
pendingRetryTimers.delete(retryTimer);
if (isQuitting() || destroyed) return;
fn();
}, delayMs);
pendingRetryTimers.add(retryTimer);
},
);
}
// EXEC-05: Promote deferred wakes after run completion
@@ -348,13 +316,14 @@ export async function handleWorkerMessage(
// Reset retry count on successful completion
if (event.status === "succeeded") {
try { db.resetTaskRetryCount(run.taskId); } catch { /* best effort */ }
handleSuccessReset(db, run.taskId);
}
const cb = createCircuitBreaker(consecutiveWakeMap, MAX_CONSECUTIVE_WAKES);
// Reset circuit breaker when a task actually progressed (successful run)
if (event.status === "succeeded") {
const wakeKey = `${run.agentId}:${run.companyId}`;
consecutiveWakeMap.delete(wakeKey);
cb.reset(run.agentId, run.companyId);
}
// Self-wake with circuit breaker to prevent infinite loops
@@ -362,12 +331,9 @@ export async function handleWorkerMessage(
try {
const actionableTasks = db.listAgentTasks(run.agentId, run.companyId, ["todo", "in_progress"]);
if (actionableTasks.length > 0) {
const wakeKey = `${run.agentId}:${run.companyId}`;
const consecutiveCount = consecutiveWakeMap.get(wakeKey) ?? 0;
if (consecutiveCount >= MAX_CONSECUTIVE_WAKES) {
logger.warn(`[self-wake] Circuit breaker: agent ${run.agentId} has self-woken ${consecutiveCount} times without progress. Pausing self-wake.`);
consecutiveWakeMap.delete(wakeKey);
const allowed = cb.tryWake(run.agentId, run.companyId);
if (!allowed) {
logger.warn(`[self-wake] Circuit breaker: agent ${run.agentId} has self-woken ${MAX_CONSECUTIVE_WAKES} times without progress. Pausing self-wake.`);
// Escalate to manager since agent is stuck
try {
const chain = db.getChainOfCommand(run.agentId);
@@ -376,7 +342,6 @@ export async function handleWorkerMessage(
}
} catch { /* best effort */ }
} else {
consecutiveWakeMap.set(wakeKey, consecutiveCount + 1);
wakeAgentIfPossible(run.agentId, run.companyId, "task_continuation");
}
}
+3 -2
View File
@@ -1,5 +1,6 @@
import { z } from "zod";
import type { AppDatabase } from "../database";
import type { DesktopEvent, RunRecord } from "@shared/types";
import type { NotificationOptions } from "../notification-service";
import type { createBudgetEnforcementService } from "../budget-enforcement-service";
import type { createSessionManagementService } from "../session-management-service";
@@ -48,7 +49,7 @@ export interface WorkerServiceDependencies {
workerPath: string;
isQuitting: () => boolean;
publishDomainChanged: () => void;
emitEvent: (event: any) => void;
emitEvent: (event: DesktopEvent) => void;
wakeAgentIfPossible: (agentId: string | null | undefined, companyId: string, trigger: string) => string | null;
dispatchRunToWorker: (params: {
runId: string;
@@ -63,7 +64,7 @@ export interface WorkerServiceDependencies {
snapshot: ReturnType<AppDatabase["listSnapshot"]>;
trigger: string;
}) => void;
processRunCompletion: (run: any, event: z.infer<typeof workerMessageSchema> & { type: "run-finished" }) => void;
processRunCompletion: (run: RunRecord, event: z.infer<typeof workerMessageSchema> & { type: "run-finished" }) => void;
onRunStarted?: (runId: string, taskId: string, agentId: string, companyId: string) => void;
onRunCompleted?: (runId: string, taskId: string, agentId: string, companyId: string, status: string, summary: string | null, errorMessage: string | null) => void;
notify: (options: NotificationOptions) => void;