update
This commit is contained in:
1 parent
09971cabc2
commit
9145572422
163 files changed
+3348
-2874
No files matched your search
@@ -0,0 +1,49 @@
|
||||
/**
|
||||
* Self-wake circuit breaker.
|
||||
*
|
||||
* Prevents infinite self-wake loops when an agent is stuck on a task.
|
||||
* After MAX_CONSECUTIVE_WAKES consecutive wakes without progress (i.e.,
|
||||
* without a successful run), self-wake is suppressed so the manager can
|
||||
* be notified instead.
|
||||
*/
|
||||
|
||||
export const MAX_CONSECUTIVE_WAKES = 5;
|
||||
|
||||
export interface CircuitBreaker {
|
||||
/** Attempt a self-wake. Returns true if allowed, false if circuit breaker trips. */
|
||||
tryWake(agentId: string, companyId: string): boolean;
|
||||
/** Reset the counter for an agent (called on task progress / successful run). */
|
||||
reset(agentId: string, companyId: string): void;
|
||||
/** Get the current consecutive-wake count for an agent. */
|
||||
getCount(agentId: string, companyId: string): number;
|
||||
}
|
||||
|
||||
export function createCircuitBreaker(
|
||||
consecutiveWakeMap: Map<string, number>,
|
||||
maxConsecutive: number = MAX_CONSECUTIVE_WAKES,
|
||||
): CircuitBreaker {
|
||||
function wakeKey(agentId: string, companyId: string): string {
|
||||
return `${agentId}:${companyId}`;
|
||||
}
|
||||
|
||||
return {
|
||||
tryWake(agentId: string, companyId: string): boolean {
|
||||
const key = wakeKey(agentId, companyId);
|
||||
const count = consecutiveWakeMap.get(key) ?? 0;
|
||||
if (count >= maxConsecutive) {
|
||||
consecutiveWakeMap.delete(key);
|
||||
return false;
|
||||
}
|
||||
consecutiveWakeMap.set(key, count + 1);
|
||||
return true;
|
||||
},
|
||||
|
||||
reset(agentId: string, companyId: string): void {
|
||||
consecutiveWakeMap.delete(wakeKey(agentId, companyId));
|
||||
},
|
||||
|
||||
getCount(agentId: string, companyId: string): number {
|
||||
return consecutiveWakeMap.get(wakeKey(agentId, companyId)) ?? 0;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
/**
|
||||
* Retry-aware failure escalation pipeline.
|
||||
*
|
||||
* State machine:
|
||||
* 1. Task fails with retry_count < MAX_RETRY_ATTEMPTS -> increment count, re-queue
|
||||
* 2. Task fails with retry_count >= MAX_RETRY_ATTEMPTS -> escalate to manager chain
|
||||
* 3. No manager in chain -> fall through to CEO
|
||||
* 4. No CEO available -> notify user
|
||||
* 5. Task succeeds -> reset retry count
|
||||
*/
|
||||
|
||||
export interface EscalationDeps {
|
||||
db: {
|
||||
getTaskRetryCount: (taskId: string) => number;
|
||||
incrementTaskRetryCount: (taskId: string) => void;
|
||||
resetTaskRetryCount: (taskId: string) => void;
|
||||
getChainOfCommand: (agentId: string) => Array<{ id: string; name: string }>;
|
||||
getCeoForCompany: (companyId: string) => { id: string; name: string } | null;
|
||||
advanceTaskStatus: (taskId: string, status: string, note: string) => unknown;
|
||||
};
|
||||
wakeAgentIfPossible: (agentId: string, companyId: string, trigger: string) => unknown;
|
||||
notify: (title: string, body: string) => void;
|
||||
logger: {
|
||||
info: (...args: unknown[]) => void;
|
||||
warn: (...args: unknown[]) => void;
|
||||
error: (...args: unknown[]) => void;
|
||||
};
|
||||
}
|
||||
|
||||
export interface EscalationResult {
|
||||
action: "retry" | "escalated" | "user_notified" | "none";
|
||||
retryScheduled?: boolean;
|
||||
}
|
||||
|
||||
export function handleFailureEscalation(
|
||||
deps: EscalationDeps,
|
||||
run: { taskId: string; agentId: string; companyId: string },
|
||||
task: { title: string } | null,
|
||||
eventStatus: "failed" | "timed_out",
|
||||
maxRetryAttempts: number,
|
||||
retryCooldownMs: number,
|
||||
scheduleRetry: (fn: () => void, delayMs: number) => void,
|
||||
): EscalationResult {
|
||||
const { db, wakeAgentIfPossible, notify, logger } = deps;
|
||||
const retryCount = db.getTaskRetryCount(run.taskId);
|
||||
|
||||
if (retryCount < maxRetryAttempts) {
|
||||
db.incrementTaskRetryCount(run.taskId);
|
||||
db.advanceTaskStatus(
|
||||
run.taskId,
|
||||
"todo",
|
||||
`Run ${eventStatus} (attempt ${retryCount + 1}/${maxRetryAttempts}). Auto-retrying after ${retryCooldownMs / 1000}s cooldown.`,
|
||||
);
|
||||
logger.info(
|
||||
`[escalation] Retrying task ${run.taskId} (attempt ${retryCount + 1}/${maxRetryAttempts}) for agent ${run.agentId}`,
|
||||
);
|
||||
scheduleRetry(() => {
|
||||
wakeAgentIfPossible(run.agentId, run.companyId, "retry_failed");
|
||||
}, retryCooldownMs);
|
||||
return { action: "retry", retryScheduled: true };
|
||||
}
|
||||
|
||||
// Retry limit reached — escalate up the chain
|
||||
db.resetTaskRetryCount(run.taskId);
|
||||
logger.warn(
|
||||
`[escalation] Task ${run.taskId} failed ${maxRetryAttempts} times. Escalating to manager.`,
|
||||
);
|
||||
|
||||
let escalated = false;
|
||||
try {
|
||||
const chain = db.getChainOfCommand(run.agentId);
|
||||
if (chain.length > 0) {
|
||||
wakeAgentIfPossible(chain[0].id, run.companyId, "report_failed");
|
||||
escalated = true;
|
||||
logger.info(
|
||||
`[escalation] Escalated task ${run.taskId} to manager ${chain[0].name} (${chain[0].id})`,
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// Chain lookup failed (agent deleted)
|
||||
}
|
||||
|
||||
if (!escalated) {
|
||||
try {
|
||||
const ceo = db.getCeoForCompany(run.companyId);
|
||||
if (ceo) {
|
||||
wakeAgentIfPossible(ceo.id, run.companyId, "report_failed");
|
||||
escalated = true;
|
||||
logger.info(
|
||||
`[escalation] Escalated task ${run.taskId} to CEO ${ceo.name} (${ceo.id})`,
|
||||
);
|
||||
}
|
||||
} catch { /* CEO lookup failed */ }
|
||||
}
|
||||
|
||||
if (!escalated) {
|
||||
const taskTitle = task?.title ?? run.taskId;
|
||||
notify(
|
||||
"Task failed -- manual intervention needed",
|
||||
`"${taskTitle}" failed ${maxRetryAttempts} times and no manager is available to handle it.`,
|
||||
);
|
||||
logger.error(
|
||||
`[escalation] Task ${run.taskId} failed with no escalation target. User notified.`,
|
||||
);
|
||||
return { action: "user_notified" };
|
||||
}
|
||||
|
||||
return { action: "escalated" };
|
||||
}
|
||||
|
||||
export function handleSuccessReset(
|
||||
db: EscalationDeps["db"],
|
||||
taskId: string,
|
||||
): void {
|
||||
try {
|
||||
db.resetTaskRetryCount(taskId);
|
||||
} catch { /* best effort */ }
|
||||
}
|
||||
@@ -80,7 +80,12 @@ const activeByAgent = new Map<string, string>();
|
||||
const activeByWorkspace = new Map<string, string>();
|
||||
const queue: RunJob[] = [];
|
||||
const cancelledRuns = new Set<string>();
|
||||
const allowUnsafeConnectorBypass = process.env.AGENTCOMPANY_ALLOW_UNSAFE_CONNECTOR_BYPASS !== "false";
|
||||
const allowUnsafeConnectorBypass =
|
||||
process.env.AGENTCOMPANY_ALLOW_UNSAFE_CONNECTOR_BYPASS === "true" &&
|
||||
process.env.NODE_ENV === "development";
|
||||
if (allowUnsafeConnectorBypass) {
|
||||
process.stderr.write("[SECURITY WARNING] AGENTCOMPANY_ALLOW_UNSAFE_CONNECTOR_BYPASS is enabled (development only) — sandbox, approvals, and permissions checks are disabled.\n");
|
||||
}
|
||||
|
||||
function post(message: ChildMessage) {
|
||||
parentPort.postMessage(message);
|
||||
@@ -133,8 +138,8 @@ function buildConnectorExecutionConfig(job: RunJob, sanitizedEnv: Record<string,
|
||||
case "gemini_local":
|
||||
return {
|
||||
...baseConfig,
|
||||
yolo: true,
|
||||
approvalMode: "yolo",
|
||||
yolo: allowUnsafeConnectorBypass,
|
||||
approvalMode: allowUnsafeConnectorBypass ? "yolo" : "suggest",
|
||||
sandbox: true,
|
||||
};
|
||||
default:
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { z } from "zod";
|
||||
import { getConnectorDefinition } from "../connectors";
|
||||
import type { WorkerServiceDependencies, workerMessageSchema } from "./worker-types";
|
||||
import { createCircuitBreaker } from "./circuit-breaker";
|
||||
import { handleFailureEscalation, handleSuccessReset } from "./failure-escalation";
|
||||
|
||||
const LEASE_RENEWAL_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
|
||||
|
||||
@@ -261,61 +263,27 @@ export async function handleWorkerMessage(
|
||||
}
|
||||
|
||||
if ((event.status === "failed" || event.status === "timed_out") && !isQuitting() && !destroyed) {
|
||||
const retryCount = db.getTaskRetryCount(run.taskId);
|
||||
|
||||
if (retryCount < MAX_RETRY_ATTEMPTS) {
|
||||
// Retry same agent after cooldown
|
||||
db.incrementTaskRetryCount(run.taskId);
|
||||
db.advanceTaskStatus(run.taskId, "todo",
|
||||
`Run ${event.status} (attempt ${retryCount + 1}/${MAX_RETRY_ATTEMPTS}). Auto-retrying after ${RETRY_COOLDOWN_MS / 1000}s cooldown.`);
|
||||
logger.info(`[escalation] Retrying task ${run.taskId} (attempt ${retryCount + 1}/${MAX_RETRY_ATTEMPTS}) for agent ${run.agentId}`);
|
||||
const retryTimer = setTimeout(() => {
|
||||
pendingRetryTimers.delete(retryTimer);
|
||||
if (isQuitting() || destroyed) return;
|
||||
wakeAgentIfPossible(run.agentId, run.companyId, "retry_failed");
|
||||
}, RETRY_COOLDOWN_MS);
|
||||
pendingRetryTimers.add(retryTimer);
|
||||
} else {
|
||||
// Retry limit reached -- escalate up the chain
|
||||
db.resetTaskRetryCount(run.taskId);
|
||||
logger.warn(`[escalation] Task ${run.taskId} failed ${MAX_RETRY_ATTEMPTS} times. Escalating to manager.`);
|
||||
|
||||
let escalated = false;
|
||||
try {
|
||||
const chain = db.getChainOfCommand(run.agentId);
|
||||
if (chain.length > 0) {
|
||||
wakeAgentIfPossible(chain[0].id, run.companyId, "report_failed");
|
||||
escalated = true;
|
||||
logger.info(`[escalation] Escalated task ${run.taskId} to manager ${chain[0].name} (${chain[0].id})`);
|
||||
}
|
||||
} catch {
|
||||
// Chain lookup failed (agent deleted)
|
||||
}
|
||||
|
||||
if (!escalated) {
|
||||
// No manager found -- try CEO
|
||||
try {
|
||||
const ceo = db.getCeoForCompany(run.companyId);
|
||||
if (ceo) {
|
||||
wakeAgentIfPossible(ceo.id, run.companyId, "report_failed");
|
||||
escalated = true;
|
||||
logger.info(`[escalation] Escalated task ${run.taskId} to CEO ${ceo.name} (${ceo.id})`);
|
||||
}
|
||||
} catch { /* CEO lookup failed */ }
|
||||
}
|
||||
|
||||
if (!escalated) {
|
||||
// No one to escalate to -- notify user
|
||||
const taskTitle = task?.title ?? run.taskId;
|
||||
notify({
|
||||
title: "Attention Required",
|
||||
body: `${taskTitle} needs manual intervention`,
|
||||
urgency: "critical",
|
||||
navigation: { section: "tasks", entityId: run.taskId },
|
||||
});
|
||||
logger.error(`[escalation] Task ${run.taskId} failed with no escalation target. User notified.`);
|
||||
}
|
||||
}
|
||||
handleFailureEscalation(
|
||||
{
|
||||
db,
|
||||
wakeAgentIfPossible,
|
||||
notify: (title, body) => notify({ title, body, urgency: "critical", navigation: { section: "tasks", entityId: run.taskId } }),
|
||||
logger,
|
||||
},
|
||||
{ taskId: run.taskId, agentId: run.agentId, companyId: run.companyId },
|
||||
task,
|
||||
event.status,
|
||||
MAX_RETRY_ATTEMPTS,
|
||||
RETRY_COOLDOWN_MS,
|
||||
(fn, delayMs) => {
|
||||
const retryTimer = setTimeout(() => {
|
||||
pendingRetryTimers.delete(retryTimer);
|
||||
if (isQuitting() || destroyed) return;
|
||||
fn();
|
||||
}, delayMs);
|
||||
pendingRetryTimers.add(retryTimer);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// EXEC-05: Promote deferred wakes after run completion
|
||||
@@ -348,13 +316,14 @@ export async function handleWorkerMessage(
|
||||
|
||||
// Reset retry count on successful completion
|
||||
if (event.status === "succeeded") {
|
||||
try { db.resetTaskRetryCount(run.taskId); } catch { /* best effort */ }
|
||||
handleSuccessReset(db, run.taskId);
|
||||
}
|
||||
|
||||
const cb = createCircuitBreaker(consecutiveWakeMap, MAX_CONSECUTIVE_WAKES);
|
||||
|
||||
// Reset circuit breaker when a task actually progressed (successful run)
|
||||
if (event.status === "succeeded") {
|
||||
const wakeKey = `${run.agentId}:${run.companyId}`;
|
||||
consecutiveWakeMap.delete(wakeKey);
|
||||
cb.reset(run.agentId, run.companyId);
|
||||
}
|
||||
|
||||
// Self-wake with circuit breaker to prevent infinite loops
|
||||
@@ -362,12 +331,9 @@ export async function handleWorkerMessage(
|
||||
try {
|
||||
const actionableTasks = db.listAgentTasks(run.agentId, run.companyId, ["todo", "in_progress"]);
|
||||
if (actionableTasks.length > 0) {
|
||||
const wakeKey = `${run.agentId}:${run.companyId}`;
|
||||
const consecutiveCount = consecutiveWakeMap.get(wakeKey) ?? 0;
|
||||
|
||||
if (consecutiveCount >= MAX_CONSECUTIVE_WAKES) {
|
||||
logger.warn(`[self-wake] Circuit breaker: agent ${run.agentId} has self-woken ${consecutiveCount} times without progress. Pausing self-wake.`);
|
||||
consecutiveWakeMap.delete(wakeKey);
|
||||
const allowed = cb.tryWake(run.agentId, run.companyId);
|
||||
if (!allowed) {
|
||||
logger.warn(`[self-wake] Circuit breaker: agent ${run.agentId} has self-woken ${MAX_CONSECUTIVE_WAKES} times without progress. Pausing self-wake.`);
|
||||
// Escalate to manager since agent is stuck
|
||||
try {
|
||||
const chain = db.getChainOfCommand(run.agentId);
|
||||
@@ -376,7 +342,6 @@ export async function handleWorkerMessage(
|
||||
}
|
||||
} catch { /* best effort */ }
|
||||
} else {
|
||||
consecutiveWakeMap.set(wakeKey, consecutiveCount + 1);
|
||||
wakeAgentIfPossible(run.agentId, run.companyId, "task_continuation");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { z } from "zod";
|
||||
import type { AppDatabase } from "../database";
|
||||
import type { DesktopEvent, RunRecord } from "@shared/types";
|
||||
import type { NotificationOptions } from "../notification-service";
|
||||
import type { createBudgetEnforcementService } from "../budget-enforcement-service";
|
||||
import type { createSessionManagementService } from "../session-management-service";
|
||||
@@ -48,7 +49,7 @@ export interface WorkerServiceDependencies {
|
||||
workerPath: string;
|
||||
isQuitting: () => boolean;
|
||||
publishDomainChanged: () => void;
|
||||
emitEvent: (event: any) => void;
|
||||
emitEvent: (event: DesktopEvent) => void;
|
||||
wakeAgentIfPossible: (agentId: string | null | undefined, companyId: string, trigger: string) => string | null;
|
||||
dispatchRunToWorker: (params: {
|
||||
runId: string;
|
||||
@@ -63,7 +64,7 @@ export interface WorkerServiceDependencies {
|
||||
snapshot: ReturnType<AppDatabase["listSnapshot"]>;
|
||||
trigger: string;
|
||||
}) => void;
|
||||
processRunCompletion: (run: any, event: z.infer<typeof workerMessageSchema> & { type: "run-finished" }) => void;
|
||||
processRunCompletion: (run: RunRecord, event: z.infer<typeof workerMessageSchema> & { type: "run-finished" }) => void;
|
||||
onRunStarted?: (runId: string, taskId: string, agentId: string, companyId: string) => void;
|
||||
onRunCompleted?: (runId: string, taskId: string, agentId: string, companyId: string, status: string, summary: string | null, errorMessage: string | null) => void;
|
||||
notify: (options: NotificationOptions) => void;
|
||||
|
||||
Reference in new issue
Block a user