refactor(runtime): unify run attempt policy

This commit is contained in:
ZacharyZhang-NY committed 2026-04-22 16:33:41 -04:00
1 parent 58a4be7fe7
commit 473486aa61
10 files changed
+314 -217

No files matched your search

+4 -4
View File
@@ -17,8 +17,8 @@ export async function handleWorkerMessage(
destroyed: boolean,
MAX_RETRY_ATTEMPTS: number,
RETRY_COOLDOWN_MS: number,
consecutiveWakeMap: Map<string, number>,
MAX_CONSECUTIVE_WAKES: number,
selfWakeAttemptMap: Map<string, number>,
MAX_CONSECUTIVE_SELF_WAKES: number,
pendingRetryTimers: Set<ReturnType<typeof setTimeout>>,
leaseRenewalMap: Map<string, number>,
) {
@@ -33,8 +33,8 @@ export async function handleWorkerMessage(
destroyed,
MAX_RETRY_ATTEMPTS,
RETRY_COOLDOWN_MS,
MAX_CONSECUTIVE_WAKES,
consecutiveWakeMap,
MAX_CONSECUTIVE_SELF_WAKES,
selfWakeAttemptMap,
pendingRetryTimers,
leaseRenewalMap,
};
+2 -2
View File
@@ -22,8 +22,8 @@ export interface HandlerContext {
destroyed: boolean;
MAX_RETRY_ATTEMPTS: number;
RETRY_COOLDOWN_MS: number;
MAX_CONSECUTIVE_WAKES: number;
consecutiveWakeMap: Map<string, number>;
MAX_CONSECUTIVE_SELF_WAKES: number;
selfWakeAttemptMap: Map<string, number>;
pendingRetryTimers: Set<ReturnType<typeof setTimeout>>;
leaseRenewalMap: Map<string, number>;
}
@@ -1,10 +1,10 @@
import type { z } from "zod";
import { createCircuitBreaker } from "../observability/circuit-breaker";
import { handleFailureEscalation, handleSuccessReset } from "../observability/failure-escalation";
import { persistRunEvent } from "../observability/run-event-recorder";
import type { workerMessageSchema } from "../entry/types";
import type { HandlerContext, HandlerDeps, WorkerMessageEvent } from "./handler-context";
import { isUsageStatusUnattributed } from "../../run-usage-status";
import { decideRunAttempt, runAttemptKey } from "../../run-attempt-policy";
type RunFinishedEvent = Extract<WorkerMessageEvent, { type: "run-finished" }>;
type RunRecord = ReturnType<HandlerDeps["db"]["runs"]["getRun"]>;
@@ -284,28 +284,35 @@ export function triggerSelfWake(
hadDeferredWakes: boolean,
): void {
const { db, logger, isQuitting, wakeAgentIfPossible } = deps;
const { destroyed, consecutiveWakeMap, MAX_CONSECUTIVE_WAKES } = context;
const { destroyed, selfWakeAttemptMap, MAX_CONSECUTIVE_SELF_WAKES } = context;
const { run } = rfCtx;
const attemptKey = runAttemptKey(run.agentId, run.companyId);
const cb = createCircuitBreaker(consecutiveWakeMap, MAX_CONSECUTIVE_WAKES);
// Reset circuit breaker when a task actually progressed (successful run)
// Reset self-wake attempts when a task actually progressed.
if (event.status === "succeeded") {
cb.reset(run.agentId, run.companyId);
selfWakeAttemptMap.delete(attemptKey);
}
// Self-wake with circuit breaker to prevent infinite loops
// Self-wake through the shared run-attempt policy.
if (hadDeferredWakes || isQuitting() || destroyed) return;
try {
const actionableTasks = db.tasks.listAgentTasks(run.agentId, run.companyId, ["todo", "in_progress"]);
const hasWorkToContinue = actionableTasks.length > 0 || hasUnfinishedTeamBuilding(db, run.agentId, run.companyId);
if (!hasWorkToContinue) return;
const consecutiveSelfWakes = selfWakeAttemptMap.get(attemptKey) ?? 0;
const decision = decideRunAttempt({
kind: "self_wake",
hasWorkToContinue,
consecutiveSelfWakes,
maxConsecutiveSelfWakes: MAX_CONSECUTIVE_SELF_WAKES,
});
const allowed = cb.tryWake(run.agentId, run.companyId);
if (!allowed) {
if (decision.action === "skip" && decision.reason === "no_work") return;
if (decision.action === "skip") {
selfWakeAttemptMap.delete(attemptKey);
logger.warn(
`[self-wake] Circuit breaker: agent ${run.agentId} has self-woken ${MAX_CONSECUTIVE_WAKES} times without progress. Pausing self-wake.`,
`[self-wake] Run-attempt policy paused agent ${run.agentId} after ${MAX_CONSECUTIVE_SELF_WAKES} self-wakes without progress.`,
);
// Escalate to manager since agent is stuck
try {
@@ -319,6 +326,7 @@ export function triggerSelfWake(
);
}
} else {
selfWakeAttemptMap.set(attemptKey, consecutiveSelfWakes + 1);
wakeAgentIfPossible(run.agentId, run.companyId, "task_continuation");
}
} catch (error) {
@@ -336,9 +344,9 @@ export function triggerSelfWake(
* waiting for the 2-minute heartbeat timer takes bootstrap from
* 10-plus minutes down to the LLM's round-trip time.
*
* The circuit breaker on the caller side still applies, so a CEO that
* The run-attempt policy on the caller side still applies, so a CEO that
* keeps completing runs without actually creating hires gets paused
* after MAX_CONSECUTIVE_WAKES attempts.
* after MAX_CONSECUTIVE_SELF_WAKES attempts.
*/
function hasUnfinishedTeamBuilding(
db: HandlerDeps["db"],
@@ -1,49 +0,0 @@
/**
* Self-wake circuit breaker.
*
* Prevents infinite self-wake loops when an agent is stuck on a task.
* After MAX_CONSECUTIVE_WAKES consecutive wakes without progress (i.e.,
* without a successful run), self-wake is suppressed so the manager can
* be notified instead.
*/
export const MAX_CONSECUTIVE_WAKES = 5;
export interface CircuitBreaker {
/** Attempt a self-wake. Returns true if allowed, false if circuit breaker trips. */
tryWake(agentId: string, companyId: string): boolean;
/** Reset the counter for an agent (called on task progress / successful run). */
reset(agentId: string, companyId: string): void;
/** Get the current consecutive-wake count for an agent. */
getCount(agentId: string, companyId: string): number;
}
export function createCircuitBreaker(
consecutiveWakeMap: Map<string, number>,
maxConsecutive: number = MAX_CONSECUTIVE_WAKES,
): CircuitBreaker {
function wakeKey(agentId: string, companyId: string): string {
return `${agentId}:${companyId}`;
}
return {
tryWake(agentId: string, companyId: string): boolean {
const key = wakeKey(agentId, companyId);
const count = consecutiveWakeMap.get(key) ?? 0;
if (count >= maxConsecutive) {
consecutiveWakeMap.delete(key);
return false;
}
consecutiveWakeMap.set(key, count + 1);
return true;
},
reset(agentId: string, companyId: string): void {
consecutiveWakeMap.delete(wakeKey(agentId, companyId));
},
getCount(agentId: string, companyId: string): number {
return consecutiveWakeMap.get(wakeKey(agentId, companyId)) ?? 0;
},
};
}
@@ -9,6 +9,8 @@
* 5. Task succeeds -> reset retry count
*/
import { decideRunAttempt } from "../../run-attempt-policy";
export interface EscalationDeps {
db: {
tasks: {
@@ -47,16 +49,21 @@ export function handleFailureEscalation(
): EscalationResult {
const { db, wakeAgentIfPossible, notify, logger } = deps;
const retryCount = db.tasks.getTaskRetryCount(run.taskId);
const retryDecision = decideRunAttempt({
kind: "task_failure",
retryCount,
maxRetryAttempts,
});
if (retryCount < maxRetryAttempts) {
if (retryDecision.action === "retry") {
db.tasks.incrementTaskRetryCount(run.taskId);
db.tasks.advanceTaskStatus(
run.taskId,
"todo",
`Run ${eventStatus} (attempt ${retryCount + 1}/${maxRetryAttempts}). Auto-retrying after ${retryCooldownMs / 1000}s cooldown.`,
`Run ${eventStatus} (attempt ${retryDecision.attemptNumber}/${maxRetryAttempts}). Auto-retrying after ${retryCooldownMs / 1000}s cooldown.`,
);
logger.info(
`[escalation] Retrying task ${run.taskId} (attempt ${retryCount + 1}/${maxRetryAttempts}) for agent ${run.agentId}`,
`[escalation] Retrying task ${run.taskId} (attempt ${retryDecision.attemptNumber}/${maxRetryAttempts}) for agent ${run.agentId}`,
);
scheduleRetry(() => {
wakeAgentIfPossible(run.agentId, run.companyId, "retry_failed");