refactor(runtime): unify run attempt policy
This commit is contained in:
1 parent
58a4be7fe7
commit
473486aa61
10 files changed
+314
-217
No files matched your search
@@ -17,8 +17,8 @@ export async function handleWorkerMessage(
|
||||
destroyed: boolean,
|
||||
MAX_RETRY_ATTEMPTS: number,
|
||||
RETRY_COOLDOWN_MS: number,
|
||||
consecutiveWakeMap: Map<string, number>,
|
||||
MAX_CONSECUTIVE_WAKES: number,
|
||||
selfWakeAttemptMap: Map<string, number>,
|
||||
MAX_CONSECUTIVE_SELF_WAKES: number,
|
||||
pendingRetryTimers: Set<ReturnType<typeof setTimeout>>,
|
||||
leaseRenewalMap: Map<string, number>,
|
||||
) {
|
||||
@@ -33,8 +33,8 @@ export async function handleWorkerMessage(
|
||||
destroyed,
|
||||
MAX_RETRY_ATTEMPTS,
|
||||
RETRY_COOLDOWN_MS,
|
||||
MAX_CONSECUTIVE_WAKES,
|
||||
consecutiveWakeMap,
|
||||
MAX_CONSECUTIVE_SELF_WAKES,
|
||||
selfWakeAttemptMap,
|
||||
pendingRetryTimers,
|
||||
leaseRenewalMap,
|
||||
};
|
||||
|
||||
@@ -22,8 +22,8 @@ export interface HandlerContext {
|
||||
destroyed: boolean;
|
||||
MAX_RETRY_ATTEMPTS: number;
|
||||
RETRY_COOLDOWN_MS: number;
|
||||
MAX_CONSECUTIVE_WAKES: number;
|
||||
consecutiveWakeMap: Map<string, number>;
|
||||
MAX_CONSECUTIVE_SELF_WAKES: number;
|
||||
selfWakeAttemptMap: Map<string, number>;
|
||||
pendingRetryTimers: Set<ReturnType<typeof setTimeout>>;
|
||||
leaseRenewalMap: Map<string, number>;
|
||||
}
|
||||
@@ -1,10 +1,10 @@
|
||||
import type { z } from "zod";
|
||||
import { createCircuitBreaker } from "../observability/circuit-breaker";
|
||||
import { handleFailureEscalation, handleSuccessReset } from "../observability/failure-escalation";
|
||||
import { persistRunEvent } from "../observability/run-event-recorder";
|
||||
import type { workerMessageSchema } from "../entry/types";
|
||||
import type { HandlerContext, HandlerDeps, WorkerMessageEvent } from "./handler-context";
|
||||
import { isUsageStatusUnattributed } from "../../run-usage-status";
|
||||
import { decideRunAttempt, runAttemptKey } from "../../run-attempt-policy";
|
||||
|
||||
type RunFinishedEvent = Extract<WorkerMessageEvent, { type: "run-finished" }>;
|
||||
type RunRecord = ReturnType<HandlerDeps["db"]["runs"]["getRun"]>;
|
||||
@@ -284,28 +284,35 @@ export function triggerSelfWake(
|
||||
hadDeferredWakes: boolean,
|
||||
): void {
|
||||
const { db, logger, isQuitting, wakeAgentIfPossible } = deps;
|
||||
const { destroyed, consecutiveWakeMap, MAX_CONSECUTIVE_WAKES } = context;
|
||||
const { destroyed, selfWakeAttemptMap, MAX_CONSECUTIVE_SELF_WAKES } = context;
|
||||
const { run } = rfCtx;
|
||||
const attemptKey = runAttemptKey(run.agentId, run.companyId);
|
||||
|
||||
const cb = createCircuitBreaker(consecutiveWakeMap, MAX_CONSECUTIVE_WAKES);
|
||||
|
||||
// Reset circuit breaker when a task actually progressed (successful run)
|
||||
// Reset self-wake attempts when a task actually progressed.
|
||||
if (event.status === "succeeded") {
|
||||
cb.reset(run.agentId, run.companyId);
|
||||
selfWakeAttemptMap.delete(attemptKey);
|
||||
}
|
||||
|
||||
// Self-wake with circuit breaker to prevent infinite loops
|
||||
// Self-wake through the shared run-attempt policy.
|
||||
if (hadDeferredWakes || isQuitting() || destroyed) return;
|
||||
|
||||
try {
|
||||
const actionableTasks = db.tasks.listAgentTasks(run.agentId, run.companyId, ["todo", "in_progress"]);
|
||||
const hasWorkToContinue = actionableTasks.length > 0 || hasUnfinishedTeamBuilding(db, run.agentId, run.companyId);
|
||||
if (!hasWorkToContinue) return;
|
||||
const consecutiveSelfWakes = selfWakeAttemptMap.get(attemptKey) ?? 0;
|
||||
const decision = decideRunAttempt({
|
||||
kind: "self_wake",
|
||||
hasWorkToContinue,
|
||||
consecutiveSelfWakes,
|
||||
maxConsecutiveSelfWakes: MAX_CONSECUTIVE_SELF_WAKES,
|
||||
});
|
||||
|
||||
const allowed = cb.tryWake(run.agentId, run.companyId);
|
||||
if (!allowed) {
|
||||
if (decision.action === "skip" && decision.reason === "no_work") return;
|
||||
|
||||
if (decision.action === "skip") {
|
||||
selfWakeAttemptMap.delete(attemptKey);
|
||||
logger.warn(
|
||||
`[self-wake] Circuit breaker: agent ${run.agentId} has self-woken ${MAX_CONSECUTIVE_WAKES} times without progress. Pausing self-wake.`,
|
||||
`[self-wake] Run-attempt policy paused agent ${run.agentId} after ${MAX_CONSECUTIVE_SELF_WAKES} self-wakes without progress.`,
|
||||
);
|
||||
// Escalate to manager since agent is stuck
|
||||
try {
|
||||
@@ -319,6 +326,7 @@ export function triggerSelfWake(
|
||||
);
|
||||
}
|
||||
} else {
|
||||
selfWakeAttemptMap.set(attemptKey, consecutiveSelfWakes + 1);
|
||||
wakeAgentIfPossible(run.agentId, run.companyId, "task_continuation");
|
||||
}
|
||||
} catch (error) {
|
||||
@@ -336,9 +344,9 @@ export function triggerSelfWake(
|
||||
* waiting for the 2-minute heartbeat timer takes bootstrap from
|
||||
* 10-plus minutes down to the LLM's round-trip time.
|
||||
*
|
||||
* The circuit breaker on the caller side still applies, so a CEO that
|
||||
* The run-attempt policy on the caller side still applies, so a CEO that
|
||||
* keeps completing runs without actually creating hires gets paused
|
||||
* after MAX_CONSECUTIVE_WAKES attempts.
|
||||
* after MAX_CONSECUTIVE_SELF_WAKES attempts.
|
||||
*/
|
||||
function hasUnfinishedTeamBuilding(
|
||||
db: HandlerDeps["db"],
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
/**
|
||||
* Self-wake circuit breaker.
|
||||
*
|
||||
* Prevents infinite self-wake loops when an agent is stuck on a task.
|
||||
* After MAX_CONSECUTIVE_WAKES consecutive wakes without progress (i.e.,
|
||||
* without a successful run), self-wake is suppressed so the manager can
|
||||
* be notified instead.
|
||||
*/
|
||||
|
||||
export const MAX_CONSECUTIVE_WAKES = 5;
|
||||
|
||||
export interface CircuitBreaker {
|
||||
/** Attempt a self-wake. Returns true if allowed, false if circuit breaker trips. */
|
||||
tryWake(agentId: string, companyId: string): boolean;
|
||||
/** Reset the counter for an agent (called on task progress / successful run). */
|
||||
reset(agentId: string, companyId: string): void;
|
||||
/** Get the current consecutive-wake count for an agent. */
|
||||
getCount(agentId: string, companyId: string): number;
|
||||
}
|
||||
|
||||
export function createCircuitBreaker(
|
||||
consecutiveWakeMap: Map<string, number>,
|
||||
maxConsecutive: number = MAX_CONSECUTIVE_WAKES,
|
||||
): CircuitBreaker {
|
||||
function wakeKey(agentId: string, companyId: string): string {
|
||||
return `${agentId}:${companyId}`;
|
||||
}
|
||||
|
||||
return {
|
||||
tryWake(agentId: string, companyId: string): boolean {
|
||||
const key = wakeKey(agentId, companyId);
|
||||
const count = consecutiveWakeMap.get(key) ?? 0;
|
||||
if (count >= maxConsecutive) {
|
||||
consecutiveWakeMap.delete(key);
|
||||
return false;
|
||||
}
|
||||
consecutiveWakeMap.set(key, count + 1);
|
||||
return true;
|
||||
},
|
||||
|
||||
reset(agentId: string, companyId: string): void {
|
||||
consecutiveWakeMap.delete(wakeKey(agentId, companyId));
|
||||
},
|
||||
|
||||
getCount(agentId: string, companyId: string): number {
|
||||
return consecutiveWakeMap.get(wakeKey(agentId, companyId)) ?? 0;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -9,6 +9,8 @@
|
||||
* 5. Task succeeds -> reset retry count
|
||||
*/
|
||||
|
||||
import { decideRunAttempt } from "../../run-attempt-policy";
|
||||
|
||||
export interface EscalationDeps {
|
||||
db: {
|
||||
tasks: {
|
||||
@@ -47,16 +49,21 @@ export function handleFailureEscalation(
|
||||
): EscalationResult {
|
||||
const { db, wakeAgentIfPossible, notify, logger } = deps;
|
||||
const retryCount = db.tasks.getTaskRetryCount(run.taskId);
|
||||
const retryDecision = decideRunAttempt({
|
||||
kind: "task_failure",
|
||||
retryCount,
|
||||
maxRetryAttempts,
|
||||
});
|
||||
|
||||
if (retryCount < maxRetryAttempts) {
|
||||
if (retryDecision.action === "retry") {
|
||||
db.tasks.incrementTaskRetryCount(run.taskId);
|
||||
db.tasks.advanceTaskStatus(
|
||||
run.taskId,
|
||||
"todo",
|
||||
`Run ${eventStatus} (attempt ${retryCount + 1}/${maxRetryAttempts}). Auto-retrying after ${retryCooldownMs / 1000}s cooldown.`,
|
||||
`Run ${eventStatus} (attempt ${retryDecision.attemptNumber}/${maxRetryAttempts}). Auto-retrying after ${retryCooldownMs / 1000}s cooldown.`,
|
||||
);
|
||||
logger.info(
|
||||
`[escalation] Retrying task ${run.taskId} (attempt ${retryCount + 1}/${maxRetryAttempts}) for agent ${run.agentId}`,
|
||||
`[escalation] Retrying task ${run.taskId} (attempt ${retryDecision.attemptNumber}/${maxRetryAttempts}) for agent ${run.agentId}`,
|
||||
);
|
||||
scheduleRetry(() => {
|
||||
wakeAgentIfPossible(run.agentId, run.companyId, "retry_failed");
|
||||
|
||||
Reference in new issue
Block a user