update
This commit is contained in:
1 parent
09971cabc2
commit
9145572422
163 files changed
+3348
-2874
No files matched your search
@@ -0,0 +1,118 @@
|
||||
/**
|
||||
* Retry-aware failure escalation pipeline.
|
||||
*
|
||||
* State machine:
|
||||
* 1. Task fails with retry_count < MAX_RETRY_ATTEMPTS -> increment count, re-queue
|
||||
* 2. Task fails with retry_count >= MAX_RETRY_ATTEMPTS -> escalate to manager chain
|
||||
* 3. No manager in chain -> fall through to CEO
|
||||
* 4. No CEO available -> notify user
|
||||
* 5. Task succeeds -> reset retry count
|
||||
*/
|
||||
|
||||
export interface EscalationDeps {
|
||||
db: {
|
||||
getTaskRetryCount: (taskId: string) => number;
|
||||
incrementTaskRetryCount: (taskId: string) => void;
|
||||
resetTaskRetryCount: (taskId: string) => void;
|
||||
getChainOfCommand: (agentId: string) => Array<{ id: string; name: string }>;
|
||||
getCeoForCompany: (companyId: string) => { id: string; name: string } | null;
|
||||
advanceTaskStatus: (taskId: string, status: string, note: string) => unknown;
|
||||
};
|
||||
wakeAgentIfPossible: (agentId: string, companyId: string, trigger: string) => unknown;
|
||||
notify: (title: string, body: string) => void;
|
||||
logger: {
|
||||
info: (...args: unknown[]) => void;
|
||||
warn: (...args: unknown[]) => void;
|
||||
error: (...args: unknown[]) => void;
|
||||
};
|
||||
}
|
||||
|
||||
export interface EscalationResult {
|
||||
action: "retry" | "escalated" | "user_notified" | "none";
|
||||
retryScheduled?: boolean;
|
||||
}
|
||||
|
||||
export function handleFailureEscalation(
|
||||
deps: EscalationDeps,
|
||||
run: { taskId: string; agentId: string; companyId: string },
|
||||
task: { title: string } | null,
|
||||
eventStatus: "failed" | "timed_out",
|
||||
maxRetryAttempts: number,
|
||||
retryCooldownMs: number,
|
||||
scheduleRetry: (fn: () => void, delayMs: number) => void,
|
||||
): EscalationResult {
|
||||
const { db, wakeAgentIfPossible, notify, logger } = deps;
|
||||
const retryCount = db.getTaskRetryCount(run.taskId);
|
||||
|
||||
if (retryCount < maxRetryAttempts) {
|
||||
db.incrementTaskRetryCount(run.taskId);
|
||||
db.advanceTaskStatus(
|
||||
run.taskId,
|
||||
"todo",
|
||||
`Run ${eventStatus} (attempt ${retryCount + 1}/${maxRetryAttempts}). Auto-retrying after ${retryCooldownMs / 1000}s cooldown.`,
|
||||
);
|
||||
logger.info(
|
||||
`[escalation] Retrying task ${run.taskId} (attempt ${retryCount + 1}/${maxRetryAttempts}) for agent ${run.agentId}`,
|
||||
);
|
||||
scheduleRetry(() => {
|
||||
wakeAgentIfPossible(run.agentId, run.companyId, "retry_failed");
|
||||
}, retryCooldownMs);
|
||||
return { action: "retry", retryScheduled: true };
|
||||
}
|
||||
|
||||
// Retry limit reached — escalate up the chain
|
||||
db.resetTaskRetryCount(run.taskId);
|
||||
logger.warn(
|
||||
`[escalation] Task ${run.taskId} failed ${maxRetryAttempts} times. Escalating to manager.`,
|
||||
);
|
||||
|
||||
let escalated = false;
|
||||
try {
|
||||
const chain = db.getChainOfCommand(run.agentId);
|
||||
if (chain.length > 0) {
|
||||
wakeAgentIfPossible(chain[0].id, run.companyId, "report_failed");
|
||||
escalated = true;
|
||||
logger.info(
|
||||
`[escalation] Escalated task ${run.taskId} to manager ${chain[0].name} (${chain[0].id})`,
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// Chain lookup failed (agent deleted)
|
||||
}
|
||||
|
||||
if (!escalated) {
|
||||
try {
|
||||
const ceo = db.getCeoForCompany(run.companyId);
|
||||
if (ceo) {
|
||||
wakeAgentIfPossible(ceo.id, run.companyId, "report_failed");
|
||||
escalated = true;
|
||||
logger.info(
|
||||
`[escalation] Escalated task ${run.taskId} to CEO ${ceo.name} (${ceo.id})`,
|
||||
);
|
||||
}
|
||||
} catch { /* CEO lookup failed */ }
|
||||
}
|
||||
|
||||
if (!escalated) {
|
||||
const taskTitle = task?.title ?? run.taskId;
|
||||
notify(
|
||||
"Task failed -- manual intervention needed",
|
||||
`"${taskTitle}" failed ${maxRetryAttempts} times and no manager is available to handle it.`,
|
||||
);
|
||||
logger.error(
|
||||
`[escalation] Task ${run.taskId} failed with no escalation target. User notified.`,
|
||||
);
|
||||
return { action: "user_notified" };
|
||||
}
|
||||
|
||||
return { action: "escalated" };
|
||||
}
|
||||
|
||||
export function handleSuccessReset(
|
||||
db: EscalationDeps["db"],
|
||||
taskId: string,
|
||||
): void {
|
||||
try {
|
||||
db.resetTaskRetryCount(taskId);
|
||||
} catch { /* best effort */ }
|
||||
}
|
||||
Reference in new issue
Block a user