119 lines
3.9 KiB
TypeScript
119 lines
3.9 KiB
TypeScript
/**
|
|
* Retry-aware failure escalation pipeline.
|
|
*
|
|
* State machine:
|
|
* 1. Task fails with retry_count < MAX_RETRY_ATTEMPTS -> increment count, re-queue
|
|
* 2. Task fails with retry_count >= MAX_RETRY_ATTEMPTS -> escalate to manager chain
|
|
* 3. No manager in chain -> fall through to CEO
|
|
* 4. No CEO available -> notify user
|
|
* 5. Task succeeds -> reset retry count
|
|
*/
|
|
|
|
export interface EscalationDeps {
|
|
db: {
|
|
getTaskRetryCount: (taskId: string) => number;
|
|
incrementTaskRetryCount: (taskId: string) => void;
|
|
resetTaskRetryCount: (taskId: string) => void;
|
|
getChainOfCommand: (agentId: string) => Array<{ id: string; name: string }>;
|
|
getCeoForCompany: (companyId: string) => { id: string; name: string } | null;
|
|
advanceTaskStatus: (taskId: string, status: string, note: string) => unknown;
|
|
};
|
|
wakeAgentIfPossible: (agentId: string, companyId: string, trigger: string) => unknown;
|
|
notify: (title: string, body: string) => void;
|
|
logger: {
|
|
info: (...args: unknown[]) => void;
|
|
warn: (...args: unknown[]) => void;
|
|
error: (...args: unknown[]) => void;
|
|
};
|
|
}
|
|
|
|
export interface EscalationResult {
|
|
action: "retry" | "escalated" | "user_notified" | "none";
|
|
retryScheduled?: boolean;
|
|
}
|
|
|
|
export function handleFailureEscalation(
|
|
deps: EscalationDeps,
|
|
run: { taskId: string; agentId: string; companyId: string },
|
|
task: { title: string } | null,
|
|
eventStatus: "failed" | "timed_out",
|
|
maxRetryAttempts: number,
|
|
retryCooldownMs: number,
|
|
scheduleRetry: (fn: () => void, delayMs: number) => void,
|
|
): EscalationResult {
|
|
const { db, wakeAgentIfPossible, notify, logger } = deps;
|
|
const retryCount = db.getTaskRetryCount(run.taskId);
|
|
|
|
if (retryCount < maxRetryAttempts) {
|
|
db.incrementTaskRetryCount(run.taskId);
|
|
db.advanceTaskStatus(
|
|
run.taskId,
|
|
"todo",
|
|
`Run ${eventStatus} (attempt ${retryCount + 1}/${maxRetryAttempts}). Auto-retrying after ${retryCooldownMs / 1000}s cooldown.`,
|
|
);
|
|
logger.info(
|
|
`[escalation] Retrying task ${run.taskId} (attempt ${retryCount + 1}/${maxRetryAttempts}) for agent ${run.agentId}`,
|
|
);
|
|
scheduleRetry(() => {
|
|
wakeAgentIfPossible(run.agentId, run.companyId, "retry_failed");
|
|
}, retryCooldownMs);
|
|
return { action: "retry", retryScheduled: true };
|
|
}
|
|
|
|
// Retry limit reached — escalate up the chain
|
|
db.resetTaskRetryCount(run.taskId);
|
|
logger.warn(
|
|
`[escalation] Task ${run.taskId} failed ${maxRetryAttempts} times. Escalating to manager.`,
|
|
);
|
|
|
|
let escalated = false;
|
|
try {
|
|
const chain = db.getChainOfCommand(run.agentId);
|
|
if (chain.length > 0) {
|
|
wakeAgentIfPossible(chain[0].id, run.companyId, "report_failed");
|
|
escalated = true;
|
|
logger.info(
|
|
`[escalation] Escalated task ${run.taskId} to manager ${chain[0].name} (${chain[0].id})`,
|
|
);
|
|
}
|
|
} catch {
|
|
// Chain lookup failed (agent deleted)
|
|
}
|
|
|
|
if (!escalated) {
|
|
try {
|
|
const ceo = db.getCeoForCompany(run.companyId);
|
|
if (ceo) {
|
|
wakeAgentIfPossible(ceo.id, run.companyId, "report_failed");
|
|
escalated = true;
|
|
logger.info(
|
|
`[escalation] Escalated task ${run.taskId} to CEO ${ceo.name} (${ceo.id})`,
|
|
);
|
|
}
|
|
} catch { /* CEO lookup failed */ }
|
|
}
|
|
|
|
if (!escalated) {
|
|
const taskTitle = task?.title ?? run.taskId;
|
|
notify(
|
|
"Task failed -- manual intervention needed",
|
|
`"${taskTitle}" failed ${maxRetryAttempts} times and no manager is available to handle it.`,
|
|
);
|
|
logger.error(
|
|
`[escalation] Task ${run.taskId} failed with no escalation target. User notified.`,
|
|
);
|
|
return { action: "user_notified" };
|
|
}
|
|
|
|
return { action: "escalated" };
|
|
}
|
|
|
|
export function handleSuccessReset(
|
|
db: EscalationDeps["db"],
|
|
taskId: string,
|
|
): void {
|
|
try {
|
|
db.resetTaskRetryCount(taskId);
|
|
} catch { /* best effort */ }
|
|
}
|