379 lines
13 KiB
TypeScript
379 lines
13 KiB
TypeScript
import type { z } from "zod";
|
|
import { handleFailureEscalation, handleSuccessReset } from "../observability/failure-escalation";
|
|
import { persistRunEvent } from "../observability/run-event-recorder";
|
|
import type { workerMessageSchema } from "../entry/types";
|
|
import type { HandlerContext, HandlerDeps, WorkerMessageEvent } from "./handler-context";
|
|
import { isUsageStatusUnattributed } from "../../run-usage-status";
|
|
import { decideRunAttempt, runAttemptKey } from "../../run-attempt-policy";
|
|
|
|
type RunFinishedEvent = Extract<WorkerMessageEvent, { type: "run-finished" }>;
|
|
type RunRecord = ReturnType<HandlerDeps["db"]["core"]["runs"]["getRun"]>;
|
|
type TaskRecord = ReturnType<HandlerDeps["db"]["core"]["entity"]["getTask"]>;
|
|
|
|
export interface RunFinishedContext {
|
|
run: RunRecord;
|
|
task: TaskRecord | null;
|
|
runOutcome: ReturnType<HandlerDeps["db"]["core"]["runs"]["finishRun"]>;
|
|
}
|
|
|
|
export function finalizeRun(
|
|
event: RunFinishedEvent,
|
|
deps: HandlerDeps,
|
|
context: HandlerContext,
|
|
): RunFinishedContext | null {
|
|
const { db, logger } = deps;
|
|
const { leaseRenewalMap } = context;
|
|
|
|
let run: RunRecord;
|
|
try {
|
|
run = db.core.runs.getRun(event.runId);
|
|
} catch (error) {
|
|
logger.warn(
|
|
`[finalizeRun] getRun(${event.runId}) failed; run may have been deleted: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
return null;
|
|
}
|
|
|
|
const unattributedCost = isUsageStatusUnattributed(event.usageStatus);
|
|
|
|
persistRunEvent(db, logger, {
|
|
companyId: run.companyId,
|
|
runId: run.id,
|
|
agentId: run.agentId,
|
|
eventType: "run_finished",
|
|
level: event.status === "succeeded" ? "info" : "error",
|
|
message: event.summary ?? event.errorMessage ?? event.status,
|
|
payload: {
|
|
status: event.status,
|
|
summary: event.summary,
|
|
errorMessage: event.errorMessage,
|
|
exitCode: event.exitCode,
|
|
signal: event.signal,
|
|
model: event.model,
|
|
usageStatus: event.usageStatus,
|
|
unattributedCost,
|
|
inputTokens: event.inputTokens,
|
|
outputTokens: event.outputTokens,
|
|
},
|
|
});
|
|
|
|
let task: TaskRecord | null = null;
|
|
if (run.taskId) {
|
|
try {
|
|
task = db.core.entity.getTask(run.taskId);
|
|
} catch (error) {
|
|
logger.warn(
|
|
`[finalizeRun] getTask(${run.taskId}) failed; task may have been deleted: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
task = null;
|
|
}
|
|
}
|
|
|
|
leaseRenewalMap.delete(event.runId);
|
|
const runOutcome = db.core.runs.finishRun({ ...event, unattributedCost });
|
|
db.core.activity.addActivity({
|
|
companyId: run.companyId,
|
|
actor: run.agentId,
|
|
action: `run.${event.status}`,
|
|
entityType: "run",
|
|
entityId: run.id,
|
|
detail: event.summary ?? event.errorMessage ?? "",
|
|
});
|
|
|
|
return { run, task, runOutcome };
|
|
}
|
|
|
|
export function releaseCompletedRunWorkspaceLock(
|
|
rfCtx: RunFinishedContext | null,
|
|
deps: Pick<HandlerDeps, "db">,
|
|
): void {
|
|
const workspaceId = rfCtx?.run.workspaceId;
|
|
if (!workspaceId) return;
|
|
deps.db.core.workspaces.releaseWorkspaceLock(workspaceId, rfCtx.run.id);
|
|
}
|
|
|
|
export function accumulateAndRotateSession(
|
|
event: RunFinishedEvent,
|
|
deps: HandlerDeps,
|
|
rfCtx: RunFinishedContext,
|
|
): void {
|
|
const { db, logger, sessionService } = deps;
|
|
const { run } = rfCtx;
|
|
|
|
if (!sessionService) return;
|
|
|
|
// Sessions are agent-scoped (post-v34 migration). The previous gate on
|
|
// `!task` silently dropped token accumulation and rotation for every
|
|
// heartbeat run — the exact failure mode that lets a taskless transcript
|
|
// grow past the model's context window. Resolve the session through the
|
|
// agent, never through the task.
|
|
let agentRecord: ReturnType<HandlerDeps["db"]["core"]["entity"]["getAgent"]>;
|
|
try {
|
|
agentRecord = db.core.entity.getAgent(run.agentId);
|
|
} catch (error) {
|
|
logger.warn(
|
|
`[session] accumulateAndRotateSession: getAgent(${run.agentId}) failed; skipping: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
return;
|
|
}
|
|
if (!agentRecord.sessionId) return;
|
|
const session = db.core.tasks.getSessionById(agentRecord.sessionId);
|
|
if (!session) return;
|
|
|
|
sessionService.accumulateTokenUsage(
|
|
session.id,
|
|
event.inputTokens ?? 0,
|
|
event.outputTokens ?? 0,
|
|
event.runId,
|
|
);
|
|
// Re-read after accumulation: evaluateRotation must see the counts
|
|
// that include this run, otherwise a run that just crossed the
|
|
// threshold waits a full additional cycle to rotate — the exact
|
|
// failure mode that let transcripts balloon past the context window.
|
|
const updatedSession = db.core.tasks.getSessionById(agentRecord.sessionId);
|
|
if (!updatedSession) return;
|
|
const rotationResult = sessionService.evaluateRotation(updatedSession);
|
|
if (rotationResult.shouldRotate) {
|
|
sessionService.rotateSession(updatedSession);
|
|
logger.info(`[session] Rotated session for agent ${run.agentId}: ${rotationResult.reason}`);
|
|
}
|
|
}
|
|
|
|
export function emitAndNotifyCompletion(
|
|
event: RunFinishedEvent,
|
|
deps: HandlerDeps,
|
|
rfCtx: RunFinishedContext,
|
|
): void {
|
|
const { db, emitEvent, notify, logger } = deps;
|
|
const { run, task, runOutcome } = rfCtx;
|
|
|
|
emitEvent({ type: "run-updated", runId: run.id, action: "completed" });
|
|
emitEvent({
|
|
type: "run-status",
|
|
runId: run.id,
|
|
status: event.status,
|
|
message: event.summary ?? event.errorMessage ?? event.status,
|
|
});
|
|
if ((event.status === "failed" || event.status === "timed_out" || event.status === "interrupted") && task) {
|
|
notify({
|
|
title: "Task Failed",
|
|
body: `${task.title} -- manual intervention needed`,
|
|
urgency: "informational",
|
|
navigation: { section: "tasks", entityId: task.id },
|
|
batchKey: "failed_runs",
|
|
});
|
|
}
|
|
|
|
let updatedAgent: ReturnType<HandlerDeps["db"]["core"]["entity"]["getAgent"]> | null = null;
|
|
try {
|
|
updatedAgent = db.core.entity.getAgent(run.agentId);
|
|
} catch (error) {
|
|
logger.warn(
|
|
`[emitAndNotifyCompletion] getAgent(${run.agentId}) failed; skipping budget-paused notification: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
updatedAgent = null;
|
|
}
|
|
|
|
if (updatedAgent && runOutcome.budgetHardStopped) {
|
|
db.core.activity.addActivity({
|
|
companyId: updatedAgent.companyId,
|
|
actor: "system",
|
|
action: "agent.auto_paused",
|
|
entityType: "agent",
|
|
entityId: updatedAgent.id,
|
|
detail: `Budget exhausted at ${updatedAgent.spentMonthlyUsd.toFixed(2)} / ${updatedAgent.budgetMonthlyUsd.toFixed(2)} USD.`,
|
|
});
|
|
notify({
|
|
title: "Agent Paused",
|
|
body: `${updatedAgent.name} paused -- budget limit reached`,
|
|
urgency: "informational",
|
|
navigation: { section: "agents", entityId: updatedAgent.id },
|
|
batchKey: "agent_paused",
|
|
});
|
|
emitEvent({ type: "agent-updated", agentId: updatedAgent.id, action: "paused" });
|
|
}
|
|
}
|
|
|
|
export function runPostCompletionAutomation(
|
|
event: RunFinishedEvent,
|
|
deps: HandlerDeps,
|
|
context: HandlerContext,
|
|
rfCtx: RunFinishedContext,
|
|
): { hadDeferredWakes: boolean } {
|
|
const { db, logger, isQuitting, wakeAgentIfPossible, processRunCompletion, onRunCompleted, notify, budgetService } = deps;
|
|
const { destroyed, MAX_RETRY_ATTEMPTS, RETRY_COOLDOWN_MS, pendingRetryTimers } = context;
|
|
const { run, task } = rfCtx;
|
|
|
|
// Policy-based incident check after run completion. Always runs once the
|
|
// run is finalized — cost is read from the cost_entries SUM inside
|
|
// finishRun/checkAndCreateIncidents, not from the worker event.
|
|
if (budgetService && event.status === "succeeded") {
|
|
budgetService.checkAndCreateIncidents(run.companyId, run.agentId, task?.projectId ?? null);
|
|
}
|
|
|
|
if ((event.status === "failed" || event.status === "timed_out") && !isQuitting() && !destroyed && run.taskId) {
|
|
handleFailureEscalation(
|
|
{
|
|
db,
|
|
wakeAgentIfPossible,
|
|
notify: (title, body) =>
|
|
notify({ title, body, urgency: "critical", navigation: { section: "tasks", entityId: run.taskId ?? "" } }),
|
|
logger,
|
|
},
|
|
{ taskId: run.taskId, agentId: run.agentId, companyId: run.companyId },
|
|
task,
|
|
event.status,
|
|
MAX_RETRY_ATTEMPTS,
|
|
RETRY_COOLDOWN_MS,
|
|
(fn, delayMs) => {
|
|
const retryTimer = setTimeout(() => {
|
|
pendingRetryTimers.delete(retryTimer);
|
|
if (isQuitting() || destroyed) return;
|
|
fn();
|
|
}, delayMs);
|
|
pendingRetryTimers.add(retryTimer);
|
|
},
|
|
);
|
|
}
|
|
|
|
// EXEC-05: Promote deferred wakes after run completion
|
|
let hadDeferredWakes = false;
|
|
if (!isQuitting() && !destroyed) {
|
|
const promoted = db.automation.wakeup.promoteWakeupRequests(run.agentId, run.companyId);
|
|
const pendingWakeTrigger = db.automation.wakeup.getPreferredWakeTrigger(run.agentId, run.companyId);
|
|
if (promoted > 0) {
|
|
logger.info(`[worker] Promoted ${promoted} deferred wake(s) for agent ${run.agentId}`);
|
|
}
|
|
if (pendingWakeTrigger) {
|
|
hadDeferredWakes = true;
|
|
logger.info(
|
|
`[worker] Processing pending wake(s) for agent ${run.agentId} with priority trigger "${pendingWakeTrigger}"`,
|
|
);
|
|
wakeAgentIfPossible(run.agentId, run.companyId, pendingWakeTrigger);
|
|
}
|
|
}
|
|
|
|
if (!isQuitting() && !destroyed) {
|
|
processRunCompletion(run, event as z.infer<typeof workerMessageSchema> & { type: "run-finished" });
|
|
}
|
|
|
|
// Run housekeeping: thread events + knowledge candidates
|
|
if (!isQuitting() && !destroyed) {
|
|
try {
|
|
onRunCompleted?.(
|
|
run.id,
|
|
run.taskId,
|
|
run.agentId,
|
|
run.companyId,
|
|
event.status,
|
|
event.summary ?? null,
|
|
event.errorMessage ?? null,
|
|
);
|
|
} catch (err) {
|
|
logger.error(`[worker] onRunCompleted housekeeping failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
}
|
|
}
|
|
|
|
// Reset retry count on successful completion
|
|
if (event.status === "succeeded" && run.taskId) {
|
|
handleSuccessReset(db, run.taskId, logger);
|
|
}
|
|
|
|
return { hadDeferredWakes };
|
|
}
|
|
|
|
export function triggerSelfWake(
|
|
event: RunFinishedEvent,
|
|
deps: HandlerDeps,
|
|
context: HandlerContext,
|
|
rfCtx: RunFinishedContext,
|
|
hadDeferredWakes: boolean,
|
|
): void {
|
|
const { db, logger, isQuitting, wakeAgentIfPossible } = deps;
|
|
const { destroyed, selfWakeAttemptMap, MAX_CONSECUTIVE_SELF_WAKES } = context;
|
|
const { run } = rfCtx;
|
|
const attemptKey = runAttemptKey(run.agentId, run.companyId);
|
|
|
|
// Reset self-wake attempts when a task actually progressed.
|
|
if (event.status === "succeeded") {
|
|
selfWakeAttemptMap.delete(attemptKey);
|
|
}
|
|
|
|
// Self-wake through the shared run-attempt policy.
|
|
if (hadDeferredWakes || isQuitting() || destroyed) return;
|
|
|
|
try {
|
|
const actionableTasks = db.core.tasks.listAgentTasks(run.agentId, run.companyId, ["todo", "in_progress"]);
|
|
const hasWorkToContinue = actionableTasks.length > 0 || hasUnfinishedTeamBuilding(db, run.agentId, run.companyId);
|
|
const consecutiveSelfWakes = selfWakeAttemptMap.get(attemptKey) ?? 0;
|
|
const decision = decideRunAttempt({
|
|
kind: "self_wake",
|
|
hasWorkToContinue,
|
|
consecutiveSelfWakes,
|
|
maxConsecutiveSelfWakes: MAX_CONSECUTIVE_SELF_WAKES,
|
|
});
|
|
|
|
if (decision.action === "skip" && decision.reason === "no_work") return;
|
|
|
|
if (decision.action === "skip") {
|
|
selfWakeAttemptMap.delete(attemptKey);
|
|
logger.warn(
|
|
`[self-wake] Run-attempt policy paused agent ${run.agentId} after ${MAX_CONSECUTIVE_SELF_WAKES} self-wakes without progress.`,
|
|
);
|
|
// Escalate to manager since agent is stuck
|
|
try {
|
|
const chain = db.core.agents.getChainOfCommand(run.agentId);
|
|
if (chain.length > 0) {
|
|
wakeAgentIfPossible(chain[0].id, run.companyId, "report_stuck");
|
|
}
|
|
} catch (error) {
|
|
logger.warn(
|
|
`[self-wake] chain-of-command lookup failed for agent ${run.agentId}: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
}
|
|
} else {
|
|
selfWakeAttemptMap.set(attemptKey, consecutiveSelfWakes + 1);
|
|
wakeAgentIfPossible(run.agentId, run.companyId, "task_continuation");
|
|
}
|
|
} catch (error) {
|
|
logger.warn(
|
|
`[self-wake] self-wake evaluation failed for agent ${run.agentId}: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Returns true when a top-level agent (CEO) still has team-building work
|
|
* in flight: the company has an active goal, zero active direct reports,
|
|
* and no pending hire approvals. Each wake in that window files at most a
|
|
* handful of hire_agent POSTs, so chaining runs via self-wake instead of
|
|
* waiting for the 2-minute heartbeat timer takes bootstrap from
|
|
* 10-plus minutes down to the LLM's round-trip time.
|
|
*
|
|
* The run-attempt policy on the caller side still applies, so a CEO that
|
|
* keeps completing runs without actually creating hires gets paused
|
|
* after MAX_CONSECUTIVE_SELF_WAKES attempts.
|
|
*/
|
|
function hasUnfinishedTeamBuilding(
|
|
db: HandlerDeps["db"],
|
|
agentId: string,
|
|
companyId: string,
|
|
): boolean {
|
|
try {
|
|
const agent = db.core.entity.getAgent(agentId);
|
|
if (agent.reportsTo !== null) return false;
|
|
const activeReports = db.core.agents
|
|
.getDirectReports(agentId, companyId)
|
|
.filter((r) => r.status !== "pending_approval" && r.status !== "terminated");
|
|
if (activeReports.length > 0) return false;
|
|
const goalCount = db.core.entity.countActiveGoalsForCompany(companyId);
|
|
if (goalCount === 0) return false;
|
|
const pendingHires = db.core.approvals
|
|
.listCompanyApprovalsByType(companyId, "hire_agent")
|
|
.filter((a) => a.state === "pending");
|
|
return pendingHires.length === 0;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|