fix(scheduler): drain pending wakeup requests
This commit is contained in:
1 parent
749ad3a231
commit
49ea508ee1
3 files changed
+191
-4
No files matched your search
@@ -159,6 +159,17 @@ export function buildWakeupMethods(
|
||||
return promoted;
|
||||
},
|
||||
|
||||
listPendingWakeupAgents(companyId: string): Array<{ agentId: string }> {
|
||||
return readAll(
|
||||
`SELECT agent_id AS agentId
|
||||
FROM wakeup_requests
|
||||
WHERE company_id = ? AND status IN ('queued', 'deferred')
|
||||
GROUP BY agent_id
|
||||
ORDER BY MIN(requested_at) ASC`,
|
||||
companyId,
|
||||
);
|
||||
},
|
||||
|
||||
listWakeupRequests(companyId: string, filters?: { status?: string; agentId?: string; limit?: number }): Array<{
|
||||
id: string; companyId: string; agentId: string; source: string; triggerDetail: string | null;
|
||||
status: string; coalescedCount: number; runId: string | null; requestedAt: string;
|
||||
|
||||
@@ -8,6 +8,7 @@ import type { OrchestrationDependencies } from "./types";
|
||||
type SchedulerPhase = "lock_recovery" | "housekeeping" | "idle_automation" | "heartbeat_cadence";
|
||||
|
||||
const TICK_INTERVAL_MS = 10_000;
|
||||
const PENDING_WAKE_RETRY_INTERVAL_MS = 120_000;
|
||||
const PHASE_INTERVAL_MS: Record<SchedulerPhase, number> = {
|
||||
lock_recovery: 60_000,
|
||||
housekeeping: 300_000,
|
||||
@@ -36,6 +37,7 @@ export function createScheduler(
|
||||
let unlockListener: (() => void) | null = null;
|
||||
const phaseLastRunAt = new Map<SchedulerPhase, number>();
|
||||
const scheduleLastFiredByCompany = new Map<string, number>();
|
||||
const pendingWakeAttemptByAgent = new Map<string, number>();
|
||||
const taskWakeAttemptByAgent = new Map<string, number>();
|
||||
const bootstrapAttemptByAgent = new Map<string, number>();
|
||||
|
||||
@@ -46,6 +48,10 @@ export function createScheduler(
|
||||
return true;
|
||||
}
|
||||
|
||||
function agentAttemptKey(companyId: string, agentId: string): string {
|
||||
return `${companyId}:${agentId}`;
|
||||
}
|
||||
|
||||
function runLockRecoveryPhase() {
|
||||
if (!hasWorkerReady()) return;
|
||||
try {
|
||||
@@ -206,13 +212,14 @@ export function createScheduler(
|
||||
}
|
||||
}
|
||||
|
||||
function runCompanyBootstrap(company: { id: string; name: string }, now: number) {
|
||||
function runCompanyBootstrap(company: { id: string; name: string }, now: number, wokenAgentIds: Set<string>) {
|
||||
const goalCount = db.core.entity.countActiveGoalsForCompany(company.id);
|
||||
const advancingCount = db.core.tasks.countAdvancingTasksForCompany(company.id);
|
||||
if (goalCount === 0 || advancingCount > 0) return;
|
||||
|
||||
const ceo = db.core.agents.getCeoForCompany(company.id);
|
||||
if (!ceo) return;
|
||||
if (wokenAgentIds.has(ceo.id)) return;
|
||||
if (hasActiveRun({ agentId: ceo.id, companyId: company.id })) return;
|
||||
|
||||
const lastHeartbeat = ceo.lastHeartbeatAt ? new Date(ceo.lastHeartbeatAt).getTime() : 0;
|
||||
@@ -232,9 +239,34 @@ export function createScheduler(
|
||||
scheduleLastFiredByCompany.set(companyId, now);
|
||||
}
|
||||
|
||||
function runAssignedTaskWakeCadence(companyId: string, now: number) {
|
||||
function runPendingWakeupCadence(companyId: string, now: number): Set<string> {
|
||||
const wokenAgentIds = new Set<string>();
|
||||
const pendingWakeupAgents = db.automation.wakeup.listPendingWakeupAgents(companyId);
|
||||
for (const { agentId } of pendingWakeupAgents) {
|
||||
if (hasActiveRun({ agentId, companyId })) continue;
|
||||
if (deps.checkpointEngine?.shouldPauseHeartbeat(companyId, agentId)) continue;
|
||||
|
||||
const attemptKey = agentAttemptKey(companyId, agentId);
|
||||
const lastAttempt = pendingWakeAttemptByAgent.get(attemptKey) ?? 0;
|
||||
if (lastAttempt > 0 && now - lastAttempt < PENDING_WAKE_RETRY_INTERVAL_MS) continue;
|
||||
|
||||
const trigger = db.automation.wakeup.getPreferredWakeTrigger(agentId, companyId);
|
||||
if (!trigger) continue;
|
||||
|
||||
pendingWakeAttemptByAgent.set(attemptKey, now);
|
||||
const runId = wakeAgentIfPossible(agentId, companyId, trigger);
|
||||
if (runId) {
|
||||
pendingWakeAttemptByAgent.delete(attemptKey);
|
||||
wokenAgentIds.add(agentId);
|
||||
}
|
||||
}
|
||||
return wokenAgentIds;
|
||||
}
|
||||
|
||||
function runAssignedTaskWakeCadence(companyId: string, now: number, wokenAgentIds: Set<string>) {
|
||||
const activeAgents = db.core.agents.listActiveAgentsForCompany(companyId);
|
||||
for (const activeAgent of activeAgents) {
|
||||
if (wokenAgentIds.has(activeAgent.id)) continue;
|
||||
if (hasActiveRun({ agentId: activeAgent.id, companyId })) continue;
|
||||
if (deps.checkpointEngine?.shouldPauseHeartbeat(companyId, activeAgent.id)) continue;
|
||||
|
||||
@@ -258,8 +290,9 @@ export function createScheduler(
|
||||
for (const company of activeCompanies) {
|
||||
try {
|
||||
runScheduleAutomation(company.id, now);
|
||||
runAssignedTaskWakeCadence(company.id, now);
|
||||
runCompanyBootstrap(company, now);
|
||||
const wokenAgentIds = runPendingWakeupCadence(company.id, now);
|
||||
runAssignedTaskWakeCadence(company.id, now, wokenAgentIds);
|
||||
runCompanyBootstrap(company, now, wokenAgentIds);
|
||||
} catch (error) {
|
||||
logAppError(logger, error, {
|
||||
kind: "recoverable",
|
||||
|
||||
@@ -199,6 +199,149 @@ describe("unified scheduler heartbeat cadence", () => {
|
||||
expect(wakeAgentIfPossible).toHaveBeenCalledWith(agentId, companyId, "timer");
|
||||
});
|
||||
|
||||
it("wakes an agent with a queued wakeup request on heartbeat cadence", () => {
|
||||
const companyId = createCompany(db);
|
||||
createConnector(db);
|
||||
const workspaceId = createWorkspace(db, companyId, tempDir);
|
||||
const agentId = createAgent(db, companyId, workspaceId);
|
||||
db.automation.wakeup.enqueueWakeupRequest(
|
||||
companyId,
|
||||
agentId,
|
||||
"subtask_completed",
|
||||
"Queued subtask completion wake",
|
||||
);
|
||||
const wakeAgentIfPossible = vi.fn();
|
||||
const scheduler = createScheduler(
|
||||
createDeps(db),
|
||||
() => false,
|
||||
wakeAgentIfPossible,
|
||||
);
|
||||
|
||||
try {
|
||||
scheduler.startAllSchedulers();
|
||||
vi.advanceTimersByTime(60_000);
|
||||
} finally {
|
||||
scheduler.stopAllSchedulers();
|
||||
}
|
||||
|
||||
expect(wakeAgentIfPossible).toHaveBeenCalledWith(agentId, companyId, "subtask_completed");
|
||||
});
|
||||
|
||||
it("throttles queued wakeup recovery attempts", () => {
|
||||
const companyId = createCompany(db);
|
||||
createConnector(db);
|
||||
const workspaceId = createWorkspace(db, companyId, tempDir);
|
||||
const agentId = createAgent(db, companyId, workspaceId);
|
||||
db.automation.wakeup.enqueueWakeupRequest(
|
||||
companyId,
|
||||
agentId,
|
||||
"subtask_completed",
|
||||
"Queued subtask completion wake",
|
||||
);
|
||||
const wakeAgentIfPossible = vi.fn();
|
||||
const scheduler = createScheduler(
|
||||
createDeps(db),
|
||||
() => false,
|
||||
wakeAgentIfPossible,
|
||||
);
|
||||
|
||||
try {
|
||||
scheduler.startAllSchedulers();
|
||||
vi.advanceTimersByTime(60_000);
|
||||
expect(wakeAgentIfPossible).toHaveBeenCalledTimes(1);
|
||||
|
||||
vi.advanceTimersByTime(60_000);
|
||||
expect(wakeAgentIfPossible).toHaveBeenCalledTimes(1);
|
||||
|
||||
vi.advanceTimersByTime(60_000);
|
||||
expect(wakeAgentIfPossible).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
scheduler.stopAllSchedulers();
|
||||
}
|
||||
});
|
||||
|
||||
it("clears queued wakeup throttle after a started run", () => {
|
||||
const companyId = createCompany(db);
|
||||
createConnector(db);
|
||||
const workspaceId = createWorkspace(db, companyId, tempDir);
|
||||
const agentId = createAgent(db, companyId, workspaceId);
|
||||
db.automation.wakeup.enqueueWakeupRequest(
|
||||
companyId,
|
||||
agentId,
|
||||
"subtask_completed",
|
||||
"Queued subtask completion wake",
|
||||
);
|
||||
let runCount = 0;
|
||||
const wakeAgentIfPossible = vi.fn((wokenAgentId: string, wokenCompanyId: string) => {
|
||||
runCount += 1;
|
||||
const runId = `run-${runCount}`;
|
||||
db.automation.wakeup.completePendingWakeupRequests(wokenAgentId, wokenCompanyId, runId);
|
||||
return runId;
|
||||
});
|
||||
const scheduler = createScheduler(
|
||||
createDeps(db),
|
||||
() => false,
|
||||
wakeAgentIfPossible,
|
||||
);
|
||||
|
||||
try {
|
||||
scheduler.startAllSchedulers();
|
||||
vi.advanceTimersByTime(60_000);
|
||||
expect(wakeAgentIfPossible).toHaveBeenCalledTimes(1);
|
||||
|
||||
db.automation.wakeup.enqueueWakeupRequest(
|
||||
companyId,
|
||||
agentId,
|
||||
"approval_resolved",
|
||||
"Queued approval resolution wake",
|
||||
);
|
||||
vi.advanceTimersByTime(60_000);
|
||||
} finally {
|
||||
scheduler.stopAllSchedulers();
|
||||
}
|
||||
|
||||
expect(wakeAgentIfPossible).toHaveBeenCalledTimes(2);
|
||||
expect(wakeAgentIfPossible).toHaveBeenLastCalledWith(agentId, companyId, "approval_resolved");
|
||||
});
|
||||
|
||||
it("lets assigned task wake run after a queued wakeup attempt fails", () => {
|
||||
const companyId = createCompany(db);
|
||||
createConnector(db);
|
||||
const workspaceId = createWorkspace(db, companyId, tempDir);
|
||||
const agentId = createAgent(db, companyId, workspaceId);
|
||||
db.automation.wakeup.enqueueWakeupRequest(
|
||||
companyId,
|
||||
agentId,
|
||||
"subtask_completed",
|
||||
"Queued subtask completion wake",
|
||||
);
|
||||
db.core.tasks.saveTask({
|
||||
companyId,
|
||||
title: "Continue after failed queued wake",
|
||||
description: "The assigned task cadence can recover the agent in the same tick.",
|
||||
assigneeAgentId: agentId,
|
||||
workspaceId,
|
||||
priority: "high",
|
||||
status: "todo",
|
||||
});
|
||||
const wakeAgentIfPossible = vi.fn(() => null);
|
||||
const scheduler = createScheduler(
|
||||
createDeps(db),
|
||||
() => false,
|
||||
wakeAgentIfPossible,
|
||||
);
|
||||
|
||||
try {
|
||||
scheduler.startAllSchedulers();
|
||||
vi.advanceTimersByTime(60_000);
|
||||
} finally {
|
||||
scheduler.stopAllSchedulers();
|
||||
}
|
||||
|
||||
expect(wakeAgentIfPossible).toHaveBeenNthCalledWith(1, agentId, companyId, "subtask_completed");
|
||||
expect(wakeAgentIfPossible).toHaveBeenNthCalledWith(2, agentId, companyId, "timer");
|
||||
});
|
||||
|
||||
it("bootstraps the CEO through the wake pipeline", () => {
|
||||
const companyId = createCompany(db);
|
||||
createConnector(db);
|
||||
|
||||
Reference in new issue
Block a user