import { powerMonitor } from "electron"; import { logAppError } from "../errors/index.js"; import { resolveApprovalDecision } from "../approval-policy"; import { dispatchAgentMessage } from "../messaging/dispatch"; import { sweepStaleReviews, type SweepStaleReviewsDeps } from "../task-quality-gates"; import type { OrchestrationDependencies } from "./types"; type SchedulerPhase = "lock_recovery" | "housekeeping" | "idle_automation" | "heartbeat_cadence"; const TICK_INTERVAL_MS = 10_000; const PENDING_WAKE_RETRY_INTERVAL_MS = 120_000; const PHASE_INTERVAL_MS: Record = { lock_recovery: 60_000, housekeeping: 300_000, idle_automation: 60_000, heartbeat_cadence: 60_000, }; export function createScheduler( deps: OrchestrationDependencies, hasActiveRun: (filter: { taskId?: string; agentId?: string; companyId: string }) => boolean, wakeAgentIfPossible: (agentId: string, companyId: string, trigger: string) => string | null | undefined, ) { const { db, logger, publishDomainChanged, dispatchAutomation, hasWorkerReady, isQuitting, notify, budgetService, } = deps; let timer: ReturnType | null = null; let resumeListener: (() => void) | null = null; let unlockListener: (() => void) | null = null; const phaseLastRunAt = new Map(); const scheduleLastFiredByCompany = new Map(); const pendingWakeAttemptByAgent = new Map(); const taskWakeAttemptByAgent = new Map(); const bootstrapAttemptByAgent = new Map(); function phaseIsDue(phase: SchedulerPhase, now: number): boolean { const lastRunAt = phaseLastRunAt.get(phase) ?? 0; if (now - lastRunAt < PHASE_INTERVAL_MS[phase]) return false; phaseLastRunAt.set(phase, now); return true; } function agentAttemptKey(companyId: string, agentId: string): string { return `${companyId}:${agentId}`; } function runLockRecoveryPhase() { if (!hasWorkerReady()) return; try { const recovered = db.recoverStaleLocks(); for (const task of recovered) { db.core.activity.addActivity({ companyId: task.companyId, actor: "system", action: "execution.lock_recovered", entityType: "task", entityId: task.taskId, detail: `Recovered stale lock on task "${task.title}" after 15min timeout`, }); logger.info(`[lock-recovery] Recovered stale lock on task ${task.taskId}: "${task.title}"`); } } catch (error) { logAppError(logger, error, { kind: "recoverable", code: "scheduler.lock_recovery.stale_lock_sweep_failed", message: "Stale task lock recovery sweep failed; next tick will retry", }); } try { const staleReleased = db.core.workspaces.releaseStaleWorkspaceLocks(); if (staleReleased > 0) { logger.info(`[lock-recovery] Released ${staleReleased} stale workspace lock(s)`); publishDomainChanged(); } } catch (error) { logAppError(logger, error, { kind: "recoverable", code: "scheduler.lock_recovery.workspace_lock_sweep_failed", message: "Stale workspace lock cleanup failed; next tick will retry", }); } } function runHousekeepingPhase() { if (!hasWorkerReady()) return; const activeCompanies = db.core.company.listActiveCompanyIds(); for (const company of activeCompanies) { try { const currentMonth = new Date().toISOString().slice(0, 7); const lastResetMonth = db.core.settings.getLastBudgetResetMonth(company.id) ?? ""; if (currentMonth !== lastResetMonth) { db.core.agents.resetMonthlyBudgets(company.id); db.core.settings.setLastBudgetResetMonth(company.id, currentMonth); logger.info(`[housekeeping] Reset monthly budgets for company ${company.name} (${currentMonth})`); } const closedIncidents = budgetService.autoCloseExpiredIncidents(company.id); if (closedIncidents > 0) { logger.info(`[housekeeping] Auto-closed ${closedIncidents} expired budget incident(s) for company ${company.name}`); } try { const sweepDeps: SweepStaleReviewsDeps = { db, dispatchAgentMessage, resolveApprovalDecision, notify, getCompanyExecutionMode: (companyId: string) => db.core.company.getCompanyExecutionMode(companyId), handleTaskStatusChange: (taskId: string, companyId: string, previousStatus: string, newStatus: string) => { dispatchAutomation("task_status_changed", { companyId, taskId, previousStatus, newStatus }); publishDomainChanged(); }, handleApprovalResolved: (approvalId: string, companyId: string, _decision: string) => { dispatchAutomation("approval_resolved", { companyId }); publishDomainChanged(); }, emitEvent: deps.emitEvent, publishDomainChanged, wakeAgentIfPossible: (agentId: string | null | undefined, companyId: string, trigger: string) => agentId != null ? wakeAgentIfPossible(agentId, companyId, trigger) ?? null : null, logger, }; const escalated = sweepStaleReviews(company.id, sweepDeps); if (escalated > 0) { logger.info(`[housekeeping] Auto-escalated ${escalated} stale review(s) for company ${company.name}`); publishDomainChanged(); } } catch (error) { logAppError(logger, error, { kind: "recoverable", code: "scheduler.housekeeping.stale_review_sweep_failed", message: `Stale review sweep failed for company ${company.id}`, context: { companyId: company.id }, }); } } catch (error) { logAppError(logger, error, { kind: "recoverable", code: "scheduler.housekeeping.company_tick_failed", message: `Housekeeping tick failed for company ${company.id}`, context: { companyId: company.id }, }); } } try { const staleWakeups = db.automation.wakeup.purgeExpiredWakeupRequests(); if (staleWakeups > 0) { logger.info(`[housekeeping] Purged ${staleWakeups} expired wakeup request(s)`); } } catch (error) { logAppError(logger, error, { kind: "recoverable", code: "scheduler.housekeeping.wakeup_purge_failed", message: "Expired wakeup request purge failed; next tick will retry", }); } try { const cutoff = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(); const purgedProjections = db.projection.inbox.purgeOrphanProjections(cutoff); if (purgedProjections > 0) { logger.info(`[housekeeping] Purged ${purgedProjections} orphan inbox projection ledger row(s)`); } } catch (error) { logAppError(logger, error, { kind: "recoverable", code: "scheduler.housekeeping.inbox_projection_purge_failed", message: "Orphan inbox projection purge failed; next tick will retry", }); } } function runIdleAutomationPhase() { if (!hasWorkerReady()) return; const activeCompanies = db.core.company.listActiveCompanyIds(); const now = Date.now(); for (const company of activeCompanies) { try { const companyAgents = db.core.agents.listActiveAgentsForCompany(company.id); for (const agent of companyAgents) { if (db.core.agents.hasAgentAssignedTasks(agent.id, company.id)) continue; const lastActive = agent.lastHeartbeatAt ? new Date(agent.lastHeartbeatAt).getTime() : 0; const idleThreshold = 20 * 60 * 1000; if (now - lastActive > idleThreshold) { dispatchAutomation("agent_idle", { companyId: company.id, agentId: agent.id, department: agent.department ?? undefined, }); } } } catch (error) { logAppError(logger, error, { kind: "recoverable", code: "scheduler.idle_automation.company_tick_failed", message: `Idle automation tick failed for company ${company.id}`, context: { companyId: company.id }, }); } } } function runCompanyBootstrap(company: { id: string; name: string }, now: number, wokenAgentIds: Set) { const goalCount = db.core.entity.countActiveGoalsForCompany(company.id); const advancingCount = db.core.tasks.countAdvancingTasksForCompany(company.id); if (goalCount === 0 || advancingCount > 0) return; const ceo = db.core.agents.getCeoForCompany(company.id); if (!ceo) return; if (wokenAgentIds.has(ceo.id)) return; if (hasActiveRun({ agentId: ceo.id, companyId: company.id })) return; const lastHeartbeat = ceo.lastHeartbeatAt ? new Date(ceo.lastHeartbeatAt).getTime() : 0; const lastAttempt = bootstrapAttemptByAgent.get(ceo.id) ?? 0; const bootstrapCooldown = 120_000; if (now - Math.max(lastHeartbeat, lastAttempt) <= bootstrapCooldown) return; bootstrapAttemptByAgent.set(ceo.id, now); logger.info(`[scheduler] Company ${company.name} has ${goalCount} goal(s) and no advancing tasks; bootstrapping CEO ${ceo.name}`); wakeAgentIfPossible(ceo.id, company.id, "bootstrap"); } function runScheduleAutomation(companyId: string, now: number) { const lastScheduleFired = scheduleLastFiredByCompany.get(companyId) ?? 0; if (now - lastScheduleFired < 60_000) return; dispatchAutomation("schedule", { companyId }); scheduleLastFiredByCompany.set(companyId, now); } function runPendingWakeupCadence(companyId: string, now: number): Set { const wokenAgentIds = new Set(); const pendingWakeupAgents = db.automation.wakeup.listPendingWakeupAgents(companyId); for (const { agentId } of pendingWakeupAgents) { if (hasActiveRun({ agentId, companyId })) continue; if (deps.checkpointEngine?.shouldPauseHeartbeat(companyId, agentId)) continue; const attemptKey = agentAttemptKey(companyId, agentId); const lastAttempt = pendingWakeAttemptByAgent.get(attemptKey) ?? 0; if (lastAttempt > 0 && now - lastAttempt < PENDING_WAKE_RETRY_INTERVAL_MS) continue; const trigger = db.automation.wakeup.getPreferredWakeTrigger(agentId, companyId); if (!trigger) continue; pendingWakeAttemptByAgent.set(attemptKey, now); const runId = wakeAgentIfPossible(agentId, companyId, trigger); if (runId) { pendingWakeAttemptByAgent.delete(attemptKey); wokenAgentIds.add(agentId); } } return wokenAgentIds; } function runAssignedTaskWakeCadence(companyId: string, now: number, wokenAgentIds: Set) { const activeAgents = db.core.agents.listActiveAgentsForCompany(companyId); for (const activeAgent of activeAgents) { if (wokenAgentIds.has(activeAgent.id)) continue; if (hasActiveRun({ agentId: activeAgent.id, companyId })) continue; if (deps.checkpointEngine?.shouldPauseHeartbeat(companyId, activeAgent.id)) continue; const lastHeartbeat = activeAgent.lastHeartbeatAt ? new Date(activeAgent.lastHeartbeatAt).getTime() : 0; const lastAttempt = taskWakeAttemptByAgent.get(activeAgent.id) ?? 0; const intervalMs = (activeAgent.heartbeatIntervalSec || 120) * 1000; if (now - Math.max(lastHeartbeat, lastAttempt) < intervalMs) continue; const actionableTasks = db.core.tasks.listAgentTasks(activeAgent.id, companyId, ["todo"]); if (!actionableTasks.some((task) => !task.activeRunId)) continue; taskWakeAttemptByAgent.set(activeAgent.id, now); wakeAgentIfPossible(activeAgent.id, companyId, "timer"); } } function runHeartbeatCadencePhase() { const activeCompanies = db.core.company.listActiveCompanyIds(); const now = Date.now(); for (const company of activeCompanies) { try { runScheduleAutomation(company.id, now); const wokenAgentIds = runPendingWakeupCadence(company.id, now); runAssignedTaskWakeCadence(company.id, now, wokenAgentIds); runCompanyBootstrap(company, now, wokenAgentIds); } catch (error) { logAppError(logger, error, { kind: "recoverable", code: "scheduler.heartbeat_cadence.company_tick_failed", message: `Heartbeat cadence failed for company ${company.id}`, context: { companyId: company.id }, }); } } } function runPhase(phase: SchedulerPhase, runner: () => void) { try { runner(); } catch (error) { logAppError(logger, error, { kind: "recoverable", code: `scheduler.${phase}.tick_failed`, message: `Scheduler phase ${phase} raised; next interval will retry`, }); } } function runTick() { if (isQuitting()) return; const now = Date.now(); if (phaseIsDue("lock_recovery", now)) runPhase("lock_recovery", runLockRecoveryPhase); if (phaseIsDue("housekeeping", now)) runPhase("housekeeping", runHousekeepingPhase); if (phaseIsDue("idle_automation", now)) runPhase("idle_automation", runIdleAutomationPhase); if (phaseIsDue("heartbeat_cadence", now)) runPhase("heartbeat_cadence", runHeartbeatCadencePhase); } function armTimer() { if (timer) clearInterval(timer); timer = setInterval(runTick, TICK_INTERVAL_MS); } function rearm(trigger: string) { if (isQuitting()) return; logger.info(`[scheduler] Re-arming unified scheduler (${trigger})`); armTimer(); runTick(); } function startAllSchedulers() { armTimer(); if (!resumeListener) { resumeListener = () => rearm("power_resume"); powerMonitor.on("resume", resumeListener); } if (!unlockListener) { unlockListener = () => rearm("screen_unlock"); powerMonitor.on("unlock-screen", unlockListener); } } function stopAllSchedulers() { if (timer) clearInterval(timer); timer = null; if (resumeListener) { powerMonitor.removeListener("resume", resumeListener); resumeListener = null; } if (unlockListener) { powerMonitor.removeListener("unlock-screen", unlockListener); unlockListener = null; } } return { startAllSchedulers, stopAllSchedulers }; }