refactor(scheduler): unify cadence phases

src/main/orchestration/scheduler.ts

@@ -1,33 +1,312 @@
+import { powerMonitor } from "electron";
+import { logAppError } from "../errors/index.js";
+import { resolveApprovalDecision } from "../approval-policy";
+import { dispatchAgentMessage } from "../messaging/dispatch";
+import { sweepStaleReviews, type SweepStaleReviewsDeps } from "../task-quality-gates";
 import type { OrchestrationDependencies } from "./types";
-import type { HeartbeatStartResult } from "./heartbeat";
-import { createHeartbeatScheduler } from "./scheduler-heartbeat";
-import { createLockRecoveryScheduler } from "./scheduler-lock-recovery";
-import { createIdleAutomationScheduler } from "./scheduler-idle-automation";
-import { createHousekeepingScheduler } from "./scheduler-housekeeping";
+
+type SchedulerPhase = "lock_recovery" | "housekeeping" | "idle_automation" | "heartbeat_cadence";
+
+const TICK_INTERVAL_MS = 10_000;
+const PHASE_INTERVAL_MS: Record<SchedulerPhase, number> = {
+  lock_recovery: 60_000,
+  housekeeping: 300_000,
+  idle_automation: 60_000,
+  heartbeat_cadence: 60_000,
+};
 
 export function createScheduler(
   deps: OrchestrationDependencies,
   hasActiveRun: (filter: { taskId?: string; agentId?: string; companyId: string }) => boolean,
-  startAgentHeartbeatRun: (agentId: string, companyId: string, trigger: string) => HeartbeatStartResult,
   wakeAgentIfPossible: (agentId: string, companyId: string, trigger: string) => string | null | undefined,
 ) {
-  const heartbeat = createHeartbeatScheduler(deps, hasActiveRun, startAgentHeartbeatRun);
-  const lockRecovery = createLockRecoveryScheduler(deps);
-  const idleAutomation = createIdleAutomationScheduler(deps);
-  const housekeeping = createHousekeepingScheduler(deps, wakeAgentIfPossible);
+  const {
+    db,
+    logger,
+    publishDomainChanged,
+    dispatchAutomation,
+    hasWorkerReady,
+    isQuitting,
+    notify,
+    budgetService,
+  } = deps;
+
+  let timer: ReturnType<typeof setInterval> | null = null;
+  let resumeListener: (() => void) | null = null;
+  let unlockListener: (() => void) | null = null;
+  const phaseLastRunAt = new Map<SchedulerPhase, number>();
+  const scheduleLastFiredByCompany = new Map<string, number>();
+  const bootstrapAttemptByAgent = new Map<string, number>();
+
+  function phaseIsDue(phase: SchedulerPhase, now: number): boolean {
+    const lastRunAt = phaseLastRunAt.get(phase) ?? 0;
+    if (now - lastRunAt < PHASE_INTERVAL_MS[phase]) return false;
+    phaseLastRunAt.set(phase, now);
+    return true;
+  }
+
+  function runLockRecoveryPhase() {
+    if (!hasWorkerReady()) return;
+    try {
+      const recovered = db.recoverStaleLocks();
+      for (const task of recovered) {
+        db.core.activity.addActivity({
+          companyId: task.companyId,
+          actor: "system",
+          action: "execution.lock_recovered",
+          entityType: "task",
+          entityId: task.taskId,
+          detail: `Recovered stale lock on task "${task.title}" after 15min timeout`,
+        });
+        logger.info(`[lock-recovery] Recovered stale lock on task ${task.taskId}: "${task.title}"`);
+      }
+    } catch (error) {
+      logAppError(logger, error, {
+        kind: "recoverable",
+        code: "scheduler.lock_recovery.stale_lock_sweep_failed",
+        message: "Stale task lock recovery sweep failed; next tick will retry",
+      });
+    }
+
+    try {
+      const staleReleased = db.core.workspaces.releaseStaleWorkspaceLocks();
+      if (staleReleased > 0) {
+        logger.info(`[lock-recovery] Released ${staleReleased} stale workspace lock(s)`);
+        publishDomainChanged();
+      }
+    } catch (error) {
+      logAppError(logger, error, {
+        kind: "recoverable",
+        code: "scheduler.lock_recovery.workspace_lock_sweep_failed",
+        message: "Stale workspace lock cleanup failed; next tick will retry",
+      });
+    }
+  }
+
+  function runHousekeepingPhase() {
+    if (!hasWorkerReady()) return;
+    const activeCompanies = db.core.company.listActiveCompanyIds();
+
+    for (const company of activeCompanies) {
+      try {
+        const currentMonth = new Date().toISOString().slice(0, 7);
+        const lastResetMonth = db.core.settings.getLastBudgetResetMonth(company.id) ?? "";
+        if (currentMonth !== lastResetMonth) {
+          db.core.agents.resetMonthlyBudgets(company.id);
+          db.core.settings.setLastBudgetResetMonth(company.id, currentMonth);
+          logger.info(`[housekeeping] Reset monthly budgets for company ${company.name} (${currentMonth})`);
+        }
+
+        const closedIncidents = budgetService.autoCloseExpiredIncidents(company.id);
+        if (closedIncidents > 0) {
+          logger.info(`[housekeeping] Auto-closed ${closedIncidents} expired budget incident(s) for company ${company.name}`);
+        }
+
+        try {
+          const sweepDeps: SweepStaleReviewsDeps = {
+            db,
+            dispatchAgentMessage,
+            resolveApprovalDecision,
+            notify,
+            getCompanyExecutionMode: (companyId: string) => db.core.company.getCompanyExecutionMode(companyId),
+            handleTaskStatusChange: (taskId: string, companyId: string, previousStatus: string, newStatus: string) => {
+              dispatchAutomation("task_status_changed", { companyId, taskId, previousStatus, newStatus });
+              publishDomainChanged();
+            },
+            handleApprovalResolved: (approvalId: string, companyId: string, _decision: string) => {
+              dispatchAutomation("approval_resolved", { companyId });
+              publishDomainChanged();
+            },
+            emitEvent: deps.emitEvent,
+            publishDomainChanged,
+            wakeAgentIfPossible: (agentId: string | null | undefined, companyId: string, trigger: string) =>
+              agentId != null ? wakeAgentIfPossible(agentId, companyId, trigger) ?? null : null,
+            logger,
+          };
+          const escalated = sweepStaleReviews(company.id, sweepDeps);
+          if (escalated > 0) {
+            logger.info(`[housekeeping] Auto-escalated ${escalated} stale review(s) for company ${company.name}`);
+            publishDomainChanged();
+          }
+        } catch (error) {
+          logAppError(logger, error, {
+            kind: "recoverable",
+            code: "scheduler.housekeeping.stale_review_sweep_failed",
+            message: `Stale review sweep failed for company ${company.id}`,
+            context: { companyId: company.id },
+          });
+        }
+      } catch (error) {
+        logAppError(logger, error, {
+          kind: "recoverable",
+          code: "scheduler.housekeeping.company_tick_failed",
+          message: `Housekeeping tick failed for company ${company.id}`,
+          context: { companyId: company.id },
+        });
+      }
+    }
+
+    try {
+      const staleWakeups = db.automation.wakeup.purgeExpiredWakeupRequests();
+      if (staleWakeups > 0) {
+        logger.info(`[housekeeping] Purged ${staleWakeups} expired wakeup request(s)`);
+      }
+    } catch (error) {
+      logAppError(logger, error, {
+        kind: "recoverable",
+        code: "scheduler.housekeeping.wakeup_purge_failed",
+        message: "Expired wakeup request purge failed; next tick will retry",
+      });
+    }
+
+    try {
+      const cutoff = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
+      const purgedProjections = db.projection.inbox.purgeOrphanProjections(cutoff);
+      if (purgedProjections > 0) {
+        logger.info(`[housekeeping] Purged ${purgedProjections} orphan inbox projection ledger row(s)`);
+      }
+    } catch (error) {
+      logAppError(logger, error, {
+        kind: "recoverable",
+        code: "scheduler.housekeeping.inbox_projection_purge_failed",
+        message: "Orphan inbox projection purge failed; next tick will retry",
+      });
+    }
+  }
+
+  function runIdleAutomationPhase() {
+    if (!hasWorkerReady()) return;
+    const activeCompanies = db.core.company.listActiveCompanyIds();
+    const now = Date.now();
+
+    for (const company of activeCompanies) {
+      try {
+        const companyAgents = db.core.agents.listActiveAgentsForCompany(company.id);
+        for (const agent of companyAgents) {
+          if (db.core.agents.hasAgentAssignedTasks(agent.id, company.id)) continue;
+          const lastActive = agent.lastHeartbeatAt ? new Date(agent.lastHeartbeatAt).getTime() : 0;
+          const idleThreshold = 20 * 60 * 1000;
+          if (now - lastActive > idleThreshold) {
+            dispatchAutomation("agent_idle", {
+              companyId: company.id,
+              agentId: agent.id,
+              department: agent.department ?? undefined,
+            });
+          }
+        }
+      } catch (error) {
+        logAppError(logger, error, {
+          kind: "recoverable",
+          code: "scheduler.idle_automation.company_tick_failed",
+          message: `Idle automation tick failed for company ${company.id}`,
+          context: { companyId: company.id },
+        });
+      }
+    }
+  }
+
+  function runCompanyBootstrap(company: { id: string; name: string }, now: number) {
+    const goalCount = db.core.entity.countActiveGoalsForCompany(company.id);
+    const advancingCount = db.core.tasks.countAdvancingTasksForCompany(company.id);
+    if (goalCount === 0 || advancingCount > 0) return;
+
+    const ceo = db.core.agents.getCeoForCompany(company.id);
+    if (!ceo) return;
+    if (hasActiveRun({ agentId: ceo.id, companyId: company.id })) return;
+
+    const lastHeartbeat = ceo.lastHeartbeatAt ? new Date(ceo.lastHeartbeatAt).getTime() : 0;
+    const lastAttempt = bootstrapAttemptByAgent.get(ceo.id) ?? 0;
+    const bootstrapCooldown = 120_000;
+    if (now - Math.max(lastHeartbeat, lastAttempt) <= bootstrapCooldown) return;
+
+    bootstrapAttemptByAgent.set(ceo.id, now);
+    logger.info(`[scheduler] Company ${company.name} has ${goalCount} goal(s) and no advancing tasks; bootstrapping CEO ${ceo.name}`);
+    wakeAgentIfPossible(ceo.id, company.id, "bootstrap");
+  }
+
+  function runScheduleAutomation(companyId: string, now: number) {
+    const lastScheduleFired = scheduleLastFiredByCompany.get(companyId) ?? 0;
+    if (now - lastScheduleFired < 60_000) return;
+    dispatchAutomation("schedule", { companyId });
+    scheduleLastFiredByCompany.set(companyId, now);
+  }
+
+  function runHeartbeatCadencePhase() {
+    const activeCompanies = db.core.company.listActiveCompanyIds();
+    const now = Date.now();
+
+    for (const company of activeCompanies) {
+      try {
+        runScheduleAutomation(company.id, now);
+        runCompanyBootstrap(company, now);
+      } catch (error) {
+        logAppError(logger, error, {
+          kind: "recoverable",
+          code: "scheduler.heartbeat_cadence.company_tick_failed",
+          message: `Heartbeat cadence failed for company ${company.id}`,
+          context: { companyId: company.id },
+        });
+      }
+    }
+  }
+
+  function runPhase(phase: SchedulerPhase, runner: () => void) {
+    try {
+      runner();
+    } catch (error) {
+      logAppError(logger, error, {
+        kind: "recoverable",
+        code: `scheduler.${phase}.tick_failed`,
+        message: `Scheduler phase ${phase} raised; next interval will retry`,
+      });
+    }
+  }
+
+  function runTick() {
+    if (isQuitting()) return;
+    const now = Date.now();
+
+    if (phaseIsDue("lock_recovery", now)) runPhase("lock_recovery", runLockRecoveryPhase);
+    if (phaseIsDue("housekeeping", now)) runPhase("housekeeping", runHousekeepingPhase);
+    if (phaseIsDue("idle_automation", now)) runPhase("idle_automation", runIdleAutomationPhase);
+    if (phaseIsDue("heartbeat_cadence", now)) runPhase("heartbeat_cadence", runHeartbeatCadencePhase);
+  }
+
+  function armTimer() {
+    if (timer) clearInterval(timer);
+    timer = setInterval(runTick, TICK_INTERVAL_MS);
+  }
+
+  function rearm(trigger: string) {
+    if (isQuitting()) return;
+    logger.info(`[scheduler] Re-arming unified scheduler (${trigger})`);
+    armTimer();
+    runTick();
+  }
 
   function startAllSchedulers() {
-    heartbeat.start();
-    lockRecovery.start();
-    idleAutomation.start();
-    housekeeping.start();
+    armTimer();
+    if (!resumeListener) {
+      resumeListener = () => rearm("power_resume");
+      powerMonitor.on("resume", resumeListener);
+    }
+    if (!unlockListener) {
+      unlockListener = () => rearm("screen_unlock");
+      powerMonitor.on("unlock-screen", unlockListener);
+    }
   }
 
   function stopAllSchedulers() {
-    heartbeat.stop();
-    lockRecovery.stop();
-    idleAutomation.stop();
-    housekeeping.stop();
+    if (timer) clearInterval(timer);
+    timer = null;
+    if (resumeListener) {
+      powerMonitor.removeListener("resume", resumeListener);
+      resumeListener = null;
+    }
+    if (unlockListener) {
+      powerMonitor.removeListener("unlock-screen", unlockListener);
+      unlockListener = null;
+    }
   }
 
   return { startAllSchedulers, stopAllSchedulers };