ZacharyZhang-NY
927bdc91f2
The July 9 run died at 07:22 when the GPT account hit its plan-side usage limit: every later request failed with "The usage limit has been reached", each failure counted against the 3-attempt retry budget, and 31 tasks were abandoned for good while the worker kept hammering 12 doomed requests a minute until the queue drained. Root fix, three parts sharing one classifier: - isTransientProviderFailure() in provider-runtime-domain recognises provider-side verdicts (usage limit, rate limit, 429, overloaded) plus the internal monthly-quota message; matching is against recorded task_runs.error_message rows, so historical damage heals retroactively. - enqueueHeartbeatRuns computes the retry budget with that predicate instead of an exact match on the internal quota string alone. - A transient failure parks the provider for a 10-minute cooldown: the heartbeat stops enqueuing its tasks and the worker leaves its queued runs unclaimed, then a single probe run re-arms or clears the park. Transitions are logged once each way. Tests: predicate unit tests, budget-exemption regression on the raw provider message, cooldown park/resume with an injected clock, claim exclusion.
101 lines
3.9 KiB
TypeScript
101 lines
3.9 KiB
TypeScript
import type { BrowserWindow } from "electron";
|
|
import type { DatabaseConnection } from "../database/connection";
|
|
import { IPC_CHANNELS } from "../../shared/ipc";
|
|
import { TaskExecutionService } from "../services/task-execution-service";
|
|
|
|
const DEFAULT_INTERVAL_MS = 5_000;
|
|
|
|
export interface TaskQueueWorkerInput {
|
|
connection: DatabaseConnection;
|
|
mainWindow: BrowserWindow;
|
|
intervalMs?: number;
|
|
}
|
|
|
|
export interface QueuedRun {
|
|
id: string;
|
|
org_id: string;
|
|
}
|
|
|
|
/** True while the operator has pulled the global autonomous handbrake. */
|
|
export function isAutonomousPaused(connection: DatabaseConnection): boolean {
|
|
const row = connection.db.prepare("select autorun_paused from app_state where id = 1")
|
|
.get() as { autorun_paused: number } | undefined;
|
|
return row?.autorun_paused === 1;
|
|
}
|
|
|
|
/**
|
|
* Atomically claims the oldest queued task_run by transitioning it
|
|
* `queued` -> `running` in one transaction; returns the claimed run or null.
|
|
* Runs whose provider sits in a transient-failure cooldown are left queued —
|
|
* they would fail on arrival, and waiting costs nothing.
|
|
*/
|
|
export function claimNextQueuedRun(connection: DatabaseConnection, excludedProviderIds: string[] = []): QueuedRun | null {
|
|
return connection.transaction(() => {
|
|
const exclusion = excludedProviderIds.length > 0
|
|
? `and provider_id not in (${excludedProviderIds.map(() => "?").join(", ")})`
|
|
: "";
|
|
const row = connection.db
|
|
.prepare(`select id, org_id from task_runs where state = 'queued' ${exclusion} order by created_at asc limit 1`)
|
|
.get(...excludedProviderIds) as QueuedRun | undefined;
|
|
if (!row) return null;
|
|
const result = connection.db
|
|
.prepare("update task_runs set state = 'running', started_at = coalesce(started_at, ?) where id = ? and state = 'queued'")
|
|
.run(new Date().toISOString(), row.id);
|
|
return result.changes === 1 ? row : null;
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Autonomous executor: every tick it claims the oldest `queued` task_run (the
|
|
* bootstrap CEO run, plus any future heartbeat-created runs) and executes its
|
|
* real agent loop. The claim is a single transaction (`queued` -> `running`,
|
|
* verified by row changes) so a run is never executed twice.
|
|
*/
|
|
export function startTaskQueueWorker(input: TaskQueueWorkerInput): () => void {
|
|
const notifyDataChanged = () => {
|
|
if (!input.mainWindow.isDestroyed()) {
|
|
input.mainWindow.webContents.send(IPC_CHANNELS.rendererEvent, { type: "snapshot-changed" });
|
|
}
|
|
};
|
|
const taskExecution = new TaskExecutionService(input.connection.db, undefined, undefined, notifyDataChanged);
|
|
const intervalMs = input.intervalMs ?? DEFAULT_INTERVAL_MS;
|
|
let running = false;
|
|
let wasPaused = false;
|
|
|
|
const tick = async (): Promise<void> => {
|
|
if (running) return;
|
|
running = true;
|
|
try {
|
|
// The operator's global pause parks the whole engine: nothing new is
|
|
// enqueued or claimed until they resume. Queued work simply waits.
|
|
const paused = isAutonomousPaused(input.connection);
|
|
if (paused !== wasPaused) {
|
|
console.info(paused ? "[task-queue] paused by the operator; no new runs start." : "[task-queue] resumed by the operator.");
|
|
wasPaused = paused;
|
|
}
|
|
if (paused) return;
|
|
const enqueued = input.connection.transaction(() => taskExecution.enqueueHeartbeatRuns());
|
|
const claimed = claimNextQueuedRun(input.connection, taskExecution.cooledProviderIds());
|
|
if (!claimed) {
|
|
if (enqueued.length > 0) notifyDataChanged();
|
|
return;
|
|
}
|
|
notifyDataChanged();
|
|
try {
|
|
await taskExecution.executeRun(claimed.org_id, claimed.id);
|
|
} catch {
|
|
// executeRun records the failure on the run row; the UI reflects it below.
|
|
}
|
|
notifyDataChanged();
|
|
} finally {
|
|
running = false;
|
|
}
|
|
};
|
|
|
|
const interval = setInterval(() => {
|
|
void tick();
|
|
}, intervalMs);
|
|
void tick();
|
|
return () => clearInterval(interval);
|
|
}
|