Split task execution: run preparation, provider cooldowns, heartbeat

Pure move to keep task-execution-service under 500 lines.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ZacharyZhang-NYandClaude Opus 5.5 committed 2026-10-08 05:27:14 -04:00
1 parent 95e607bea7
commit 12e2d9dcd5
4 files changed
+381 -329

No files matched your search

+144
View File
@@ -0,0 +1,144 @@
import type { DatabaseSync } from "node:sqlite";
import { currentProviderUsageWindow, isTransientProviderFailure } from "../../shared/provider-runtime-domain";
import { createId } from "../../shared/ids";
import type { ProviderCooldowns } from "./provider-cooldowns";
import { assertExecutionApproval, createIsolationContext, loadWorkspace, requireAgentProvider, requireAssignedAgent, requireTask } from "./task-run-preparation";
const TASK_RETRY_BUDGET = 3;
/** Enqueues follow-up runs for open tasks each worker tick. */
export class HeartbeatScheduler {
/** Providers currently parked on their monthly quota, for transition logging. */
private quotaBlockedProviders = new Map<string, string>();
/** Tasks already logged as budget-spent, so the park is announced once, not every 5s tick. */
private spentTasksLogged = new Set<string>();
constructor(private readonly db: DatabaseSync, private readonly cooldowns: ProviderCooldowns) {}
/**
* Heartbeat: keeps every open assigned task of a heartbeat-enabled agent
* moving. A task is enqueued whenever it has no run in flight and its
* status is still open — a completed run that left the task open gets a
* follow-up session (the prompt carries the prior-run count), because a
* task is finished when it reaches done, not when a run happens to end.
* The retry budget bounds this: at most TASK_RETRY_BUDGET non-transient
* runs per task, then it parks for a human or manager to intervene.
*
* Three rules keep the loop self-healing when a provider stops serving:
* tasks on a provider over its monthly window are parked (not enqueued into
* certain failure), tasks on a provider inside a transient-failure cooldown
* wait it out, and transient failures (usage/rate limits, overload) never
* count toward the retry budget. Raising a limit — or the provider's own
* window resetting — therefore resumes work without a restart.
*/
enqueueHeartbeatRuns(): string[] {
const window = currentProviderUsageWindow();
const rows = this.db.prepare(`
select t.id as task_id, t.org_id as org_id, p.id as provider_id
from tasks t
join agents a on a.id = t.assignee_agent_id and a.org_id = t.org_id
join providers p on p.id = a.model_provider_id and p.org_id = t.org_id
left join provider_usage_windows u
on u.org_id = p.org_id and u.provider_id = p.id and u.window_start = ?
where t.status in ('backlog', 'in_progress')
and a.heartbeat_enabled = 1
and a.lifecycle = 'active'
and p.enabled = 1
and coalesce(u.request_count, 0) < p.monthly_request_limit
and not exists (select 1 from task_runs r where r.task_id = t.id and r.state in ('queued', 'running'))
`).all(window.start) as Array<{ task_id: string; org_id: string; provider_id: string }>;
this.observeQuotaBlocks(window.start);
const cooling = new Set(this.cooldowns.cooledProviderIds());
const attemptMessages = this.db.prepare("select error_message from task_runs where task_id = ?");
const created: string[] = [];
for (const row of rows) {
if (cooling.has(row.provider_id)) continue;
const attempts = (attemptMessages.all(row.task_id) as Array<{ error_message: string | null }>)
.filter((run) => !isTransientProviderFailure(run.error_message)).length;
if (attempts >= TASK_RETRY_BUDGET) {
this.observeSpentTask(row.task_id);
continue;
}
try {
created.push(this.enqueueRun(row.org_id, row.task_id));
} catch {
// Skip tasks that cannot be prepared yet (e.g. provider not fully configured).
}
}
return created;
}
/**
* A task that spent its whole retry budget without reaching done parks
* silently otherwise — one log line makes the stall diagnosable from the
* main-process log instead of looking like an idle queue.
*/
private observeSpentTask(taskId: string): void {
if (this.spentTasksLogged.has(taskId)) return;
this.spentTasksLogged.add(taskId);
const task = this.db.prepare("select title from tasks where id = ?").get(taskId) as { title: string } | undefined;
console.warn(`[task-queue] task "${task?.title ?? taskId}" spent its ${TASK_RETRY_BUDGET}-run budget without reaching done; it stays parked until someone closes or reassigns it.`);
}
/**
* Logs monthly-quota transitions (blocked <-> unblocked) exactly once per
* state change so a parked queue is diagnosable from the main-process log
* without spamming a line every 5s tick.
*/
private observeQuotaBlocks(windowStart: string): void {
const rows = this.db.prepare(`
select p.id as id, p.label as label
from providers p
join provider_usage_windows u
on u.org_id = p.org_id and u.provider_id = p.id and u.window_start = ?
where p.enabled = 1 and u.request_count >= p.monthly_request_limit
`).all(windowStart) as Array<{ id: string; label: string }>;
const current = new Map(rows.map((row) => [row.id, row.label]));
for (const [id, label] of current) {
if (!this.quotaBlockedProviders.has(id)) {
console.warn(`[task-queue] provider "${label}" hit its monthly request limit; its tasks are parked until the limit is raised or the window resets.`);
}
}
for (const [id, label] of this.quotaBlockedProviders) {
if (!current.has(id)) {
console.info(`[task-queue] provider "${label}" is under its monthly request limit again; parked tasks resume.`);
}
}
this.quotaBlockedProviders = current;
}
private enqueueRun(orgId: string, taskId: string): string {
const task = requireTask(this.db, orgId, taskId);
// Autonomous execution honours the same gate as a manual run: a high-risk or
// filesystem task whose approval is still pending (or was declined) is left
// unqueued and retried once the operator approves it, instead of running
// unsupervised. enqueueHeartbeatRuns swallows the throw and moves on.
assertExecutionApproval(this.db, orgId, task);
const agent = requireAssignedAgent(this.db, orgId, task);
const provider = requireAgentProvider(this.db, orgId, agent);
const workspace = loadWorkspace(this.db, task.workspace_id);
const isolation = createIsolationContext(this.db, orgId, task, agent, provider, workspace);
const runId = createId("run");
const now = new Date().toISOString();
this.db.prepare(`
insert into task_runs(
id, org_id, task_id, agent_id, provider_id, provider_request_id,
execution_context_id, service_identity_id, workspace_mount_json, secret_scope_json,
trigger, state, created_at, started_at, completed_at, error_message
) values (?, ?, ?, ?, ?, null, ?, ?, ?, ?, 'heartbeat', ?, ?, null, null, null)
`).run(
runId,
orgId,
task.id,
agent.id,
provider.id,
isolation.executionContextId,
isolation.serviceIdentityId,
isolation.workspaceMountJson,
isolation.secretScopeJson,
"queued",
now,
);
return runId;
}
}