Q130: every open task says why it is not moving, when it resumes, and who must act

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ZacharyZhang-NYandClaude Opus 5.5 committed 2026-10-08 21:27:29 -04:00
1 parent 271a83a7bc
commit 074a1fc8e3
14 files changed
+255 -7

No files matched your search

+12 -2
View File
@@ -85,13 +85,23 @@ export class HeartbeatScheduler {
try {
created.push(this.enqueueRun(row.org_id, row.task_id));
pacedAgents.add(row.agent_id);
} catch {
// Skip tasks that cannot be prepared yet (e.g. provider not fully configured).
this.db.prepare("update tasks set enqueue_error = null where id = ? and enqueue_error is not null").run(row.task_id);
} catch (error) {
// A task that cannot be prepared yet (approval pending, setup missing) stays open; its task shows why until it queues.
this.recordEnqueueError(row.task_id, error instanceof Error ? error.message : String(error));
}
}
return created;
}
/** Records on the task why it could not be queued, logging and notifying only when the reason changes. */
private recordEnqueueError(taskId: string, message: string): void {
const changed = this.db.prepare("update tasks set enqueue_error = ? where id = ? and enqueue_error is not ?").run(message, taskId, message);
if (changed.changes === 0) return;
console.warn(`[task-queue] task ${taskId} could not be queued: ${message}`);
this.notifyDataChanged();
}
/** Records once, on the task itself, that it spent its run budget without reaching done. */
private markStalled(orgId: string, taskId: string, why: string): void {
const reason = `Stalled: ${why}. Edit, reassign or reopen the task to give it a fresh budget.`;
+3 -1
View File
@@ -4,6 +4,7 @@ import { PROVIDER_CATALOG } from "../../shared/provider-catalog";
import { IntegrationKeyStore } from "./integration-key-store";
import { currentProviderUsageWindow } from "../../shared/provider-runtime-domain";
import { buildUsageStats } from "./usage-stats";
import { taskHolds } from "./task-holds";
type Row = Record<string, unknown>;
type ProviderUsageWindowRow = Pick<
@@ -49,8 +50,9 @@ export class SnapshotService {
const activeOrgId = this.resolveActiveOrgId(organizations.map((org) => org.id));
const agents = list<AppSnapshot["agents"][number]>(this.db, "agents", "created_at asc")
.map((agent) => ({ ...agent, heartbeatEnabled: toBool(agent.heartbeatEnabled) }));
const holds = taskHolds(this.db, new Date());
const tasks = list<AppSnapshot["tasks"][number]>(this.db, "tasks", "created_at asc")
.map((task) => ({ ...task, requiresFilesystem: toBool(task.requiresFilesystem) }));
.map((task) => ({ ...task, requiresFilesystem: toBool(task.requiresFilesystem), hold: holds.get(task.id) ?? null }));
const workspaces = list<AppSnapshot["workspaces"][number]>(this.db, "workspaces", "created_at asc")
.map((workspace) => ({ ...workspace, isPrimary: toBool(workspace.isPrimary) }));
const providers = list<AppSnapshot["providers"][number]>(this.db, "providers", "label asc")
+2 -1
View File
@@ -69,7 +69,8 @@ export function budgetBlock(db: DatabaseSync, agentId: string, paid: boolean, no
const agentSpend = (db.prepare("select coalesce(sum(cost_cents), 0) as cents from provider_requests where agent_id = ? and created_at >= ?")
.get(agentId, since) as { cents: number }).cents;
if (agentSpend >= agent.budget_cents) {
return `${agent.name} has spent its monthly budget (${dollars(agentSpend)} of ${dollars(agent.budget_cents)}). Raise the budget or wait for next month.`;
const next = agent.budget_cents === 0 ? "Raise the budget to let it run." : "Raise the budget or wait for next month.";
return `${agent.name} has spent its monthly budget (${dollars(agentSpend)} of ${dollars(agent.budget_cents)}). ${next}`;
}
const department = db.prepare("select monthly_budget_limit_cents from org_departments where org_id = ? and name = ?")
.get(agent.org_id, agent.department) as { monthly_budget_limit_cents: number } | undefined;
+76
View File
@@ -0,0 +1,76 @@
import type { DatabaseSync } from "node:sqlite";
import type { TaskHold } from "../../shared/domain";
import { currentProviderUsageWindow } from "../../shared/provider-runtime-domain";
import { budgetBlock, isPriced, providerPrices } from "./spend";
interface OpenTaskRow {
id: string;
stalled_reason: string | null;
enqueue_error: string | null;
agent_id: string | null;
provider_id: string | null;
provider_label: string | null;
request_limit: number;
request_count: number;
cooldown_until_ms: number | null;
cooldown_label: string | null;
}
/** Why each open task without a run in flight is not moving, when it resumes on its own, and who must act otherwise. */
export function taskHolds(db: DatabaseSync, now: Date): Map<string, TaskHold> {
const window = currentProviderUsageWindow(now);
const rows = db.prepare(`
select t.id, t.stalled_reason, t.enqueue_error, a.id as agent_id, p.id as provider_id, p.label as provider_label,
p.monthly_request_limit as request_limit, coalesce(u.request_count, 0) as request_count,
c.until_ms as cooldown_until_ms, c.label as cooldown_label
from tasks t
left join agents a on a.id = t.assignee_agent_id
left join providers p on p.id = a.model_provider_id
left join provider_usage_windows u on u.org_id = p.org_id and u.provider_id = p.id and u.window_start = ?
left join provider_cooldowns c on c.provider_id = p.id
where t.status in ('backlog', 'in_progress')
and not exists (select 1 from task_runs r where r.task_id = t.id and r.state = 'running')
`).all(window.start) as unknown as OpenTaskRow[];
const budgets = new Map<string, MoneyBlock | null>();
// A zero cap stays spent after the monthly reset, so only raising it lets the work resume.
const zeroCap = db.prepare(`
select a.budget_cents = 0 or coalesce(d.monthly_budget_limit_cents, 1) = 0 as zero
from agents a left join org_departments d on d.org_id = a.org_id and d.name = a.department where a.id = ?
`);
const holds = new Map<string, TaskHold>();
for (const row of rows) {
const hold = holdOf(row, now, window.end, (agentId, providerId) => {
let block = budgets.get(agentId);
if (block === undefined) {
const reason = budgetBlock(db, agentId, isPriced(providerPrices(db, providerId)), now);
block = reason ? { reason, renews: (zeroCap.get(agentId) as { zero: number }).zero === 0 } : null;
budgets.set(agentId, block);
}
return block;
});
if (hold) holds.set(row.id, hold);
}
return holds;
}
interface MoneyBlock {
reason: string;
/** False when a zero cap means the next month starts spent too. */
renews: boolean;
}
function holdOf(row: OpenTaskRow, now: Date, windowEnd: string, budget: (agentId: string, providerId: string) => MoneyBlock | null): TaskHold | null {
if (row.stalled_reason) return { reason: row.stalled_reason, resumesAt: null, waitingOn: "operator" };
if (row.enqueue_error) return { reason: `Could not start: ${row.enqueue_error}`, resumesAt: null, waitingOn: "operator" };
if (!row.agent_id || !row.provider_id) return null;
if (row.request_count >= row.request_limit) {
if (row.request_limit === 0) return { reason: `${row.provider_label} has a monthly request limit of 0. Raise the limit to let its work run.`, resumesAt: null, waitingOn: "operator" };
return { reason: `${row.provider_label} used its monthly request limit (${row.request_limit}). Raise the limit or wait for the next window.`, resumesAt: windowEnd, waitingOn: "operator" };
}
const spent = budget(row.agent_id, row.provider_id);
if (spent) return { reason: spent.reason, resumesAt: spent.renews ? windowEnd : null, waitingOn: "operator" };
if (row.cooldown_until_ms !== null && row.cooldown_until_ms > now.getTime()) {
return { reason: `${row.cooldown_label} reported a passing failure; its work waits out a cooldown.`, resumesAt: new Date(row.cooldown_until_ms).toISOString(), waitingOn: "provider" };
}
return null;
}
+1 -1
View File
@@ -50,7 +50,7 @@ export class TaskMutations {
}
const result = this.db.prepare(`
update tasks set title = ?, description = ?, assignee_agent_id = ?, risk = ?, priority = ?, parent_task_id = ?, updated_at = ?,
stalled_reason = null, run_budget_floor = (select coalesce(max(rowid), 0) from task_runs)
stalled_reason = null, enqueue_error = null, run_budget_floor = (select coalesce(max(rowid), 0) from task_runs)
where id = ? and org_id = ?
`).run(input.title.trim(), input.description.trim(), input.assigneeAgentId, input.risk, input.priority, input.parentTaskId, now, input.taskId, input.orgId);
if (result.changes === 0) throw new Error("Task was not found.");