Q130: every open task says why it is not moving, when it resumes, and who must act
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
271a83a7bc
commit
074a1fc8e3
14 files changed
+255
-7
No files matched your search
@@ -85,13 +85,23 @@ export class HeartbeatScheduler {
|
||||
try {
|
||||
created.push(this.enqueueRun(row.org_id, row.task_id));
|
||||
pacedAgents.add(row.agent_id);
|
||||
} catch {
|
||||
// Skip tasks that cannot be prepared yet (e.g. provider not fully configured).
|
||||
this.db.prepare("update tasks set enqueue_error = null where id = ? and enqueue_error is not null").run(row.task_id);
|
||||
} catch (error) {
|
||||
// A task that cannot be prepared yet (approval pending, setup missing) stays open; its task shows why until it queues.
|
||||
this.recordEnqueueError(row.task_id, error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
}
|
||||
return created;
|
||||
}
|
||||
|
||||
/** Records on the task why it could not be queued, logging and notifying only when the reason changes. */
|
||||
private recordEnqueueError(taskId: string, message: string): void {
|
||||
const changed = this.db.prepare("update tasks set enqueue_error = ? where id = ? and enqueue_error is not ?").run(message, taskId, message);
|
||||
if (changed.changes === 0) return;
|
||||
console.warn(`[task-queue] task ${taskId} could not be queued: ${message}`);
|
||||
this.notifyDataChanged();
|
||||
}
|
||||
|
||||
/** Records once, on the task itself, that it spent its run budget without reaching done. */
|
||||
private markStalled(orgId: string, taskId: string, why: string): void {
|
||||
const reason = `Stalled: ${why}. Edit, reassign or reopen the task to give it a fresh budget.`;
|
||||
|
||||
@@ -4,6 +4,7 @@ import { PROVIDER_CATALOG } from "../../shared/provider-catalog";
|
||||
import { IntegrationKeyStore } from "./integration-key-store";
|
||||
import { currentProviderUsageWindow } from "../../shared/provider-runtime-domain";
|
||||
import { buildUsageStats } from "./usage-stats";
|
||||
import { taskHolds } from "./task-holds";
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
type ProviderUsageWindowRow = Pick<
|
||||
@@ -49,8 +50,9 @@ export class SnapshotService {
|
||||
const activeOrgId = this.resolveActiveOrgId(organizations.map((org) => org.id));
|
||||
const agents = list<AppSnapshot["agents"][number]>(this.db, "agents", "created_at asc")
|
||||
.map((agent) => ({ ...agent, heartbeatEnabled: toBool(agent.heartbeatEnabled) }));
|
||||
const holds = taskHolds(this.db, new Date());
|
||||
const tasks = list<AppSnapshot["tasks"][number]>(this.db, "tasks", "created_at asc")
|
||||
.map((task) => ({ ...task, requiresFilesystem: toBool(task.requiresFilesystem) }));
|
||||
.map((task) => ({ ...task, requiresFilesystem: toBool(task.requiresFilesystem), hold: holds.get(task.id) ?? null }));
|
||||
const workspaces = list<AppSnapshot["workspaces"][number]>(this.db, "workspaces", "created_at asc")
|
||||
.map((workspace) => ({ ...workspace, isPrimary: toBool(workspace.isPrimary) }));
|
||||
const providers = list<AppSnapshot["providers"][number]>(this.db, "providers", "label asc")
|
||||
|
||||
@@ -69,7 +69,8 @@ export function budgetBlock(db: DatabaseSync, agentId: string, paid: boolean, no
|
||||
const agentSpend = (db.prepare("select coalesce(sum(cost_cents), 0) as cents from provider_requests where agent_id = ? and created_at >= ?")
|
||||
.get(agentId, since) as { cents: number }).cents;
|
||||
if (agentSpend >= agent.budget_cents) {
|
||||
return `${agent.name} has spent its monthly budget (${dollars(agentSpend)} of ${dollars(agent.budget_cents)}). Raise the budget or wait for next month.`;
|
||||
const next = agent.budget_cents === 0 ? "Raise the budget to let it run." : "Raise the budget or wait for next month.";
|
||||
return `${agent.name} has spent its monthly budget (${dollars(agentSpend)} of ${dollars(agent.budget_cents)}). ${next}`;
|
||||
}
|
||||
const department = db.prepare("select monthly_budget_limit_cents from org_departments where org_id = ? and name = ?")
|
||||
.get(agent.org_id, agent.department) as { monthly_budget_limit_cents: number } | undefined;
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import type { TaskHold } from "../../shared/domain";
|
||||
import { currentProviderUsageWindow } from "../../shared/provider-runtime-domain";
|
||||
import { budgetBlock, isPriced, providerPrices } from "./spend";
|
||||
|
||||
interface OpenTaskRow {
|
||||
id: string;
|
||||
stalled_reason: string | null;
|
||||
enqueue_error: string | null;
|
||||
agent_id: string | null;
|
||||
provider_id: string | null;
|
||||
provider_label: string | null;
|
||||
request_limit: number;
|
||||
request_count: number;
|
||||
cooldown_until_ms: number | null;
|
||||
cooldown_label: string | null;
|
||||
}
|
||||
|
||||
/** Why each open task without a run in flight is not moving, when it resumes on its own, and who must act otherwise. */
|
||||
export function taskHolds(db: DatabaseSync, now: Date): Map<string, TaskHold> {
|
||||
const window = currentProviderUsageWindow(now);
|
||||
const rows = db.prepare(`
|
||||
select t.id, t.stalled_reason, t.enqueue_error, a.id as agent_id, p.id as provider_id, p.label as provider_label,
|
||||
p.monthly_request_limit as request_limit, coalesce(u.request_count, 0) as request_count,
|
||||
c.until_ms as cooldown_until_ms, c.label as cooldown_label
|
||||
from tasks t
|
||||
left join agents a on a.id = t.assignee_agent_id
|
||||
left join providers p on p.id = a.model_provider_id
|
||||
left join provider_usage_windows u on u.org_id = p.org_id and u.provider_id = p.id and u.window_start = ?
|
||||
left join provider_cooldowns c on c.provider_id = p.id
|
||||
where t.status in ('backlog', 'in_progress')
|
||||
and not exists (select 1 from task_runs r where r.task_id = t.id and r.state = 'running')
|
||||
`).all(window.start) as unknown as OpenTaskRow[];
|
||||
const budgets = new Map<string, MoneyBlock | null>();
|
||||
// A zero cap stays spent after the monthly reset, so only raising it lets the work resume.
|
||||
const zeroCap = db.prepare(`
|
||||
select a.budget_cents = 0 or coalesce(d.monthly_budget_limit_cents, 1) = 0 as zero
|
||||
from agents a left join org_departments d on d.org_id = a.org_id and d.name = a.department where a.id = ?
|
||||
`);
|
||||
const holds = new Map<string, TaskHold>();
|
||||
for (const row of rows) {
|
||||
const hold = holdOf(row, now, window.end, (agentId, providerId) => {
|
||||
let block = budgets.get(agentId);
|
||||
if (block === undefined) {
|
||||
const reason = budgetBlock(db, agentId, isPriced(providerPrices(db, providerId)), now);
|
||||
block = reason ? { reason, renews: (zeroCap.get(agentId) as { zero: number }).zero === 0 } : null;
|
||||
budgets.set(agentId, block);
|
||||
}
|
||||
return block;
|
||||
});
|
||||
if (hold) holds.set(row.id, hold);
|
||||
}
|
||||
return holds;
|
||||
}
|
||||
|
||||
interface MoneyBlock {
|
||||
reason: string;
|
||||
/** False when a zero cap means the next month starts spent too. */
|
||||
renews: boolean;
|
||||
}
|
||||
|
||||
function holdOf(row: OpenTaskRow, now: Date, windowEnd: string, budget: (agentId: string, providerId: string) => MoneyBlock | null): TaskHold | null {
|
||||
if (row.stalled_reason) return { reason: row.stalled_reason, resumesAt: null, waitingOn: "operator" };
|
||||
if (row.enqueue_error) return { reason: `Could not start: ${row.enqueue_error}`, resumesAt: null, waitingOn: "operator" };
|
||||
if (!row.agent_id || !row.provider_id) return null;
|
||||
if (row.request_count >= row.request_limit) {
|
||||
if (row.request_limit === 0) return { reason: `${row.provider_label} has a monthly request limit of 0. Raise the limit to let its work run.`, resumesAt: null, waitingOn: "operator" };
|
||||
return { reason: `${row.provider_label} used its monthly request limit (${row.request_limit}). Raise the limit or wait for the next window.`, resumesAt: windowEnd, waitingOn: "operator" };
|
||||
}
|
||||
const spent = budget(row.agent_id, row.provider_id);
|
||||
if (spent) return { reason: spent.reason, resumesAt: spent.renews ? windowEnd : null, waitingOn: "operator" };
|
||||
if (row.cooldown_until_ms !== null && row.cooldown_until_ms > now.getTime()) {
|
||||
return { reason: `${row.cooldown_label} reported a passing failure; its work waits out a cooldown.`, resumesAt: new Date(row.cooldown_until_ms).toISOString(), waitingOn: "provider" };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -50,7 +50,7 @@ export class TaskMutations {
|
||||
}
|
||||
const result = this.db.prepare(`
|
||||
update tasks set title = ?, description = ?, assignee_agent_id = ?, risk = ?, priority = ?, parent_task_id = ?, updated_at = ?,
|
||||
stalled_reason = null, run_budget_floor = (select coalesce(max(rowid), 0) from task_runs)
|
||||
stalled_reason = null, enqueue_error = null, run_budget_floor = (select coalesce(max(rowid), 0) from task_runs)
|
||||
where id = ? and org_id = ?
|
||||
`).run(input.title.trim(), input.description.trim(), input.assigneeAgentId, input.risk, input.priority, input.parentTaskId, now, input.taskId, input.orgId);
|
||||
if (result.changes === 0) throw new Error("Task was not found.");
|
||||
|
||||
Reference in new issue
Block a user