Q130: every open task says why it is not moving, when it resumes, and who must act
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
271a83a7bc
commit
074a1fc8e3
14 files changed
+255
-7
No files matched your search
@@ -0,0 +1,7 @@
|
||||
// Why the heartbeat last failed to queue an open task; cleared when it next queues it.
|
||||
export const taskEnqueueErrorSchema = `
|
||||
alter table tasks add column enqueue_error text;
|
||||
|
||||
insert into schema_migrations(version, name, applied_at)
|
||||
values (93, 'task_enqueue_error', datetime('now'));
|
||||
`;
|
||||
@@ -91,6 +91,7 @@ import { fileVersionsSchema } from "./migrations/089-file-versions";
|
||||
import { erasureReceiptsSchema } from "./migrations/090-erasure-receipts";
|
||||
import { requestPurposeSchema } from "./migrations/091-request-purpose";
|
||||
import { runActionTargetsSchema } from "./migrations/092-run-action-targets";
|
||||
import { taskEnqueueErrorSchema } from "./migrations/093-task-enqueue-error";
|
||||
|
||||
interface Migration {
|
||||
version: number;
|
||||
@@ -191,6 +192,7 @@ const MIGRATIONS: Migration[] = [
|
||||
{ version: 90, name: "erasure_receipts", sql: erasureReceiptsSchema },
|
||||
{ version: 91, name: "request_purpose", sql: requestPurposeSchema },
|
||||
{ version: 92, name: "run_action_targets", sql: runActionTargetsSchema },
|
||||
{ version: 93, name: "task_enqueue_error", sql: taskEnqueueErrorSchema },
|
||||
];
|
||||
|
||||
const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version;
|
||||
|
||||
@@ -85,13 +85,23 @@ export class HeartbeatScheduler {
|
||||
try {
|
||||
created.push(this.enqueueRun(row.org_id, row.task_id));
|
||||
pacedAgents.add(row.agent_id);
|
||||
} catch {
|
||||
// Skip tasks that cannot be prepared yet (e.g. provider not fully configured).
|
||||
this.db.prepare("update tasks set enqueue_error = null where id = ? and enqueue_error is not null").run(row.task_id);
|
||||
} catch (error) {
|
||||
// A task that cannot be prepared yet (approval pending, setup missing) stays open; its task shows why until it queues.
|
||||
this.recordEnqueueError(row.task_id, error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
}
|
||||
return created;
|
||||
}
|
||||
|
||||
/** Records on the task why it could not be queued, logging and notifying only when the reason changes. */
|
||||
private recordEnqueueError(taskId: string, message: string): void {
|
||||
const changed = this.db.prepare("update tasks set enqueue_error = ? where id = ? and enqueue_error is not ?").run(message, taskId, message);
|
||||
if (changed.changes === 0) return;
|
||||
console.warn(`[task-queue] task ${taskId} could not be queued: ${message}`);
|
||||
this.notifyDataChanged();
|
||||
}
|
||||
|
||||
/** Records once, on the task itself, that it spent its run budget without reaching done. */
|
||||
private markStalled(orgId: string, taskId: string, why: string): void {
|
||||
const reason = `Stalled: ${why}. Edit, reassign or reopen the task to give it a fresh budget.`;
|
||||
|
||||
@@ -4,6 +4,7 @@ import { PROVIDER_CATALOG } from "../../shared/provider-catalog";
|
||||
import { IntegrationKeyStore } from "./integration-key-store";
|
||||
import { currentProviderUsageWindow } from "../../shared/provider-runtime-domain";
|
||||
import { buildUsageStats } from "./usage-stats";
|
||||
import { taskHolds } from "./task-holds";
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
type ProviderUsageWindowRow = Pick<
|
||||
@@ -49,8 +50,9 @@ export class SnapshotService {
|
||||
const activeOrgId = this.resolveActiveOrgId(organizations.map((org) => org.id));
|
||||
const agents = list<AppSnapshot["agents"][number]>(this.db, "agents", "created_at asc")
|
||||
.map((agent) => ({ ...agent, heartbeatEnabled: toBool(agent.heartbeatEnabled) }));
|
||||
const holds = taskHolds(this.db, new Date());
|
||||
const tasks = list<AppSnapshot["tasks"][number]>(this.db, "tasks", "created_at asc")
|
||||
.map((task) => ({ ...task, requiresFilesystem: toBool(task.requiresFilesystem) }));
|
||||
.map((task) => ({ ...task, requiresFilesystem: toBool(task.requiresFilesystem), hold: holds.get(task.id) ?? null }));
|
||||
const workspaces = list<AppSnapshot["workspaces"][number]>(this.db, "workspaces", "created_at asc")
|
||||
.map((workspace) => ({ ...workspace, isPrimary: toBool(workspace.isPrimary) }));
|
||||
const providers = list<AppSnapshot["providers"][number]>(this.db, "providers", "label asc")
|
||||
|
||||
@@ -69,7 +69,8 @@ export function budgetBlock(db: DatabaseSync, agentId: string, paid: boolean, no
|
||||
const agentSpend = (db.prepare("select coalesce(sum(cost_cents), 0) as cents from provider_requests where agent_id = ? and created_at >= ?")
|
||||
.get(agentId, since) as { cents: number }).cents;
|
||||
if (agentSpend >= agent.budget_cents) {
|
||||
return `${agent.name} has spent its monthly budget (${dollars(agentSpend)} of ${dollars(agent.budget_cents)}). Raise the budget or wait for next month.`;
|
||||
const next = agent.budget_cents === 0 ? "Raise the budget to let it run." : "Raise the budget or wait for next month.";
|
||||
return `${agent.name} has spent its monthly budget (${dollars(agentSpend)} of ${dollars(agent.budget_cents)}). ${next}`;
|
||||
}
|
||||
const department = db.prepare("select monthly_budget_limit_cents from org_departments where org_id = ? and name = ?")
|
||||
.get(agent.org_id, agent.department) as { monthly_budget_limit_cents: number } | undefined;
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import type { TaskHold } from "../../shared/domain";
|
||||
import { currentProviderUsageWindow } from "../../shared/provider-runtime-domain";
|
||||
import { budgetBlock, isPriced, providerPrices } from "./spend";
|
||||
|
||||
interface OpenTaskRow {
|
||||
id: string;
|
||||
stalled_reason: string | null;
|
||||
enqueue_error: string | null;
|
||||
agent_id: string | null;
|
||||
provider_id: string | null;
|
||||
provider_label: string | null;
|
||||
request_limit: number;
|
||||
request_count: number;
|
||||
cooldown_until_ms: number | null;
|
||||
cooldown_label: string | null;
|
||||
}
|
||||
|
||||
/** Why each open task without a run in flight is not moving, when it resumes on its own, and who must act otherwise. */
|
||||
export function taskHolds(db: DatabaseSync, now: Date): Map<string, TaskHold> {
|
||||
const window = currentProviderUsageWindow(now);
|
||||
const rows = db.prepare(`
|
||||
select t.id, t.stalled_reason, t.enqueue_error, a.id as agent_id, p.id as provider_id, p.label as provider_label,
|
||||
p.monthly_request_limit as request_limit, coalesce(u.request_count, 0) as request_count,
|
||||
c.until_ms as cooldown_until_ms, c.label as cooldown_label
|
||||
from tasks t
|
||||
left join agents a on a.id = t.assignee_agent_id
|
||||
left join providers p on p.id = a.model_provider_id
|
||||
left join provider_usage_windows u on u.org_id = p.org_id and u.provider_id = p.id and u.window_start = ?
|
||||
left join provider_cooldowns c on c.provider_id = p.id
|
||||
where t.status in ('backlog', 'in_progress')
|
||||
and not exists (select 1 from task_runs r where r.task_id = t.id and r.state = 'running')
|
||||
`).all(window.start) as unknown as OpenTaskRow[];
|
||||
const budgets = new Map<string, MoneyBlock | null>();
|
||||
// A zero cap stays spent after the monthly reset, so only raising it lets the work resume.
|
||||
const zeroCap = db.prepare(`
|
||||
select a.budget_cents = 0 or coalesce(d.monthly_budget_limit_cents, 1) = 0 as zero
|
||||
from agents a left join org_departments d on d.org_id = a.org_id and d.name = a.department where a.id = ?
|
||||
`);
|
||||
const holds = new Map<string, TaskHold>();
|
||||
for (const row of rows) {
|
||||
const hold = holdOf(row, now, window.end, (agentId, providerId) => {
|
||||
let block = budgets.get(agentId);
|
||||
if (block === undefined) {
|
||||
const reason = budgetBlock(db, agentId, isPriced(providerPrices(db, providerId)), now);
|
||||
block = reason ? { reason, renews: (zeroCap.get(agentId) as { zero: number }).zero === 0 } : null;
|
||||
budgets.set(agentId, block);
|
||||
}
|
||||
return block;
|
||||
});
|
||||
if (hold) holds.set(row.id, hold);
|
||||
}
|
||||
return holds;
|
||||
}
|
||||
|
||||
interface MoneyBlock {
|
||||
reason: string;
|
||||
/** False when a zero cap means the next month starts spent too. */
|
||||
renews: boolean;
|
||||
}
|
||||
|
||||
function holdOf(row: OpenTaskRow, now: Date, windowEnd: string, budget: (agentId: string, providerId: string) => MoneyBlock | null): TaskHold | null {
|
||||
if (row.stalled_reason) return { reason: row.stalled_reason, resumesAt: null, waitingOn: "operator" };
|
||||
if (row.enqueue_error) return { reason: `Could not start: ${row.enqueue_error}`, resumesAt: null, waitingOn: "operator" };
|
||||
if (!row.agent_id || !row.provider_id) return null;
|
||||
if (row.request_count >= row.request_limit) {
|
||||
if (row.request_limit === 0) return { reason: `${row.provider_label} has a monthly request limit of 0. Raise the limit to let its work run.`, resumesAt: null, waitingOn: "operator" };
|
||||
return { reason: `${row.provider_label} used its monthly request limit (${row.request_limit}). Raise the limit or wait for the next window.`, resumesAt: windowEnd, waitingOn: "operator" };
|
||||
}
|
||||
const spent = budget(row.agent_id, row.provider_id);
|
||||
if (spent) return { reason: spent.reason, resumesAt: spent.renews ? windowEnd : null, waitingOn: "operator" };
|
||||
if (row.cooldown_until_ms !== null && row.cooldown_until_ms > now.getTime()) {
|
||||
return { reason: `${row.cooldown_label} reported a passing failure; its work waits out a cooldown.`, resumesAt: new Date(row.cooldown_until_ms).toISOString(), waitingOn: "provider" };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -50,7 +50,7 @@ export class TaskMutations {
|
||||
}
|
||||
const result = this.db.prepare(`
|
||||
update tasks set title = ?, description = ?, assignee_agent_id = ?, risk = ?, priority = ?, parent_task_id = ?, updated_at = ?,
|
||||
stalled_reason = null, run_budget_floor = (select coalesce(max(rowid), 0) from task_runs)
|
||||
stalled_reason = null, enqueue_error = null, run_budget_floor = (select coalesce(max(rowid), 0) from task_runs)
|
||||
where id = ? and org_id = ?
|
||||
`).run(input.title.trim(), input.description.trim(), input.assigneeAgentId, input.risk, input.priority, input.parentTaskId, now, input.taskId, input.orgId);
|
||||
if (result.changes === 0) throw new Error("Task was not found.");
|
||||
|
||||
@@ -31,6 +31,11 @@ export const zh: Record<string, string> = {
|
||||
"Writing": "写作方式",
|
||||
"superseded": "已被取代",
|
||||
"stalled": "已停滞",
|
||||
"needs you": "待你处理",
|
||||
"waiting": "等待中",
|
||||
"Resumes on its own at {time}.": "将于 {time} 自动恢复。",
|
||||
"Waiting on you, or resumes on its own at {time}.": "等待你处理,或于 {time} 自动恢复。",
|
||||
"Waiting on you.": "等待你处理。",
|
||||
"Resume": "恢复",
|
||||
"Missing this run": "本次运行缺少的能力",
|
||||
"Disconnect": "断开连接",
|
||||
|
||||
@@ -338,6 +338,11 @@ function TaskCard({
|
||||
<div className="row" style={{ flexWrap: "wrap", gap: 6, minWidth: 0 }}>
|
||||
{project ? <Pill>{project.name}</Pill> : <Pill>{capitalize(task.risk)}</Pill>}
|
||||
{task.stalledReason && task.status !== "done" && <Pill tone="red">{t("stalled")}</Pill>}
|
||||
{task.hold && !task.stalledReason && (
|
||||
<span title={task.hold.reason} style={{ display: "inline-flex" }}>
|
||||
<Pill tone={task.hold.waitingOn === "operator" ? "red" : "amber"}>{task.hold.waitingOn === "operator" ? t("needs you") : t("waiting")}</Pill>
|
||||
</span>
|
||||
)}
|
||||
{task.status !== "done" && latestRun && (
|
||||
<span style={{ flexShrink: 0, display: "inline-flex" }}>
|
||||
<Pill tone={runTone(latestRun.state)}>{prettyRunState(latestRun.state)}</Pill>
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { TaskHoldNotice } from "./TaskHoldNotice";
|
||||
import { useState, type FormEvent } from "react";
|
||||
import { Avatar, Button, Field, Icon, IconButton, Pill, Segmented } from "../../shared/ui";
|
||||
import type {
|
||||
@@ -142,7 +143,7 @@ function OverviewTab({ task, assignee, project, goal, workspace, provider, runs
|
||||
}) {
|
||||
return (
|
||||
<>
|
||||
{task.stalledReason && <div className="form-error" role="alert" style={{ marginBottom: 12 }}>{task.stalledReason}</div>}
|
||||
{task.hold && <TaskHoldNotice hold={task.hold} />}
|
||||
{task.description && (
|
||||
<p style={{ margin: "0 0 14px", color: "var(--text-2)", fontSize: 14, lineHeight: 1.55, whiteSpace: "pre-wrap" }}>{task.description}</p>
|
||||
)}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import type { TaskHold } from "../../../../shared/domain";
|
||||
import { t } from "../../i18n/i18n";
|
||||
|
||||
function when(iso: string): string {
|
||||
return new Date(iso).toLocaleString([], { month: "short", day: "numeric", hour: "2-digit", minute: "2-digit", hour12: false });
|
||||
}
|
||||
|
||||
/** Why an open task is not moving, when it resumes on its own, and whether it waits on the operator. */
|
||||
export function TaskHoldNotice({ hold }: { hold: TaskHold }) {
|
||||
const next = hold.waitingOn === "provider"
|
||||
? t("Resumes on its own at {time}.", { time: when(hold.resumesAt) })
|
||||
: hold.resumesAt
|
||||
? t("Waiting on you, or resumes on its own at {time}.", { time: when(hold.resumesAt) })
|
||||
: t("Waiting on you.");
|
||||
const color = hold.waitingOn === "operator" ? "var(--red)" : "var(--amber)";
|
||||
return (
|
||||
<div role="status" style={{ marginBottom: 12, fontSize: 13, color, lineHeight: 1.5 }}>
|
||||
<div>{hold.reason}</div>
|
||||
<div style={{ fontWeight: 600 }}>{next}</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -87,6 +87,11 @@ export interface AgentRecord {
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
/** What stops an open task: the reason, when it resumes on its own (null: only when someone acts), and who that is. */
|
||||
export type TaskHold =
|
||||
| { reason: string; waitingOn: "provider"; resumesAt: string }
|
||||
| { reason: string; waitingOn: "operator"; resumesAt: string | null };
|
||||
|
||||
export interface TaskRecord {
|
||||
id: string;
|
||||
orgId: string;
|
||||
@@ -104,6 +109,10 @@ export interface TaskRecord {
|
||||
writingMode: WritingMode;
|
||||
/** Set when the task spent its run budget without reaching done. */
|
||||
stalledReason: string | null;
|
||||
/** Why the heartbeat last failed to queue the task; cleared once it queues. */
|
||||
enqueueError: string | null;
|
||||
/** Why the open task is not moving now, or null when nothing holds it. */
|
||||
hold: TaskHold | null;
|
||||
/** Runs at or below this task_runs rowid no longer count toward the run budget. */
|
||||
runBudgetFloor: number;
|
||||
createdAt: string;
|
||||
|
||||
Reference in new issue
Block a user