refactor(database): move task runtime ownership to leases

This commit is contained in:
ZacharyZhang-NY committed 2026-04-24 00:18:51 -04:00
1 parent 1f4af89be0
commit f8eb57ad8c
21 files changed
+419 -162

No files matched your search

+3 -3
View File
@@ -15,8 +15,8 @@ export interface StaleRunDb {
interruptRunningRuns(ts: string): number;
/** Mark tasks associated with interrupted runs as 'blocked'. */
blockInterruptedTasks(ts: string): void;
/** Clear active_run_id on tasks whose run is now interrupted. */
clearActiveRunIds(): void;
/** Delete task lease rows whose run is now interrupted. */
clearInterruptedTaskLeases(): void;
/** Reset all running agents to idle. */
resetAgentsToIdle(ts: string): void;
/** Release all workspace locks. */
@@ -51,7 +51,7 @@ export function interruptStaleRunsLogic(
if (interruptedCount > 0) {
db.blockInterruptedTasks(ts);
}
db.clearActiveRunIds();
db.clearInterruptedTaskLeases();
db.resetAgentsToIdle(ts);
db.releaseAllWorkspaceLocks();
});
+1 -1
View File
@@ -53,7 +53,7 @@ export function buildEntityCrudMethods(
getTask(id: string): TaskRecord {
const row = getDb()
.prepare(
"select id, company_id as companyId, project_id as projectId, goal_id as goalId, parent_id as parentId, title, description, assignee_agent_id as assigneeAgentId, workspace_id as workspaceId, status, priority, active_run_id as activeRunId, cost_usd as costUsd, task_type as taskType, requires_user_review as requiresUserReview, requires_filesystem as requiresFilesystem, started_at as startedAt, completed_at as completedAt, cancelled_at as cancelledAt, created_at as createdAt, updated_at as updatedAt from tasks where id = ?",
"select id, company_id as companyId, project_id as projectId, goal_id as goalId, parent_id as parentId, title, description, assignee_agent_id as assigneeAgentId, workspace_id as workspaceId, status, priority, (select l.run_id from task_leases l where l.task_id = tasks.id) as activeRunId, cost_usd as costUsd, task_type as taskType, requires_user_review as requiresUserReview, requires_filesystem as requiresFilesystem, started_at as startedAt, completed_at as completedAt, cancelled_at as cancelledAt, created_at as createdAt, updated_at as updatedAt from tasks where id = ?",
)
.get(id) as TaskRecord | undefined;
if (!row) throw new Error(`Task not found: ${id}`);
@@ -0,0 +1,121 @@
import type { DatabaseSync } from "node:sqlite";
import { STALE_LOCK_TIMEOUT_MS } from "../../constants";
import { columnExists, nowIso, tableExists } from "./types";
export const TASK_LEASE_SCHEMA_VERSION = 57;
export const TASK_LEASE_SCHEMA_NAME = "task_leases_runtime_ownership";
const LEASE_SECONDS = Math.floor(STALE_LOCK_TIMEOUT_MS / 1000);
export function migrateTaskLeasesV57(db: DatabaseSync): void {
installTaskLeasesTable(db);
backfillTaskLeases(db);
dropTaskOwnershipColumns(db);
db
.prepare("insert or replace into schema_migrations (version, name, applied_at) values (?, ?, ?)")
.run(TASK_LEASE_SCHEMA_VERSION, TASK_LEASE_SCHEMA_NAME, nowIso());
assertTaskLeasesV57(db);
}
export function assertTaskLeasesV57(db: DatabaseSync): void {
if (!tableExists(db, "task_leases")) {
throw new Error("Schema v57 drift: task_leases table is missing.");
}
if (!tableExists(db, "tasks")) {
throw new Error("Schema v57 drift: tasks table is missing.");
}
const leaseColumns = db
.prepare("pragma table_info(task_leases)")
.all()
.map((row) => row as { name: string; type: string; notnull: number; pk: number });
const expectedLeaseColumns = [
["task_id", "TEXT", 0, 1],
["company_id", "TEXT", 1, 0],
["run_id", "TEXT", 1, 0],
["agent_id", "TEXT", 1, 0],
["acquired_at", "TEXT", 1, 0],
["renewed_at", "TEXT", 1, 0],
["expires_at", "TEXT", 1, 0],
["created_at", "TEXT", 1, 0],
["updated_at", "TEXT", 1, 0],
];
const actualLeaseColumns = leaseColumns.map((column) => [
column.name,
column.type.toUpperCase(),
column.notnull,
column.pk,
]);
if (JSON.stringify(actualLeaseColumns) !== JSON.stringify(expectedLeaseColumns)) {
throw new Error("Schema v57 drift: task_leases columns do not match the canonical schema.");
}
const columns = new Set(
db.prepare("pragma table_info(tasks)").all().map((row) => (row as { name: string }).name),
);
for (const column of ["active_run_id", "checkout_run_id", "execution_locked_at", "active_session_id"]) {
if (columns.has(column)) {
throw new Error(`Schema v57 drift: tasks.${column} should be stored in task_leases.`);
}
}
}
function installTaskLeasesTable(db: DatabaseSync): void {
db.exec(`
create table if not exists task_leases (
task_id text primary key references tasks(id) on delete cascade,
company_id text not null,
run_id text not null unique references runs(id) on delete cascade,
agent_id text not null references agents(id) on delete cascade,
acquired_at text not null,
renewed_at text not null,
expires_at text not null,
created_at text not null,
updated_at text not null
);
create index if not exists idx_task_leases_company_agent on task_leases(company_id, agent_id);
create index if not exists idx_task_leases_expires_at on task_leases(expires_at);
`);
}
function backfillTaskLeases(db: DatabaseSync): void {
if (!tableExists(db, "tasks") || !tableExists(db, "runs")) return;
if (!columnExists(db, "tasks", "checkout_run_id") && !columnExists(db, "tasks", "active_run_id")) return;
const runExpr = [
columnExists(db, "tasks", "checkout_run_id") ? "t.checkout_run_id" : "NULL",
columnExists(db, "tasks", "active_run_id") ? "t.active_run_id" : "NULL",
].join(", ");
const lockExpr = columnExists(db, "tasks", "execution_locked_at")
? "coalesce(t.execution_locked_at, t.updated_at)"
: "t.updated_at";
const ts = nowIso();
db.exec(`
insert or ignore into task_leases (
task_id, company_id, run_id, agent_id, acquired_at, renewed_at, expires_at, created_at, updated_at
)
select t.id,
t.company_id,
coalesce(${runExpr}) as run_id,
r.agent_id,
${lockExpr} as acquired_at,
${lockExpr} as renewed_at,
strftime('%Y-%m-%dT%H:%M:%fZ', ${lockExpr}, '+${LEASE_SECONDS} seconds') as expires_at,
'${ts}' as created_at,
'${ts}' as updated_at
from tasks t
join runs r on r.id = coalesce(${runExpr})
where coalesce(${runExpr}) is not null
and r.status in ('queued', 'running')
`);
}
function dropTaskOwnershipColumns(db: DatabaseSync): void {
if (!tableExists(db, "tasks")) return;
for (const column of ["active_run_id", "checkout_run_id", "execution_locked_at", "active_session_id"]) {
if (columnExists(db, "tasks", column)) {
db.exec(`alter table tasks drop column ${column}`);
}
}
}
+30 -23
View File
@@ -215,35 +215,26 @@ export function buildRunMethods(
input.status, input.summary ?? "", input.errorMessage, input.exitCode,
input.signal, input.model, runCostUsd,
input.unattributedCost ? 1 : 0, usageStatus, ts, ts, input.runId,
);
);
if (run.taskId !== null) {
getDb().prepare("update tasks set active_run_id = null, updated_at = ? where id = ?").run(ts, run.taskId);
getDb()
.prepare("UPDATE tasks SET checkout_run_id = NULL, execution_locked_at = NULL, updated_at = ? WHERE id = ? AND checkout_run_id = ?")
.run(ts, run.taskId, input.runId);
.prepare("delete from task_leases where task_id = ? and run_id = ?")
.run(run.taskId, input.runId);
if (runNeverStarted) {
// Roll the task back to `todo` when no *other* live run claims
// it. Invariant: `task.status = 'in_progress'` ⇒ a queued or
// running run owns it. We drop the previous
// `checkout_run_id IS NULL AND active_run_id IS NULL` guard
// because the conditional UPDATE two lines up can miss (race:
// checkout_run_id already swapped to a newer run id), which
// would leave the task silently stuck at in_progress. The
// NOT EXISTS clause is the canonical "no live owner" check.
// running run owns it through `task_leases`.
getDb()
.prepare(
`UPDATE tasks
SET status = 'todo',
checkout_run_id = NULL,
execution_locked_at = NULL,
updated_at = ?
WHERE id = ?
AND status = 'in_progress'
AND NOT EXISTS (
SELECT 1 FROM runs r
WHERE r.task_id = tasks.id
AND r.id != ?
AND r.status IN ('queued', 'running')
SELECT 1 FROM task_leases l
WHERE l.task_id = tasks.id
AND l.run_id != ?
)`,
)
.run(ts, run.taskId, input.runId);
@@ -291,10 +282,9 @@ export function buildRunMethods(
)
.run(summary, "aborted_before_usage", ts, ts, runId);
if (taskId !== null) {
getDb().prepare("update tasks set active_run_id = null, updated_at = ? where id = ?").run(ts, taskId);
getDb()
.prepare("UPDATE tasks SET checkout_run_id = NULL, execution_locked_at = NULL, updated_at = ? WHERE id = ? AND checkout_run_id = ?")
.run(ts, taskId, runId);
.prepare("delete from task_leases where task_id = ? and run_id = ?")
.run(taskId, runId);
}
if (workspaceId) {
getDb()
@@ -308,12 +298,29 @@ export function buildRunMethods(
},
hasActiveRun(filter: { companyId: string; taskId?: string; agentId?: string }) {
if (filter.taskId) {
const whereClauses = [
"l.company_id = ?",
"l.task_id = ?",
"r.status in ('queued', 'running')",
];
const params: SQLInputValue[] = [filter.companyId, filter.taskId];
if (filter.agentId) {
whereClauses.push("r.agent_id = ?");
params.push(filter.agentId);
}
const row = getDb().prepare(
`select 1 as present
from task_leases l
join runs r on r.id = l.run_id
where ${whereClauses.join(" and ")}
limit 1`,
).get(...params) as { present: number } | undefined;
return Boolean(row?.present);
}
const whereClauses = ["company_id = ?", "status in ('queued', 'running')"];
const params: SQLInputValue[] = [filter.companyId];
if (filter.taskId) {
whereClauses.push("task_id = ?");
params.push(filter.taskId);
}
if (filter.agentId) {
whereClauses.push("agent_id = ?");
params.push(filter.agentId);
+20 -2
View File
@@ -9,11 +9,18 @@ import {
markBaselineSchema,
upgradeLegacyProfileToBaselineV56,
} from "./migrations/legacy-to-v56";
import {
TASK_LEASE_SCHEMA_VERSION,
assertTaskLeasesV57,
migrateTaskLeasesV57,
} from "./migrations/v57-task-leases";
import {
type ReadAll,
type Transaction,
} from "./migrations/types";
const LATEST_SCHEMA_VERSION = TASK_LEASE_SCHEMA_VERSION;
export function initializeDatabase(
db: DatabaseSync,
ra: ReadAll,
@@ -23,8 +30,16 @@ export function initializeDatabase(
db.exec("create table if not exists schema_migrations (version integer primary key, name text not null, applied_at text not null)");
const currentVersion = currentSchemaVersion(db);
if (currentVersion > BASELINE_SCHEMA_VERSION) {
throw new Error(`Profile schema version ${currentVersion} is newer than this app baseline ${BASELINE_SCHEMA_VERSION}.`);
if (currentVersion > LATEST_SCHEMA_VERSION) {
throw new Error(`Profile schema version ${currentVersion} is newer than this app schema ${LATEST_SCHEMA_VERSION}.`);
}
if (currentVersion === LATEST_SCHEMA_VERSION) {
tx(() => {
seedSettings(db);
assertTaskLeasesV57(db);
});
return;
}
if (currentVersion === BASELINE_SCHEMA_VERSION) {
@@ -33,6 +48,7 @@ export function initializeDatabase(
markBaselineSchema(db);
seedSettings(db);
assertBaselineSchema(db);
migrateTaskLeasesV57(db);
});
return;
}
@@ -44,6 +60,7 @@ export function initializeDatabase(
upgradeLegacyProfileToBaselineV56({ db, readAll: ra });
seedSettings(db);
assertBaselineSchema(db);
migrateTaskLeasesV57(db);
});
return;
}
@@ -53,6 +70,7 @@ export function initializeDatabase(
markBaselineSchema(db);
seedSettings(db);
assertBaselineSchema(db);
migrateTaskLeasesV57(db);
});
}
+1 -1
View File
@@ -83,7 +83,7 @@ export function buildSnapshotMethods(
...companyArgs,
),
tasks: readAll<TaskRecord>(
`select id, company_id as companyId, project_id as projectId, goal_id as goalId, parent_id as parentId, title, description, assignee_agent_id as assigneeAgentId, workspace_id as workspaceId, status, priority, active_run_id as activeRunId, cost_usd as costUsd, task_type as taskType, requires_user_review as requiresUserReview, requires_filesystem as requiresFilesystem, started_at as startedAt, completed_at as completedAt, cancelled_at as cancelledAt, created_at as createdAt, updated_at as updatedAt from tasks${companyWhere} order by created_at desc limit 2000`,
`select id, company_id as companyId, project_id as projectId, goal_id as goalId, parent_id as parentId, title, description, assignee_agent_id as assigneeAgentId, workspace_id as workspaceId, status, priority, (select l.run_id from task_leases l where l.task_id = tasks.id) as activeRunId, cost_usd as costUsd, task_type as taskType, requires_user_review as requiresUserReview, requires_filesystem as requiresFilesystem, started_at as startedAt, completed_at as completedAt, cancelled_at as cancelledAt, created_at as createdAt, updated_at as updatedAt from tasks${companyWhere} order by created_at desc limit 2000`,
...companyArgs,
).map((record) => ({ ...record, requiresUserReview: toBoolean(record.requiresUserReview), requiresFilesystem: toBoolean(record.requiresFilesystem), startedAt: record.startedAt ?? null, completedAt: record.completedAt ?? null, cancelledAt: record.cancelledAt ?? null })) as TaskRecord[],
approvals: readAll<ApprovalRecord>(
+44 -11
View File
@@ -1,16 +1,25 @@
import type { DatabaseSync } from "node:sqlite";
import { STALE_LOCK_TIMEOUT_MS } from "../constants";
function isoFromMs(ms: number): string {
return new Date(ms).toISOString();
}
function assertRunExists(db: DatabaseSync, runId: string): void {
function loadRunForCheckout(
db: DatabaseSync,
taskId: string,
runId: string,
): { companyId: string; agentId: string; taskId: string | null } {
const row = db
.prepare("SELECT 1 AS present FROM runs WHERE id = ?")
.get(runId) as { present: number } | undefined;
.prepare("SELECT company_id as companyId, agent_id as agentId, task_id as taskId FROM runs WHERE id = ?")
.get(runId) as { companyId: string; agentId: string; taskId: string | null } | undefined;
if (!row) {
throw new Error(`Run not found for task checkout: ${runId}`);
}
if (row.taskId !== null && row.taskId !== taskId) {
throw new Error(`Run ${runId} is bound to task ${row.taskId}, not ${taskId}.`);
}
return row;
}
export function checkoutTaskInTransaction(
@@ -18,26 +27,50 @@ export function checkoutTaskInTransaction(
taskId: string,
runId: string,
): boolean {
assertRunExists(db, runId);
const run = loadRunForCheckout(db, taskId, runId);
const nowMs = Date.now();
const ts = isoFromMs(nowMs);
const result = db
.prepare(
`UPDATE tasks
SET checkout_run_id = ?,
active_run_id = ?,
execution_locked_at = ?,
status = 'in_progress',
SET status = 'in_progress',
started_at = coalesce(started_at, ?),
updated_at = ?
WHERE id = ?
AND checkout_run_id IS NULL
AND status IN ('todo', 'backlog')`,
AND company_id = ?
AND status IN ('todo', 'backlog')
AND NOT EXISTS (
SELECT 1 FROM task_leases
WHERE task_leases.task_id = tasks.id
)`,
)
.run(runId, runId, ts, ts, taskId);
.run(ts, ts, taskId, run.companyId);
if (result.changes === 0) {
return false;
}
if (run.taskId === null) {
db.prepare("update runs set task_id = ?, updated_at = ? where id = ? and task_id is null").run(taskId, ts, runId);
}
db
.prepare(
`insert into task_leases (
task_id, company_id, run_id, agent_id, acquired_at, renewed_at, expires_at, created_at, updated_at
) values (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
)
.run(
taskId,
run.companyId,
runId,
run.agentId,
ts,
ts,
isoFromMs(nowMs + STALE_LOCK_TIMEOUT_MS),
ts,
ts,
);
return true;
}
+2 -2
View File
@@ -25,7 +25,7 @@ export function buildTaskQueryMethods(
): TaskRecord[] {
const placeholders = statuses.map(() => "?").join(",");
return readAll<TaskRecord>(
`select id, company_id as companyId, project_id as projectId, goal_id as goalId, parent_id as parentId, title, description, assignee_agent_id as assigneeAgentId, workspace_id as workspaceId, status, priority, active_run_id as activeRunId, cost_usd as costUsd, task_type as taskType, created_at as createdAt, updated_at as updatedAt
`select id, company_id as companyId, project_id as projectId, goal_id as goalId, parent_id as parentId, title, description, assignee_agent_id as assigneeAgentId, workspace_id as workspaceId, status, priority, (select l.run_id from task_leases l where l.task_id = tasks.id) as activeRunId, cost_usd as costUsd, task_type as taskType, created_at as createdAt, updated_at as updatedAt
from tasks
where company_id = ? and assignee_agent_id = ? and status in (${placeholders})
order by
@@ -91,7 +91,7 @@ export function buildTaskQueryMethods(
): TaskListByStatusesRow[] {
const placeholders = statuses.map(() => "?").join(", ");
return readAll<TaskListByStatusesRow & { requiresUserReview: number | boolean }>(
`select id, company_id as companyId, project_id as projectId, goal_id as goalId, parent_id as parentId, title, description, assignee_agent_id as assigneeAgentId, workspace_id as workspaceId, status, priority, active_run_id as activeRunId, cost_usd as costUsd, requires_user_review as requiresUserReview, started_at as startedAt, completed_at as completedAt, cancelled_at as cancelledAt, created_at as createdAt, updated_at as updatedAt from tasks where company_id = ? and status in (${placeholders}) order by created_at desc`,
`select id, company_id as companyId, project_id as projectId, goal_id as goalId, parent_id as parentId, title, description, assignee_agent_id as assigneeAgentId, workspace_id as workspaceId, status, priority, (select l.run_id from task_leases l where l.task_id = tasks.id) as activeRunId, cost_usd as costUsd, requires_user_review as requiresUserReview, started_at as startedAt, completed_at as completedAt, cancelled_at as cancelledAt, created_at as createdAt, updated_at as updatedAt from tasks where company_id = ? and status in (${placeholders}) order by created_at desc`,
companyId,
...statuses,
).map((t) => ({
+3 -3
View File
@@ -7,9 +7,9 @@ import { type ReadAll, type Transaction, nowIso } from "./helpers";
* Agent-scoped session lifecycle. Sessions belong to agents, not
* tasks — `agents.session_id` is the canonical pointer, and
* `task_sessions` rows are looked up by their own primary key only.
* The pre-migration `tasks.active_session_id` / `task_sessions.task_id`
* columns were dropped by v41; any read that still needs a session
* given a task id resolves it through the task's assignee agent.
* Legacy task/session pointer columns were dropped by v41; any read
* that still needs a session given a task id resolves it through the
* task's assignee agent.
*/
export function buildTaskSessionMethods(
getDb: () => DatabaseSync,
-7
View File
@@ -37,12 +37,5 @@ export function buildTaskStatusTransition(
} else {
sets.push("cancelled_at = NULL");
}
if (newStatus !== "in_progress") {
sets.push("checkout_run_id = NULL");
}
if (newStatus === "todo" || newStatus === "backlog") {
sets.push("active_run_id = NULL");
}
return { sets, values };
}
+2 -7
View File
@@ -158,13 +158,8 @@ export function buildTaskCrudMethods(
)
.run("Task deleted.", "aborted_before_usage", ts, ts, run.id);
getDb()
.prepare("update tasks set active_run_id = null, updated_at = ? where id = ?")
.run(ts, run.taskId);
getDb()
.prepare(
"UPDATE tasks SET checkout_run_id = NULL, execution_locked_at = NULL, updated_at = ? WHERE id = ? AND checkout_run_id = ?",
)
.run(ts, run.taskId, run.id);
.prepare("delete from task_leases where task_id = ? and run_id = ?")
.run(run.taskId, run.id);
if (run.workspaceId) {
getDb()
.prepare(
+11 -8
View File
@@ -10,6 +10,7 @@ import { chooseBestTaskAssignee } from "./task-assignment";
import { checkoutTaskInTransaction } from "./task-checkout";
import { guardInReviewHasReviewer } from "./task-review-guard";
import { assertValidTaskStatus, buildTaskStatusTransition } from "./task-status-rules";
import { STALE_LOCK_TIMEOUT_MS } from "../constants";
export function buildTaskStatusMethods(
getDb: () => DatabaseSync,
@@ -52,6 +53,9 @@ export function buildTaskStatusMethods(
`UPDATE tasks SET ${transition.sets.join(", ")} WHERE id = ?`,
)
.run(...transition.values, taskId);
if (newStatus !== "in_progress") {
getDb().prepare("delete from task_leases where task_id = ?").run(taskId);
}
});
if (comment) {
@@ -77,7 +81,8 @@ export function buildTaskStatusMethods(
`select t.id, t.company_id as companyId, t.project_id as projectId, t.goal_id as goalId,
t.parent_id as parentId, t.title, t.description,
t.assignee_agent_id as assigneeAgentId, t.workspace_id as workspaceId,
t.status, t.priority, t.active_run_id as activeRunId,
t.status, t.priority,
(select l.run_id from task_leases l where l.task_id = t.id) as activeRunId,
t.cost_usd as costUsd,
t.created_at as createdAt, t.updated_at as updatedAt
from tasks t
@@ -94,21 +99,19 @@ export function buildTaskStatusMethods(
},
releaseTaskLock(taskId: string, runId: string): void {
const ts = nowIso();
getDb()
.prepare(
"UPDATE tasks SET checkout_run_id = NULL, execution_locked_at = NULL, updated_at = ? WHERE id = ? AND checkout_run_id = ?",
)
.run(ts, taskId, runId);
.prepare("delete from task_leases where task_id = ? and run_id = ?")
.run(taskId, runId);
},
renewTaskLease(taskId: string, runId: string): boolean {
const ts = nowIso();
const expiresAt = new Date(Date.now() + STALE_LOCK_TIMEOUT_MS).toISOString();
const result = getDb()
.prepare(
"UPDATE tasks SET execution_locked_at = ?, updated_at = ? WHERE id = ? AND checkout_run_id = ?",
"update task_leases set renewed_at = ?, expires_at = ?, updated_at = ? where task_id = ? and run_id = ?",
)
.run(ts, ts, taskId, runId);
.run(ts, expiresAt, ts, taskId, runId);
return result.changes > 0;
},