feat(04-01): implement CAS checkout, lock release, stale recovery, and migration 16

- Rewrite checkoutTask to use runId-based CAS with checkout_run_id IS NULL guard
- Add releaseTaskLock and recoverStaleLocks methods
- Add lock release in finishRun and cancelRunForDeletion exit paths
- Add migration 16: pending_wakes to wakeup_requests cutover with DROP TABLE
- Replace all pending_wakes references with wakeup_requests
- Add wakeup lifecycle CRUD: enqueue, claim, complete, defer, promote, list
- Update api-server, orchestration-service, worker-service callers
- Update existing database.test.ts for new checkoutTask signature
This commit is contained in:
ZacharyZhang-NY committed 2026-03-26 18:56:54 -04:00
1 parent 02e87dc06d
commit 08935b3379
5 files changed
+326 -71

No files matched your search

+1 -1
View File
@@ -355,7 +355,7 @@ export class AgentApiServer {
return;
}
const previousStatus = task.status;
const success = this.db.checkoutTask(params.issueId!, agentId);
const success = this.db.checkoutTask(params.issueId!, token.runId);
if (!success) {
this.json(res, 409, { error: "Task cannot be checked out — already assigned or not in valid status" });
return;
+283 -43
View File
@@ -112,6 +112,8 @@ function taskPriorityOrderSql(columnName: string) {
end`;
}
export const STALE_LOCK_TIMEOUT_MS = 15 * 60 * 1000; // 15 minutes (D-01)
export class AppDatabase {
readonly profileDir: string;
readonly dbPath: string;
@@ -341,6 +343,31 @@ export class AppDatabase {
create index if not exists idx_pending_wakes_agent on pending_wakes(agent_id);
`);
this.db.exec(`
CREATE TABLE IF NOT EXISTS wakeup_requests (
id TEXT PRIMARY KEY,
company_id TEXT NOT NULL,
agent_id TEXT NOT NULL,
source TEXT NOT NULL,
trigger_detail TEXT,
reason TEXT,
payload TEXT,
status TEXT NOT NULL DEFAULT 'queued',
coalesced_count INTEGER NOT NULL DEFAULT 0,
requested_by_actor_type TEXT,
requested_by_actor_id TEXT,
idempotency_key TEXT,
run_id TEXT,
requested_at TEXT NOT NULL,
claimed_at TEXT,
finished_at TEXT,
error TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_wakeup_requests_agent ON wakeup_requests(agent_id, company_id, status);
`);
this.migrateSecretsTable();
this.ensureOptionalColumns();
@@ -363,7 +390,7 @@ export class AppDatabase {
this.seedConnectors();
this._lastInterruptedCount = this.interruptStaleRuns();
this.purgeExpiredRunLogs();
this.purgeExpiredPendingWakes();
this.purgeExpiredWakeupRequests();
}
close() {
@@ -1323,6 +1350,9 @@ export class AppDatabase {
input.runId,
);
this.db.prepare("update tasks set active_run_id = null, updated_at = ? where id = ?").run(ts, run.taskId);
this.db
.prepare("UPDATE tasks SET checkout_run_id = NULL, execution_locked_at = NULL, updated_at = ? WHERE id = ? AND checkout_run_id = ?")
.run(ts, run.taskId, input.runId);
if (typeof input.costUsd === "number" && Number.isFinite(input.costUsd)) {
this.db
.prepare("update tasks set cost_usd = cost_usd + ?, updated_at = ? where id = ?")
@@ -1361,6 +1391,9 @@ export class AppDatabase {
)
.run(summary, ts, ts, runId);
this.db.prepare("update tasks set active_run_id = null, updated_at = ? where id = ?").run(ts, taskId);
this.db
.prepare("UPDATE tasks SET checkout_run_id = NULL, execution_locked_at = NULL, updated_at = ? WHERE id = ? AND checkout_run_id = ?")
.run(ts, taskId, runId);
if (workspaceId) {
this.db
.prepare(
@@ -1403,7 +1436,7 @@ export class AppDatabase {
} catch (err) {
if (!(err instanceof Error) || !err.message.includes("no such table")) throw err;
}
this.db.prepare("delete from pending_wakes where company_id = ?").run(companyId);
this.db.prepare("DELETE FROM wakeup_requests WHERE company_id = ?").run(companyId);
this.db.prepare("delete from cost_entries where company_id = ?").run(companyId);
this.db.prepare("delete from activity where company_id = ?").run(companyId);
this.db.prepare("delete from task_comments where company_id = ?").run(companyId);
@@ -1458,8 +1491,8 @@ export class AppDatabase {
this.db.prepare("update documents set reviewer_agent_id = null where reviewer_agent_id = ? and company_id = ?").run(id, companyId);
// Clean up knowledge entries authored by this agent
this.db.prepare("update knowledge_base set author_agent_id = null where author_agent_id = ? and company_id = ?").run(id, companyId);
// Remove pending wakes for this agent
this.db.prepare("delete from pending_wakes where agent_id = ? and company_id = ?").run(id, companyId);
// Remove wakeup requests for this agent
this.db.prepare("DELETE FROM wakeup_requests WHERE agent_id = ? AND company_id = ?").run(id, companyId);
this.db.prepare("delete from agents where id = ? and company_id = ?").run(id, companyId);
});
}
@@ -1793,19 +1826,54 @@ export class AppDatabase {
).map((record) => ({ ...record, heartbeatEnabled: toBoolean(record.heartbeatEnabled) })) as AgentRecord[];
}
checkoutTask(taskId: string, agentId: string): boolean {
checkoutTask(taskId: string, runId: string): boolean {
const ts = nowIso();
const result = this.db
.prepare(
`update tasks
set assignee_agent_id = ?, status = 'in_progress', updated_at = ?
where id = ?
and status in ('todo', 'backlog')
and (assignee_agent_id is null or assignee_agent_id = ?)`,
`UPDATE tasks
SET checkout_run_id = ?, execution_locked_at = ?, status = 'in_progress', updated_at = ?
WHERE id = ?
AND checkout_run_id IS NULL
AND status IN ('todo', 'backlog')`,
)
.run(agentId, nowIso(), taskId, agentId);
.run(runId, ts, ts, taskId);
return result.changes > 0;
}
releaseTaskLock(taskId: string, runId: string): void {
this.db
.prepare(
"UPDATE tasks SET checkout_run_id = NULL, execution_locked_at = NULL, updated_at = ? WHERE id = ? AND checkout_run_id = ?",
)
.run(nowIso(), taskId, runId);
}
recoverStaleLocks(): Array<{ taskId: string; companyId: string; title: string }> {
const cutoff = new Date(Date.now() - STALE_LOCK_TIMEOUT_MS).toISOString();
const stale = this.readAll<{ id: string; companyId: string; title: string }>(
`SELECT id, company_id AS companyId, title
FROM tasks
WHERE execution_locked_at IS NOT NULL
AND execution_locked_at < ?
AND status = 'in_progress'`,
cutoff,
);
const ts = nowIso();
for (const task of stale) {
this.db
.prepare(
`UPDATE tasks
SET checkout_run_id = NULL,
execution_locked_at = NULL,
status = 'todo',
updated_at = ?
WHERE id = ?`,
)
.run(ts, task.id);
}
return stale.map((t) => ({ taskId: t.id, companyId: t.companyId, title: t.title }));
}
listAgentTasks(agentId: string, companyId: string, statuses: string[]): TaskRecord[] {
const placeholders = statuses.map(() => "?").join(",");
return this.readAll<TaskRecord>(
@@ -1871,36 +1939,172 @@ export class AppDatabase {
).map((record) => ({ ...record, heartbeatEnabled: toBoolean(record.heartbeatEnabled) })) as AgentRecord[];
}
enqueuePendingWake(companyId: string, agentId: string, trigger: string) {
const existing = this.db
.prepare("select id from pending_wakes where company_id = ? and agent_id = ? limit 1")
.get(companyId, agentId) as { id: string } | undefined;
enqueueWakeupRequest(companyId: string, agentId: string, trigger: string, detail: string): void {
const ts = nowIso();
if (existing) {
this.db
.prepare("update pending_wakes set trigger_reason = ?, created_at = ? where id = ?")
.run(trigger, ts, existing.id);
return;
}
this.db
.prepare(
"insert into pending_wakes (id, company_id, agent_id, trigger_reason, created_at) values (?, ?, ?, ?, ?)",
)
.run(randomUUID(), companyId, agentId, trigger, ts);
}
consumePendingWakes(agentId: string): Array<{ id: string; companyId: string; trigger: string }> {
let rows: Array<{ id: string; companyId: string; trigger: string }> = [];
this.transaction(() => {
rows = this.readAll<{ id: string; companyId: string; trigger: string }>(
"select id, company_id as companyId, trigger_reason as trigger from pending_wakes where agent_id = ? order by created_at asc",
agentId,
);
if (rows.length > 0) {
this.db.prepare("delete from pending_wakes where agent_id = ?").run(agentId);
// D-07: Per-agent scope -- check for existing queued or deferred wake for this agent
const existing = this.db
.prepare(
`SELECT id, trigger_detail, coalesced_count
FROM wakeup_requests
WHERE agent_id = ? AND company_id = ? AND status IN ('queued', 'deferred')
LIMIT 1`,
)
.get(agentId, companyId) as
| { id: string; trigger_detail: string | null; coalesced_count: number }
| undefined;
if (existing) {
// D-06: Coalesce -- append trigger to JSON array
const triggers: Array<{ trigger: string; at: string; detail: string }> = existing.trigger_detail
? JSON.parse(existing.trigger_detail)
: [];
triggers.push({ trigger, at: ts, detail });
this.db
.prepare(
`UPDATE wakeup_requests
SET trigger_detail = ?,
coalesced_count = coalesced_count + 1,
updated_at = ?
WHERE id = ?`,
)
.run(JSON.stringify(triggers), ts, existing.id);
} else {
// New wakeup request
const triggerDetail = JSON.stringify([{ trigger, at: ts, detail }]);
this.db
.prepare(
`INSERT INTO wakeup_requests
(id, company_id, agent_id, source, trigger_detail, status, coalesced_count, requested_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, 'queued', 0, ?, ?, ?)`,
)
.run(randomUUID(), companyId, agentId, trigger, triggerDetail, ts, ts, ts);
}
});
return rows;
}
claimWakeupRequest(agentId: string, companyId: string, runId: string): { id: string; triggerDetail: string | null } | null {
let result: { id: string; triggerDetail: string | null } | null = null;
this.transaction(() => {
const row = this.db
.prepare(
`SELECT id, trigger_detail AS triggerDetail
FROM wakeup_requests
WHERE agent_id = ? AND company_id = ? AND status = 'queued'
ORDER BY requested_at ASC
LIMIT 1`,
)
.get(agentId, companyId) as { id: string; triggerDetail: string | null } | undefined;
if (!row) return;
const ts = nowIso();
this.db
.prepare(
`UPDATE wakeup_requests SET status = 'claimed', claimed_at = ?, run_id = ?, updated_at = ? WHERE id = ?`,
)
.run(ts, runId, ts, row.id);
result = row;
});
return result;
}
completeWakeupRequest(wakeupId: string): void {
const ts = nowIso();
this.db
.prepare(
`UPDATE wakeup_requests SET status = 'completed', finished_at = ?, updated_at = ? WHERE id = ? AND status = 'claimed'`,
)
.run(ts, ts, wakeupId);
}
deferWakeupRequest(agentId: string, companyId: string): void {
const ts = nowIso();
this.db
.prepare(
`UPDATE wakeup_requests SET status = 'deferred', updated_at = ? WHERE agent_id = ? AND company_id = ? AND status = 'queued'`,
)
.run(ts, agentId, companyId);
}
promoteWakeupRequests(agentId: string, companyId: string): number {
let promoted = 0;
this.transaction(() => {
const queued = this.db
.prepare(
`SELECT id, trigger_detail FROM wakeup_requests
WHERE agent_id = ? AND company_id = ? AND status = 'queued' LIMIT 1`,
)
.get(agentId, companyId) as { id: string; trigger_detail: string | null } | undefined;
const deferred = this.readAll<{ id: string; trigger_detail: string | null; coalesced_count: number }>(
`SELECT id, trigger_detail, coalesced_count FROM wakeup_requests
WHERE agent_id = ? AND company_id = ? AND status = 'deferred'`,
agentId, companyId,
);
if (deferred.length === 0) return;
const ts = nowIso();
if (queued) {
// Merge deferred triggers into existing queued wake
const queuedTriggers: Array<{ trigger: string; at: string; detail: string }> = queued.trigger_detail ? JSON.parse(queued.trigger_detail) : [];
let addedCount = 0;
for (const d of deferred) {
const dTriggers: Array<{ trigger: string; at: string; detail: string }> = d.trigger_detail ? JSON.parse(d.trigger_detail) : [];
queuedTriggers.push(...dTriggers);
addedCount += d.coalesced_count + 1;
}
this.db.prepare(
`UPDATE wakeup_requests SET trigger_detail = ?, coalesced_count = coalesced_count + ?, updated_at = ? WHERE id = ?`,
).run(JSON.stringify(queuedTriggers), addedCount, ts, queued.id);
// Delete the merged deferred rows
for (const d of deferred) {
this.db.prepare("DELETE FROM wakeup_requests WHERE id = ?").run(d.id);
}
promoted = deferred.length;
} else {
// No queued wake -- promote first deferred to queued, merge rest
const [first, ...rest] = deferred;
this.db.prepare(
`UPDATE wakeup_requests SET status = 'queued', updated_at = ? WHERE id = ?`,
).run(ts, first!.id);
if (rest.length > 0) {
const firstTriggers: Array<{ trigger: string; at: string; detail: string }> = first!.trigger_detail ? JSON.parse(first!.trigger_detail) : [];
let addedCount = 0;
for (const d of rest) {
const dTriggers: Array<{ trigger: string; at: string; detail: string }> = d.trigger_detail ? JSON.parse(d.trigger_detail) : [];
firstTriggers.push(...dTriggers);
addedCount += d.coalesced_count + 1;
}
this.db.prepare(
`UPDATE wakeup_requests SET trigger_detail = ?, coalesced_count = coalesced_count + ?, updated_at = ? WHERE id = ?`,
).run(JSON.stringify(firstTriggers), addedCount, ts, first!.id);
for (const d of rest) {
this.db.prepare("DELETE FROM wakeup_requests WHERE id = ?").run(d.id);
}
}
promoted = deferred.length;
}
});
return promoted;
}
listWakeupRequests(companyId: string, filters?: { status?: string; agentId?: string; limit?: number }): Array<{
id: string; companyId: string; agentId: string; source: string; triggerDetail: string | null;
status: string; coalescedCount: number; runId: string | null; requestedAt: string;
claimedAt: string | null; finishedAt: string | null; createdAt: string; updatedAt: string;
}> {
const conditions = ["company_id = ?"];
const params: unknown[] = [companyId];
if (filters?.status) { conditions.push("status = ?"); params.push(filters.status); }
if (filters?.agentId) { conditions.push("agent_id = ?"); params.push(filters.agentId); }
const limit = filters?.limit ?? 50;
return this.readAll(
`SELECT id, company_id AS companyId, agent_id AS agentId, source, trigger_detail AS triggerDetail,
status, coalesced_count AS coalescedCount, run_id AS runId, requested_at AS requestedAt,
claimed_at AS claimedAt, finished_at AS finishedAt, created_at AS createdAt, updated_at AS updatedAt
FROM wakeup_requests
WHERE ${conditions.join(" AND ")}
ORDER BY requested_at DESC
LIMIT ?`,
...params, limit,
);
}
exportBackup(targetDir: string) {
@@ -2026,10 +2230,10 @@ export class AppDatabase {
}
}
purgeExpiredPendingWakes() {
// Remove pending wakes older than 2 hours — if agent hasn't woken by then, the wake is stale
purgeExpiredWakeupRequests() {
// Remove completed wakeup requests older than 2 hours -- only clean completed, not active ones
const cutoff = new Date(Date.now() - 2 * 60 * 60 * 1000).toISOString();
const result = this.db.prepare("delete from pending_wakes where created_at < ?").run(cutoff);
const result = this.db.prepare("DELETE FROM wakeup_requests WHERE created_at < ? AND status IN ('completed')").run(cutoff);
return Number(result.changes);
}
@@ -2187,6 +2391,8 @@ export class AppDatabase {
addColumnIfMissing("alter table tasks add column requires_user_review integer not null default 0");
addColumnIfMissing("alter table tasks add column retry_count integer not null default 0");
addColumnIfMissing("alter table tasks add column last_failure_at text");
addColumnIfMissing("alter table tasks add column checkout_run_id text");
addColumnIfMissing("alter table tasks add column execution_locked_at text");
this.db.exec(`
create table if not exists task_comments (
id text primary key,
@@ -2979,7 +3185,7 @@ export class AppDatabase {
.map((row) => row.version),
);
const migrations: Array<{ version: number; name: string; sql: string }> = [
const migrations: Array<{ version: number; name: string; sql?: string; fn?: () => void }> = [
{
version: 1,
name: "add_workspace_lock_lease",
@@ -3197,14 +3403,48 @@ export class AppDatabase {
create index if not exists idx_agent_message_reads_agent on agent_message_reads(agent_id);
`,
},
{
version: 16,
name: "migrate_pending_wakes_to_wakeup_requests",
fn: () => {
// D-08: Copy existing pending_wakes rows into wakeup_requests with 'queued' status
const existingWakes = this.readAll<{
id: string;
companyId: string;
agentId: string;
triggerReason: string;
createdAt: string;
}>(
`SELECT id, company_id AS companyId, agent_id AS agentId, trigger_reason AS triggerReason, created_at AS createdAt
FROM pending_wakes`,
);
const ts = nowIso();
for (const wake of existingWakes) {
const triggerDetail = JSON.stringify([{ trigger: wake.triggerReason, at: wake.createdAt, detail: wake.triggerReason }]);
this.db
.prepare(
`INSERT OR IGNORE INTO wakeup_requests
(id, company_id, agent_id, source, trigger_detail, status, coalesced_count, requested_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, 'queued', 0, ?, ?, ?)`,
)
.run(wake.id, wake.companyId, wake.agentId, wake.triggerReason, triggerDetail, wake.createdAt, ts, ts);
}
// Drop the old table
this.db.exec("DROP TABLE IF EXISTS pending_wakes");
},
},
];
for (const migration of migrations) {
if (applied.has(migration.version)) continue;
try {
this.transaction(() => {
for (const stmt of migration.sql.split(";").map((s) => s.trim()).filter(Boolean)) {
this.db.prepare(stmt).run();
if (migration.fn) {
migration.fn();
} else if (migration.sql) {
for (const stmt of migration.sql.split(";").map((s) => s.trim()).filter(Boolean)) {
this.db.prepare(stmt).run();
}
}
this.db
.prepare("insert into schema_migrations (version, name, applied_at) values (?, ?, ?)")
+4 -4
View File
@@ -473,7 +473,7 @@ export function createOrchestrationService({
const agentBusy = hasActiveRun({ agentId, companyId });
const runId = startAgentHeartbeatRun(agentId, companyId, trigger);
if (!runId && agentBusy) {
db.enqueuePendingWake(companyId, agentId, trigger);
db.enqueueWakeupRequest(companyId, agentId, trigger, trigger);
}
return runId;
}
@@ -689,9 +689,9 @@ export function createOrchestrationService({
logger.info(`[heartbeat] Released ${staleReleased} stale workspace lock(s)`);
publishDomainChanged();
}
const stalePendingWakes = db.purgeExpiredPendingWakes();
if (stalePendingWakes > 0) {
logger.info(`[heartbeat] Purged ${stalePendingWakes} stale pending wake(s)`);
const staleWakeups = db.purgeExpiredWakeupRequests();
if (staleWakeups > 0) {
logger.info(`[heartbeat] Purged ${staleWakeups} expired wakeup request(s)`);
}
} catch (error) {
logger.error(`[heartbeat] Stale lock cleanup failed: ${error instanceof Error ? error.message : String(error)}`);
+14 -7
View File
@@ -145,9 +145,14 @@ export function createWorkerService({
}
for (const agentId of affectedAgentIds) {
const pendingWakes = db.consumePendingWakes(agentId);
for (const wake of pendingWakes) {
wakeAgentIfPossible(agentId, wake.companyId, wake.trigger);
// Promote any deferred wakes back to queued, then process queued wakes
const agentRuns = snapshot.runs.filter((r) => r.agentId === agentId);
for (const companyId of new Set(agentRuns.map((r) => r.companyId))) {
db.promoteWakeupRequests(agentId, companyId);
const wakes = db.listWakeupRequests(companyId, { agentId, status: "queued" });
for (const wake of wakes) {
wakeAgentIfPossible(agentId, wake.companyId, wake.source);
}
}
}
@@ -381,10 +386,12 @@ export function createWorkerService({
}
}
const pendingWakes = db.consumePendingWakes(run.agentId);
if (pendingWakes.length > 0 && !isQuitting() && !destroyed) {
for (const wake of pendingWakes) {
wakeAgentIfPossible(run.agentId, wake.companyId, wake.trigger);
// Promote any deferred wakes back to queued after run completes
db.promoteWakeupRequests(run.agentId, run.companyId);
const queuedWakes = db.listWakeupRequests(run.companyId, { agentId: run.agentId, status: "queued" });
if (queuedWakes.length > 0 && !isQuitting() && !destroyed) {
for (const wake of queuedWakes) {
wakeAgentIfPossible(run.agentId, wake.companyId, wake.source);
}
}
+24 -16
View File
@@ -164,7 +164,7 @@ describe("AppDatabase", () => {
});
});
it("atomically checks out unclaimed work to a single agent", () => {
it("atomically checks out unclaimed work via CAS with run ID", () => {
const companyId = db.saveCompany({
name: "Checkout Company",
description: "Atomic checkout validation",
@@ -191,18 +191,12 @@ describe("AppDatabase", () => {
capabilities: "Leads",
budgetMonthlyUsd: 100,
});
const contributorId = db.saveAgent({
companyId,
name: "Contributor",
role: "IC",
title: "Contributor",
status: "active",
reportsTo: leadId,
connectorId: "codex_local",
workspaceId,
model: null,
capabilities: "Executes",
budgetMonthlyUsd: 100,
db.saveConnector({
id: "codex_local",
command: "codex",
model: "gpt-5",
envBindingText: "",
notes: "Test connector",
});
const taskId = db.saveTask({
companyId,
@@ -214,11 +208,25 @@ describe("AppDatabase", () => {
status: "todo",
});
expect(db.checkoutTask(taskId, leadId)).toBe(true);
expect(db.checkoutTask(taskId, contributorId)).toBe(false);
const { id: runId1 } = db.createRun({
companyId,
taskId,
agentId: leadId,
workspaceId,
connectorId: "codex_local",
});
const { id: runId2 } = db.createRun({
companyId,
taskId,
agentId: leadId,
workspaceId,
connectorId: "codex_local",
});
expect(db.checkoutTask(taskId, runId1)).toBe(true);
expect(db.checkoutTask(taskId, runId2)).toBe(false);
const task = db.getTask(taskId);
expect(task.assigneeAgentId).toBe(leadId);
expect(task.status).toBe("in_progress");
});