fix(db): strip implicit data mutations from startup
`init()` ran three implicit data-repair queries on every boot: `reconcileOrphanedTasks`, `reconcileUnreviewableTasks`, and a secrets-table migration whose fallback glued legacy rows onto a synthetic zero UUID when no current company existed. The first two papered over state-machine leaks at init time — hiding the real bug and making startup observably mutate production data; the third silently connected ciphertext to a non-existent company with no UI visibility and no deletion path. Root fix: the startup path is now allowed exactly one mutation — `interruptStaleRuns`, which is crash recovery (running-at-boot = worker died mid-turn) and cannot be expressed anywhere else. The dead-letter `in_review` case moves to the transition point: `tasks-status.guardInReviewHasReviewer` refuses to advance a task into `in_review` when its assignee is a top-level agent (`reports_to IS NULL`) and the task is not flagged for user review. Callers either pre-flag the task (the CEO-completion path in `automation-run-completion.advanceToInReview` now does this) or the transition is rejected. No more silent resurrection at boot; the failure is visible and actionable at source. The orphan-task reconcile disappears entirely — the `finishRun` guard already enforces the invariant at the one place a run can leave the in-progress state, and `recoverStaleLocks` handles stuck executions at runtime. The secrets migration now throws when neither `settings.current_company_id` nor any company row exists, rather than binding legacy ciphertext to `00000000-0000-0000-0000-000000000000`. Legacy rows stay on the old table until the operator creates a real company, at which point the upgrade succeeds deterministically. Tests rewritten: the reconciliation regression suite now asserts the transition-time guard and the lone `interruptStaleRuns` behavior. `resolve-workspace.test.ts` catches up with the exclusive-by-default lock policy committed earlier.
This commit is contained in:
1 parent
cdf783c0b8
commit
7f23f40e87
6 files changed
+94
-208
No files matched your search
@@ -177,6 +177,15 @@ function advanceToInReview(
|
||||
statusMessage: string,
|
||||
): void {
|
||||
const { db, logger, publishDomainChanged } = deps;
|
||||
// The tasks-status guard refuses an `in_review` transition for a
|
||||
// top-level assignee (reports_to IS NULL) unless the task is
|
||||
// flagged for user review. Auto-completed runs owned by the CEO
|
||||
// routinely trip that path — pre-flag them so the transition
|
||||
// succeeds and the human becomes the reviewer.
|
||||
const agent = db.getAgent(run.agentId);
|
||||
if (agent.reportsTo === null) {
|
||||
db.setTaskRequiresUserReview(taskId, true);
|
||||
}
|
||||
db.advanceTaskStatus(taskId, "in_review", statusMessage);
|
||||
callbacks.handleTaskStatusChange(taskId, run.companyId, "in_progress", "in_review");
|
||||
const company = db.listSnapshot().companies.find((c) => c.id === run.companyId);
|
||||
@@ -184,7 +193,6 @@ function advanceToInReview(
|
||||
const importantDocs = findImportantDocsForRun(db, run.id, run.companyId);
|
||||
if (importantDocs.length > 0) {
|
||||
db.setTaskRequiresUserReview(taskId, true);
|
||||
const agent = db.getAgent(run.agentId);
|
||||
if (agent.heartbeatEnabled) {
|
||||
db.setHeartbeat(agent.id, false, agent.heartbeatIntervalSec || 120);
|
||||
logger.info(`[deliverable-review] Paused heartbeat for ${agent.name} — deliverable flagged for review.`);
|
||||
|
||||
@@ -164,93 +164,17 @@ export class AppDatabaseBase {
|
||||
initializeDatabase(this.db, ra, tx);
|
||||
this.connectorsCrud.seedConnectors();
|
||||
this.modelCapabilities.seedModelCapabilities(MODEL_CAPABILITY_SEED);
|
||||
// Crash-recovery: a run still at `running` on boot means the worker
|
||||
// died mid-turn. Flip it to `interrupted`, surface the related task
|
||||
// as `blocked` for operator attention, and release workspace locks.
|
||||
// This is the only mutation the startup path is permitted to make;
|
||||
// all other "data-repair" passes have moved to explicit runtime
|
||||
// state-machine guards (see `tasks-status.ts`) or maintenance IPC.
|
||||
this._lastInterruptedCount = this.interruptStaleRuns();
|
||||
this._lastOrphanResetCount = this.reconcileOrphanedTasks();
|
||||
this._lastUnreviewableResetCount = this.reconcileUnreviewableTasks();
|
||||
this.settings.purgeExpiredRunLogs();
|
||||
this.wakeup.purgeExpiredWakeupRequests();
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset tasks stuck at `in_progress` with no live run.
|
||||
*
|
||||
* Invariant: `task.status = 'in_progress'` ⇒ there exists a run `r`
|
||||
* where `r.status IN ('queued', 'running')` referenced by this task
|
||||
* (via checkout_run_id or active_run_id).
|
||||
*
|
||||
* Whenever the invariant breaks — because a dispatch failed, a worker
|
||||
* crashed, or a pre-fix finishRun cleared the run refs but not the
|
||||
* status — the task becomes un-reclaimable: `checkoutTask`'s guard
|
||||
* requires `status IN ('todo','backlog')`, and heartbeat loops forever
|
||||
* logging "lost checkout race". One query at boot drains every such
|
||||
* orphan, regardless of which of the two reference columns happens to
|
||||
* be null. The runtime `finishRun` path enforces the same invariant
|
||||
* going forward, so this should stabilise at zero rows changed.
|
||||
*/
|
||||
protected reconcileOrphanedTasks(): number {
|
||||
const result = this.db
|
||||
.prepare(
|
||||
`UPDATE tasks
|
||||
SET status = 'todo',
|
||||
checkout_run_id = NULL,
|
||||
execution_locked_at = NULL,
|
||||
active_run_id = NULL,
|
||||
updated_at = ?
|
||||
WHERE status = 'in_progress'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM runs r
|
||||
WHERE (r.id = tasks.checkout_run_id OR r.id = tasks.active_run_id)
|
||||
AND r.status IN ('queued', 'running')
|
||||
)`,
|
||||
)
|
||||
.run(new Date().toISOString());
|
||||
const changes = Number(result.changes);
|
||||
if (changes > 0) {
|
||||
|
||||
console.info(`[init] Reconciled ${changes} orphaned in_progress task(s) → todo.`);
|
||||
}
|
||||
return changes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset tasks stuck at `in_review` with no reachable reviewer.
|
||||
*
|
||||
* A task enters `in_review` when an assignee submits work for approval.
|
||||
* The reviewer is the assignee's manager (walk up via `reports_to`),
|
||||
* or the human operator when the task is flagged
|
||||
* `requires_user_review = 1`. When the assignee is a top-level agent
|
||||
* (CEO/board, `reports_to IS NULL`) and the task does not require
|
||||
* user review, *no one* can review it — the task becomes a dead
|
||||
* letter, and the CEO heartbeat falls into a loop re-proposing its
|
||||
* `approve_ceo_strategy` without ever recovering.
|
||||
*
|
||||
* This observed pattern cost us a full debugging session: CEO
|
||||
* self-assigned a large task, moved it to `in_review`, then spent
|
||||
* every heartbeat cycle re-emitting the same approval. Demoting the
|
||||
* task to `todo` breaks the loop — the CEO sees un-reviewable work
|
||||
* and correctly falls through to the hire path instead.
|
||||
*/
|
||||
protected reconcileUnreviewableTasks(): number {
|
||||
const result = this.db
|
||||
.prepare(
|
||||
`UPDATE tasks
|
||||
SET status = 'todo', updated_at = ?
|
||||
WHERE status = 'in_review'
|
||||
AND requires_user_review = 0
|
||||
AND assignee_agent_id IS NOT NULL
|
||||
AND assignee_agent_id IN (
|
||||
SELECT id FROM agents WHERE reports_to IS NULL
|
||||
)`,
|
||||
)
|
||||
.run(new Date().toISOString());
|
||||
const changes = Number(result.changes);
|
||||
if (changes > 0) {
|
||||
|
||||
console.info(`[init] Reconciled ${changes} un-reviewable in_review task(s) → todo.`);
|
||||
}
|
||||
return changes;
|
||||
}
|
||||
|
||||
close() { this.db.close(); }
|
||||
|
||||
// ─── Settings ──────────────────────────────────────────────────────────────
|
||||
@@ -435,11 +359,7 @@ export class AppDatabaseBase {
|
||||
|
||||
// ─── Internal infrastructure ───────────────────────────────────────────────
|
||||
protected _lastInterruptedCount = 0;
|
||||
protected _lastOrphanResetCount = 0;
|
||||
protected _lastUnreviewableResetCount = 0;
|
||||
get lastInterruptedCount(): number { return this._lastInterruptedCount; }
|
||||
get lastOrphanResetCount(): number { return this._lastOrphanResetCount; }
|
||||
get lastUnreviewableResetCount(): number { return this._lastUnreviewableResetCount; }
|
||||
|
||||
interruptStaleRuns(): number {
|
||||
const ts = nowIso();
|
||||
|
||||
@@ -74,10 +74,19 @@ export function migrateSecretsTable(db: DatabaseSync, readAll: ReadAll) {
|
||||
db.prepare("create unique index if not exists secrets_company_name_idx on secrets(company_id, name)").run();
|
||||
return;
|
||||
}
|
||||
const fallbackCompanyId =
|
||||
((db.prepare("select value from settings where key = 'current_company_id'").get() as { value: string } | undefined)?.value) ??
|
||||
((db.prepare("select id from companies order by created_at asc limit 1").get() as { id: string } | undefined)?.id) ??
|
||||
"00000000-0000-0000-0000-000000000000";
|
||||
// Legacy profiles predate the per-company secrets table. Rebind each
|
||||
// row to the current company; if neither `current_company_id` nor any
|
||||
// company row exists we refuse rather than attach secrets to a
|
||||
// synthetic zero UUID that references nothing — that would leak
|
||||
// ciphertext with no deletion path and no UI visibility.
|
||||
const currentCompany = db.prepare("select value from settings where key = 'current_company_id'").get() as { value: string } | undefined;
|
||||
const firstCompany = db.prepare("select id from companies order by created_at asc limit 1").get() as { id: string } | undefined;
|
||||
const fallbackCompanyId = currentCompany?.value ?? firstCompany?.id;
|
||||
if (!fallbackCompanyId) {
|
||||
throw new Error(
|
||||
"Cannot migrate legacy secrets table: no current_company_id is set and no company rows exist. Create a company before upgrading, or drop the legacy secrets table if the rows are stale.",
|
||||
);
|
||||
}
|
||||
db.exec(`
|
||||
alter table secrets rename to secrets_legacy;
|
||||
create table secrets (id text primary key, company_id text not null, name text not null, description text not null, backend text not null, ciphertext blob not null, updated_at text not null, created_at text not null, unique(company_id, name));
|
||||
|
||||
@@ -37,6 +37,37 @@ export function buildTaskStatusMethods(
|
||||
"cancelled",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Guard against the "in_review dead letter": a task whose assignee is
|
||||
* a top-level agent (reports_to IS NULL, the DB invariant CEO marker)
|
||||
* and that does not require user review has no reachable reviewer —
|
||||
* no manager to walk up to, and no human gate. Without this guard the
|
||||
* CEO heartbeat loops forever re-proposing the same approval.
|
||||
*
|
||||
* The right root fix is to reject the transition at source: a caller
|
||||
* that moves such a task into `in_review` must first set
|
||||
* requires_user_review=1 (escalate to human) or route differently
|
||||
* (advance to done, hire a reviewer, bounce to todo). Previous
|
||||
* versions silently resurrected these tasks at every DB init; this
|
||||
* replaces the startup side-effect with a clear runtime error.
|
||||
*/
|
||||
function guardInReviewHasReviewer(taskId: string): void {
|
||||
const row = readAll<{ requiresUserReview: number; reportsTo: string | null }>(
|
||||
`select t.requires_user_review as requiresUserReview,
|
||||
a.reports_to as reportsTo
|
||||
from tasks t
|
||||
left join agents a on a.id = t.assignee_agent_id
|
||||
where t.id = ?`,
|
||||
taskId,
|
||||
)[0];
|
||||
if (!row) return;
|
||||
if (row.requiresUserReview === 1) return;
|
||||
if (row.reportsTo !== null) return;
|
||||
throw new Error(
|
||||
`Task ${taskId} has a top-level assignee and no reviewer: cannot enter in_review without requires_user_review=1.`,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
advanceTaskStatus(
|
||||
taskId: string,
|
||||
@@ -52,6 +83,7 @@ export function buildTaskStatusMethods(
|
||||
if (!task) throw new Error(`Task not found: ${taskId}`);
|
||||
const previousStatus = task.status;
|
||||
if (previousStatus === newStatus) return { previousStatus };
|
||||
if (newStatus === "in_review") guardInReviewHasReviewer(taskId);
|
||||
const ts = nowIso();
|
||||
|
||||
// Status side-effect timestamps (Paperclip pattern)
|
||||
|
||||
Reference in new issue
Block a user