Recover interrupted runs and requests at boot

A task_run still marked running at startup has no executor - the
single-instance app that owned it is gone - yet nothing failed it, the
queue worker only claims queued rows, and the heartbeat excludes tasks
with a running run, so one interrupted shutdown wedged the company
forever. Boot now fails orphaned running runs and their in-flight
foreground provider requests (auditably), which hands the task back to
the heartbeat for an automatic retry. Background provider requests are
untouched; the background worker resumes those by external id.
This commit is contained in:
ZacharyZhang-NY committed 2026-06-09 21:16:42 -04:00
1 parent 4e47da4d6b
commit f15ed8cb6f
3 files changed
+129

No files matched your search

@@ -0,0 +1,37 @@
import type { DatabaseSync } from "node:sqlite";
import { AuditService } from "../services/audit-service";
const INTERRUPTED_MESSAGE = "Interrupted by application shutdown before completion.";
/**
* A single app instance executes runs in-process, so any task_run still
* `running` at boot was interrupted by a previous shutdown and has no
* executor. Marking it failed — together with its in-flight foreground
* provider requests — returns the task to the heartbeat, which enqueues a
* fresh run within one tick. Background provider requests are left alone:
* the provider background worker resumes those from their external id.
*/
export function recoverInterruptedWork(db: DatabaseSync): void {
const now = new Date().toISOString();
const audit = new AuditService(db);
const runs = db.prepare("select id, org_id, task_id from task_runs where state = 'running'")
.all() as Array<{ id: string; org_id: string; task_id: string }>;
for (const run of runs) {
db.prepare("update task_runs set state = 'failed', completed_at = ?, error_message = ? where id = ?")
.run(now, INTERRUPTED_MESSAGE, run.id);
audit.record({
orgId: run.org_id,
entityType: "task_run",
entityId: run.id,
actorType: "system",
actorId: null,
action: "failed",
payload: { taskId: run.task_id, reason: "interrupted_by_shutdown" },
});
}
db.prepare(`
update provider_requests
set state = 'failed', completed_at = ?, error_message = ?
where execution_mode = 'foreground' and state in ('pending', 'streaming')
`).run(now, INTERRUPTED_MESSAGE);
}
+2
View File
@@ -5,6 +5,7 @@ import { DatabaseConnection } from "./database/connection";
import { createMainWindow } from "./app/create-window";
import { registerIpcHandlers } from "./app/ipc-handlers";
import { startProviderBackgroundWorker } from "./app/provider-background-worker";
import { recoverInterruptedWork } from "./app/startup-recovery";
import { startTaskQueueWorker } from "./app/task-queue-worker";
const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -17,6 +18,7 @@ if (!hasSingleInstanceLock) {
app.whenReady().then(() => {
const profileDir = process.env.AGENTCOMPANY_NEW_PROFILE_DIR ?? join(app.getPath("userData"), "profile");
const connection = new DatabaseConnection(join(profileDir, "agentcompany.db"));
connection.transaction(() => recoverInterruptedWork(connection.db));
const win = createMainWindow({
preloadPath: join(__dirname, "../preload/index.js"),
rendererDist: join(__dirname, "../renderer"),