Recover interrupted runs and requests at boot
A task_run still marked running at startup has no executor - the single-instance app that owned it is gone - yet nothing failed it, the queue worker only claims queued rows, and the heartbeat excludes tasks with a running run, so one interrupted shutdown wedged the company forever. Boot now fails orphaned running runs and their in-flight foreground provider requests (auditably), which hands the task back to the heartbeat for an automatic retry. Background provider requests are untouched; the background worker resumes those by external id.
This commit is contained in:
1 parent
4e47da4d6b
commit
f15ed8cb6f
3 files changed
+129
No files matched your search
@@ -0,0 +1,37 @@
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { AuditService } from "../services/audit-service";
|
||||
|
||||
const INTERRUPTED_MESSAGE = "Interrupted by application shutdown before completion.";
|
||||
|
||||
/**
|
||||
* A single app instance executes runs in-process, so any task_run still
|
||||
* `running` at boot was interrupted by a previous shutdown and has no
|
||||
* executor. Marking it failed — together with its in-flight foreground
|
||||
* provider requests — returns the task to the heartbeat, which enqueues a
|
||||
* fresh run within one tick. Background provider requests are left alone:
|
||||
* the provider background worker resumes those from their external id.
|
||||
*/
|
||||
export function recoverInterruptedWork(db: DatabaseSync): void {
|
||||
const now = new Date().toISOString();
|
||||
const audit = new AuditService(db);
|
||||
const runs = db.prepare("select id, org_id, task_id from task_runs where state = 'running'")
|
||||
.all() as Array<{ id: string; org_id: string; task_id: string }>;
|
||||
for (const run of runs) {
|
||||
db.prepare("update task_runs set state = 'failed', completed_at = ?, error_message = ? where id = ?")
|
||||
.run(now, INTERRUPTED_MESSAGE, run.id);
|
||||
audit.record({
|
||||
orgId: run.org_id,
|
||||
entityType: "task_run",
|
||||
entityId: run.id,
|
||||
actorType: "system",
|
||||
actorId: null,
|
||||
action: "failed",
|
||||
payload: { taskId: run.task_id, reason: "interrupted_by_shutdown" },
|
||||
});
|
||||
}
|
||||
db.prepare(`
|
||||
update provider_requests
|
||||
set state = 'failed', completed_at = ?, error_message = ?
|
||||
where execution_mode = 'foreground' and state in ('pending', 'streaming')
|
||||
`).run(now, INTERRUPTED_MESSAGE);
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import { DatabaseConnection } from "./database/connection";
|
||||
import { createMainWindow } from "./app/create-window";
|
||||
import { registerIpcHandlers } from "./app/ipc-handlers";
|
||||
import { startProviderBackgroundWorker } from "./app/provider-background-worker";
|
||||
import { recoverInterruptedWork } from "./app/startup-recovery";
|
||||
import { startTaskQueueWorker } from "./app/task-queue-worker";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
@@ -17,6 +18,7 @@ if (!hasSingleInstanceLock) {
|
||||
app.whenReady().then(() => {
|
||||
const profileDir = process.env.AGENTCOMPANY_NEW_PROFILE_DIR ?? join(app.getPath("userData"), "profile");
|
||||
const connection = new DatabaseConnection(join(profileDir, "agentcompany.db"));
|
||||
connection.transaction(() => recoverInterruptedWork(connection.db));
|
||||
const win = createMainWindow({
|
||||
preloadPath: join(__dirname, "../preload/index.js"),
|
||||
rendererDist: join(__dirname, "../renderer"),
|
||||
|
||||
Reference in new issue
Block a user