Stale chat wakes expire instead of resuming days later

Live incident: the founder DMed the CEO and got silence — the reply run
sat 13th in a FIFO queue behind chat wakes from four days ago, which the
reopened app was faithfully executing (necro-replies that woke yet more
agents). A wake run is a conversation reply; past one hour the moment is
gone. expireStaleChatWakes (chat-agent-wake-service) fails queued
chat_mention runs older than the TTL and closes their vehicle tasks;
the worker tick runs it before claiming, with one log line. Real work
(heartbeat/kickoff triggers) never expires. Red-green tested.
This commit is contained in:
ZacharyZhang-NY committed 2026-07-14 20:44:37 -04:00
1 parent c7eafb2836
commit 39aa292d09
3 files changed
+143

No files matched your search

+9
View File
@@ -1,6 +1,7 @@
import type { BrowserWindow } from "electron";
import type { DatabaseConnection } from "../database/connection";
import { IPC_CHANNELS } from "../../shared/ipc";
import { expireStaleChatWakes } from "../services/chat-agent-wake-service";
import { OrganizationService } from "../services/organization-service";
import { spawnGoalReviews } from "../services/goal-review-service";
import { TaskExecutionService } from "../services/task-execution-service";
@@ -77,6 +78,14 @@ export function startTaskQueueWorker(input: TaskQueueWorkerInput): () => void {
wasPaused = paused;
}
if (paused) return;
// Stale conversation wakes die before anything is claimed: replying to
// a thread hours later is noise, and a backlog of them would starve
// fresh messages (the founder's included) for the whole grind.
const expired = input.connection.transaction(() => expireStaleChatWakes(input.connection.db, new Date().toISOString()));
if (expired > 0) {
console.info(`[task-queue] expired ${expired} stale chat wake run(s); newer messages re-wake their agents.`);
notifyDataChanged();
}
// Goal checkpoints first, so a review task created this tick is
// eligible for the same tick's heartbeat enqueue below.
const reviews = input.connection.transaction(() => spawnGoalReviews(input.connection.db, organization));
@@ -28,6 +28,41 @@ export interface AgentWakeResult {
runId: string;
}
/**
* How long a queued chat wake stays claimable. A wake run is a conversation
* reply; once the moment has passed — the app sat closed overnight, or the
* queue spent an hour behind other work — answering the stale thread reads
* as noise, burns provider budget, and its reply wakes yet more agents off a
* dead conversation. Expiring it costs nothing: any newer message re-wakes
* the same agent fresh.
*/
export const CHAT_WAKE_TTL_MS = 60 * 60_000;
/**
* Fails every queued chat-mention run older than the TTL and closes its
* vehicle task (the reply was the task's whole deliverable — see
* closeChatWakeTask). Returns how many wakes were expired.
*/
export function expireStaleChatWakes(db: DatabaseSync, now: string): number {
const cutoff = new Date(new Date(now).getTime() - CHAT_WAKE_TTL_MS).toISOString();
const stale = db.prepare(`
select id, org_id, task_id from task_runs
where state = 'queued' and trigger = 'chat_mention' and created_at < ?
`).all(cutoff) as Array<{ id: string; org_id: string; task_id: string }>;
if (stale.length === 0) return 0;
const organization = new OrganizationService(db);
for (const run of stale) {
db.prepare("update task_runs set state = 'failed', completed_at = ?, error_message = ? where id = ?")
.run(now, "Expired: the conversation moved on before this reply could start. A newer message re-wakes the agent.", run.id);
const task = db.prepare("select status from tasks where id = ? and org_id = ?")
.get(run.task_id, run.org_id) as { status: string } | undefined;
if (task && (task.status === TASK_STATUS.BACKLOG || task.status === TASK_STATUS.IN_PROGRESS)) {
organization.updateTaskStatus(run.org_id, run.task_id, TASK_STATUS.DONE);
}
}
return stale.length;
}
export class ChatAgentWakeService {
private readonly organization: OrganizationService;