Chat-wake tasks live and die with their reply run
Of the 202 tasks stranded in the backlog on July 9, 200 were "Reply to ..." wake vehicles: every agent @mention creates a task plus a queued run, the reply goes out, and nothing ever closed the task — a task with a completed run is never enqueued again, so each one became a permanent zombie row. On top of that, every message in an active thread minted a fresh wake even when the target already had one queued, multiplying runs that would each read the same conversation. The lifecycle contract, made explicit: - The reply run IS the wake task's deliverable. Completing a chat_mention run now closes its task (backlog/in_progress -> done, via the audited status path). Tasks an agent moved to review are left alone. - One pending wake per (agent, thread): wakeFromMention returns the queued run instead of minting a duplicate — that run reads the thread's newest messages when it executes anyway. - Migration 054 heals existing rows: wake tasks whose runs are all spent close as done; in-flight wakes keep working. This also stops Fix 1's retry-budget healing from resurrecting 31 stale replies on next launch. The chat-governance loop test now completes each pending wake run between posts, matching how the exchange actually interleaves in production.
This commit is contained in:
1 parent
927bdc91f2
commit
2526633bfb
9 files changed
+149
-4
No files matched your search
@@ -0,0 +1,16 @@
|
|||||||
|
export const closeStaleWakeTasksSchema = `
|
||||||
|
-- Chat-wake tasks are conversation vehicles: the reply run is their whole
|
||||||
|
-- deliverable. Runs used to complete without closing them, stranding every
|
||||||
|
-- "Reply to ..." row in the backlog forever (a task with a completed run is
|
||||||
|
-- never enqueued again). Close each wake task whose runs have all been
|
||||||
|
-- spent; run completion closes them from now on. In-flight wakes (a queued
|
||||||
|
-- or running run) keep working normally.
|
||||||
|
update tasks
|
||||||
|
set status = 'done', updated_at = datetime('now')
|
||||||
|
where status in ('backlog', 'in_progress')
|
||||||
|
and exists (select 1 from task_runs r where r.task_id = tasks.id and r.trigger = 'chat_mention')
|
||||||
|
and not exists (select 1 from task_runs r where r.task_id = tasks.id and r.state in ('queued', 'running'));
|
||||||
|
|
||||||
|
insert into schema_migrations(version, name, applied_at)
|
||||||
|
values (54, 'close_stale_wake_tasks', datetime('now'));
|
||||||
|
`;
|
||||||
@@ -52,6 +52,7 @@ import { financeSalesDepartmentsSchema } from "./migrations/050-finance-sales-de
|
|||||||
import { purgeRuntimeSnapshotBloatSchema } from "./migrations/051-purge-runtime-snapshot-bloat";
|
import { purgeRuntimeSnapshotBloatSchema } from "./migrations/051-purge-runtime-snapshot-bloat";
|
||||||
import { avatarImagesSchema } from "./migrations/052-avatar-images";
|
import { avatarImagesSchema } from "./migrations/052-avatar-images";
|
||||||
import { autonomousPauseSchema } from "./migrations/053-autonomous-pause";
|
import { autonomousPauseSchema } from "./migrations/053-autonomous-pause";
|
||||||
|
import { closeStaleWakeTasksSchema } from "./migrations/054-close-stale-wake-tasks";
|
||||||
|
|
||||||
interface Migration {
|
interface Migration {
|
||||||
version: number;
|
version: number;
|
||||||
@@ -113,6 +114,7 @@ const MIGRATIONS: Migration[] = [
|
|||||||
{ version: 51, name: "purge_runtime_snapshot_bloat", sql: purgeRuntimeSnapshotBloatSchema },
|
{ version: 51, name: "purge_runtime_snapshot_bloat", sql: purgeRuntimeSnapshotBloatSchema },
|
||||||
{ version: 52, name: "avatar_images", sql: avatarImagesSchema },
|
{ version: 52, name: "avatar_images", sql: avatarImagesSchema },
|
||||||
{ version: 53, name: "autonomous_pause", sql: autonomousPauseSchema },
|
{ version: 53, name: "autonomous_pause", sql: autonomousPauseSchema },
|
||||||
|
{ version: 54, name: "close_stale_wake_tasks", sql: closeStaleWakeTasksSchema },
|
||||||
];
|
];
|
||||||
|
|
||||||
const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version;
|
const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version;
|
||||||
|
|||||||
@@ -37,6 +37,11 @@ export class ChatAgentWakeService {
|
|||||||
|
|
||||||
wakeFromMention(input: SendMessageInput, agentId: string, messageId: string, threadId: string, now: string): AgentWakeResult {
|
wakeFromMention(input: SendMessageInput, agentId: string, messageId: string, threadId: string, now: string): AgentWakeResult {
|
||||||
const agent = this.requireRunnableAgent(input.orgId, agentId, "Agent mention");
|
const agent = this.requireRunnableAgent(input.orgId, agentId, "Agent mention");
|
||||||
|
// One pending wake per (agent, thread): the queued run reads the thread's
|
||||||
|
// newest messages when it executes, so waking again before it starts would
|
||||||
|
// spend a whole run to answer the same conversation twice.
|
||||||
|
const pending = this.pendingWakeRun(input.orgId, agentId, threadId);
|
||||||
|
if (pending) return { targetId: agentId, taskId: pending.taskId, runId: pending.runId };
|
||||||
const workspace = this.workspaceForAgent(input.orgId, agent.workspace_id);
|
const workspace = this.workspaceForAgent(input.orgId, agent.workspace_id);
|
||||||
const from = this.principalName(input.orgId, input.senderType, input.senderId);
|
const from = this.principalName(input.orgId, input.senderType, input.senderId);
|
||||||
const space = this.spaceRow(input.orgId, input.spaceId);
|
const space = this.spaceRow(input.orgId, input.spaceId);
|
||||||
@@ -81,6 +86,16 @@ export class ChatAgentWakeService {
|
|||||||
return { targetId: agent.id, taskId, runId };
|
return { targetId: agent.id, taskId, runId };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private pendingWakeRun(orgId: string, agentId: string, threadId: string): { taskId: string; runId: string } | null {
|
||||||
|
const row = this.db.prepare(`
|
||||||
|
select id, task_id from task_runs
|
||||||
|
where org_id = ? and agent_id = ? and state = 'queued' and trigger = 'chat_mention'
|
||||||
|
and json_extract(secret_scope_json, '$.sourceThreadId') = ?
|
||||||
|
limit 1
|
||||||
|
`).get(orgId, agentId, threadId) as { id: string; task_id: string } | undefined;
|
||||||
|
return row ? { taskId: row.task_id, runId: row.id } : null;
|
||||||
|
}
|
||||||
|
|
||||||
private requireAgentMention(mentions: ResolvedMention[]): string {
|
private requireAgentMention(mentions: ResolvedMention[]): string {
|
||||||
const target = mentions.find((mention) => mention.target.type === "agent")?.target;
|
const target = mentions.find((mention) => mention.target.type === "agent")?.target;
|
||||||
if (!target || target.type !== "agent") throw new Error("Slash command /agent requires an agent mention.");
|
if (!target || target.type !== "agent") throw new Error("Slash command /agent requires an agent mention.");
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import type { DatabaseSync } from "node:sqlite";
|
import type { DatabaseSync } from "node:sqlite";
|
||||||
import type { RunTaskInput } from "../../shared/commands";
|
import type { RunTaskInput } from "../../shared/commands";
|
||||||
import { APPROVAL_STATE, RISK_LEVEL } from "../../shared/domain";
|
import { APPROVAL_STATE, RISK_LEVEL, TASK_STATUS } from "../../shared/domain";
|
||||||
import { currentProviderUsageWindow, isTransientProviderFailure } from "../../shared/provider-runtime-domain";
|
import { currentProviderUsageWindow, isTransientProviderFailure } from "../../shared/provider-runtime-domain";
|
||||||
import { createId } from "../../shared/ids";
|
import { createId } from "../../shared/ids";
|
||||||
import { AuditService } from "./audit-service";
|
import { AuditService } from "./audit-service";
|
||||||
@@ -524,6 +524,23 @@ export class TaskExecutionService {
|
|||||||
set state = ?, provider_request_id = ?, completed_at = ?, error_message = null
|
set state = ?, provider_request_id = ?, completed_at = ?, error_message = null
|
||||||
where id = ? and org_id = ?
|
where id = ? and org_id = ?
|
||||||
`).run(TASK_RUN_STATE.COMPLETED, providerRequestId, new Date().toISOString(), runId, orgId);
|
`).run(TASK_RUN_STATE.COMPLETED, providerRequestId, new Date().toISOString(), runId, orgId);
|
||||||
|
this.closeChatWakeTask(orgId, runId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A chat-wake task exists only to carry its reply run — the reply is the
|
||||||
|
* whole deliverable. The run completing therefore closes the task; left
|
||||||
|
* open it would strand forever (a task with a completed run is never
|
||||||
|
* enqueued again) and pile zombie "Reply to ..." rows onto the board.
|
||||||
|
*/
|
||||||
|
private closeChatWakeTask(orgId: string, runId: string): void {
|
||||||
|
const run = this.db.prepare("select task_id, trigger from task_runs where id = ? and org_id = ?")
|
||||||
|
.get(runId, orgId) as { task_id: string; trigger: string } | undefined;
|
||||||
|
if (!run || run.trigger !== "chat_mention") return;
|
||||||
|
const task = this.db.prepare("select status from tasks where id = ? and org_id = ?")
|
||||||
|
.get(run.task_id, orgId) as { status: string } | undefined;
|
||||||
|
if (!task || (task.status !== TASK_STATUS.BACKLOG && task.status !== TASK_STATUS.IN_PROGRESS)) return;
|
||||||
|
this.organization.updateTaskStatus(orgId, run.task_id, TASK_STATUS.DONE);
|
||||||
}
|
}
|
||||||
|
|
||||||
private failRun(orgId: string, runId: string, providerRequestId: string | null, errorMessage: string): void {
|
private failRun(orgId: string, runId: string, providerRequestId: string | null, errorMessage: string): void {
|
||||||
|
|||||||
@@ -98,6 +98,12 @@ describe("inter-agent reply", () => {
|
|||||||
// Execute Dana's woken run: she replies in the CEO's thread.
|
// Execute Dana's woken run: she replies in the CEO's thread.
|
||||||
await taskExecution.executeRun(orgId, wake!.id);
|
await taskExecution.executeRun(orgId, wake!.id);
|
||||||
|
|
||||||
|
// The reply was the wake task's whole deliverable, so completing the run closes it.
|
||||||
|
const wakeTask = connection.db.prepare(
|
||||||
|
"select t.status from tasks t join task_runs r on r.task_id = t.id where r.id = ?",
|
||||||
|
).get(wake!.id) as { status: string };
|
||||||
|
expect(wakeTask).toEqual({ status: "done" });
|
||||||
|
|
||||||
const engineering = connection.db.prepare("select id from chat_spaces where org_id = ? and name = 'engineering'").get(orgId) as { id: string };
|
const engineering = connection.db.prepare("select id from chat_spaces where org_id = ? and name = 'engineering'").get(orgId) as { id: string };
|
||||||
const messages = snapshot.load().chatMessages.filter((m) => m.orgId === orgId && m.spaceId === engineering.id);
|
const messages = snapshot.load().chatMessages.filter((m) => m.orgId === orgId && m.spaceId === engineering.id);
|
||||||
const senders = new Set(messages.map((m) => m.senderId));
|
const senders = new Set(messages.map((m) => m.senderId));
|
||||||
|
|||||||
@@ -59,5 +59,26 @@ describe("thread participant wake", () => {
|
|||||||
const runsAfterReply = connection.db.prepare("select count(*) as c from task_runs where org_id = ? and trigger = 'chat_mention'")
|
const runsAfterReply = connection.db.prepare("select count(*) as c from task_runs where org_id = ? and trigger = 'chat_mention'")
|
||||||
.get(orgId) as { c: number };
|
.get(orgId) as { c: number };
|
||||||
expect(runsAfterReply.c).toBeGreaterThan(runsBeforeReply.c);
|
expect(runsAfterReply.c).toBeGreaterThan(runsBeforeReply.c);
|
||||||
|
|
||||||
|
// A second human message in the same thread coalesces into the still-queued
|
||||||
|
// wake instead of spending another run on the same conversation.
|
||||||
|
connection.transaction(() => {
|
||||||
|
chat.sendMessage({
|
||||||
|
orgId,
|
||||||
|
spaceId: (connection.db.prepare("select id from chat_spaces where org_id = ? and name = 'general'").get(orgId) as { id: string }).id,
|
||||||
|
threadId: threadId!,
|
||||||
|
senderType: "employee",
|
||||||
|
senderId: employeeId,
|
||||||
|
bodyMd: "One more thing: schedule the rollout for Friday.",
|
||||||
|
attachmentFileIds: [],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
const runsAfterSecondReply = connection.db.prepare("select count(*) as c from task_runs where org_id = ? and trigger = 'chat_mention'")
|
||||||
|
.get(orgId) as { c: number };
|
||||||
|
expect(runsAfterSecondReply.c).toBe(runsAfterReply.c);
|
||||||
|
const wakeTasks = connection.db.prepare(
|
||||||
|
"select count(distinct r.task_id) as c from task_runs r where r.org_id = ? and r.trigger = 'chat_mention'",
|
||||||
|
).get(orgId) as { c: number };
|
||||||
|
expect(wakeTasks.c).toBe(1);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -402,12 +402,22 @@ describe("chat governance", () => {
|
|||||||
"select count(*) as n from task_runs where org_id = ? and agent_id = ? and trigger = 'chat_mention'",
|
"select count(*) as n from task_runs where org_id = ? and agent_id = ? and trigger = 'chat_mention'",
|
||||||
).get(orgId, targetId) as { n: number }).n;
|
).get(orgId, targetId) as { n: number }).n;
|
||||||
|
|
||||||
// Alice (an agent) keeps @mentioning Bob in one thread. Each post is an
|
// Simulates the worker finishing Bob's pending reply run, the state a real
|
||||||
// autonomous turn; only the first batch wakes Bob, then the budget is spent.
|
// exchange is in when the next message lands (a still-queued run coalesces).
|
||||||
|
const finishPendingWakeRuns = (): void => {
|
||||||
|
connection.db.prepare(
|
||||||
|
"update task_runs set state = 'completed', completed_at = datetime('now') where org_id = ? and agent_id = ? and state = 'queued'",
|
||||||
|
).run(orgId, targetId);
|
||||||
|
};
|
||||||
|
|
||||||
|
// Alice (an agent) keeps @mentioning Bob in one thread, and Bob's reply run
|
||||||
|
// completes before each next post. Only the first batch of turns wakes Bob,
|
||||||
|
// then the autonomous budget is spent.
|
||||||
connection.transaction(() => {
|
connection.transaction(() => {
|
||||||
const first = chat.sendMessage({ orgId, spaceId, threadId: null, senderType: "agent", senderId: speakerId, bodyMd: "@Bob turn 1", attachmentFileIds: [] });
|
const first = chat.sendMessage({ orgId, spaceId, threadId: null, senderType: "agent", senderId: speakerId, bodyMd: "@Bob turn 1", attachmentFileIds: [] });
|
||||||
threadId = (connection.db.prepare("select thread_id from chat_messages where id = ?").get(first) as { thread_id: string }).thread_id;
|
threadId = (connection.db.prepare("select thread_id from chat_messages where id = ?").get(first) as { thread_id: string }).thread_id;
|
||||||
for (let turn = 2; turn <= 40; turn += 1) {
|
for (let turn = 2; turn <= 40; turn += 1) {
|
||||||
|
finishPendingWakeRuns();
|
||||||
chat.sendMessage({ orgId, spaceId, threadId, senderType: "agent", senderId: speakerId, bodyMd: `@Bob turn ${turn}`, attachmentFileIds: [] });
|
chat.sendMessage({ orgId, spaceId, threadId, senderType: "agent", senderId: speakerId, bodyMd: `@Bob turn ${turn}`, attachmentFileIds: [] });
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -419,7 +429,9 @@ describe("chat governance", () => {
|
|||||||
// A human re-enters the thread: their message wakes Bob and grants the
|
// A human re-enters the thread: their message wakes Bob and grants the
|
||||||
// agents a fresh budget, so the next agent turn wakes him again.
|
// agents a fresh budget, so the next agent turn wakes him again.
|
||||||
connection.transaction(() => {
|
connection.transaction(() => {
|
||||||
|
finishPendingWakeRuns();
|
||||||
chat.sendMessage({ orgId, spaceId, threadId, senderType: "employee", senderId: employeeId, bodyMd: "Keep going @Bob", attachmentFileIds: [] });
|
chat.sendMessage({ orgId, spaceId, threadId, senderType: "employee", senderId: employeeId, bodyMd: "Keep going @Bob", attachmentFileIds: [] });
|
||||||
|
finishPendingWakeRuns();
|
||||||
chat.sendMessage({ orgId, spaceId, threadId, senderType: "agent", senderId: speakerId, bodyMd: "@Bob resuming", attachmentFileIds: [] });
|
chat.sendMessage({ orgId, spaceId, threadId, senderType: "agent", senderId: speakerId, bodyMd: "@Bob resuming", attachmentFileIds: [] });
|
||||||
});
|
});
|
||||||
expect(wakeRunCount()).toBeGreaterThan(capped);
|
expect(wakeRunCount()).toBeGreaterThan(capped);
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
import { randomUUID } from "node:crypto";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { DatabaseSync } from "node:sqlite";
|
||||||
|
import { describe, expect, test } from "vitest";
|
||||||
|
import { closeStaleWakeTasksSchema } from "../../src/main/database/migrations/054-close-stale-wake-tasks";
|
||||||
|
|
||||||
|
function createDatabase(): DatabaseSync {
|
||||||
|
const db = new DatabaseSync(join(tmpdir(), `agentcompany-wake-migration-${randomUUID()}.db`));
|
||||||
|
db.exec(`
|
||||||
|
create table schema_migrations(version integer primary key, name text not null, applied_at text not null);
|
||||||
|
create table tasks(id text primary key, status text not null, updated_at text not null);
|
||||||
|
create table task_runs(id text primary key, task_id text not null, trigger text not null, state text not null);
|
||||||
|
insert into schema_migrations(version, name, applied_at) values (53, 'autonomous_pause', datetime('now'));
|
||||||
|
`);
|
||||||
|
return db;
|
||||||
|
}
|
||||||
|
|
||||||
|
function seedTask(db: DatabaseSync, taskId: string, status: string, runs: Array<{ trigger: string; state: string }>): void {
|
||||||
|
db.prepare("insert into tasks(id, status, updated_at) values (?, ?, datetime('now'))").run(taskId, status);
|
||||||
|
for (const run of runs) {
|
||||||
|
db.prepare("insert into task_runs(id, task_id, trigger, state) values (?, ?, ?, ?)")
|
||||||
|
.run(`run_${randomUUID()}`, taskId, run.trigger, run.state);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("close stale wake tasks migration", () => {
|
||||||
|
test("closes spent chat-wake tasks and leaves everything else alone", () => {
|
||||||
|
const db = createDatabase();
|
||||||
|
// Zombie: reply went out but the task was never closed.
|
||||||
|
seedTask(db, "tsk_replied", "backlog", [{ trigger: "chat_mention", state: "completed" }]);
|
||||||
|
// Zombie: every attempt failed on the provider limit; the conversation has moved on.
|
||||||
|
seedTask(db, "tsk_starved", "backlog", [
|
||||||
|
{ trigger: "chat_mention", state: "failed" },
|
||||||
|
{ trigger: "heartbeat", state: "failed" },
|
||||||
|
{ trigger: "heartbeat", state: "failed" },
|
||||||
|
]);
|
||||||
|
// In-flight wake: its queued run must keep working.
|
||||||
|
seedTask(db, "tsk_pending", "backlog", [{ trigger: "chat_mention", state: "queued" }]);
|
||||||
|
// Real work is not a wake vehicle and is never touched.
|
||||||
|
seedTask(db, "tsk_real", "backlog", [{ trigger: "heartbeat", state: "completed" }]);
|
||||||
|
|
||||||
|
db.exec(closeStaleWakeTasksSchema);
|
||||||
|
|
||||||
|
const statuses = Object.fromEntries(
|
||||||
|
(db.prepare("select id, status from tasks").all() as Array<{ id: string; status: string }>).map((row) => [row.id, row.status]),
|
||||||
|
);
|
||||||
|
expect(statuses).toEqual({
|
||||||
|
tsk_replied: "done",
|
||||||
|
tsk_starved: "done",
|
||||||
|
tsk_pending: "backlog",
|
||||||
|
tsk_real: "backlog",
|
||||||
|
});
|
||||||
|
expect(db.prepare("select version from schema_migrations order by version desc limit 1").get()).toEqual({ version: 54 });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -24,7 +24,7 @@ describe("domain services", () => {
|
|||||||
const connection = createConnection();
|
const connection = createConnection();
|
||||||
|
|
||||||
const rows = connection.db.prepare("select version from schema_migrations order by version asc").all() as Array<{ version: number }>;
|
const rows = connection.db.prepare("select version from schema_migrations order by version asc").all() as Array<{ version: number }>;
|
||||||
expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53]);
|
expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54]);
|
||||||
connection.close();
|
connection.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user