From 8c70111c9383740a571c650d1cc68f6bc6926095 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Thu, 8 Oct 2026 05:34:17 -0400 Subject: [PATCH] Q25: runs leave outcomes and failure causes as searchable memory Co-Authored-By: Claude Opus 5.5 --- src/main/services/run-memory.ts | 22 +++++ src/main/services/task-execution-service.ts | 5 + src/main/services/task-prompts.ts | 6 ++ tests/integration/run-memory.test.ts | 103 ++++++++++++++++++++ 4 files changed, 136 insertions(+) create mode 100644 src/main/services/run-memory.ts create mode 100644 tests/integration/run-memory.test.ts diff --git a/src/main/services/run-memory.ts b/src/main/services/run-memory.ts new file mode 100644 index 00000000..7512b099 --- /dev/null +++ b/src/main/services/run-memory.ts @@ -0,0 +1,22 @@ +import type { DatabaseSync } from "node:sqlite"; +import { KnowledgeService } from "./knowledge-service"; + +const MEMORY_CHARS = 2_000; + +/** Keeps what a finished or failed run of real work concluded, so later runs can find it. */ +export function recordRunMemory( + db: DatabaseSync, + input: { orgId: string; runId: string; taskTitle: string; outcome: "completed" | "failed"; text: string }, +): void { + const text = input.text.trim(); + if (!text) return; + new KnowledgeService(db).create({ + orgId: input.orgId, + topic: `${input.outcome === "completed" ? "Outcome" : "Failure"}: ${input.taskTitle}`, + category: input.outcome === "completed" ? "run_outcome" : "failure_cause", + content: text.length > MEMORY_CHARS ? `${text.slice(0, MEMORY_CHARS)}…` : text, + importance: "normal", + sourceType: "task_run", + sourceId: input.runId, + }); +} diff --git a/src/main/services/task-execution-service.ts b/src/main/services/task-execution-service.ts index 271fc89b..600013bb 100644 --- a/src/main/services/task-execution-service.ts +++ b/src/main/services/task-execution-service.ts @@ -23,6 +23,7 @@ import { } from "./task-prompts"; import { renderTaskHandoff } from "./agent-task-handoff"; import { relevantKnowledgeLines } from "./agent-knowledge-tools"; +import { recordRunMemory } from "./run-memory"; import { createRunContextRecorder, createRunMessagePoller, loadChatWakeContext } from "./chat-wake-context"; import { runningWorkspaceWriter } from "./workspace-write-lock"; import { inputBudgetTokens, skillBudgetTokens } from "./context-budget"; @@ -126,6 +127,8 @@ export class TaskExecutionService { action: "failed", payload: { taskId: run.task_id, agentId: run.agent_id, providerId: run.provider_id, stage: "preparation", errorMessage: message }, }); + const task = this.db.prepare("select title, origin from tasks where id = ?").get(run.task_id) as { title: string; origin: string | null } | undefined; + if (task && task.origin === null) recordRunMemory(this.db, { orgId, runId, taskTitle: task.title, outcome: "failed", text: message }); throw error; } return this.executeAgentRun(prepared); @@ -220,6 +223,7 @@ export class TaskExecutionService { } this.completeRun(orgId, runId, result.lastRequestId); this.cooldowns.clear(provider); + if (task.origin === null) recordRunMemory(this.db, { orgId, runId, taskTitle: task.title, outcome: "completed", text: result.finalText }); this.audit.record({ orgId, entityType: "task_run", @@ -244,6 +248,7 @@ export class TaskExecutionService { const message = error instanceof Error ? error.message : String(error); this.failRun(orgId, runId, providerRequestId, message); if (isTransientProviderFailure(message)) this.cooldowns.park(provider, message); + else if (task.origin === null) recordRunMemory(this.db, { orgId, runId, taskTitle: task.title, outcome: "failed", text: message }); this.audit.record({ orgId, entityType: "task_run", diff --git a/src/main/services/task-prompts.ts b/src/main/services/task-prompts.ts index 22e8a270..db7b1aeb 100644 --- a/src/main/services/task-prompts.ts +++ b/src/main/services/task-prompts.ts @@ -1,5 +1,8 @@ import { canonicalMentionHandle } from "../../shared/mention-handles"; +const RECORD_LEARNINGS_INSTRUCTION = + "Before you finish, record anything the company should remember — decisions, preferences, why something failed, conclusions you verified — with record_knowledge."; + export interface TaskExecutionTaskRow { id: string; org_id: string; @@ -226,6 +229,8 @@ export function buildTaskPrompt(input: { input.chatContext.fresh || "(nothing new since your last reply)", "", `You have been brought into this thread. Reply to the people here in your own voice — answer what they asked, give your honest take, push back if you disagree, or hand work over. Stay in the thread by passing thread_id="${input.chatContext.threadId}" to send_message. This is a conversation between colleagues, not a status report; only change a task's status if the work itself actually moved.`, + "", + RECORD_LEARNINGS_INSTRUCTION, ); return lines.join("\n"); } @@ -254,6 +259,7 @@ export function buildTaskPrompt(input: { `First, read_messages to check if anyone has said something you should respond to — reply to them like a colleague, not a report. Then do the work: write real output with write_deliverable, raise decisions and blockers with ${managerReference} in a direct message (send_message with to=), and keep team-wide topics in channels. Hand files over with /handoff and attachment_file_ids. Keep conversation going — ask questions, share your thinking, push back if something seems wrong. Before you finish: if the work is genuinely done, call update_task_status(task_id="${input.task.id}", status="done"); if it needs another session, move it to "in_progress" and say in the thread what remains — an open task will be run again.`, ); } + lines.push("", RECORD_LEARNINGS_INSTRUCTION); return lines.join("\n"); } diff --git a/tests/integration/run-memory.test.ts b/tests/integration/run-memory.test.ts new file mode 100644 index 00000000..c69fc128 --- /dev/null +++ b/tests/integration/run-memory.test.ts @@ -0,0 +1,103 @@ +import { randomUUID } from "node:crypto"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, test } from "vitest"; +import { DatabaseConnection } from "../../src/main/database/connection"; +import { OnboardingService } from "../../src/main/services/onboarding-service"; +import { ProviderService } from "../../src/main/services/provider-service"; +import { providerAdapters } from "../../src/main/services/provider-adapters"; +import { TaskExecutionService } from "../../src/main/services/task-execution-service"; +import { ChatService } from "../../src/main/services/chat-service"; +import { ensureCeoDirectSpace } from "../../src/main/services/ceo-direct-space"; +import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker"; +import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn"; + +const testVault = { + encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), + decryptSecret: (ciphertext: Buffer) => ciphertext.toString("utf8"), +}; + +async function company(reply: (prompt: string) => AgentTurnResult) { + const prompts: string[] = []; + const turn = async (input: ProviderToolTurnInput): Promise => { + const first = input.messages.find((message) => message.role === "user"); + const prompt = first && first.role === "user" ? first.text : ""; + prompts.push(prompt); + return reply(prompt); + }; + const connection = new DatabaseConnection(join(tmpdir(), `ac-memory-${randomUUID()}.db`)); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn); + const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); + const execution = new TaskExecutionService(connection.db, providers); + let orgId = ""; + let ceoId = ""; + connection.transaction(() => { + const summary = onboarding.bootstrap({ + companyName: "FlowPilot", companyDescription: "CSM analytics.", employeeName: "Yang", workspacePath: "", + executionMode: "ai_ceo", autoApproveHires: true, reviewDeliverables: false, + goalTitle: "Ship the MVP.", goalDescription: "A working product.", + providerKey: "openai", providerSecret: "sk-test-key", providerModel: "gpt-5.4", + }); + orgId = summary.orgId; + ceoId = summary.ceoAgentId; + }); + const memories = () => connection.db.prepare("select topic, category, content, source_type from knowledge_entries where org_id = ? order by created_at") + .all(orgId) as Array<{ topic: string; category: string; content: string; source_type: string }>; + return { connection, execution, orgId, ceoId, prompts, memories }; +} + +describe("run memory", () => { + test("a finished run of real work leaves its conclusion; prompts ask agents to record learnings", async () => { + const h = await company(() => ({ text: "Chose Postgres for bookings; MySQL lacked the JSON features we need.", toolCalls: [], rawResponseJson: "{}" })); + const kickoff = claimNextQueuedRun(h.connection)!; + await h.execution.executeRun(h.orgId, kickoff.id); + expect(h.prompts[0]).toContain("record anything the company should remember"); + expect(h.memories()).toEqual([{ + topic: expect.stringMatching(/^Outcome: /), category: "run_outcome", + content: "Chose Postgres for bookings; MySQL lacked the JSON features we need.", source_type: "task_run", + }]); + h.connection.close(); + }); + + test("a real failure is remembered; a transient provider failure is not", async () => { + let failure = "Workspace build failed: missing package.json"; + const h = await company(() => { throw new Error(failure); }); + const first = claimNextQueuedRun(h.connection)!; + await expect(h.execution.executeRun(h.orgId, first.id)).rejects.toThrow(failure); + expect(h.memories().map((entry) => [entry.category, entry.content])).toEqual([["failure_cause", failure]]); + + failure = "429 rate limit exceeded"; + h.connection.db.prepare("update task_runs set state = 'queued' where id = ?").run(first.id); + const second = claimNextQueuedRun(h.connection)!; + await expect(h.execution.executeRun(h.orgId, second.id)).rejects.toThrow(failure); + expect(h.memories()).toHaveLength(1); + h.connection.close(); + }); + + test("a run that fails before it starts is remembered too", async () => { + const h = await company(() => ({ text: "unused", toolCalls: [], rawResponseJson: "{}" })); + const kickoff = claimNextQueuedRun(h.connection)!; + h.connection.db.prepare("update tasks set assignee_agent_id = null where id = (select task_id from task_runs where id = ?)").run(kickoff.id); + await expect(h.execution.executeRun(h.orgId, kickoff.id)).rejects.toThrow("Task was reassigned after this run was queued"); + expect(h.memories().map((entry) => entry.category)).toEqual(["failure_cause"]); + expect(h.memories()[0].content).toContain("Task was reassigned after this run was queued"); + h.connection.close(); + }); + + test("chat replies are not auto-recorded", async () => { + const h = await company((prompt) => ({ text: /thread_id="/.test(prompt) ? "Hiring two engineers this week." : "Kickoff noted.", toolCalls: [], rawResponseJson: "{}" })); + const kickoff = claimNextQueuedRun(h.connection)!; + await h.execution.executeRun(h.orgId, kickoff.id); + const before = h.memories().length; + const founder = (h.connection.db.prepare("select id from employees where org_id = ?").get(h.orgId) as { id: string }).id; + new ChatService(h.connection.db).sendMessage({ + orgId: h.orgId, spaceId: ensureCeoDirectSpace(h.connection.db, h.orgId)!, threadId: null, + senderType: "employee", senderId: founder, bodyMd: "Hiring plan?", attachmentFileIds: [], + }); + const wake = h.connection.db.prepare("select id from task_runs where agent_id = ? and trigger = 'chat_mention' and state = 'queued'").get(h.ceoId) as { id: string }; + await h.execution.executeRun(h.orgId, wake.id); + expect(h.prompts.at(-1)).toContain("record anything the company should remember"); + expect(h.memories()).toHaveLength(before); + h.connection.close(); + }); +});