Q129: a run with failed actions it never redid does not count as completed

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ZacharyZhang-NYandClaude Opus 5.5 committed 2026-10-08 21:15:47 -04:00
1 parent 6afd7bef2a
commit 271a83a7bc
7 files changed
+209 -23

No files matched your search

+51 -3
View File
@@ -1,4 +1,5 @@
import type { DatabaseSync } from "node:sqlite";
import { posix } from "node:path";
import { createId } from "../../shared/ids";
import type { ToolCall } from "./provider-tool-turn";
@@ -10,6 +11,15 @@ const READ_ONLY_TOOLS = new Set([
const JOURNAL_LINES = 40;
const FIELD_CHARS = 160;
const DECLINED_LINES = 5;
/** The arguments that name what an action acted on (retry controls such as confirm_not_done excluded); a tool not listed is identified by all of its arguments. */
const TARGET_ARGUMENTS: Record<string, string[]> = {
write_deliverable: ["path"], create_document: ["title", "doc_type"], update_document: ["doc_id"], create_task: ["title"],
update_task_status: ["task_id"], hire_agent: ["name"], grant_connector: ["agent_name", "toolkit"], revoke_connector: ["agent_name", "toolkit"],
grant_skill: ["agent_name", "skill"], revoke_skill: ["agent_name", "skill"], record_knowledge: ["topic"], revise_knowledge: ["id"],
update_goal: ["goal_id"], complete_goal: [], run_command: ["command"], composio_run_action: ["action", "account", "arguments"],
};
/** A result line that reports failure: a refused or broken call, or a command that exited non-zero. */
const FAILED_RESULT = (row: string) => `(${row}.result_excerpt like 'ERROR%' or (${row}.result_excerpt like 'exit code %' and ${row}.result_excerpt <> 'exit code 0'))`;
/** What a held deliverable would become, over a `held_deliverables` row alias: one file, one existing document, or a new document by type and title. */
const heldTarget = (row: string) => `(case ${row}.kind when 'file' then ${row}.workspace_root || '/' || ${row}.relative_path when 'document_update' then ${row}.document_id else ${row}.doc_type || ':' || ${row}.title end)`;
@@ -17,10 +27,41 @@ const heldTarget = (row: string) => `(case ${row}.kind when 'file' then ${row}.w
export function recordRunAction(db: DatabaseSync, input: { orgId: string; runId: string; taskId: string; call: ToolCall; output: string }): void {
if (READ_ONLY_TOOLS.has(input.call.name)) return;
db.prepare(`
insert into run_actions(id, org_id, run_id, task_id, tool_name, arguments_excerpt, result_excerpt, created_at)
values (?, ?, ?, ?, ?, ?, ?, ?)
insert into run_actions(id, org_id, run_id, task_id, tool_name, arguments_excerpt, result_excerpt, target, created_at)
values (?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(createId("rac"), input.orgId, input.runId, input.taskId, input.call.name,
clip(JSON.stringify(input.call.arguments)), clip(input.output.split("\n")[0] ?? ""), new Date().toISOString());
clip(JSON.stringify(input.call.arguments)), clip(input.output.split("\n")[0] ?? ""), actionTarget(input.call), new Date().toISOString());
}
/** What the call acted on, or null when its arguments could not be read. */
function actionTarget(call: ToolCall): string | null {
if (call.argumentsError) return null;
const keys = TARGET_ARGUMENTS[call.name];
if (!keys) return JSON.stringify(canonical(call.arguments));
return JSON.stringify(keys.map((key) => targetValue(key, call.arguments[key])));
}
/** An argument as the tools read it: text trimmed, paths in one spelling, nested values in one key order. */
function targetValue(key: string, value: unknown): unknown {
if (typeof value !== "string") return canonical(value ?? null);
return key === "path" ? posix.normalize(value.trim().replaceAll("\\", "/")) : value.trim();
}
/** Why a session that ended normally still failed: actions that failed and were not later done successfully on the same target (any target when unknown). */
export function unresolvedActionFailure(db: DatabaseSync, runId: string): string | null {
const failed = db.prepare(`
select a.tool_name, a.result_excerpt from run_actions a
where a.run_id = ? and ${FAILED_RESULT("a")}
and not exists (
select 1 from run_actions later
where later.run_id = a.run_id and later.tool_name = a.tool_name and later.rowid > a.rowid and not ${FAILED_RESULT("later")}
and (a.target is null or later.target = a.target)
)
order by a.rowid
`).all(runId) as Array<{ tool_name: string; result_excerpt: string }>;
if (failed.length === 0) return null;
const list = failed.map((action) => `${action.tool_name} (${action.result_excerpt})`).join("; ");
return `The session ended with ${failed.length} failed action${failed.length === 1 ? "" : "s"} it did not redo: ${list}`;
}
/** What earlier sessions (including an interrupted attempt of this run) already did on the task, and where the last one failed. */
@@ -58,3 +99,10 @@ export function runJournalLines(db: DatabaseSync, taskId: string): string[] {
function clip(text: string): string {
return text.length > FIELD_CHARS ? `${text.slice(0, FIELD_CHARS)}...` : text;
}
/** The same JSON value with object keys sorted, so equal arguments serialize equally. */
function canonical(value: unknown): unknown {
if (Array.isArray(value)) return value.map(canonical);
if (value === null || typeof value !== "object") return value;
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical((value as Record<string, unknown>)[key])]));
}