Q129: a run with failed actions it never redid does not count as completed
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6afd7bef2a
commit
271a83a7bc
7 files changed
+209
-23
No files matched your search
@@ -1,4 +1,5 @@
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { posix } from "node:path";
|
||||
import { createId } from "../../shared/ids";
|
||||
import type { ToolCall } from "./provider-tool-turn";
|
||||
|
||||
@@ -10,6 +11,15 @@ const READ_ONLY_TOOLS = new Set([
|
||||
const JOURNAL_LINES = 40;
|
||||
const FIELD_CHARS = 160;
|
||||
const DECLINED_LINES = 5;
|
||||
/** The arguments that name what an action acted on (retry controls such as confirm_not_done excluded); a tool not listed is identified by all of its arguments. */
|
||||
const TARGET_ARGUMENTS: Record<string, string[]> = {
|
||||
write_deliverable: ["path"], create_document: ["title", "doc_type"], update_document: ["doc_id"], create_task: ["title"],
|
||||
update_task_status: ["task_id"], hire_agent: ["name"], grant_connector: ["agent_name", "toolkit"], revoke_connector: ["agent_name", "toolkit"],
|
||||
grant_skill: ["agent_name", "skill"], revoke_skill: ["agent_name", "skill"], record_knowledge: ["topic"], revise_knowledge: ["id"],
|
||||
update_goal: ["goal_id"], complete_goal: [], run_command: ["command"], composio_run_action: ["action", "account", "arguments"],
|
||||
};
|
||||
/** A result line that reports failure: a refused or broken call, or a command that exited non-zero. */
|
||||
const FAILED_RESULT = (row: string) => `(${row}.result_excerpt like 'ERROR%' or (${row}.result_excerpt like 'exit code %' and ${row}.result_excerpt <> 'exit code 0'))`;
|
||||
/** What a held deliverable would become, over a `held_deliverables` row alias: one file, one existing document, or a new document by type and title. */
|
||||
const heldTarget = (row: string) => `(case ${row}.kind when 'file' then ${row}.workspace_root || '/' || ${row}.relative_path when 'document_update' then ${row}.document_id else ${row}.doc_type || ':' || ${row}.title end)`;
|
||||
|
||||
@@ -17,10 +27,41 @@ const heldTarget = (row: string) => `(case ${row}.kind when 'file' then ${row}.w
|
||||
export function recordRunAction(db: DatabaseSync, input: { orgId: string; runId: string; taskId: string; call: ToolCall; output: string }): void {
|
||||
if (READ_ONLY_TOOLS.has(input.call.name)) return;
|
||||
db.prepare(`
|
||||
insert into run_actions(id, org_id, run_id, task_id, tool_name, arguments_excerpt, result_excerpt, created_at)
|
||||
values (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
insert into run_actions(id, org_id, run_id, task_id, tool_name, arguments_excerpt, result_excerpt, target, created_at)
|
||||
values (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(createId("rac"), input.orgId, input.runId, input.taskId, input.call.name,
|
||||
clip(JSON.stringify(input.call.arguments)), clip(input.output.split("\n")[0] ?? ""), new Date().toISOString());
|
||||
clip(JSON.stringify(input.call.arguments)), clip(input.output.split("\n")[0] ?? ""), actionTarget(input.call), new Date().toISOString());
|
||||
}
|
||||
|
||||
/** What the call acted on, or null when its arguments could not be read. */
|
||||
function actionTarget(call: ToolCall): string | null {
|
||||
if (call.argumentsError) return null;
|
||||
const keys = TARGET_ARGUMENTS[call.name];
|
||||
if (!keys) return JSON.stringify(canonical(call.arguments));
|
||||
return JSON.stringify(keys.map((key) => targetValue(key, call.arguments[key])));
|
||||
}
|
||||
|
||||
/** An argument as the tools read it: text trimmed, paths in one spelling, nested values in one key order. */
|
||||
function targetValue(key: string, value: unknown): unknown {
|
||||
if (typeof value !== "string") return canonical(value ?? null);
|
||||
return key === "path" ? posix.normalize(value.trim().replaceAll("\\", "/")) : value.trim();
|
||||
}
|
||||
|
||||
/** Why a session that ended normally still failed: actions that failed and were not later done successfully on the same target (any target when unknown). */
|
||||
export function unresolvedActionFailure(db: DatabaseSync, runId: string): string | null {
|
||||
const failed = db.prepare(`
|
||||
select a.tool_name, a.result_excerpt from run_actions a
|
||||
where a.run_id = ? and ${FAILED_RESULT("a")}
|
||||
and not exists (
|
||||
select 1 from run_actions later
|
||||
where later.run_id = a.run_id and later.tool_name = a.tool_name and later.rowid > a.rowid and not ${FAILED_RESULT("later")}
|
||||
and (a.target is null or later.target = a.target)
|
||||
)
|
||||
order by a.rowid
|
||||
`).all(runId) as Array<{ tool_name: string; result_excerpt: string }>;
|
||||
if (failed.length === 0) return null;
|
||||
const list = failed.map((action) => `${action.tool_name} (${action.result_excerpt})`).join("; ");
|
||||
return `The session ended with ${failed.length} failed action${failed.length === 1 ? "" : "s"} it did not redo: ${list}`;
|
||||
}
|
||||
|
||||
/** What earlier sessions (including an interrupted attempt of this run) already did on the task, and where the last one failed. */
|
||||
@@ -58,3 +99,10 @@ export function runJournalLines(db: DatabaseSync, taskId: string): string[] {
|
||||
function clip(text: string): string {
|
||||
return text.length > FIELD_CHARS ? `${text.slice(0, FIELD_CHARS)}...` : text;
|
||||
}
|
||||
|
||||
/** The same JSON value with object keys sorted, so equal arguments serialize equally. */
|
||||
function canonical(value: unknown): unknown {
|
||||
if (Array.isArray(value)) return value.map(canonical);
|
||||
if (value === null || typeof value !== "object") return value;
|
||||
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical((value as Record<string, unknown>)[key])]));
|
||||
}
|
||||
Reference in new issue
Block a user