Q97: external calls that report failure are failed calls
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a9620eaa5a
commit
989127bed3
5 files changed
+39
-11
No files matched your search
@@ -8,6 +8,7 @@ const FETCH_TIMEOUT_MS = 30_000;
|
||||
const MAX_REDIRECTS = 3;
|
||||
const MAX_RESPONSE_BYTES = 2_000_000;
|
||||
const SEARCH_RESULT_LIMIT = 8;
|
||||
const STATUS_EXCERPT_CHARS = 300;
|
||||
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AgentCompany/1.0";
|
||||
|
||||
/** Subset of the fetch contract the tools rely on; injectable so tests stay off the network. */
|
||||
@@ -126,6 +127,9 @@ async function guardedFetch(rawUrl: string, fetcher: WebFetcher): Promise<Guarde
|
||||
continue;
|
||||
}
|
||||
const text = await readBodyCapped(response);
|
||||
if (response.status < 200 || response.status >= 300) {
|
||||
throw new Error(`${url.toString()} answered with status ${response.status}: ${statusExcerpt(response.headers.get("content-type") ?? "", text)}`);
|
||||
}
|
||||
return {
|
||||
status: response.status,
|
||||
contentType: response.headers.get("content-type") ?? "",
|
||||
@@ -136,6 +140,12 @@ async function guardedFetch(rawUrl: string, fetcher: WebFetcher): Promise<Guarde
|
||||
throw new Error(`Too many redirects (more than ${MAX_REDIRECTS}).`);
|
||||
}
|
||||
|
||||
/** The start of an error page, as one line, so the agent sees why without the whole page. */
|
||||
function statusExcerpt(contentType: string, text: string): string {
|
||||
const plain = (looksLikeHtml(contentType, text) ? htmlToText(text) : text).replace(/\s+/g, " ").trim();
|
||||
return plain ? plain.slice(0, STATUS_EXCERPT_CHARS) : "(empty body)";
|
||||
}
|
||||
|
||||
function parseHttpUrl(rawUrl: string): URL {
|
||||
let url: URL;
|
||||
try {
|
||||
|
||||
@@ -26,10 +26,8 @@ export interface ComposioConnectedAccount {
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface ComposioToolExecutionResult {
|
||||
data: Record<string, unknown>;
|
||||
error: string | null;
|
||||
}
|
||||
/** Composio answers 200 for business failures too; only `successful` tells them apart. */
|
||||
export type ComposioToolExecutionResult = { successful: true; data: unknown } | { successful: false; error: string };
|
||||
|
||||
const MAX_RESPONSE_BYTES = 2_000_000;
|
||||
|
||||
@@ -142,10 +140,9 @@ export class ComposioClient {
|
||||
};
|
||||
if (input.connectedAccountId) body.connected_account_id = input.connectedAccountId;
|
||||
const response = await this.request("POST", url, body);
|
||||
return {
|
||||
data: typeof response === "object" && response !== null ? response as Record<string, unknown> : {},
|
||||
error: null,
|
||||
};
|
||||
if (typeof response.successful !== "boolean") throw new Error(`Composio returned no execution status for ${input.toolName}.`);
|
||||
if (response.successful) return { successful: true, data: response.data };
|
||||
return { successful: false, error: typeof response.error === "string" && response.error.trim() ? response.error.trim() : "Composio gave no reason." };
|
||||
}
|
||||
|
||||
private async request(
|
||||
|
||||
@@ -114,7 +114,10 @@ async function runAction(db: DatabaseSync, apiKey: string, orgId: string, agentI
|
||||
const input = typeof args.arguments === "object" && args.arguments !== null && !Array.isArray(args.arguments) ? args.arguments as Record<string, unknown> : null;
|
||||
if (!input) return "ERROR: arguments must be an object matching the action's input.";
|
||||
const result = await new ComposioClient(apiKey).executeTool({ toolName: action.name, arguments: input, userId: getComposioUserId(db, orgId), connectedAccountId: account.id });
|
||||
if (result.error) return `ERROR: ${result.error}`;
|
||||
if (!result.successful) {
|
||||
console.warn(`[connectors] ${action.name} failed: ${result.error.replace(/\s+/g, " ")}`);
|
||||
return `ERROR: ${action.name} failed: ${result.error}`;
|
||||
}
|
||||
return JSON.stringify(result.data);
|
||||
}
|
||||
|
||||
|
||||
@@ -62,6 +62,13 @@ describe("agent web tools", () => {
|
||||
expect(output).toContain("status 200");
|
||||
});
|
||||
|
||||
test("a page or search answered with an error status is a failed call, not data", async () => {
|
||||
const missing = await executeWebTool(call("fetch_url", { url: "http://93.184.216.34/gone" }), fetcherReturning([{ status: 404, body: "<h1>Not Found</h1>" }]).fetcher);
|
||||
expect(missing).toBe("ERROR: http://93.184.216.34/gone answered with status 404: Not Found");
|
||||
const limited = await executeWebTool(call("web_search", { query: "pricing" }), fetcherReturning([{ status: 429, body: "slow down" }]).fetcher);
|
||||
expect(limited).toMatch(/^ERROR: https:\/\/html\.duckduckgo\.com\/html\/\?q=pricing answered with status 429: slow down$/);
|
||||
});
|
||||
|
||||
test("fetch_url follows public redirects but refuses a hop into a private range", async () => {
|
||||
const followed = fetcherReturning([
|
||||
{ status: 302, headers: { location: "http://93.184.216.35/final" } },
|
||||
|
||||
@@ -7,6 +7,7 @@ import { describe, expect, test, vi } from "vitest";
|
||||
const requested: string[] = [];
|
||||
const written: string[] = [];
|
||||
const pages = new Map<string, Record<string, unknown>>();
|
||||
const executions = new Map<string, Record<string, unknown>>();
|
||||
|
||||
vi.mock("electron", () => ({
|
||||
net: {
|
||||
@@ -18,7 +19,7 @@ vi.mock("electron", () => ({
|
||||
request.end = () => {
|
||||
requested.push(url);
|
||||
const query = new URL(url).searchParams;
|
||||
const page = url.includes("/tools/execute/") ? { data: { number: 42 }, successful: true } : pages.get(`${query.get("toolkit_slug")}:${query.get("cursor") ?? ""}`);
|
||||
const page = url.includes("/tools/execute/") ? executions.get(url.split("/tools/execute/")[1]) ?? { data: { number: 42 }, error: null, successful: true } : pages.get(`${query.get("toolkit_slug")}:${query.get("cursor") ?? ""}`);
|
||||
const response = new EventEmitter() as EventEmitter & { statusCode: number };
|
||||
response.statusCode = page ? 200 : 404;
|
||||
request.emit("response", response);
|
||||
@@ -86,13 +87,23 @@ describe("agents reach every action through search and run", () => {
|
||||
const found = JSON.parse(await h.call("composio_find_actions", { query: "create pull request" })) as { matches: Array<{ action: string; input: unknown }>; searched: number };
|
||||
expect(found.searched).toBe(101);
|
||||
expect(found.matches[0]).toMatchObject({ action: "GITHUB_CREATE_A_PULL_REQUEST", input: { properties: { title: { type: "string" } }, required: ["title"] } });
|
||||
expect(await h.call("composio_run_action", { action: "GITHUB_CREATE_A_PULL_REQUEST", arguments: { title: "Fix" } })).toBe(JSON.stringify({ data: { number: 42 }, successful: true }));
|
||||
expect(await h.call("composio_run_action", { action: "GITHUB_CREATE_A_PULL_REQUEST", arguments: { title: "Fix" } })).toBe(JSON.stringify({ number: 42 }));
|
||||
expect(written.at(-1)).toContain("/tools/execute/GITHUB_CREATE_A_PULL_REQUEST");
|
||||
expect(written.at(-1)).toContain("\"connected_account_id\":\"ca_gh\"");
|
||||
expect(await h.call("composio_run_action", { action: "SLACK_SEND_MESSAGE", arguments: {} })).toContain("is not an action of your connected services (github)");
|
||||
h.connection.close();
|
||||
});
|
||||
|
||||
test("an action Composio reports as failed, or without a status, is a failed call", async () => {
|
||||
pages.set("github:", { items: [tool("GITHUB_MERGE"), tool("GITHUB_ODD")], next_cursor: null });
|
||||
executions.set("GITHUB_MERGE", { data: { message: "Not Found" }, error: "Repository not found", successful: false });
|
||||
executions.set("GITHUB_ODD", { data: { ok: 1 } });
|
||||
const h = company("ak_failed");
|
||||
expect(await h.call("composio_run_action", { action: "GITHUB_MERGE", arguments: {} })).toBe("ERROR: GITHUB_MERGE failed: Repository not found");
|
||||
expect(await h.call("composio_run_action", { action: "GITHUB_ODD", arguments: {} })).toBe("ERROR: Composio returned no execution status for GITHUB_ODD.");
|
||||
h.connection.close();
|
||||
});
|
||||
|
||||
test("a grant revoked while the catalog loads stops the action, and a search is not journaled as work", async () => {
|
||||
pages.set("github:", { items: [tool("GITHUB_STAR_A_REPOSITORY")], next_cursor: null });
|
||||
const h = company("ak_revoke");
|
||||
|
||||
Reference in new issue
Block a user