Q21: fetch_url pages long pages and continues where it stopped
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1a0da25202
commit
4d6c74a21e
2 files changed
+19
-12
No files matched your search
@@ -2,11 +2,11 @@ import { lookup } from "node:dns/promises";
|
|||||||
import { isIP } from "node:net";
|
import { isIP } from "node:net";
|
||||||
import { requireString } from "./agent-tool-args";
|
import { requireString } from "./agent-tool-args";
|
||||||
import type { ToolCall, ToolDefinition } from "./provider-tool-turn";
|
import type { ToolCall, ToolDefinition } from "./provider-tool-turn";
|
||||||
|
import { PAGE_TOOL_PROPERTIES, pageText, readPageArgs, type PageRequest } from "./text-paging";
|
||||||
|
|
||||||
const FETCH_TIMEOUT_MS = 30_000;
|
const FETCH_TIMEOUT_MS = 30_000;
|
||||||
const MAX_REDIRECTS = 3;
|
const MAX_REDIRECTS = 3;
|
||||||
const MAX_RESPONSE_BYTES = 2_000_000;
|
const MAX_RESPONSE_BYTES = 2_000_000;
|
||||||
const PAGE_CHAR_LIMIT = 20_000;
|
|
||||||
const SEARCH_RESULT_LIMIT = 8;
|
const SEARCH_RESULT_LIMIT = 8;
|
||||||
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AgentCompany/1.0";
|
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AgentCompany/1.0";
|
||||||
|
|
||||||
@@ -39,12 +39,13 @@ export const WEB_TOOLS: ToolDefinition[] = [
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "fetch_url",
|
name: "fetch_url",
|
||||||
description: "Fetch a public web page and return its readable text. Use after web_search, or on any URL you were given. Only public hosts are reachable.",
|
description: "Fetch a public web page and return its readable text — whole, a range of lines, or only the lines matching a search. A long page tells you which start_line continues it. Use after web_search, or on any URL you were given. Only public hosts are reachable.",
|
||||||
parameters: {
|
parameters: {
|
||||||
type: "object",
|
type: "object",
|
||||||
additionalProperties: false,
|
additionalProperties: false,
|
||||||
properties: {
|
properties: {
|
||||||
url: { type: "string", description: "Absolute http(s) URL to fetch." },
|
url: { type: "string", description: "Absolute http(s) URL to fetch." },
|
||||||
|
...PAGE_TOOL_PROPERTIES,
|
||||||
},
|
},
|
||||||
required: ["url"],
|
required: ["url"],
|
||||||
},
|
},
|
||||||
@@ -63,7 +64,7 @@ export async function executeWebTool(call: ToolCall, fetcher: WebFetcher = fetch
|
|||||||
case "web_search":
|
case "web_search":
|
||||||
return await webSearch(requireString(call.arguments.query, "query"), fetcher);
|
return await webSearch(requireString(call.arguments.query, "query"), fetcher);
|
||||||
case "fetch_url":
|
case "fetch_url":
|
||||||
return await fetchUrl(requireString(call.arguments.url, "url"), fetcher);
|
return await fetchUrl(requireString(call.arguments.url, "url"), readPageArgs(call.arguments), fetcher);
|
||||||
default:
|
default:
|
||||||
throw new Error(`Unknown web tool "${call.name}".`);
|
throw new Error(`Unknown web tool "${call.name}".`);
|
||||||
}
|
}
|
||||||
@@ -85,14 +86,11 @@ async function webSearch(query: string, fetcher: WebFetcher): Promise<string> {
|
|||||||
.join("\n\n");
|
.join("\n\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchUrl(rawUrl: string, fetcher: WebFetcher): Promise<string> {
|
async function fetchUrl(rawUrl: string, page: PageRequest, fetcher: WebFetcher): Promise<string> {
|
||||||
const body = await guardedFetch(rawUrl, fetcher);
|
const body = await guardedFetch(rawUrl, fetcher);
|
||||||
const text = looksLikeHtml(body.contentType, body.text) ? htmlToText(body.text) : body.text.trim();
|
const text = looksLikeHtml(body.contentType, body.text) ? htmlToText(body.text) : body.text.trim();
|
||||||
if (!text) return `${body.finalUrl} returned an empty page (status ${body.status}).`;
|
if (!text) return `${body.finalUrl} returned an empty page (status ${body.status}).`;
|
||||||
const truncated = text.length > PAGE_CHAR_LIMIT
|
return `${body.finalUrl} (status ${body.status})\n\n${pageText(body.finalUrl, text, page)}`;
|
||||||
? `${text.slice(0, PAGE_CHAR_LIMIT)}\n\n[truncated: showing the first ${PAGE_CHAR_LIMIT} of ${text.length} characters]`
|
|
||||||
: text;
|
|
||||||
return `${body.finalUrl} (status ${body.status})\n\n${truncated}`;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
interface GuardedResponse {
|
interface GuardedResponse {
|
||||||
|
|||||||
@@ -79,10 +79,19 @@ describe("agent web tools", () => {
|
|||||||
expect(bounced.requested).toHaveLength(1);
|
expect(bounced.requested).toHaveLength(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("fetch_url truncates oversized pages with an honest note", async () => {
|
test("fetch_url pages long pages and continues where the last page stopped", async () => {
|
||||||
const { fetcher } = fetcherReturning([{ body: `<p>${"word ".repeat(9000)}</p>` }]);
|
const paragraphs = Array.from({ length: 900 }, (_, index) => `<p>Paragraph ${index + 1} ${"word ".repeat(20)}</p>`).join("");
|
||||||
const output = await executeWebTool(call("fetch_url", { url: "http://93.184.216.34/big" }), fetcher);
|
const first = await executeWebTool(call("fetch_url", { url: "http://93.184.216.34/big" }), fetcherReturning([{ body: paragraphs }]).fetcher);
|
||||||
expect(output).toContain("[truncated: showing the first");
|
const next = /\[more: pass start_line=(\d+) to continue\]/.exec(first);
|
||||||
|
expect(next).not.toBeNull();
|
||||||
|
expect(first).not.toContain("Paragraph 900 ");
|
||||||
|
const rest = await executeWebTool(
|
||||||
|
call("fetch_url", { url: "http://93.184.216.34/big", start_line: Number(next![1]), line_count: 5000 }),
|
||||||
|
fetcherReturning([{ body: paragraphs }]).fetcher,
|
||||||
|
);
|
||||||
|
expect(rest).toMatch(/\[lines \d+-\d+ of \d+ in http:\/\/93\.184\.216\.34\/big\]/);
|
||||||
|
const searched = await executeWebTool(call("fetch_url", { url: "http://93.184.216.34/big", search: "Paragraph 900 " }), fetcherReturning([{ body: paragraphs }]).fetcher);
|
||||||
|
expect(searched).toContain("[1 matching lines for \"Paragraph 900\" in http://93.184.216.34/big]\n900: Paragraph 900 word");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("web_search parses DuckDuckGo results and unwraps redirect hrefs", async () => {
|
test("web_search parses DuckDuckGo results and unwraps redirect hrefs", async () => {
|
||||||
|
|||||||
Reference in new issue
Block a user