From 4d6c74a21e7e8551f9f0227a75fa406e455b25a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Thu, 8 Oct 2026 05:01:28 -0400 Subject: [PATCH] Q21: fetch_url pages long pages and continues where it stopped Co-Authored-By: Claude Opus 5.5 --- src/main/services/agent-web-tools.ts | 14 ++++++-------- tests/unit/agent-web-tools.test.ts | 17 +++++++++++++---- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/main/services/agent-web-tools.ts b/src/main/services/agent-web-tools.ts index 2e2cf894..a07a6538 100644 --- a/src/main/services/agent-web-tools.ts +++ b/src/main/services/agent-web-tools.ts @@ -2,11 +2,11 @@ import { lookup } from "node:dns/promises"; import { isIP } from "node:net"; import { requireString } from "./agent-tool-args"; import type { ToolCall, ToolDefinition } from "./provider-tool-turn"; +import { PAGE_TOOL_PROPERTIES, pageText, readPageArgs, type PageRequest } from "./text-paging"; const FETCH_TIMEOUT_MS = 30_000; const MAX_REDIRECTS = 3; const MAX_RESPONSE_BYTES = 2_000_000; -const PAGE_CHAR_LIMIT = 20_000; const SEARCH_RESULT_LIMIT = 8; const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AgentCompany/1.0"; @@ -39,12 +39,13 @@ export const WEB_TOOLS: ToolDefinition[] = [ }, { name: "fetch_url", - description: "Fetch a public web page and return its readable text. Use after web_search, or on any URL you were given. Only public hosts are reachable.", + description: "Fetch a public web page and return its readable text — whole, a range of lines, or only the lines matching a search. A long page tells you which start_line continues it. Use after web_search, or on any URL you were given. Only public hosts are reachable.", parameters: { type: "object", additionalProperties: false, properties: { url: { type: "string", description: "Absolute http(s) URL to fetch." }, + ...PAGE_TOOL_PROPERTIES, }, required: ["url"], }, @@ -63,7 +64,7 @@ export async function executeWebTool(call: ToolCall, fetcher: WebFetcher = fetch case "web_search": return await webSearch(requireString(call.arguments.query, "query"), fetcher); case "fetch_url": - return await fetchUrl(requireString(call.arguments.url, "url"), fetcher); + return await fetchUrl(requireString(call.arguments.url, "url"), readPageArgs(call.arguments), fetcher); default: throw new Error(`Unknown web tool "${call.name}".`); } @@ -85,14 +86,11 @@ async function webSearch(query: string, fetcher: WebFetcher): Promise { .join("\n\n"); } -async function fetchUrl(rawUrl: string, fetcher: WebFetcher): Promise { +async function fetchUrl(rawUrl: string, page: PageRequest, fetcher: WebFetcher): Promise { const body = await guardedFetch(rawUrl, fetcher); const text = looksLikeHtml(body.contentType, body.text) ? htmlToText(body.text) : body.text.trim(); if (!text) return `${body.finalUrl} returned an empty page (status ${body.status}).`; - const truncated = text.length > PAGE_CHAR_LIMIT - ? `${text.slice(0, PAGE_CHAR_LIMIT)}\n\n[truncated: showing the first ${PAGE_CHAR_LIMIT} of ${text.length} characters]` - : text; - return `${body.finalUrl} (status ${body.status})\n\n${truncated}`; + return `${body.finalUrl} (status ${body.status})\n\n${pageText(body.finalUrl, text, page)}`; } interface GuardedResponse { diff --git a/tests/unit/agent-web-tools.test.ts b/tests/unit/agent-web-tools.test.ts index 4eb270a4..9532925a 100644 --- a/tests/unit/agent-web-tools.test.ts +++ b/tests/unit/agent-web-tools.test.ts @@ -79,10 +79,19 @@ describe("agent web tools", () => { expect(bounced.requested).toHaveLength(1); }); - test("fetch_url truncates oversized pages with an honest note", async () => { - const { fetcher } = fetcherReturning([{ body: `

${"word ".repeat(9000)}

` }]); - const output = await executeWebTool(call("fetch_url", { url: "http://93.184.216.34/big" }), fetcher); - expect(output).toContain("[truncated: showing the first"); + test("fetch_url pages long pages and continues where the last page stopped", async () => { + const paragraphs = Array.from({ length: 900 }, (_, index) => `

Paragraph ${index + 1} ${"word ".repeat(20)}

`).join(""); + const first = await executeWebTool(call("fetch_url", { url: "http://93.184.216.34/big" }), fetcherReturning([{ body: paragraphs }]).fetcher); + const next = /\[more: pass start_line=(\d+) to continue\]/.exec(first); + expect(next).not.toBeNull(); + expect(first).not.toContain("Paragraph 900 "); + const rest = await executeWebTool( + call("fetch_url", { url: "http://93.184.216.34/big", start_line: Number(next![1]), line_count: 5000 }), + fetcherReturning([{ body: paragraphs }]).fetcher, + ); + expect(rest).toMatch(/\[lines \d+-\d+ of \d+ in http:\/\/93\.184\.216\.34\/big\]/); + const searched = await executeWebTool(call("fetch_url", { url: "http://93.184.216.34/big", search: "Paragraph 900 " }), fetcherReturning([{ body: paragraphs }]).fetcher); + expect(searched).toContain("[1 matching lines for \"Paragraph 900\" in http://93.184.216.34/big]\n900: Paragraph 900 word"); }); test("web_search parses DuckDuckGo results and unwraps redirect hrefs", async () => {