Q21: fetch_url pages long pages and continues where it stopped

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/main/services/agent-web-tools.ts

@@ -2,11 +2,11 @@ import { lookup } from "node:dns/promises";
 import { isIP } from "node:net";
 import { requireString } from "./agent-tool-args";
 import type { ToolCall, ToolDefinition } from "./provider-tool-turn";
+import { PAGE_TOOL_PROPERTIES, pageText, readPageArgs, type PageRequest } from "./text-paging";
 
 const FETCH_TIMEOUT_MS = 30_000;
 const MAX_REDIRECTS = 3;
 const MAX_RESPONSE_BYTES = 2_000_000;
-const PAGE_CHAR_LIMIT = 20_000;
 const SEARCH_RESULT_LIMIT = 8;
 const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AgentCompany/1.0";
 
@@ -39,12 +39,13 @@ export const WEB_TOOLS: ToolDefinition[] = [
   },
   {
     name: "fetch_url",
-    description: "Fetch a public web page and return its readable text. Use after web_search, or on any URL you were given. Only public hosts are reachable.",
+    description: "Fetch a public web page and return its readable text — whole, a range of lines, or only the lines matching a search. A long page tells you which start_line continues it. Use after web_search, or on any URL you were given. Only public hosts are reachable.",
     parameters: {
       type: "object",
       additionalProperties: false,
       properties: {
         url: { type: "string", description: "Absolute http(s) URL to fetch." },
+        ...PAGE_TOOL_PROPERTIES,
       },
       required: ["url"],
     },
@@ -63,7 +64,7 @@ export async function executeWebTool(call: ToolCall, fetcher: WebFetcher = fetch
       case "web_search":
         return await webSearch(requireString(call.arguments.query, "query"), fetcher);
       case "fetch_url":
-        return await fetchUrl(requireString(call.arguments.url, "url"), fetcher);
+        return await fetchUrl(requireString(call.arguments.url, "url"), readPageArgs(call.arguments), fetcher);
       default:
         throw new Error(`Unknown web tool "${call.name}".`);
     }
@@ -85,14 +86,11 @@ async function webSearch(query: string, fetcher: WebFetcher): Promise<string> {
     .join("\n\n");
 }
 
-async function fetchUrl(rawUrl: string, fetcher: WebFetcher): Promise<string> {
+async function fetchUrl(rawUrl: string, page: PageRequest, fetcher: WebFetcher): Promise<string> {
   const body = await guardedFetch(rawUrl, fetcher);
   const text = looksLikeHtml(body.contentType, body.text) ? htmlToText(body.text) : body.text.trim();
   if (!text) return `${body.finalUrl} returned an empty page (status ${body.status}).`;
-  const truncated = text.length > PAGE_CHAR_LIMIT
-    ? `${text.slice(0, PAGE_CHAR_LIMIT)}\n\n[truncated: showing the first ${PAGE_CHAR_LIMIT} of ${text.length} characters]`
-    : text;
-  return `${body.finalUrl} (status ${body.status})\n\n${truncated}`;
+  return `${body.finalUrl} (status ${body.status})\n\n${pageText(body.finalUrl, text, page)}`;
 }
 
 interface GuardedResponse {

tests/unit/agent-web-tools.test.ts

@@ -79,10 +79,19 @@ describe("agent web tools", () => {
     expect(bounced.requested).toHaveLength(1);
   });
 
-  test("fetch_url truncates oversized pages with an honest note", async () => {
-    const { fetcher } = fetcherReturning([{ body: `<p>${"word ".repeat(9000)}</p>` }]);
-    const output = await executeWebTool(call("fetch_url", { url: "http://93.184.216.34/big" }), fetcher);
-    expect(output).toContain("[truncated: showing the first");
+  test("fetch_url pages long pages and continues where the last page stopped", async () => {
+    const paragraphs = Array.from({ length: 900 }, (_, index) => `<p>Paragraph ${index + 1} ${"word ".repeat(20)}</p>`).join("");
+    const first = await executeWebTool(call("fetch_url", { url: "http://93.184.216.34/big" }), fetcherReturning([{ body: paragraphs }]).fetcher);
+    const next = /\[more: pass start_line=(\d+) to continue\]/.exec(first);
+    expect(next).not.toBeNull();
+    expect(first).not.toContain("Paragraph 900 ");
+    const rest = await executeWebTool(
+      call("fetch_url", { url: "http://93.184.216.34/big", start_line: Number(next![1]), line_count: 5000 }),
+      fetcherReturning([{ body: paragraphs }]).fetcher,
+    );
+    expect(rest).toMatch(/\[lines \d+-\d+ of \d+ in http:\/\/93\.184\.216\.34\/big\]/);
+    const searched = await executeWebTool(call("fetch_url", { url: "http://93.184.216.34/big", search: "Paragraph 900 " }), fetcherReturning([{ body: paragraphs }]).fetcher);
+    expect(searched).toContain("[1 matching lines for \"Paragraph 900\" in http://93.184.216.34/big]\n900: Paragraph 900 word");
   });
 
   test("web_search parses DuckDuckGo results and unwraps redirect hrefs", async () => {