Every agent gets the live web: web_search and fetch_url by default
"Be ashamed to guess APIs blindly; be proud to look things up carefully" — until now the agents had no way to look anything up: no search, no fetch, so market research, API usage, and competitive claims were all invented from model memory. agent-web-tools.ts, carried by every agent in every run (same pattern as the workspace tools, no grant needed): - web_search: keyless DuckDuckGo HTML backend; parses titles, unwraps the /l/?uddg= redirect hrefs, pairs snippets; reports honestly when nothing parses instead of fabricating results. Verified against the live endpoint. - fetch_url: readable-text extraction (dependency-free HTML-to-text with entity decoding), 20k-char page cap, 2MB body cap read in chunks, 30s timeout. - SSRF guard on both: hostnames resolve before connecting and loopback, private, link-local, CGNAT, and reserved ranges are refused — re-checked on every redirect hop, so a public page cannot bounce the agent into 127.0.0.1 or the cloud metadata address. IPv6 and v4-mapped forms covered. - The fetcher is injectable, so the test suite covers the guard, redirect policy, truncation, extraction, and the DuckDuckGo parser without touching the network.
This commit is contained in:
ZacharyZhang-NY committed
1 parent
26de3129b1
commit
80e70649c3
4 files changed
+405
-2
No files matched your search
@@ -0,0 +1,290 @@
|
||||
import { lookup } from "node:dns/promises";
|
||||
import { isIP } from "node:net";
|
||||
import { requireString } from "./agent-tool-args";
|
||||
import type { ToolCall, ToolDefinition } from "./provider-tool-turn";
|
||||
|
||||
const FETCH_TIMEOUT_MS = 30_000;
|
||||
const MAX_REDIRECTS = 3;
|
||||
const MAX_RESPONSE_BYTES = 2_000_000;
|
||||
const PAGE_CHAR_LIMIT = 20_000;
|
||||
const SEARCH_RESULT_LIMIT = 8;
|
||||
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AgentCompany/1.0";
|
||||
|
||||
/** Subset of the fetch contract the tools rely on; injectable so tests stay off the network. */
|
||||
export type WebFetcher = (url: string, init: { headers: Record<string, string>; redirect: "manual"; signal: AbortSignal }) => Promise<Response>;
|
||||
|
||||
/**
|
||||
* Every agent carries live web access by default: web_search (keyless
|
||||
* DuckDuckGo HTML backend) and fetch_url (readable text extraction). The
|
||||
* doctrine is "be ashamed to guess, be proud to look things up" — an agent
|
||||
* that can browse verifies APIs and facts instead of inventing them.
|
||||
*
|
||||
* These tools run on the operator's machine, so every request is guarded
|
||||
* against SSRF: hostnames resolve before connecting and private, loopback,
|
||||
* link-local, and reserved ranges are refused — re-checked on every redirect
|
||||
* hop.
|
||||
*/
|
||||
export const WEB_TOOLS: ToolDefinition[] = [
|
||||
{
|
||||
name: "web_search",
|
||||
description: "Search the live web. Use it to look up APIs, libraries, prices, competitors, and current facts instead of guessing. Returns titles, URLs, and snippets — follow up with fetch_url to read a promising result in full.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
query: { type: "string", description: "The search query." },
|
||||
},
|
||||
required: ["query"],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "fetch_url",
|
||||
description: "Fetch a public web page and return its readable text. Use after web_search, or on any URL you were given. Only public hosts are reachable.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
url: { type: "string", description: "Absolute http(s) URL to fetch." },
|
||||
},
|
||||
required: ["url"],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const WEB_TOOL_NAMES = new Set(WEB_TOOLS.map((tool) => tool.name));
|
||||
|
||||
export function isWebTool(name: string): boolean {
|
||||
return WEB_TOOL_NAMES.has(name);
|
||||
}
|
||||
|
||||
export async function executeWebTool(call: ToolCall, fetcher: WebFetcher = fetch): Promise<string> {
|
||||
try {
|
||||
switch (call.name) {
|
||||
case "web_search":
|
||||
return await webSearch(requireString(call.arguments.query, "query"), fetcher);
|
||||
case "fetch_url":
|
||||
return await fetchUrl(requireString(call.arguments.url, "url"), fetcher);
|
||||
default:
|
||||
throw new Error(`Unknown web tool "${call.name}".`);
|
||||
}
|
||||
} catch (error) {
|
||||
return `ERROR: ${error instanceof Error ? error.message : String(error)}`;
|
||||
}
|
||||
}
|
||||
|
||||
async function webSearch(query: string, fetcher: WebFetcher): Promise<string> {
|
||||
const url = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
|
||||
const body = await guardedFetch(url, fetcher);
|
||||
const results = parseSearchResults(body.text);
|
||||
if (results.length === 0) {
|
||||
throw new Error("The search returned no parseable results — the backend may have blocked the request or changed its markup. Try fetch_url on a known site instead.");
|
||||
}
|
||||
return results
|
||||
.slice(0, SEARCH_RESULT_LIMIT)
|
||||
.map((result, index) => `${index + 1}. ${result.title}\n ${result.url}${result.snippet ? `\n ${result.snippet}` : ""}`)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
async function fetchUrl(rawUrl: string, fetcher: WebFetcher): Promise<string> {
|
||||
const body = await guardedFetch(rawUrl, fetcher);
|
||||
const text = looksLikeHtml(body.contentType, body.text) ? htmlToText(body.text) : body.text.trim();
|
||||
if (!text) return `${body.finalUrl} returned an empty page (status ${body.status}).`;
|
||||
const truncated = text.length > PAGE_CHAR_LIMIT
|
||||
? `${text.slice(0, PAGE_CHAR_LIMIT)}\n\n[truncated: showing the first ${PAGE_CHAR_LIMIT} of ${text.length} characters]`
|
||||
: text;
|
||||
return `${body.finalUrl} (status ${body.status})\n\n${truncated}`;
|
||||
}
|
||||
|
||||
interface GuardedResponse {
|
||||
status: number;
|
||||
contentType: string;
|
||||
finalUrl: string;
|
||||
text: string;
|
||||
}
|
||||
|
||||
/** Fetches with the SSRF guard applied to the initial URL and every redirect hop. */
|
||||
async function guardedFetch(rawUrl: string, fetcher: WebFetcher): Promise<GuardedResponse> {
|
||||
let current = rawUrl;
|
||||
for (let hop = 0; hop <= MAX_REDIRECTS; hop += 1) {
|
||||
const url = parseHttpUrl(current);
|
||||
await assertPublicHost(url.hostname);
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetcher(url.toString(), {
|
||||
headers: { "user-agent": USER_AGENT, accept: "text/html,application/xhtml+xml,text/plain,*/*" },
|
||||
redirect: "manual",
|
||||
signal: controller.signal,
|
||||
});
|
||||
} catch (error) {
|
||||
throw new Error(`Fetching ${url.hostname} failed: ${error instanceof Error ? error.message : String(error)}`);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
const location = response.headers.get("location");
|
||||
if (response.status >= 300 && response.status < 400 && location) {
|
||||
current = new URL(location, url).toString();
|
||||
continue;
|
||||
}
|
||||
const text = await readBodyCapped(response);
|
||||
return {
|
||||
status: response.status,
|
||||
contentType: response.headers.get("content-type") ?? "",
|
||||
finalUrl: url.toString(),
|
||||
text,
|
||||
};
|
||||
}
|
||||
throw new Error(`Too many redirects (more than ${MAX_REDIRECTS}).`);
|
||||
}
|
||||
|
||||
function parseHttpUrl(rawUrl: string): URL {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(rawUrl);
|
||||
} catch {
|
||||
throw new Error(`"${rawUrl}" is not a valid absolute URL.`);
|
||||
}
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
||||
throw new Error(`Only http(s) URLs are allowed, got "${url.protocol}".`);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
/** Refuses hosts that resolve to loopback, private, link-local, or otherwise non-public addresses. */
|
||||
async function assertPublicHost(hostname: string): Promise<void> {
|
||||
const bare = hostname.replace(/^\[|\]$/g, "");
|
||||
if (isIP(bare)) {
|
||||
if (isPrivateAddress(bare)) throw new Error(`Refusing to fetch non-public address "${bare}".`);
|
||||
return;
|
||||
}
|
||||
let addresses: Array<{ address: string }>;
|
||||
try {
|
||||
addresses = await lookup(bare, { all: true });
|
||||
} catch {
|
||||
throw new Error(`Could not resolve host "${bare}".`);
|
||||
}
|
||||
for (const { address } of addresses) {
|
||||
if (isPrivateAddress(address)) throw new Error(`Refusing to fetch "${bare}" — it resolves to the non-public address ${address}.`);
|
||||
}
|
||||
}
|
||||
|
||||
function isPrivateAddress(ip: string): boolean {
|
||||
if (isIP(ip) === 6) {
|
||||
const lower = ip.toLowerCase();
|
||||
const mapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/.exec(lower);
|
||||
if (mapped) return isPrivateAddress(mapped[1]);
|
||||
return lower === "::" || lower === "::1" || lower.startsWith("fc") || lower.startsWith("fd") || lower.startsWith("fe8") || lower.startsWith("fe9") || lower.startsWith("fea") || lower.startsWith("feb");
|
||||
}
|
||||
const octets = ip.split(".").map(Number);
|
||||
if (octets.length !== 4 || octets.some((octet) => Number.isNaN(octet))) return true;
|
||||
const [a, b] = octets;
|
||||
return (
|
||||
a === 0 || a === 10 || a === 127 ||
|
||||
(a === 100 && b >= 64 && b <= 127) ||
|
||||
(a === 169 && b === 254) ||
|
||||
(a === 172 && b >= 16 && b <= 31) ||
|
||||
(a === 192 && b === 168) ||
|
||||
(a === 198 && (b === 18 || b === 19)) ||
|
||||
a >= 224
|
||||
);
|
||||
}
|
||||
|
||||
async function readBodyCapped(response: Response): Promise<string> {
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) return "";
|
||||
const decoder = new TextDecoder("utf-8", { fatal: false });
|
||||
let text = "";
|
||||
let bytes = 0;
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
bytes += value.byteLength;
|
||||
text += decoder.decode(value, { stream: true });
|
||||
if (bytes >= MAX_RESPONSE_BYTES) {
|
||||
await reader.cancel();
|
||||
break;
|
||||
}
|
||||
}
|
||||
return text + decoder.decode();
|
||||
}
|
||||
|
||||
function looksLikeHtml(contentType: string, text: string): boolean {
|
||||
if (contentType.includes("html")) return true;
|
||||
return /<\s*(!doctype|html|body|div|p)\b/i.test(text.slice(0, 2000));
|
||||
}
|
||||
|
||||
/** Dependency-free readable-text extraction: good enough to study a page, not a browser. */
|
||||
export function htmlToText(html: string): string {
|
||||
const withoutBlocks = html
|
||||
.replace(/<script[\s\S]*?<\/script\s*>/gi, " ")
|
||||
.replace(/<style[\s\S]*?<\/style\s*>/gi, " ")
|
||||
.replace(/<noscript[\s\S]*?<\/noscript\s*>/gi, " ")
|
||||
.replace(/<!--[\s\S]*?-->/g, " ");
|
||||
const withBreaks = withoutBlocks
|
||||
.replace(/<\/(p|div|li|h[1-6]|tr|section|article|blockquote|pre)\s*>/gi, "\n")
|
||||
.replace(/<(br|hr)\s*\/?\s*>/gi, "\n");
|
||||
const withoutTags = withBreaks.replace(/<[^>]+>/g, " ");
|
||||
return decodeEntities(withoutTags)
|
||||
.split("\n")
|
||||
.map((line) => line.replace(/[ \t]+/g, " ").trim())
|
||||
.filter((line, index, lines) => line.length > 0 || (index > 0 && lines[index - 1].length > 0))
|
||||
.join("\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function decodeEntities(text: string): string {
|
||||
const named: Record<string, string> = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", mdash: "—", ndash: "–", hellip: "…", rsquo: "'", lsquo: "'", rdquo: '"', ldquo: '"' };
|
||||
return text
|
||||
.replace(/&#x([0-9a-f]+);/gi, (_, hex: string) => safeCodePoint(parseInt(hex, 16)))
|
||||
.replace(/&#(\d+);/g, (_, dec: string) => safeCodePoint(parseInt(dec, 10)))
|
||||
.replace(/&([a-z]+);/gi, (match, name: string) => named[name.toLowerCase()] ?? match);
|
||||
}
|
||||
|
||||
function safeCodePoint(code: number): string {
|
||||
return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : "";
|
||||
}
|
||||
|
||||
interface SearchResult {
|
||||
title: string;
|
||||
url: string;
|
||||
snippet: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses DuckDuckGo's HTML endpoint. Result links carry class "result__a";
|
||||
* their hrefs are usually /l/?uddg=<encoded-target> redirects, which are
|
||||
* unwrapped to the real URL. Snippets carry class "result__snippet".
|
||||
*/
|
||||
export function parseSearchResults(html: string): SearchResult[] {
|
||||
const results: SearchResult[] = [];
|
||||
const anchorPattern = /<a[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi;
|
||||
const snippetPattern = /class="[^"]*result__snippet[^"]*"[^>]*>([\s\S]*?)<\/(a|td|div|span)>/gi;
|
||||
const snippets: string[] = [];
|
||||
for (let match = snippetPattern.exec(html); match; match = snippetPattern.exec(html)) {
|
||||
snippets.push(htmlToText(match[1]));
|
||||
}
|
||||
for (let match = anchorPattern.exec(html); match; match = anchorPattern.exec(html)) {
|
||||
const target = unwrapDuckDuckGoHref(decodeEntities(match[1]));
|
||||
if (!target) continue;
|
||||
results.push({
|
||||
title: htmlToText(match[2]) || target,
|
||||
url: target,
|
||||
snippet: snippets[results.length] ?? "",
|
||||
});
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
function unwrapDuckDuckGoHref(href: string): string | null {
|
||||
const absolute = href.startsWith("//") ? `https:${href}` : href;
|
||||
try {
|
||||
const url = new URL(absolute, "https://duckduckgo.com");
|
||||
const wrapped = url.searchParams.get("uddg");
|
||||
if (wrapped) return decodeURIComponent(wrapped);
|
||||
if (url.hostname.endsWith("duckduckgo.com") && url.pathname.startsWith("/l/")) return null;
|
||||
return url.protocol === "http:" || url.protocol === "https:" ? url.toString() : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user