diff --git a/src/main/services/agent-run-loop.ts b/src/main/services/agent-run-loop.ts index 035fb32c..cd2bf93f 100644 --- a/src/main/services/agent-run-loop.ts +++ b/src/main/services/agent-run-loop.ts @@ -1,5 +1,6 @@ import type { ProviderService } from "./provider-service"; import { AGENT_TOOLS, dispatchAgentTool, type AgentToolContext, type AgentToolDeps } from "./agent-tools"; +import { executeWebTool, isWebTool, WEB_TOOLS } from "./agent-web-tools"; import { executeWorkspaceTool, isWorkspaceTool, WORKSPACE_TOOLS } from "./agent-workspace-tools"; import { canonicalMentionHandle } from "../../shared/mention-handles"; import type { AgentTurnMessage, ToolDefinition } from "./provider-tool-turn"; @@ -38,7 +39,7 @@ export interface AgentRunLoopDeps { * iteration cap is hit (which throws rather than silently truncating). */ export async function runAgentLoop(deps: AgentRunLoopDeps, input: AgentRunLoopInput): Promise { - const allTools = [...AGENT_TOOLS, ...WORKSPACE_TOOLS, ...(input.externalTools ?? [])]; + const allTools = [...AGENT_TOOLS, ...WORKSPACE_TOOLS, ...WEB_TOOLS, ...(input.externalTools ?? [])]; const messages: AgentTurnMessage[] = [{ role: "user", text: input.initialUserTurn }]; let toolCallCount = 0; let lastRequestId = ""; @@ -73,6 +74,8 @@ export async function runAgentLoop(deps: AgentRunLoopDeps, input: AgentRunLoopIn output = await executeComposioTool(deps.tools.db, input.orgId, input.toolContext.agent.id, call); } else if (isWorkspaceTool(call.name)) { output = await executeWorkspaceTool(input.toolContext, call); + } else if (isWebTool(call.name)) { + output = await executeWebTool(call); } else { output = dispatchAgentTool(deps.tools, input.toolContext, call, { announceTaskAssignments: shouldAnnounceTaskAssignment(call, turn.toolCalls), diff --git a/src/main/services/agent-web-tools.ts b/src/main/services/agent-web-tools.ts new file mode 100644 index 00000000..2e2cf894 --- /dev/null +++ b/src/main/services/agent-web-tools.ts @@ -0,0 +1,290 @@ +import { lookup } from "node:dns/promises"; +import { isIP } from "node:net"; +import { requireString } from "./agent-tool-args"; +import type { ToolCall, ToolDefinition } from "./provider-tool-turn"; + +const FETCH_TIMEOUT_MS = 30_000; +const MAX_REDIRECTS = 3; +const MAX_RESPONSE_BYTES = 2_000_000; +const PAGE_CHAR_LIMIT = 20_000; +const SEARCH_RESULT_LIMIT = 8; +const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AgentCompany/1.0"; + +/** Subset of the fetch contract the tools rely on; injectable so tests stay off the network. */ +export type WebFetcher = (url: string, init: { headers: Record; redirect: "manual"; signal: AbortSignal }) => Promise; + +/** + * Every agent carries live web access by default: web_search (keyless + * DuckDuckGo HTML backend) and fetch_url (readable text extraction). The + * doctrine is "be ashamed to guess, be proud to look things up" — an agent + * that can browse verifies APIs and facts instead of inventing them. + * + * These tools run on the operator's machine, so every request is guarded + * against SSRF: hostnames resolve before connecting and private, loopback, + * link-local, and reserved ranges are refused — re-checked on every redirect + * hop. + */ +export const WEB_TOOLS: ToolDefinition[] = [ + { + name: "web_search", + description: "Search the live web. Use it to look up APIs, libraries, prices, competitors, and current facts instead of guessing. Returns titles, URLs, and snippets — follow up with fetch_url to read a promising result in full.", + parameters: { + type: "object", + additionalProperties: false, + properties: { + query: { type: "string", description: "The search query." }, + }, + required: ["query"], + }, + }, + { + name: "fetch_url", + description: "Fetch a public web page and return its readable text. Use after web_search, or on any URL you were given. Only public hosts are reachable.", + parameters: { + type: "object", + additionalProperties: false, + properties: { + url: { type: "string", description: "Absolute http(s) URL to fetch." }, + }, + required: ["url"], + }, + }, +]; + +const WEB_TOOL_NAMES = new Set(WEB_TOOLS.map((tool) => tool.name)); + +export function isWebTool(name: string): boolean { + return WEB_TOOL_NAMES.has(name); +} + +export async function executeWebTool(call: ToolCall, fetcher: WebFetcher = fetch): Promise { + try { + switch (call.name) { + case "web_search": + return await webSearch(requireString(call.arguments.query, "query"), fetcher); + case "fetch_url": + return await fetchUrl(requireString(call.arguments.url, "url"), fetcher); + default: + throw new Error(`Unknown web tool "${call.name}".`); + } + } catch (error) { + return `ERROR: ${error instanceof Error ? error.message : String(error)}`; + } +} + +async function webSearch(query: string, fetcher: WebFetcher): Promise { + const url = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`; + const body = await guardedFetch(url, fetcher); + const results = parseSearchResults(body.text); + if (results.length === 0) { + throw new Error("The search returned no parseable results — the backend may have blocked the request or changed its markup. Try fetch_url on a known site instead."); + } + return results + .slice(0, SEARCH_RESULT_LIMIT) + .map((result, index) => `${index + 1}. ${result.title}\n ${result.url}${result.snippet ? `\n ${result.snippet}` : ""}`) + .join("\n\n"); +} + +async function fetchUrl(rawUrl: string, fetcher: WebFetcher): Promise { + const body = await guardedFetch(rawUrl, fetcher); + const text = looksLikeHtml(body.contentType, body.text) ? htmlToText(body.text) : body.text.trim(); + if (!text) return `${body.finalUrl} returned an empty page (status ${body.status}).`; + const truncated = text.length > PAGE_CHAR_LIMIT + ? `${text.slice(0, PAGE_CHAR_LIMIT)}\n\n[truncated: showing the first ${PAGE_CHAR_LIMIT} of ${text.length} characters]` + : text; + return `${body.finalUrl} (status ${body.status})\n\n${truncated}`; +} + +interface GuardedResponse { + status: number; + contentType: string; + finalUrl: string; + text: string; +} + +/** Fetches with the SSRF guard applied to the initial URL and every redirect hop. */ +async function guardedFetch(rawUrl: string, fetcher: WebFetcher): Promise { + let current = rawUrl; + for (let hop = 0; hop <= MAX_REDIRECTS; hop += 1) { + const url = parseHttpUrl(current); + await assertPublicHost(url.hostname); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); + let response: Response; + try { + response = await fetcher(url.toString(), { + headers: { "user-agent": USER_AGENT, accept: "text/html,application/xhtml+xml,text/plain,*/*" }, + redirect: "manual", + signal: controller.signal, + }); + } catch (error) { + throw new Error(`Fetching ${url.hostname} failed: ${error instanceof Error ? error.message : String(error)}`); + } finally { + clearTimeout(timer); + } + const location = response.headers.get("location"); + if (response.status >= 300 && response.status < 400 && location) { + current = new URL(location, url).toString(); + continue; + } + const text = await readBodyCapped(response); + return { + status: response.status, + contentType: response.headers.get("content-type") ?? "", + finalUrl: url.toString(), + text, + }; + } + throw new Error(`Too many redirects (more than ${MAX_REDIRECTS}).`); +} + +function parseHttpUrl(rawUrl: string): URL { + let url: URL; + try { + url = new URL(rawUrl); + } catch { + throw new Error(`"${rawUrl}" is not a valid absolute URL.`); + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error(`Only http(s) URLs are allowed, got "${url.protocol}".`); + } + return url; +} + +/** Refuses hosts that resolve to loopback, private, link-local, or otherwise non-public addresses. */ +async function assertPublicHost(hostname: string): Promise { + const bare = hostname.replace(/^\[|\]$/g, ""); + if (isIP(bare)) { + if (isPrivateAddress(bare)) throw new Error(`Refusing to fetch non-public address "${bare}".`); + return; + } + let addresses: Array<{ address: string }>; + try { + addresses = await lookup(bare, { all: true }); + } catch { + throw new Error(`Could not resolve host "${bare}".`); + } + for (const { address } of addresses) { + if (isPrivateAddress(address)) throw new Error(`Refusing to fetch "${bare}" — it resolves to the non-public address ${address}.`); + } +} + +function isPrivateAddress(ip: string): boolean { + if (isIP(ip) === 6) { + const lower = ip.toLowerCase(); + const mapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/.exec(lower); + if (mapped) return isPrivateAddress(mapped[1]); + return lower === "::" || lower === "::1" || lower.startsWith("fc") || lower.startsWith("fd") || lower.startsWith("fe8") || lower.startsWith("fe9") || lower.startsWith("fea") || lower.startsWith("feb"); + } + const octets = ip.split(".").map(Number); + if (octets.length !== 4 || octets.some((octet) => Number.isNaN(octet))) return true; + const [a, b] = octets; + return ( + a === 0 || a === 10 || a === 127 || + (a === 100 && b >= 64 && b <= 127) || + (a === 169 && b === 254) || + (a === 172 && b >= 16 && b <= 31) || + (a === 192 && b === 168) || + (a === 198 && (b === 18 || b === 19)) || + a >= 224 + ); +} + +async function readBodyCapped(response: Response): Promise { + const reader = response.body?.getReader(); + if (!reader) return ""; + const decoder = new TextDecoder("utf-8", { fatal: false }); + let text = ""; + let bytes = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + bytes += value.byteLength; + text += decoder.decode(value, { stream: true }); + if (bytes >= MAX_RESPONSE_BYTES) { + await reader.cancel(); + break; + } + } + return text + decoder.decode(); +} + +function looksLikeHtml(contentType: string, text: string): boolean { + if (contentType.includes("html")) return true; + return /<\s*(!doctype|html|body|div|p)\b/i.test(text.slice(0, 2000)); +} + +/** Dependency-free readable-text extraction: good enough to study a page, not a browser. */ +export function htmlToText(html: string): string { + const withoutBlocks = html + .replace(//gi, " ") + .replace(//gi, " ") + .replace(//gi, " ") + .replace(//g, " "); + const withBreaks = withoutBlocks + .replace(/<\/(p|div|li|h[1-6]|tr|section|article|blockquote|pre)\s*>/gi, "\n") + .replace(/<(br|hr)\s*\/?\s*>/gi, "\n"); + const withoutTags = withBreaks.replace(/<[^>]+>/g, " "); + return decodeEntities(withoutTags) + .split("\n") + .map((line) => line.replace(/[ \t]+/g, " ").trim()) + .filter((line, index, lines) => line.length > 0 || (index > 0 && lines[index - 1].length > 0)) + .join("\n") + .trim(); +} + +function decodeEntities(text: string): string { + const named: Record = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", mdash: "—", ndash: "–", hellip: "…", rsquo: "'", lsquo: "'", rdquo: '"', ldquo: '"' }; + return text + .replace(/&#x([0-9a-f]+);/gi, (_, hex: string) => safeCodePoint(parseInt(hex, 16))) + .replace(/&#(\d+);/g, (_, dec: string) => safeCodePoint(parseInt(dec, 10))) + .replace(/&([a-z]+);/gi, (match, name: string) => named[name.toLowerCase()] ?? match); +} + +function safeCodePoint(code: number): string { + return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : ""; +} + +interface SearchResult { + title: string; + url: string; + snippet: string; +} + +/** + * Parses DuckDuckGo's HTML endpoint. Result links carry class "result__a"; + * their hrefs are usually /l/?uddg= redirects, which are + * unwrapped to the real URL. Snippets carry class "result__snippet". + */ +export function parseSearchResults(html: string): SearchResult[] { + const results: SearchResult[] = []; + const anchorPattern = /]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi; + const snippetPattern = /class="[^"]*result__snippet[^"]*"[^>]*>([\s\S]*?)<\/(a|td|div|span)>/gi; + const snippets: string[] = []; + for (let match = snippetPattern.exec(html); match; match = snippetPattern.exec(html)) { + snippets.push(htmlToText(match[1])); + } + for (let match = anchorPattern.exec(html); match; match = anchorPattern.exec(html)) { + const target = unwrapDuckDuckGoHref(decodeEntities(match[1])); + if (!target) continue; + results.push({ + title: htmlToText(match[2]) || target, + url: target, + snippet: snippets[results.length] ?? "", + }); + } + return results; +} + +function unwrapDuckDuckGoHref(href: string): string | null { + const absolute = href.startsWith("//") ? `https:${href}` : href; + try { + const url = new URL(absolute, "https://duckduckgo.com"); + const wrapped = url.searchParams.get("uddg"); + if (wrapped) return decodeURIComponent(wrapped); + if (url.hostname.endsWith("duckduckgo.com") && url.pathname.startsWith("/l/")) return null; + return url.protocol === "http:" || url.protocol === "https:" ? url.toString() : null; + } catch { + return null; + } +} diff --git a/src/main/services/task-prompts.ts b/src/main/services/task-prompts.ts index 01bbdea3..2d123086 100644 --- a/src/main/services/task-prompts.ts +++ b/src/main/services/task-prompts.ts @@ -94,7 +94,7 @@ export function buildAgentInstructions(input: AgentInstructionContext): string { `You are ${agent.name}, ${agent.title || agent.role} in ${agent.department} at ${input.companyName}. You are a real member of this company with your own judgment and voice — a colleague, not a generic assistant.`, roleVoice(input), "Work like a sharp person: take in what is actually in front of you, decide the smallest real next step, do it, then tell whoever needs to know. Do not narrate busywork.", - `Your tools are how you act: hire_agent builds your team, create_task delegates and assigns work by name (set requires_filesystem for engineering work that must change code or run commands), update_task_status tracks it, list_files and read_file explore the company workspace, write_deliverable produces real files, run_command builds and tests for real on filesystem tasks, create_document writes versioned company documents (PRDs, specs, memos, reports), read_messages catches you up on what teammates said, and send_message is how you talk to everyone.`, + `Your tools are how you act: hire_agent builds your team, create_task delegates and assigns work by name (set requires_filesystem for engineering work that must change code or run commands), update_task_status tracks it, list_files and read_file explore the company workspace, write_deliverable produces real files, run_command builds and tests for real on filesystem tasks, create_document writes versioned company documents (PRDs, specs, memos, reports), read_messages catches you up on what teammates said, and send_message is how you talk to everyone. web_search and fetch_url reach the live web — look up APIs, docs, and facts instead of guessing them.`, ...(input.language === "zh" ? ["公司的工作语言是简体中文:你写的每一条消息、任务标题与描述、文档和交付物都用中文表达(代码、命令、API 名称等技术标识保持原样)。工具名称和工具参数的字段名仍按定义使用英文。"] : []), diff --git a/tests/unit/agent-web-tools.test.ts b/tests/unit/agent-web-tools.test.ts new file mode 100644 index 00000000..4eb270a4 --- /dev/null +++ b/tests/unit/agent-web-tools.test.ts @@ -0,0 +1,110 @@ +import { randomUUID } from "node:crypto"; +import { describe, expect, test } from "vitest"; +import { executeWebTool, htmlToText, isWebTool, parseSearchResults, WEB_TOOLS, type WebFetcher } from "../../src/main/services/agent-web-tools"; +import type { ToolCall } from "../../src/main/services/provider-tool-turn"; + +function call(name: string, args: Record): ToolCall { + return { id: `call_${randomUUID()}`, name, arguments: args }; +} + +function fetcherReturning(responses: Array<{ status?: number; headers?: Record; body?: string }>): { fetcher: WebFetcher; requested: string[] } { + const requested: string[] = []; + const fetcher: WebFetcher = async (url) => { + requested.push(url); + const next = responses.shift() ?? { status: 404, body: "not found" }; + return new Response(next.body ?? "", { status: next.status ?? 200, headers: next.headers ?? { "content-type": "text/html" } }); + }; + return { fetcher, requested }; +} + +const DDG_PAGE = ` + +`; + +describe("agent web tools", () => { + test("registers both tools and recognises them by name", () => { + expect(WEB_TOOLS.map((tool) => tool.name)).toEqual(["web_search", "fetch_url"]); + expect(isWebTool("web_search")).toBe(true); + expect(isWebTool("run_command")).toBe(false); + }); + + test("fetch_url refuses non-public hosts without ever connecting", async () => { + const { fetcher, requested } = fetcherReturning([]); + for (const url of [ + "http://127.0.0.1/admin", + "http://10.0.0.8/x", + "http://192.168.1.1/router", + "http://169.254.169.254/latest/meta-data", + "http://172.20.3.4/internal", + "http://[::1]/x", + "file:///etc/passwd", + ]) { + const output = await executeWebTool(call("fetch_url", { url }), fetcher); + expect(output, url).toContain("ERROR"); + } + expect(requested).toEqual([]); + }); + + test("fetch_url extracts readable text from HTML", async () => { + const { fetcher } = fetcherReturning([{ + body: "

Pricing & Plans

From $9/mo

", + }]); + const output = await executeWebTool(call("fetch_url", { url: "http://93.184.216.34/pricing" }), fetcher); + expect(output).toContain("Pricing & Plans"); + expect(output).toContain("From $9/mo"); + expect(output).not.toContain("alert(1)"); + expect(output).toContain("status 200"); + }); + + test("fetch_url follows public redirects but refuses a hop into a private range", async () => { + const followed = fetcherReturning([ + { status: 302, headers: { location: "http://93.184.216.35/final" } }, + { body: "

landed

" }, + ]); + const output = await executeWebTool(call("fetch_url", { url: "http://93.184.216.34/start" }), followed.fetcher); + expect(output).toContain("landed"); + expect(followed.requested).toHaveLength(2); + + const bounced = fetcherReturning([ + { status: 302, headers: { location: "http://169.254.169.254/creds" } }, + ]); + const blocked = await executeWebTool(call("fetch_url", { url: "http://93.184.216.34/start" }), bounced.fetcher); + expect(blocked).toContain("ERROR"); + expect(bounced.requested).toHaveLength(1); + }); + + test("fetch_url truncates oversized pages with an honest note", async () => { + const { fetcher } = fetcherReturning([{ body: `

${"word ".repeat(9000)}

` }]); + const output = await executeWebTool(call("fetch_url", { url: "http://93.184.216.34/big" }), fetcher); + expect(output).toContain("[truncated: showing the first"); + }); + + test("web_search parses DuckDuckGo results and unwraps redirect hrefs", async () => { + const { fetcher } = fetcherReturning([{ body: DDG_PAGE }]); + const output = await executeWebTool(call("web_search", { query: "example docs" }), fetcher); + expect(output).toContain("Example Docs"); + expect(output).toContain("https://example.com/docs"); + expect(output).toContain("The official documentation for Example."); + expect(output).toContain("https://other.example.org/page"); + }); + + test("web_search reports honestly when nothing parses", async () => { + const { fetcher } = fetcherReturning([{ body: "blocked" }]); + const output = await executeWebTool(call("web_search", { query: "anything" }), fetcher); + expect(output).toContain("ERROR"); + expect(output).toContain("no parseable results"); + }); + + test("htmlToText and parseSearchResults are exported pure helpers", () => { + expect(htmlToText("

a b

c © d

")).toBe("a b\nc © d"); + const results = parseSearchResults(DDG_PAGE); + expect(results).toHaveLength(2); + expect(results[0]).toMatchObject({ title: "Example Docs", url: "https://example.com/docs" }); + }); +});