/** * Data returned for each skill found in a GitHub repository. */ export interface GitHubSkillData { skillMdPath: string; markdown: string; inventory: string[]; owner: string; repo: string; } interface Logger { info: (...args: unknown[]) => void; warn: (...args: unknown[]) => void; error: (...args: unknown[]) => void; } /** 2 MB — generous upper bound for a single SKILL.md (markdown) file. */ export const MAX_SKILL_BYTES = 2 * 1024 * 1024; /** 32 MB — aggregate upper bound for all SKILL.md content from a single skill pack. */ export const MAX_TOTAL_BYTES = 32 * 1024 * 1024; /** 10 MB — upper bound for the recursive git tree JSON response. */ const MAX_TREE_BYTES = 10 * 1024 * 1024; /** * Read a fetch Response body with a byte-size ceiling. * Throws if the body exceeds `maxBytes`. */ async function readResponseWithLimit(response: Response, maxBytes: number): Promise { const contentLength = response.headers.get("Content-Length"); if (contentLength !== null && Number(contentLength) > maxBytes) { throw new Error(`Response too large: ${contentLength} bytes (max ${maxBytes})`); } const reader = response.body?.getReader(); if (!reader) { // Fallback: body not streamable (e.g. older Node fetch polyfill) const text = await response.text(); if (text.length > maxBytes) { throw new Error(`Response body too large: ${text.length} bytes (max ${maxBytes})`); } return text; } const decoder = new TextDecoder(); let totalBytes = 0; let result = ""; for (;;) { const { done, value } = await reader.read(); if (done) break; totalBytes += value.byteLength; if (totalBytes > maxBytes) { await reader.cancel(); throw new Error(`Response body exceeded ${maxBytes} bytes`); } result += decoder.decode(value, { stream: true }); } result += decoder.decode(); return result; } /** * Resolve the latest commit SHA for a given ref (branch, tag, or HEAD). * Returns the 40-character hex SHA on success, null on error. */ export async function resolveGitHubCommitSha( owner: string, repo: string, ref?: string, logger?: Logger, ): Promise { const target = ref || "HEAD"; const url = `https://api.github.com/repos/${owner}/${repo}/commits/${encodeURIComponent(target)}`; try { const response = await fetch(url, { headers: { Accept: "application/vnd.github.sha" }, }); if (!response.ok) { if (response.status === 403) { const resetHeader = response.headers.get("X-RateLimit-Reset"); logger?.warn( `GitHub API rate limit hit for ${owner}/${repo}`, resetHeader ? `resets at ${new Date(Number(resetHeader) * 1000).toISOString()}` : "", ); } else { logger?.warn(`GitHub commit resolve failed: ${response.status} for ${owner}/${repo}@${target}`); } return null; } const sha = (await readResponseWithLimit(response, MAX_SKILL_BYTES)).trim(); return /^[0-9a-f]{40}$/i.test(sha) ? sha : null; } catch (err) { logger?.error("GitHub commit resolve network error:", err); return null; } } /** * Fetch all SKILL.md files from a GitHub repository via the REST API. * * 1. GET the recursive git tree to find all SKILL.md paths * 2. Optionally filter to a specific skill slug directory * 3. Download each SKILL.md via raw.githubusercontent.com * 4. Collect sibling files as file inventory */ export async function fetchGitHubSkills( owner: string, repo: string, ref: string, requestedSlug: string | null, logger?: Logger, ): Promise<{ skills: GitHubSkillData[]; commitSha: string }> { const treeUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${encodeURIComponent(ref)}?recursive=1`; try { const treeResponse = await fetch(treeUrl, { headers: { Accept: "application/json" }, }); if (!treeResponse.ok) { logger?.warn(`GitHub tree fetch failed: ${treeResponse.status} for ${owner}/${repo}@${ref}`); return { skills: [], commitSha: ref }; } const treeData = JSON.parse(await readResponseWithLimit(treeResponse, MAX_TREE_BYTES)) as { tree?: Array<{ path: string; type: string }>; sha?: string; }; const treeSha = typeof treeData.sha === "string" ? treeData.sha : ref; const allEntries = (treeData.tree ?? []).filter( (entry) => entry.type === "blob" && typeof entry.path === "string", ); const allPaths = allEntries.map((entry) => entry.path); // Find SKILL.md files (at root or nested) let skillMdPaths = allPaths.filter( (p) => p.endsWith("/SKILL.md") || p === "SKILL.md", ); // Filter to requested slug directory if provided if (requestedSlug) { skillMdPaths = skillMdPaths.filter((p) => { const dir = p === "SKILL.md" ? "" : p.slice(0, p.lastIndexOf("/")); const dirName = dir.split("/").pop() ?? ""; return dirName.toLowerCase() === requestedSlug.toLowerCase(); }); } if (skillMdPaths.length === 0) { logger?.info(`No SKILL.md files found in ${owner}/${repo}@${ref}`); return { skills: [], commitSha: treeSha }; } const skills: GitHubSkillData[] = []; let totalBytes = 0; for (const skillMdPath of skillMdPaths) { try { // Download content via raw.githubusercontent.com (no API rate limit) const rawUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${skillMdPath}`; const contentResponse = await fetch(rawUrl); if (!contentResponse.ok) { logger?.warn(`Failed to download ${rawUrl}: ${contentResponse.status}`); continue; } const remainingBudget = MAX_TOTAL_BYTES - totalBytes; const perFileCap = Math.min(MAX_SKILL_BYTES, Math.max(0, remainingBudget)); if (perFileCap === 0) { throw new Error( `Skill pack exceeded aggregate limit of ${MAX_TOTAL_BYTES} bytes`, ); } const markdown = await readResponseWithLimit(contentResponse, perFileCap); totalBytes += Buffer.byteLength(markdown, "utf-8"); if (totalBytes > MAX_TOTAL_BYTES) { throw new Error( `Skill pack exceeded aggregate limit of ${MAX_TOTAL_BYTES} bytes`, ); } // Collect sibling files in the same directory (or subdirectories) const skillDir = skillMdPath === "SKILL.md" ? "" : skillMdPath.slice(0, skillMdPath.lastIndexOf("/")); const prefix = skillDir ? `${skillDir}/` : ""; const inventory = allPaths.filter( (p) => p !== skillMdPath && (prefix ? p.startsWith(prefix) : !p.includes("/")), ); skills.push({ skillMdPath, markdown, inventory, owner, repo, }); } catch (err) { logger?.error(`Error downloading skill ${skillMdPath}:`, err); continue; } } return { skills, commitSha: treeSha }; } catch (err) { logger?.error(`GitHub tree fetch network error for ${owner}/${repo}:`, err); return { skills: [], commitSha: ref }; } }