chore: drop non-code junk (pi-mono-reference, runtime state, root duplicates)

Remove files that should never have been tracked:

- pi-mono-reference/: 688-file copy of an upstream monorepo, zero imports,
  used only as offline human reference
- build/electron-builder.runtime.json: environment-specific build artifact
  with hardcoded local paths
- state/cancel-signal-state.json: runtime state written by the app
- prompt.txt, logo.svg (root): exact duplicates of the canonical copies
  under .planning/archive/ and assets/

Add state/ to .gitignore so future runtime state stays out.
This commit is contained in:
ZacharyZhang-NY committed 2026-04-17 22:37:39 -04:00
1 parent 34ca5b77f1
commit 1f7230fe87
691 files changed
+3 -190434

No files matched your search

@@ -1,260 +0,0 @@
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { complete, stream } from "../src/stream.js";
import type { Api, Context, Model, StreamOptions } from "../src/types.js";
type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;
import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.js";
import { hasBedrockCredentials } from "./bedrock-utils.js";
import { resolveApiKey } from "./oauth.js";
// Resolve OAuth tokens at module level (async, runs before tests)
const [geminiCliToken, openaiCodexToken] = await Promise.all([
resolveApiKey("google-gemini-cli"),
resolveApiKey("openai-codex"),
]);
async function testAbortSignal<TApi extends Api>(llm: Model<TApi>, options: StreamOptionsWithExtras = {}) {
const context: Context = {
messages: [
{
role: "user",
content: "What is 15 + 27? Think step by step. Then list 50 first names.",
timestamp: Date.now(),
},
],
systemPrompt: "You are a helpful assistant.",
};
let abortFired = false;
let text = "";
const controller = new AbortController();
const response = await stream(llm, context, { ...options, signal: controller.signal });
for await (const event of response) {
if (abortFired) return;
if (event.type === "text_delta" || event.type === "thinking_delta") {
text += event.delta;
}
if (text.length >= 50) {
controller.abort();
abortFired = true;
}
}
const msg = await response.result();
// If we get here without throwing, the abort didn't work
expect(msg.stopReason).toBe("aborted");
expect(msg.content.length).toBeGreaterThan(0);
context.messages.push(msg);
context.messages.push({
role: "user",
content: "Please continue, but only generate 5 names.",
timestamp: Date.now(),
});
const followUp = await complete(llm, context, options);
expect(followUp.stopReason).toBe("stop");
expect(followUp.content.length).toBeGreaterThan(0);
}
async function testImmediateAbort<TApi extends Api>(llm: Model<TApi>, options: StreamOptionsWithExtras = {}) {
const controller = new AbortController();
controller.abort();
const context: Context = {
messages: [{ role: "user", content: "Hello", timestamp: Date.now() }],
};
const response = await complete(llm, context, { ...options, signal: controller.signal });
expect(response.stopReason).toBe("aborted");
}
async function testAbortThenNewMessage<TApi extends Api>(llm: Model<TApi>, options: StreamOptionsWithExtras = {}) {
// First request: abort immediately before any response content arrives
const controller = new AbortController();
controller.abort();
const context: Context = {
messages: [{ role: "user", content: "Hello, how are you?", timestamp: Date.now() }],
};
const abortedResponse = await complete(llm, context, { ...options, signal: controller.signal });
expect(abortedResponse.stopReason).toBe("aborted");
// The aborted message has empty content since we aborted before anything arrived
expect(abortedResponse.content.length).toBe(0);
// Add the aborted assistant message to context (this is what happens in the real coding agent)
context.messages.push(abortedResponse);
// Second request: send a new message - this should work even with the aborted message in context
context.messages.push({
role: "user",
content: "What is 2 + 2?",
timestamp: Date.now(),
});
const followUp = await complete(llm, context, options);
expect(followUp.stopReason).toBe("stop");
expect(followUp.content.length).toBeGreaterThan(0);
}
describe("AI Providers Abort Tests", () => {
describe.skipIf(!process.env.GEMINI_API_KEY)("Google Provider Abort", () => {
const llm = getModel("google", "gemini-2.5-flash");
it("should abort mid-stream", { retry: 3 }, async () => {
await testAbortSignal(llm, { thinking: { enabled: true } });
});
it("should handle immediate abort", { retry: 3 }, async () => {
await testImmediateAbort(llm, { thinking: { enabled: true } });
});
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions Provider Abort", () => {
const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!;
void _compat;
const llm: Model<"openai-completions"> = {
...baseModel,
api: "openai-completions",
};
it("should abort mid-stream", { retry: 3 }, async () => {
await testAbortSignal(llm);
});
it("should handle immediate abort", { retry: 3 }, async () => {
await testImmediateAbort(llm);
});
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses Provider Abort", () => {
const llm = getModel("openai", "gpt-5-mini");
it("should abort mid-stream", { retry: 3 }, async () => {
await testAbortSignal(llm);
});
it("should handle immediate abort", { retry: 3 }, async () => {
await testImmediateAbort(llm);
});
});
describe.skipIf(!hasAzureOpenAICredentials())("Azure OpenAI Responses Provider Abort", () => {
const llm = getModel("azure-openai-responses", "gpt-4o-mini");
const azureDeploymentName = resolveAzureDeploymentName(llm.id);
const azureOptions = azureDeploymentName ? { azureDeploymentName } : {};
it("should abort mid-stream", { retry: 3 }, async () => {
await testAbortSignal(llm, azureOptions);
});
it("should handle immediate abort", { retry: 3 }, async () => {
await testImmediateAbort(llm, azureOptions);
});
});
describe.skipIf(!process.env.ANTHROPIC_OAUTH_TOKEN)("Anthropic Provider Abort", () => {
const llm = getModel("anthropic", "claude-opus-4-1-20250805");
it("should abort mid-stream", { retry: 3 }, async () => {
await testAbortSignal(llm, { thinkingEnabled: true, thinkingBudgetTokens: 2048 });
});
it("should handle immediate abort", { retry: 3 }, async () => {
await testImmediateAbort(llm, { thinkingEnabled: true, thinkingBudgetTokens: 2048 });
});
});
describe.skipIf(!process.env.MISTRAL_API_KEY)("Mistral Provider Abort", () => {
const llm = getModel("mistral", "devstral-medium-latest");
it("should abort mid-stream", { retry: 3 }, async () => {
await testAbortSignal(llm);
});
it("should handle immediate abort", { retry: 3 }, async () => {
await testImmediateAbort(llm);
});
});
describe.skipIf(!process.env.MINIMAX_API_KEY)("MiniMax Provider Abort", () => {
const llm = getModel("minimax", "MiniMax-M2.7");
it("should abort mid-stream", { retry: 3 }, async () => {
await testAbortSignal(llm);
});
it("should handle immediate abort", { retry: 3 }, async () => {
await testImmediateAbort(llm);
});
});
describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider Abort", () => {
const llm = getModel("kimi-coding", "kimi-k2-thinking");
it("should abort mid-stream", { retry: 3 }, async () => {
await testAbortSignal(llm);
});
it("should handle immediate abort", { retry: 3 }, async () => {
await testImmediateAbort(llm);
});
});
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider Abort", () => {
const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash");
it("should abort mid-stream", { retry: 3 }, async () => {
await testAbortSignal(llm);
});
it("should handle immediate abort", { retry: 3 }, async () => {
await testImmediateAbort(llm);
});
});
// Google Gemini CLI / Antigravity share the same provider, so one test covers both
describe("Google Gemini CLI Provider Abort", () => {
it.skipIf(!geminiCliToken)("should abort mid-stream", { retry: 3 }, async () => {
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
await testAbortSignal(llm, { apiKey: geminiCliToken });
});
it.skipIf(!geminiCliToken)("should handle immediate abort", { retry: 3 }, async () => {
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
await testImmediateAbort(llm, { apiKey: geminiCliToken });
});
});
describe("OpenAI Codex Provider Abort", () => {
it.skipIf(!openaiCodexToken)("should abort mid-stream", { retry: 3 }, async () => {
const llm = getModel("openai-codex", "gpt-5.2-codex");
await testAbortSignal(llm, { apiKey: openaiCodexToken });
});
it.skipIf(!openaiCodexToken)("should handle immediate abort", { retry: 3 }, async () => {
const llm = getModel("openai-codex", "gpt-5.2-codex");
await testImmediateAbort(llm, { apiKey: openaiCodexToken });
});
});
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider Abort", () => {
const llm = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0");
it("should abort mid-stream", { retry: 3 }, async () => {
await testAbortSignal(llm, { reasoning: "medium" });
});
it("should handle immediate abort", { retry: 3 }, async () => {
await testImmediateAbort(llm);
});
it("should handle abort then new message", { retry: 3 }, async () => {
await testAbortThenNewMessage(llm);
});
});
});
@@ -1,99 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { loginAnthropic, refreshAnthropicToken } from "../src/utils/oauth/anthropic.js";
function jsonResponse(body: unknown, status: number = 200): Response {
return new Response(JSON.stringify(body), {
status,
headers: {
"Content-Type": "application/json",
},
});
}
function getUrl(input: unknown): string {
if (typeof input === "string") {
return input;
}
if (input instanceof URL) {
return input.toString();
}
if (input instanceof Request) {
return input.url;
}
throw new Error(`Unsupported fetch input: ${String(input)}`);
}
function getJsonBody(init?: RequestInit): Record<string, string> {
if (typeof init?.body !== "string") {
throw new Error(`Expected string request body, got ${typeof init?.body}`);
}
return JSON.parse(init.body) as Record<string, string>;
}
describe.sequential("Anthropic OAuth", () => {
afterEach(() => {
vi.unstubAllGlobals();
});
it("keeps the localhost redirect_uri for manual callback login", async () => {
let authUrl = "";
const fetchMock = vi.fn(async (input: unknown, init?: RequestInit): Promise<Response> => {
expect(getUrl(input)).toBe("https://platform.claude.com/v1/oauth/token");
expect(init?.method).toBe("POST");
const body = getJsonBody(init);
expect(body.grant_type).toBe("authorization_code");
expect(body.code).toBe("manual-code");
expect(body.redirect_uri).toBe("http://localhost:53692/callback");
return jsonResponse({
access_token: "access-token",
refresh_token: "refresh-token",
expires_in: 3600,
});
});
vi.stubGlobal("fetch", fetchMock);
const credentials = await loginAnthropic({
onAuth: (info) => {
authUrl = info.url;
},
onPrompt: async () => "",
onManualCodeInput: async () => {
const url = new URL(authUrl);
const state = url.searchParams.get("state");
const redirectUri = url.searchParams.get("redirect_uri");
if (!state || !redirectUri) {
throw new Error("Missing OAuth state or redirect_uri in auth URL");
}
return `${redirectUri}?code=manual-code&state=${state}`;
},
});
expect(credentials.access).toBe("access-token");
expect(credentials.refresh).toBe("refresh-token");
expect(fetchMock).toHaveBeenCalledOnce();
});
it("omits scope from refresh token requests", async () => {
const fetchMock = vi.fn(async (input: unknown, init?: RequestInit): Promise<Response> => {
expect(getUrl(input)).toBe("https://platform.claude.com/v1/oauth/token");
expect(init?.method).toBe("POST");
const body = getJsonBody(init);
expect(body.grant_type).toBe("refresh_token");
expect(body.client_id).toBeTruthy();
expect(body.refresh_token).toBe("refresh-token");
expect(body).not.toHaveProperty("scope");
return jsonResponse({
access_token: "new-access-token",
refresh_token: "new-refresh-token",
expires_in: 3600,
});
});
vi.stubGlobal("fetch", fetchMock);
const credentials = await refreshAnthropicToken("refresh-token");
expect(credentials.access).toBe("new-access-token");
expect(credentials.refresh).toBe("new-refresh-token");
expect(fetchMock).toHaveBeenCalledOnce();
});
});
@@ -1,72 +0,0 @@
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { streamSimple } from "../src/stream.js";
import type { Context } from "../src/types.js";
interface AnthropicThinkingPayload {
thinking?: { type: string };
output_config?: { effort?: string };
}
function makeContext(): Context {
return {
systemPrompt: "You are a precise assistant. Follow the user's instructions exactly.",
messages: [
{
role: "user",
content:
"Compute 48291 * 7317 and 90844 - 17729, add the results, and determine whether the sum is divisible by 11. Reply with exactly this format and nothing else: sum=<sum>; divisibleBy11=<yes|no>",
timestamp: Date.now(),
},
],
};
}
describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic Opus 4.7 smoke", () => {
it("streams Claude Opus 4.7 with reasoning enabled", { retry: 2, timeout: 30000 }, async () => {
const model = getModel("anthropic", "claude-opus-4-7");
let capturedPayload: AnthropicThinkingPayload | undefined;
const s = streamSimple(model, makeContext(), {
reasoning: "high",
maxTokens: 1024,
onPayload: (payload) => {
capturedPayload = payload as AnthropicThinkingPayload;
return payload;
},
});
let sawThinking = false;
for await (const event of s) {
if (event.type === "thinking_start" || event.type === "thinking_delta" || event.type === "thinking_end") {
sawThinking = true;
}
}
const response = await s.result();
expect(response.stopReason, response.errorMessage).toBe("stop");
expect(response.errorMessage).toBeFalsy();
expect(capturedPayload?.thinking).toEqual({ type: "adaptive" });
expect(capturedPayload?.output_config).toEqual({ effort: "high" });
expect(sawThinking).toBe(true);
const thinkingBlock = response.content.find((block) => block.type === "thinking");
expect(thinkingBlock?.type).toBe("thinking");
if (!thinkingBlock || thinkingBlock.type !== "thinking") {
throw new Error("Expected thinking block from Claude Opus 4.7");
}
expect(typeof thinkingBlock.thinkingSignature).toBe("string");
const thinkingSignature = thinkingBlock.thinkingSignature;
if (!thinkingSignature) {
throw new Error("Expected thinking signature from Claude Opus 4.7");
}
expect(thinkingSignature.length).toBeGreaterThan(0);
const text = response.content
.filter((block) => block.type === "text")
.map((block) => block.text)
.join("")
.trim();
expect(text).toBe("sum=353418362; divisibleBy11=yes");
});
});
@@ -1,152 +0,0 @@
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { streamSimple } from "../src/stream.js";
import type { Context, Model, SimpleStreamOptions } from "../src/types.js";
interface AnthropicThinkingPayload {
thinking?: { type: string; budget_tokens?: number };
output_config?: { effort?: string };
}
function makePayloadCaptureContext(): Context {
return {
messages: [{ role: "user", content: "Hello", timestamp: Date.now() }],
};
}
async function capturePayload(
model: Model<"anthropic-messages">,
options?: SimpleStreamOptions,
): Promise<AnthropicThinkingPayload> {
let capturedPayload: AnthropicThinkingPayload | undefined;
const payloadCaptureModel: Model<"anthropic-messages"> = {
...model,
baseUrl: "http://127.0.0.1:9",
};
const s = streamSimple(payloadCaptureModel, makePayloadCaptureContext(), {
...options,
apiKey: "fake-key",
onPayload: (payload) => {
capturedPayload = payload as AnthropicThinkingPayload;
return payload;
},
});
await s.result();
if (!capturedPayload) {
throw new Error("Expected payload to be captured before request failure");
}
return capturedPayload;
}
interface RunResult {
thinkingEventCount: number;
thinkingCharCount: number;
text: string;
contentTypes: string[];
}
function makeE2EContext(): Context {
return {
systemPrompt: "You are a precise assistant. Follow the requested output format exactly.",
messages: [
{
role: "user",
content:
"Before replying, carefully solve 36863 * 5279 internally. Then reply with the word pong repeated exactly 40 times, separated by single spaces. Do not add any other text.",
timestamp: Date.now(),
},
],
};
}
function countPongs(text: string): number {
return text.match(/\bpong\b/gi)?.length ?? 0;
}
async function runWithoutReasoning(model: Model<"anthropic-messages">): Promise<RunResult> {
const s = streamSimple(model, makeE2EContext(), {
temperature: 0,
maxTokens: 160,
});
let thinkingEventCount = 0;
let thinkingCharCount = 0;
for await (const event of s) {
if (event.type === "thinking_start" || event.type === "thinking_end") {
thinkingEventCount += 1;
}
if (event.type === "thinking_delta") {
thinkingEventCount += 1;
thinkingCharCount += event.delta.length;
}
}
const response = await s.result();
expect(response.stopReason, response.errorMessage).toBe("stop");
const text = response.content
.filter((block) => block.type === "text")
.map((block) => block.text)
.join("")
.trim();
return {
thinkingEventCount,
thinkingCharCount,
text,
contentTypes: response.content.map((block) => block.type),
};
}
describe("Anthropic thinking disable payload", () => {
it("sends thinking.type=disabled for budget-based reasoning models when thinking is off", async () => {
const payload = await capturePayload(getModel("anthropic", "claude-sonnet-4-5"));
expect(payload.thinking).toEqual({ type: "disabled" });
expect(payload.output_config).toBeUndefined();
});
it("sends thinking.type=disabled for adaptive reasoning models when thinking is off", async () => {
const payload = await capturePayload(getModel("anthropic", "claude-opus-4-6"));
expect(payload.thinking).toEqual({ type: "disabled" });
expect(payload.output_config).toBeUndefined();
});
it("sends thinking.type=disabled for Claude Opus 4.7 when thinking is off", async () => {
const payload = await capturePayload(getModel("anthropic", "claude-opus-4-7"));
expect(payload.thinking).toEqual({ type: "disabled" });
expect(payload.output_config).toBeUndefined();
});
it("uses adaptive thinking for Claude Opus 4.7 when reasoning is enabled", async () => {
const payload = await capturePayload(getModel("anthropic", "claude-opus-4-7"), { reasoning: "high" });
expect(payload.thinking).toEqual({ type: "adaptive" });
expect(payload.output_config).toEqual({ effort: "high" });
});
it("maps xhigh reasoning to effort=xhigh for Claude Opus 4.7", async () => {
const payload = await capturePayload(getModel("anthropic", "claude-opus-4-7"), { reasoning: "xhigh" });
expect(payload.thinking).toEqual({ type: "adaptive" });
expect(payload.output_config).toEqual({ effort: "xhigh" });
});
});
describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic thinking disable E2E", () => {
it("disables thinking for Claude reasoning models", { retry: 2, timeout: 30000 }, async () => {
const result = await runWithoutReasoning(getModel("anthropic", "claude-sonnet-4-5"));
expect(result.thinkingEventCount).toBe(0);
expect(result.thinkingCharCount).toBe(0);
expect(result.contentTypes).not.toContain("thinking");
expect(countPongs(result.text)).toBeGreaterThanOrEqual(35);
});
});
@@ -1,205 +0,0 @@
import { Type } from "@sinclair/typebox";
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { stream } from "../src/stream.js";
import type { Context, Tool } from "../src/types.js";
import { resolveApiKey } from "./oauth.js";
const oauthToken = await resolveApiKey("anthropic");
/**
* Tests for Anthropic OAuth tool name normalization.
*
* When using Claude Code OAuth, tool names must match CC's canonical casing.
* The normalization should:
* 1. Convert tool names that match CC tools (case-insensitive) to CC casing on outbound
* 2. Convert tool names back to the original casing on inbound
*
* This is a simple case-insensitive lookup, NOT a mapping of different names.
* e.g., "todowrite" -> "TodoWrite" -> "todowrite" (round-trip works)
*
* The old `find -> Glob` mapping was WRONG because:
* - Outbound: "find" -> "Glob"
* - Inbound: "Glob" -> ??? (no tool named "glob" in context.tools, only "find")
* - Result: tool call has name "Glob" but no tool exists with that name
*/
describe.skipIf(!oauthToken)("Anthropic OAuth tool name normalization", () => {
const model = getModel("anthropic", "claude-sonnet-4-20250514");
it("should normalize user-defined tool matching CC name (todowrite -> TodoWrite -> todowrite)", async () => {
// User defines a tool named "todowrite" (lowercase)
// CC has "TodoWrite" - this should round-trip correctly
const todoTool: Tool = {
name: "todowrite",
description: "Write a todo item",
parameters: Type.Object({
task: Type.String({ description: "The task to add" }),
}),
};
const context: Context = {
systemPrompt: "You are a helpful assistant. Use the todowrite tool when asked to add todos.",
messages: [
{
role: "user",
content: "Add a todo: buy milk. Use the todowrite tool.",
timestamp: Date.now(),
},
],
tools: [todoTool],
};
const s = stream(model, context, { apiKey: oauthToken });
let toolCallName: string | undefined;
for await (const event of s) {
if (event.type === "toolcall_end") {
const toolCall = event.partial.content[event.contentIndex];
if (toolCall.type === "toolCall") {
toolCallName = toolCall.name;
}
}
}
const response = await s.result();
expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("toolUse");
// The tool call should come back with the ORIGINAL name "todowrite", not "TodoWrite"
expect(toolCallName).toBe("todowrite");
});
it("should handle pi's built-in tools (read, write, edit, bash)", async () => {
// Pi's tools use lowercase names, CC uses PascalCase
const readTool: Tool = {
name: "read",
description: "Read a file",
parameters: Type.Object({
path: Type.String({ description: "File path" }),
}),
};
const context: Context = {
systemPrompt: "You are a helpful assistant. Use the read tool to read files.",
messages: [
{
role: "user",
content: "Read the file /tmp/test.txt using the read tool.",
timestamp: Date.now(),
},
],
tools: [readTool],
};
const s = stream(model, context, { apiKey: oauthToken });
let toolCallName: string | undefined;
for await (const event of s) {
if (event.type === "toolcall_end") {
const toolCall = event.partial.content[event.contentIndex];
if (toolCall.type === "toolCall") {
toolCallName = toolCall.name;
}
}
}
const response = await s.result();
expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("toolUse");
// The tool call should come back with the ORIGINAL name "read", not "Read"
expect(toolCallName).toBe("read");
});
it("should NOT map find to Glob - find is not a CC tool name", async () => {
// Pi has a "find" tool, CC has "Glob" - these are DIFFERENT tools
// The old code incorrectly mapped find -> Glob, which broke the round-trip
// because there's no tool named "glob" in context.tools
const findTool: Tool = {
name: "find",
description: "Find files by pattern",
parameters: Type.Object({
pattern: Type.String({ description: "Glob pattern" }),
}),
};
const context: Context = {
systemPrompt: "You are a helpful assistant. Use the find tool to search for files.",
messages: [
{
role: "user",
content: "Find all .ts files using the find tool.",
timestamp: Date.now(),
},
],
tools: [findTool],
};
const s = stream(model, context, { apiKey: oauthToken });
let toolCallName: string | undefined;
for await (const event of s) {
if (event.type === "toolcall_end") {
const toolCall = event.partial.content[event.contentIndex];
if (toolCall.type === "toolCall") {
toolCallName = toolCall.name;
}
}
}
const response = await s.result();
expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("toolUse");
// With the BROKEN find -> Glob mapping:
// - Sent as "Glob" to Anthropic
// - Received back as "Glob"
// - fromClaudeCodeName("Glob", tools) looks for tool.name.toLowerCase() === "glob"
// - No match (tool is named "find"), returns "Glob"
// - Test fails: toolCallName is "Glob" instead of "find"
//
// With the CORRECT implementation (no find->Glob mapping):
// - Sent as "find" to Anthropic (no CC tool named "Find")
// - Received back as "find"
// - Test passes: toolCallName is "find"
expect(toolCallName).toBe("find");
});
it("should handle custom tools that don't match any CC tool names", async () => {
// A completely custom tool should pass through unchanged
const customTool: Tool = {
name: "my_custom_tool",
description: "A custom tool",
parameters: Type.Object({
input: Type.String({ description: "Input value" }),
}),
};
const context: Context = {
systemPrompt: "You are a helpful assistant. Use my_custom_tool when asked.",
messages: [
{
role: "user",
content: "Use my_custom_tool with input 'hello'.",
timestamp: Date.now(),
},
],
tools: [customTool],
};
const s = stream(model, context, { apiKey: oauthToken });
let toolCallName: string | undefined;
for await (const event of s) {
if (event.type === "toolcall_end") {
const toolCall = event.partial.content[event.contentIndex];
if (toolCall.type === "toolCall") {
toolCallName = toolCall.name;
}
}
}
const response = await s.result();
expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("toolUse");
// Custom tool names should pass through unchanged
expect(toolCallName).toBe("my_custom_tool");
});
});
@@ -1,28 +0,0 @@
/**
* Utility functions for Azure OpenAI tests
*/
function parseDeploymentNameMap(value: string | undefined): Map<string, string> {
const map = new Map<string, string>();
if (!value) return map;
for (const entry of value.split(",")) {
const trimmed = entry.trim();
if (!trimmed) continue;
const [modelId, deploymentName] = trimmed.split("=", 2);
if (!modelId || !deploymentName) continue;
map.set(modelId.trim(), deploymentName.trim());
}
return map;
}
export function hasAzureOpenAICredentials(): boolean {
const hasKey = !!process.env.AZURE_OPENAI_API_KEY;
const hasBaseUrl = !!(process.env.AZURE_OPENAI_BASE_URL || process.env.AZURE_OPENAI_RESOURCE_NAME);
return hasKey && hasBaseUrl;
}
export function resolveAzureDeploymentName(modelId: string): string | undefined {
const mapValue = process.env.AZURE_OPENAI_DEPLOYMENT_NAME_MAP;
if (!mapValue) return undefined;
return parseDeploymentNameMap(mapValue).get(modelId);
}
@@ -1,66 +0,0 @@
/**
* A test suite to ensure all configured Amazon Bedrock models are usable.
*
* This is here to make sure we got correct model identifiers from models.dev and other sources.
* Because Amazon Bedrock requires cross-region inference in some models,
* plain model identifiers are not always usable and it requires tweaking of model identifiers to use cross-region inference.
* See https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-support.html#inference-profiles-support-system for more details.
*
* This test suite is not enabled by default unless AWS credentials and `BEDROCK_EXTENSIVE_MODEL_TEST` environment variables are set.
* This test suite takes ~2 minutes to run. Because not all models are available in all regions,
* it's recommended to use `us-west-2` region for best coverage for running this test suite.
*
* You can run this test suite with:
* ```bash
* $ AWS_REGION=us-west-2 BEDROCK_EXTENSIVE_MODEL_TEST=1 AWS_PROFILE=... npm test -- ./test/bedrock-models.test.ts
* ```
*/
import { describe, expect, it } from "vitest";
import { getModels } from "../src/models.js";
import { complete } from "../src/stream.js";
import type { Context } from "../src/types.js";
import { hasBedrockCredentials } from "./bedrock-utils.js";
describe("Amazon Bedrock Models", () => {
const models = getModels("amazon-bedrock");
it("should get all available Bedrock models", () => {
expect(models.length).toBeGreaterThan(0);
console.log(`Found ${models.length} Bedrock models`);
});
if (hasBedrockCredentials() && process.env.BEDROCK_EXTENSIVE_MODEL_TEST) {
for (const model of models) {
it(`should make a simple request with ${model.id}`, { timeout: 10_000 }, async () => {
const context: Context = {
systemPrompt: "You are a helpful assistant. Be extremely concise.",
messages: [
{
role: "user",
content: "Reply with exactly: 'OK'",
timestamp: Date.now(),
},
],
};
const response = await complete(model, context);
expect(response.role).toBe("assistant");
expect(response.content).toBeTruthy();
expect(response.content.length).toBeGreaterThan(0);
expect(response.usage.input + response.usage.cacheRead).toBeGreaterThan(0);
expect(response.usage.output).toBeGreaterThan(0);
expect(response.errorMessage).toBeFalsy();
const textContent = response.content
.filter((b) => b.type === "text")
.map((b) => (b.type === "text" ? b.text : ""))
.join("")
.trim();
expect(textContent).toBeTruthy();
console.log(`${model.id}: ${textContent.substring(0, 100)}`);
});
}
}
});
@@ -1,78 +0,0 @@
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { streamBedrock } from "../src/providers/amazon-bedrock.js";
import type { Context, Model, SimpleStreamOptions } from "../src/types.js";
interface BedrockThinkingPayload {
additionalModelRequestFields?: {
thinking?: { type: string; budget_tokens?: number };
output_config?: { effort?: string };
anthropic_beta?: string[];
};
}
function makeContext(): Context {
return {
messages: [{ role: "user", content: "Hello", timestamp: Date.now() }],
};
}
async function capturePayload(
model: Model<"bedrock-converse-stream">,
options?: SimpleStreamOptions,
): Promise<BedrockThinkingPayload> {
let capturedPayload: BedrockThinkingPayload | undefined;
const s = streamBedrock(model, makeContext(), {
...options,
reasoning: options?.reasoning ?? "high",
signal: AbortSignal.abort(),
onPayload: (payload) => {
capturedPayload = payload as BedrockThinkingPayload;
return payload;
},
});
for await (const event of s) {
if (event.type === "error") {
break;
}
}
if (!capturedPayload) {
throw new Error("Expected Bedrock payload to be captured before request abort");
}
return capturedPayload;
}
describe("Bedrock thinking payload", () => {
it("uses adaptive thinking for Claude Opus 4.7 when reasoning is enabled", async () => {
const baseModel = getModel("amazon-bedrock", "global.anthropic.claude-opus-4-6-v1");
const model: Model<"bedrock-converse-stream"> = {
...baseModel,
id: "global.anthropic.claude-opus-4-7-v1",
name: "Claude Opus 4.7 (Global)",
};
const payload = await capturePayload(model);
expect(payload.additionalModelRequestFields?.thinking).toEqual({ type: "adaptive" });
expect(payload.additionalModelRequestFields?.output_config).toEqual({ effort: "high" });
expect(payload.additionalModelRequestFields?.anthropic_beta).toBeUndefined();
});
it("maps xhigh reasoning to effort=xhigh for Claude Opus 4.7", async () => {
const baseModel = getModel("amazon-bedrock", "global.anthropic.claude-opus-4-6-v1");
const model: Model<"bedrock-converse-stream"> = {
...baseModel,
id: "global.anthropic.claude-opus-4-7-v1",
name: "Claude Opus 4.7 (Global)",
};
const payload = await capturePayload(model, { reasoning: "xhigh" });
expect(payload.additionalModelRequestFields?.thinking).toEqual({ type: "adaptive" });
expect(payload.additionalModelRequestFields?.output_config).toEqual({ effort: "xhigh" });
expect(payload.additionalModelRequestFields?.anthropic_beta).toBeUndefined();
});
});
@@ -1,18 +0,0 @@
/**
* Utility functions for Amazon Bedrock tests
*/
/**
* Check if any valid AWS credentials are configured for Bedrock.
* Returns true if any of the following are set:
* - AWS_PROFILE (named profile from ~/.aws/credentials)
* - AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY (IAM keys)
* - AWS_BEARER_TOKEN_BEDROCK (Bedrock API key)
*/
export function hasBedrockCredentials(): boolean {
return !!(
process.env.AWS_PROFILE ||
(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) ||
process.env.AWS_BEARER_TOKEN_BEDROCK
);
}
@@ -1,335 +0,0 @@
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { stream } from "../src/stream.js";
import type { Context } from "../src/types.js";
describe("Cache Retention (PI_CACHE_RETENTION)", () => {
const originalEnv = process.env.PI_CACHE_RETENTION;
beforeEach(() => {
delete process.env.PI_CACHE_RETENTION;
});
afterEach(() => {
if (originalEnv !== undefined) {
process.env.PI_CACHE_RETENTION = originalEnv;
} else {
delete process.env.PI_CACHE_RETENTION;
}
});
const context: Context = {
systemPrompt: "You are a helpful assistant.",
messages: [{ role: "user", content: "Hello", timestamp: Date.now() }],
};
describe("Anthropic Provider", () => {
it.skipIf(!process.env.ANTHROPIC_API_KEY)(
"should use default cache TTL (no ttl field) when PI_CACHE_RETENTION is not set",
async () => {
const model = getModel("anthropic", "claude-3-5-haiku-20241022");
let capturedPayload: any = null;
const s = stream(model, context, {
onPayload: (payload) => {
capturedPayload = payload;
},
});
// Consume the stream to trigger the request
for await (const _ of s) {
// Just consume
}
expect(capturedPayload).not.toBeNull();
// System prompt should have cache_control without ttl
expect(capturedPayload.system).toBeDefined();
expect(capturedPayload.system[0].cache_control).toEqual({ type: "ephemeral" });
},
);
it.skipIf(!process.env.ANTHROPIC_API_KEY)("should use 1h cache TTL when PI_CACHE_RETENTION=long", async () => {
process.env.PI_CACHE_RETENTION = "long";
const model = getModel("anthropic", "claude-3-5-haiku-20241022");
let capturedPayload: any = null;
const s = stream(model, context, {
onPayload: (payload) => {
capturedPayload = payload;
},
});
// Consume the stream to trigger the request
for await (const _ of s) {
// Just consume
}
expect(capturedPayload).not.toBeNull();
// System prompt should have cache_control with ttl: "1h"
expect(capturedPayload.system).toBeDefined();
expect(capturedPayload.system[0].cache_control).toEqual({ type: "ephemeral", ttl: "1h" });
});
it("should not add ttl when baseUrl is not api.anthropic.com", async () => {
process.env.PI_CACHE_RETENTION = "long";
// Create a model with a different baseUrl (simulating a proxy)
const baseModel = getModel("anthropic", "claude-3-5-haiku-20241022");
const proxyModel = {
...baseModel,
baseUrl: "https://my-proxy.example.com/v1",
};
let capturedPayload: any = null;
// We can't actually make the request (no proxy), but we can verify the payload
// by using a mock or checking the logic directly
// For this test, we'll import the helper directly
// Since we can't easily test this without mocking, we'll skip the actual API call
// and just verify the helper logic works correctly
const { streamAnthropic } = await import("../src/providers/anthropic.js");
try {
const s = streamAnthropic(proxyModel, context, {
apiKey: "fake-key",
onPayload: (payload) => {
capturedPayload = payload;
},
});
// This will fail since we're using a fake key and fake proxy, but the payload should be captured
for await (const event of s) {
if (event.type === "error") break;
}
} catch {
// Expected to fail
}
// The payload should have been captured before the error
if (capturedPayload) {
// System prompt should have cache_control WITHOUT ttl (proxy URL)
expect(capturedPayload.system[0].cache_control).toEqual({ type: "ephemeral" });
}
});
it("should omit cache_control when cacheRetention is none", async () => {
const baseModel = getModel("anthropic", "claude-3-5-haiku-20241022");
let capturedPayload: any = null;
const { streamAnthropic } = await import("../src/providers/anthropic.js");
try {
const s = streamAnthropic(baseModel, context, {
apiKey: "fake-key",
cacheRetention: "none",
onPayload: (payload) => {
capturedPayload = payload;
},
});
for await (const event of s) {
if (event.type === "error") break;
}
} catch {
// Expected to fail
}
expect(capturedPayload).not.toBeNull();
expect(capturedPayload.system[0].cache_control).toBeUndefined();
});
it("should add cache_control to string user messages", async () => {
const baseModel = getModel("anthropic", "claude-3-5-haiku-20241022");
let capturedPayload: any = null;
const { streamAnthropic } = await import("../src/providers/anthropic.js");
try {
const s = streamAnthropic(baseModel, context, {
apiKey: "fake-key",
onPayload: (payload) => {
capturedPayload = payload;
},
});
for await (const event of s) {
if (event.type === "error") break;
}
} catch {
// Expected to fail
}
expect(capturedPayload).not.toBeNull();
const lastMessage = capturedPayload.messages[capturedPayload.messages.length - 1];
expect(Array.isArray(lastMessage.content)).toBe(true);
const lastBlock = lastMessage.content[lastMessage.content.length - 1];
expect(lastBlock.cache_control).toEqual({ type: "ephemeral" });
});
it("should set 1h cache TTL when cacheRetention is long", async () => {
const baseModel = getModel("anthropic", "claude-3-5-haiku-20241022");
let capturedPayload: any = null;
const { streamAnthropic } = await import("../src/providers/anthropic.js");
try {
const s = streamAnthropic(baseModel, context, {
apiKey: "fake-key",
cacheRetention: "long",
onPayload: (payload) => {
capturedPayload = payload;
},
});
for await (const event of s) {
if (event.type === "error") break;
}
} catch {
// Expected to fail
}
expect(capturedPayload).not.toBeNull();
expect(capturedPayload.system[0].cache_control).toEqual({ type: "ephemeral", ttl: "1h" });
});
});
describe("OpenAI Responses Provider", () => {
it.skipIf(!process.env.OPENAI_API_KEY)(
"should not set prompt_cache_retention when PI_CACHE_RETENTION is not set",
async () => {
const model = getModel("openai", "gpt-4o-mini");
let capturedPayload: any = null;
const s = stream(model, context, {
onPayload: (payload) => {
capturedPayload = payload;
},
});
// Consume the stream to trigger the request
for await (const _ of s) {
// Just consume
}
expect(capturedPayload).not.toBeNull();
expect(capturedPayload.prompt_cache_retention).toBeUndefined();
},
);
it.skipIf(!process.env.OPENAI_API_KEY)(
"should set prompt_cache_retention to 24h when PI_CACHE_RETENTION=long",
async () => {
process.env.PI_CACHE_RETENTION = "long";
const model = getModel("openai", "gpt-4o-mini");
let capturedPayload: any = null;
const s = stream(model, context, {
onPayload: (payload) => {
capturedPayload = payload;
},
});
// Consume the stream to trigger the request
for await (const _ of s) {
// Just consume
}
expect(capturedPayload).not.toBeNull();
expect(capturedPayload.prompt_cache_retention).toBe("24h");
},
);
it("should not set prompt_cache_retention when baseUrl is not api.openai.com", async () => {
process.env.PI_CACHE_RETENTION = "long";
// Create a model with a different baseUrl (simulating a proxy)
const baseModel = getModel("openai", "gpt-4o-mini");
const proxyModel = {
...baseModel,
baseUrl: "https://my-proxy.example.com/v1",
};
let capturedPayload: any = null;
const { streamOpenAIResponses } = await import("../src/providers/openai-responses.js");
try {
const s = streamOpenAIResponses(proxyModel, context, {
apiKey: "fake-key",
onPayload: (payload) => {
capturedPayload = payload;
},
});
// This will fail since we're using a fake key and fake proxy, but the payload should be captured
for await (const event of s) {
if (event.type === "error") break;
}
} catch {
// Expected to fail
}
// The payload should have been captured before the error
if (capturedPayload) {
expect(capturedPayload.prompt_cache_retention).toBeUndefined();
}
});
it("should omit prompt_cache_key when cacheRetention is none", async () => {
const model = getModel("openai", "gpt-4o-mini");
let capturedPayload: any = null;
const { streamOpenAIResponses } = await import("../src/providers/openai-responses.js");
try {
const s = streamOpenAIResponses(model, context, {
apiKey: "fake-key",
cacheRetention: "none",
sessionId: "session-1",
onPayload: (payload) => {
capturedPayload = payload;
},
});
for await (const event of s) {
if (event.type === "error") break;
}
} catch {
// Expected to fail
}
expect(capturedPayload).not.toBeNull();
expect(capturedPayload.prompt_cache_key).toBeUndefined();
expect(capturedPayload.prompt_cache_retention).toBeUndefined();
});
it("should set prompt_cache_retention when cacheRetention is long", async () => {
const model = getModel("openai", "gpt-4o-mini");
let capturedPayload: any = null;
const { streamOpenAIResponses } = await import("../src/providers/openai-responses.js");
try {
const s = streamOpenAIResponses(model, context, {
apiKey: "fake-key",
cacheRetention: "long",
sessionId: "session-2",
onPayload: (payload) => {
capturedPayload = payload;
},
});
for await (const event of s) {
if (event.type === "error") break;
}
} catch {
// Expected to fail
}
expect(capturedPayload).not.toBeNull();
expect(capturedPayload.prompt_cache_key).toBe("session-2");
expect(capturedPayload.prompt_cache_retention).toBe("24h");
});
});
});
@@ -1,714 +0,0 @@
/**
* Test context overflow error handling across providers.
*
* Context overflow occurs when the input (prompt + history) exceeds
* the model's context window. This is different from output token limits.
*
* Expected behavior: All providers should return stopReason: "error"
* with an errorMessage that indicates the context was too large,
* OR (for z.ai) return successfully with usage.input > contextWindow.
*
* The isContextOverflow() function must return true for all providers.
*/
import type { ChildProcess } from "child_process";
import { execSync, spawn } from "child_process";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { complete } from "../src/stream.js";
import type { AssistantMessage, Context, Model, Usage } from "../src/types.js";
import { isContextOverflow } from "../src/utils/overflow.js";
import { hasAzureOpenAICredentials } from "./azure-utils.js";
import { hasBedrockCredentials } from "./bedrock-utils.js";
import { resolveApiKey } from "./oauth.js";
// Resolve OAuth tokens at module level (async, runs before tests)
const oauthTokens = await Promise.all([
resolveApiKey("github-copilot"),
resolveApiKey("google-gemini-cli"),
resolveApiKey("google-antigravity"),
resolveApiKey("openai-codex"),
]);
const [githubCopilotToken, geminiCliToken, antigravityToken, openaiCodexToken] = oauthTokens;
// Lorem ipsum paragraph for realistic token estimation
const LOREM_IPSUM = `Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. `;
// Generate a string that will exceed the context window
// Using chars/4 as token estimate (works better with varied text than repeated chars)
function generateOverflowContent(contextWindow: number): string {
const targetTokens = contextWindow + 10000; // Exceed by 10k tokens
const targetChars = targetTokens * 4 * 1.5;
const repetitions = Math.ceil(targetChars / LOREM_IPSUM.length);
return LOREM_IPSUM.repeat(repetitions);
}
interface OverflowResult {
provider: string;
model: string;
contextWindow: number;
stopReason: string;
errorMessage: string | undefined;
usage: Usage;
hasUsageData: boolean;
response: AssistantMessage;
}
async function testContextOverflow(model: Model<any>, apiKey: string): Promise<OverflowResult> {
const overflowContent = generateOverflowContent(model.contextWindow);
const context: Context = {
systemPrompt: "You are a helpful assistant.",
messages: [
{
role: "user",
content: overflowContent,
timestamp: Date.now(),
},
],
};
const response = await complete(model, context, { apiKey });
const hasUsageData = response.usage.input > 0 || response.usage.cacheRead > 0;
return {
provider: model.provider,
model: model.id,
contextWindow: model.contextWindow,
stopReason: response.stopReason,
errorMessage: response.errorMessage,
usage: response.usage,
hasUsageData,
response,
};
}
function logResult(result: OverflowResult) {
console.log(`\n${result.provider} / ${result.model}:`);
console.log(` contextWindow: ${result.contextWindow}`);
console.log(` stopReason: ${result.stopReason}`);
console.log(` errorMessage: ${result.errorMessage}`);
console.log(` usage: ${JSON.stringify(result.usage)}`);
console.log(` hasUsageData: ${result.hasUsageData}`);
}
// =============================================================================
// Anthropic
// Expected pattern: "prompt is too long: X tokens > Y maximum"
// =============================================================================
describe("Context overflow error handling", () => {
describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic (API Key)", () => {
it("claude-3-5-haiku - should detect overflow via isContextOverflow", async () => {
const model = getModel("anthropic", "claude-3-5-haiku-20241022");
const result = await testContextOverflow(model, process.env.ANTHROPIC_API_KEY!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(result.errorMessage).toMatch(/prompt is too long/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
});
describe.skipIf(!process.env.ANTHROPIC_OAUTH_TOKEN)("Anthropic (OAuth)", () => {
it("claude-sonnet-4 - should detect overflow via isContextOverflow", async () => {
const model = getModel("anthropic", "claude-sonnet-4-20250514");
const result = await testContextOverflow(model, process.env.ANTHROPIC_OAUTH_TOKEN!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(result.errorMessage).toMatch(/prompt is too long/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
});
// =============================================================================
// GitHub Copilot (OAuth)
// Tests both OpenAI and Anthropic models via Copilot
// =============================================================================
describe("GitHub Copilot (OAuth)", () => {
// OpenAI model via Copilot
it.skipIf(!githubCopilotToken)(
"gpt-4o - should detect overflow via isContextOverflow",
async () => {
const model = getModel("github-copilot", "gpt-4o");
const result = await testContextOverflow(model, githubCopilotToken!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(result.errorMessage).toMatch(/exceeds the limit of \d+/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
},
120000,
);
// Anthropic model via Copilot
it.skipIf(!githubCopilotToken)(
"claude-sonnet-4 - should detect overflow via isContextOverflow",
async () => {
const model = getModel("github-copilot", "claude-sonnet-4");
const result = await testContextOverflow(model, githubCopilotToken!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(result.errorMessage).toMatch(/exceeds the limit of \d+|input is too long/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
},
120000,
);
});
// =============================================================================
// OpenAI
// Expected pattern: "exceeds the context window"
// =============================================================================
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions", () => {
it("gpt-4o-mini - should detect overflow via isContextOverflow", async () => {
const model = { ...getModel("openai", "gpt-4o-mini") };
model.api = "openai-completions" as any;
const result = await testContextOverflow(model, process.env.OPENAI_API_KEY!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(result.errorMessage).toMatch(/maximum context length/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses", () => {
it("gpt-4o - should detect overflow via isContextOverflow", async () => {
const model = getModel("openai", "gpt-4o");
const result = await testContextOverflow(model, process.env.OPENAI_API_KEY!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(result.errorMessage).toMatch(/exceeds the context window/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
});
describe.skipIf(!hasAzureOpenAICredentials())("Azure OpenAI Responses", () => {
it("gpt-4o-mini - should detect overflow via isContextOverflow", async () => {
const model = getModel("azure-openai-responses", "gpt-4o-mini");
const result = await testContextOverflow(model, process.env.AZURE_OPENAI_API_KEY!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(result.errorMessage).toMatch(/context|maximum/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
});
// =============================================================================
// Google
// Expected pattern: "input token count (X) exceeds the maximum"
// =============================================================================
describe.skipIf(!process.env.GEMINI_API_KEY)("Google", () => {
it("gemini-2.0-flash - should detect overflow via isContextOverflow", async () => {
const model = getModel("google", "gemini-2.0-flash");
const result = await testContextOverflow(model, process.env.GEMINI_API_KEY!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(result.errorMessage).toMatch(/input token count.*exceeds the maximum/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
});
// =============================================================================
// Google Gemini CLI (OAuth)
// Uses same API as Google, expects same error pattern
// =============================================================================
describe("Google Gemini CLI (OAuth)", () => {
it.skipIf(!geminiCliToken)(
"gemini-2.5-flash - should detect overflow via isContextOverflow",
async () => {
const model = getModel("google-gemini-cli", "gemini-2.5-flash");
const result = await testContextOverflow(model, geminiCliToken!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(result.errorMessage).toMatch(/input token count.*exceeds the maximum/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
},
120000,
);
});
// =============================================================================
// Google Antigravity (OAuth)
// Tests both Gemini and Anthropic models via Antigravity
// =============================================================================
describe("Google Antigravity (OAuth)", () => {
// Gemini model
it.skipIf(!antigravityToken)(
"gemini-3-flash - should detect overflow via isContextOverflow",
async () => {
const model = getModel("google-antigravity", "gemini-3-flash");
const result = await testContextOverflow(model, antigravityToken!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(result.errorMessage).toMatch(/input token count.*exceeds the maximum/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
},
120000,
);
// Anthropic model via Antigravity
it.skipIf(!antigravityToken)(
"claude-sonnet-4-5 - should detect overflow via isContextOverflow",
async () => {
const model = getModel("google-antigravity", "claude-sonnet-4-5");
const result = await testContextOverflow(model, antigravityToken!);
logResult(result);
expect(result.stopReason).toBe("error");
// Anthropic models return "prompt is too long" pattern
expect(result.errorMessage).toMatch(/prompt is too long/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
},
120000,
);
});
// =============================================================================
// OpenAI Codex (OAuth)
// Uses ChatGPT Plus/Pro subscription via OAuth
// =============================================================================
describe("OpenAI Codex (OAuth)", () => {
it.skipIf(!openaiCodexToken)(
"gpt-5.2-codex - should detect overflow via isContextOverflow",
async () => {
const model = getModel("openai-codex", "gpt-5.2-codex");
const result = await testContextOverflow(model, openaiCodexToken!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
},
120000,
);
});
// =============================================================================
// Amazon Bedrock
// Expected pattern: "Input is too long for requested model"
// =============================================================================
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock", () => {
it("claude-sonnet-4-5 - should detect overflow via isContextOverflow", async () => {
const model = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0");
const result = await testContextOverflow(model, "");
logResult(result);
expect(result.stopReason).toBe("error");
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
});
// =============================================================================
// xAI
// Expected pattern: "maximum prompt length is X but the request contains Y"
// =============================================================================
describe.skipIf(!process.env.XAI_API_KEY)("xAI", () => {
it("grok-3-fast - should detect overflow via isContextOverflow", async () => {
const model = getModel("xai", "grok-3-fast");
const result = await testContextOverflow(model, process.env.XAI_API_KEY!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(result.errorMessage).toMatch(/maximum prompt length is \d+/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
});
// =============================================================================
// Groq
// Expected pattern: "reduce the length of the messages"
// =============================================================================
describe.skipIf(!process.env.GROQ_API_KEY)("Groq", () => {
it("llama-3.3-70b-versatile - should detect overflow via isContextOverflow", async () => {
const model = getModel("groq", "llama-3.3-70b-versatile");
const result = await testContextOverflow(model, process.env.GROQ_API_KEY!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(result.errorMessage).toMatch(/reduce the length of the messages/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
});
// =============================================================================
// Cerebras
// Expected: 400/413 status code with no body
// =============================================================================
describe.skipIf(!process.env.CEREBRAS_API_KEY)("Cerebras", () => {
it("qwen-3-235b - should detect overflow via isContextOverflow", async () => {
const model = getModel("cerebras", "qwen-3-235b-a22b-instruct-2507");
const result = await testContextOverflow(model, process.env.CEREBRAS_API_KEY!);
logResult(result);
expect(result.stopReason).toBe("error");
// Cerebras returns status code with no body (400, 413, or 429 for token rate limit)
expect(result.errorMessage).toMatch(/4(00|13|29).*\(no body\)/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
});
// =============================================================================
// Hugging Face
// Uses OpenAI-compatible Inference Router
// =============================================================================
describe.skipIf(!process.env.HF_TOKEN)("Hugging Face", () => {
it("Kimi-K2.5 - should detect overflow via isContextOverflow", async () => {
const model = getModel("huggingface", "moonshotai/Kimi-K2.5");
const result = await testContextOverflow(model, process.env.HF_TOKEN!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
});
// =============================================================================
// z.ai
// Special case: may return explicit overflow error text, may accept overflow silently,
// or may rate limit instead
// =============================================================================
describe.skipIf(!process.env.ZAI_API_KEY)("z.ai", () => {
it("glm-4.5-flash - should detect overflow via isContextOverflow when z.ai reports it", async () => {
const model = getModel("zai", "glm-4.5-flash");
const result = await testContextOverflow(model, process.env.ZAI_API_KEY!);
logResult(result);
// z.ai behavior is inconsistent:
// - Sometimes returns explicit overflow error text via non-standard finish_reason handling
// - Sometimes accepts overflow and returns successfully with usage.input > contextWindow
// - Sometimes returns rate limit error
if (result.stopReason === "error") {
if (result.errorMessage?.match(/model_context_window_exceeded/i)) {
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
} else {
console.log(" z.ai returned non-overflow error (possibly rate limited), skipping overflow detection");
}
} else if (result.stopReason === "stop") {
if (result.hasUsageData && result.usage.input > model.contextWindow) {
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
} else {
console.log(" z.ai returned stop without overflow usage data, skipping overflow detection");
}
}
}, 120000);
});
// =============================================================================
// Mistral
// =============================================================================
describe.skipIf(!process.env.MISTRAL_API_KEY)("Mistral", () => {
it("devstral-medium-latest - should detect overflow via isContextOverflow", async () => {
const model = getModel("mistral", "devstral-medium-latest");
const result = await testContextOverflow(model, process.env.MISTRAL_API_KEY!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(result.errorMessage).toMatch(/too large for model with \d+ maximum context length/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
});
// =============================================================================
// MiniMax
// Expected pattern: TBD - need to test actual error message
// =============================================================================
describe.skipIf(!process.env.MINIMAX_API_KEY)("MiniMax", () => {
it("MiniMax-M2.7 - should detect overflow via isContextOverflow", async () => {
const model = getModel("minimax", "MiniMax-M2.7");
const result = await testContextOverflow(model, process.env.MINIMAX_API_KEY!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
});
// =============================================================================
// Kimi For Coding
// =============================================================================
describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding", () => {
it("kimi-k2-thinking - should detect overflow via isContextOverflow", async () => {
const model = getModel("kimi-coding", "kimi-k2-thinking");
const result = await testContextOverflow(model, process.env.KIMI_API_KEY!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
});
// =============================================================================
// Vercel AI Gateway - Unified API for multiple providers
// =============================================================================
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway", () => {
it("google/gemini-2.5-flash via AI Gateway - should detect overflow via isContextOverflow", async () => {
const model = getModel("vercel-ai-gateway", "google/gemini-2.5-flash");
const result = await testContextOverflow(model, process.env.AI_GATEWAY_API_KEY!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
});
// =============================================================================
// OpenRouter - Multiple backend providers
// Expected pattern: "maximum context length is X tokens"
// =============================================================================
describe.skipIf(!process.env.OPENROUTER_API_KEY)("OpenRouter", () => {
// Anthropic backend
it("anthropic/claude-sonnet-4 via OpenRouter - should detect overflow via isContextOverflow", async () => {
const model = getModel("openrouter", "anthropic/claude-sonnet-4");
const result = await testContextOverflow(model, process.env.OPENROUTER_API_KEY!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(result.errorMessage).toMatch(/maximum context length is \d+ tokens/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
// DeepSeek backend
it("deepseek/deepseek-v3.2 via OpenRouter - should detect overflow via isContextOverflow", async () => {
const model = getModel("openrouter", "deepseek/deepseek-v3.2");
const result = await testContextOverflow(model, process.env.OPENROUTER_API_KEY!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(result.errorMessage).toMatch(/maximum context length is \d+ tokens/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
// Mistral backend
it("mistralai/mistral-large-2512 via OpenRouter - should detect overflow via isContextOverflow", async () => {
const model = getModel("openrouter", "mistralai/mistral-large-2512");
const result = await testContextOverflow(model, process.env.OPENROUTER_API_KEY!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(result.errorMessage).toMatch(/maximum context length is \d+ tokens/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
// Google backend
it("google/gemini-2.5-flash via OpenRouter - should detect overflow via isContextOverflow", async () => {
const model = getModel("openrouter", "google/gemini-2.5-flash");
const result = await testContextOverflow(model, process.env.OPENROUTER_API_KEY!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(result.errorMessage).toMatch(/maximum context length is \d+ tokens/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
// Meta/Llama backend
it("meta-llama/llama-4-maverick via OpenRouter - should detect overflow via isContextOverflow", async () => {
const model = getModel("openrouter", "meta-llama/llama-4-maverick");
const result = await testContextOverflow(model, process.env.OPENROUTER_API_KEY!);
logResult(result);
expect(result.stopReason).toBe("error");
expect(result.errorMessage).toMatch(/maximum context length is \d+ tokens/i);
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
});
// =============================================================================
// Ollama (local)
// =============================================================================
// Check if ollama is installed and local LLM tests are enabled
let ollamaInstalled = false;
if (!process.env.PI_NO_LOCAL_LLM) {
try {
execSync("which ollama", { stdio: "ignore" });
ollamaInstalled = true;
} catch {
ollamaInstalled = false;
}
}
describe.skipIf(!ollamaInstalled)("Ollama (local)", () => {
let ollamaProcess: ChildProcess | null = null;
let model: Model<"openai-completions">;
beforeAll(async () => {
// Check if model is available, if not pull it
try {
execSync("ollama list | grep -q 'gpt-oss:20b'", { stdio: "ignore" });
} catch {
console.log("Pulling gpt-oss:20b model for Ollama overflow tests...");
try {
execSync("ollama pull gpt-oss:20b", { stdio: "inherit" });
} catch (_e) {
console.warn("Failed to pull gpt-oss:20b model, tests will be skipped");
return;
}
}
// Start ollama server
ollamaProcess = spawn("ollama", ["serve"], {
detached: false,
stdio: "ignore",
});
// Wait for server to be ready
await new Promise<void>((resolve) => {
const checkServer = async () => {
try {
const response = await fetch("http://localhost:11434/api/tags");
if (response.ok) {
resolve();
} else {
setTimeout(checkServer, 500);
}
} catch {
setTimeout(checkServer, 500);
}
};
setTimeout(checkServer, 1000);
});
model = {
id: "gpt-oss:20b",
api: "openai-completions",
provider: "ollama",
baseUrl: "http://localhost:11434/v1",
reasoning: true,
input: ["text"],
contextWindow: 128000,
maxTokens: 16000,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
name: "Ollama GPT-OSS 20B",
};
}, 60000);
afterAll(() => {
if (ollamaProcess) {
ollamaProcess.kill("SIGTERM");
ollamaProcess = null;
}
});
it("gpt-oss:20b - should detect overflow via isContextOverflow (ollama silently truncates)", async () => {
const result = await testContextOverflow(model, "ollama");
logResult(result);
// Ollama silently truncates input instead of erroring
// It returns stopReason "stop" with truncated usage
// We cannot detect overflow via error message, only via usage comparison
if (result.stopReason === "stop" && result.hasUsageData) {
// Ollama truncated - check if reported usage is less than what we sent
// This is a "silent overflow" - we can detect it if we know expected input size
console.log(" Ollama silently truncated input to", result.usage.input, "tokens");
// For now, we accept this behavior - Ollama doesn't give us a way to detect overflow
} else if (result.stopReason === "error") {
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}
}, 300000); // 5 min timeout for local model
});
// =============================================================================
// LM Studio (local) - Skip if not running or local LLM tests disabled
// =============================================================================
let lmStudioRunning = false;
if (!process.env.PI_NO_LOCAL_LLM) {
try {
execSync("curl -s --max-time 1 http://localhost:1234/v1/models > /dev/null", { stdio: "ignore" });
lmStudioRunning = true;
} catch {
lmStudioRunning = false;
}
}
describe.skipIf(!lmStudioRunning)("LM Studio (local)", () => {
it("should detect overflow via isContextOverflow", async () => {
const model: Model<"openai-completions"> = {
id: "local-model",
api: "openai-completions",
provider: "lm-studio",
baseUrl: "http://localhost:1234/v1",
reasoning: false,
input: ["text"],
contextWindow: 8192,
maxTokens: 2048,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
name: "LM Studio Local Model",
};
const result = await testContextOverflow(model, "lm-studio");
logResult(result);
expect(result.stopReason).toBe("error");
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
});
// =============================================================================
// llama.cpp server (local) - Skip if not running or not exposing /v1/completions
// =============================================================================
let llamaCppRunning = false;
if (!process.env.PI_NO_LOCAL_LLM) {
try {
execSync("curl -s --max-time 1 http://localhost:8081/health > /dev/null", { stdio: "ignore" });
const probeStatus = execSync(
'curl -s --max-time 1 -o /dev/null -w \'%{http_code}\' -X POST http://localhost:8081/v1/completions -H \'content-type: application/json\' -d \'{"model":"local-model","prompt":"ping","max_tokens":1}\'',
{ encoding: "utf8" },
).trim();
llamaCppRunning = probeStatus !== "404" && probeStatus !== "405" && probeStatus !== "000";
} catch {
llamaCppRunning = false;
}
}
describe.skipIf(!llamaCppRunning)("llama.cpp (local)", () => {
it("should detect overflow via isContextOverflow", async () => {
// Using small context (4096) to match server --ctx-size setting
const model: Model<"openai-completions"> = {
id: "local-model",
api: "openai-completions",
provider: "llama.cpp",
baseUrl: "http://localhost:8081/v1",
reasoning: false,
input: ["text"],
contextWindow: 4096,
maxTokens: 2048,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
name: "llama.cpp Local Model",
};
const result = await testContextOverflow(model, "llama.cpp");
logResult(result);
expect(result.stopReason).toBe("error");
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
}, 120000);
});
});
@@ -1,458 +0,0 @@
/**
* Cross-Provider Handoff Test
*
* Tests that contexts generated by one provider/model can be consumed by another.
* This catches issues like:
* - Tool call ID format incompatibilities (e.g., OpenAI Codex pipe characters)
* - Thinking block transformation issues
* - Message format incompatibilities
*
* Strategy:
* 1. beforeAll: For each provider/model, generate a "small context" (if not cached):
* - User message asking to use a tool
* - Assistant response with thinking + tool call
* - Tool result
* - Final assistant response
*
* 2. Test: For each target provider/model:
* - Concatenate ALL other contexts into one
* - Ask the model to "say hi"
* - If it fails, there's a compatibility issue
*
* Fixtures are generated fresh on each run.
*/
import { Type } from "@sinclair/typebox";
import { writeFileSync } from "fs";
import { beforeAll, describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { completeSimple, getEnvApiKey } from "../src/stream.js";
import type { Api, AssistantMessage, Message, Model, Tool, ToolResultMessage } from "../src/types.js";
import { hasAzureOpenAICredentials } from "./azure-utils.js";
import { resolveApiKey } from "./oauth.js";
// Simple tool for testing
const testToolSchema = Type.Object({
value: Type.Number({ description: "A number to double" }),
});
const testTool: Tool<typeof testToolSchema> = {
name: "double_number",
description: "Doubles a number and returns the result",
parameters: testToolSchema,
};
// Provider/model pairs to test
interface ProviderModelPair {
provider: string;
model: string;
label: string;
apiOverride?: Api;
}
const PROVIDER_MODEL_PAIRS: ProviderModelPair[] = [
// Anthropic
{ provider: "anthropic", model: "claude-sonnet-4-5", label: "anthropic-claude-sonnet-4-5" },
// Google
{ provider: "google", model: "gemini-3-flash-preview", label: "google-gemini-3-flash-preview" },
// OpenAI
{
provider: "openai",
model: "gpt-4o-mini",
label: "openai-completions-gpt-4o-mini",
apiOverride: "openai-completions",
},
{ provider: "openai", model: "gpt-5-mini", label: "openai-responses-gpt-5-mini" },
{ provider: "azure-openai-responses", model: "gpt-4o-mini", label: "azure-openai-responses-gpt-4o-mini" },
// OpenAI Codex
{ provider: "openai-codex", model: "gpt-5.2-codex", label: "openai-codex-gpt-5.2-codex" },
// Google Antigravity
{ provider: "google-antigravity", model: "gemini-3-flash", label: "antigravity-gemini-3-flash" },
{ provider: "google-antigravity", model: "claude-sonnet-4-5", label: "antigravity-claude-sonnet-4-5" },
// GitHub Copilot
{ provider: "github-copilot", model: "claude-sonnet-4.5", label: "copilot-claude-sonnet-4.5" },
{ provider: "github-copilot", model: "gpt-5.1-codex", label: "copilot-gpt-5.1-codex" },
{ provider: "github-copilot", model: "gemini-3-flash-preview", label: "copilot-gemini-3-flash-preview" },
{ provider: "github-copilot", model: "grok-code-fast-1", label: "copilot-grok-code-fast-1" },
// Amazon Bedrock
{
provider: "amazon-bedrock",
model: "global.anthropic.claude-sonnet-4-5-20250929-v1:0",
label: "bedrock-claude-sonnet-4-5",
},
// xAI
{ provider: "xai", model: "grok-code-fast-1", label: "xai-grok-code-fast-1" },
// Cerebras
{ provider: "cerebras", model: "zai-glm-4.7", label: "cerebras-zai-glm-4.7" },
// Groq
{ provider: "groq", model: "openai/gpt-oss-120b", label: "groq-gpt-oss-120b" },
// Hugging Face
{ provider: "huggingface", model: "moonshotai/Kimi-K2.5", label: "huggingface-kimi-k2.5" },
// Kimi For Coding
{ provider: "kimi-coding", model: "kimi-k2-thinking", label: "kimi-coding-k2-thinking" },
// Mistral
{ provider: "mistral", model: "devstral-medium-latest", label: "mistral-devstral-medium" },
// MiniMax
{ provider: "minimax", model: "MiniMax-M2.7", label: "minimax-m2.7" },
// OpenCode Zen
{ provider: "opencode", model: "big-pickle", label: "zen-big-pickle" },
{ provider: "opencode", model: "claude-sonnet-4-5", label: "zen-claude-sonnet-4-5" },
{ provider: "opencode", model: "gemini-3-flash", label: "zen-gemini-3-flash" },
{ provider: "opencode", model: "glm-4.7-free", label: "zen-glm-4.7-free" },
{ provider: "opencode", model: "gpt-5.2-codex", label: "zen-gpt-5.2-codex" },
{ provider: "opencode", model: "minimax-m2.1-free", label: "zen-minimax-m2.1-free" },
// OpenCode Go
{ provider: "opencode-go", model: "kimi-k2.5", label: "go-kimi-k2.5" },
{ provider: "opencode-go", model: "minimax-m2.5", label: "go-minimax-m2.5" },
];
// Cached context structure
interface CachedContext {
label: string;
provider: string;
model: string;
api: Api;
messages: Message[];
generatedAt: string;
}
/**
* Get API key for provider - checks OAuth storage first, then env vars
*/
async function getApiKey(provider: string): Promise<string | undefined> {
const oauthKey = await resolveApiKey(provider);
if (oauthKey) return oauthKey;
return getEnvApiKey(provider);
}
/**
* Synchronous check for API key availability (env vars only, for skipIf)
*/
function hasApiKey(provider: string): boolean {
if (provider === "azure-openai-responses") {
return hasAzureOpenAICredentials();
}
return !!getEnvApiKey(provider);
}
/**
* Check if any provider has API keys available (for skipIf at describe level)
*/
function hasAnyApiKey(): boolean {
return PROVIDER_MODEL_PAIRS.some((pair) => hasApiKey(pair.provider));
}
function dumpFailurePayload(params: { label: string; error: string; payload?: unknown; messages: Message[] }): void {
const filename = `/tmp/pi-handoff-${params.label}-${Date.now()}.json`;
const body = {
label: params.label,
error: params.error,
payload: params.payload,
messages: params.messages,
};
writeFileSync(filename, JSON.stringify(body, null, 2));
console.log(`Wrote failure payload to ${filename}`);
}
/**
* Generate a context from a provider/model pair.
* Makes a real API call to get authentic tool call IDs and thinking blocks.
*/
async function generateContext(
pair: ProviderModelPair,
apiKey: string,
): Promise<{ messages: Message[]; api: Api } | null> {
const baseModel = (getModel as (p: string, m: string) => Model<Api> | undefined)(pair.provider, pair.model);
if (!baseModel) {
console.log(` Model not found: ${pair.provider}/${pair.model}`);
return null;
}
const model: Model<Api> = pair.apiOverride ? { ...baseModel, api: pair.apiOverride } : baseModel;
const userMessage: Message = {
role: "user",
content: "Please double the number 21 using the double_number tool.",
timestamp: Date.now(),
};
const supportsReasoning = model.reasoning === true;
let lastPayload: unknown;
let assistantResponse: AssistantMessage;
try {
assistantResponse = await completeSimple(
model,
{
systemPrompt: "You are a helpful assistant. Use the provided tool to complete the task.",
messages: [userMessage],
tools: [testTool],
},
{
apiKey,
reasoning: supportsReasoning ? "high" : undefined,
onPayload: (payload) => {
lastPayload = payload;
},
},
);
} catch (error) {
const msg = error instanceof Error ? error.message : String(error);
console.log(` Initial request failed: ${msg}`);
dumpFailurePayload({
label: `${pair.label}-initial`,
error: msg,
payload: lastPayload,
messages: [userMessage],
});
return null;
}
if (assistantResponse.stopReason === "error") {
console.log(` Initial request error: ${assistantResponse.errorMessage}`);
dumpFailurePayload({
label: `${pair.label}-initial`,
error: assistantResponse.errorMessage || "Unknown error",
payload: lastPayload,
messages: [userMessage],
});
return null;
}
const toolCall = assistantResponse.content.find((c) => c.type === "toolCall");
if (!toolCall || toolCall.type !== "toolCall") {
console.log(` No tool call in response (stopReason: ${assistantResponse.stopReason})`);
return {
messages: [userMessage, assistantResponse],
api: model.api,
};
}
console.log(` Tool call ID: ${toolCall.id}`);
const toolResult: ToolResultMessage = {
role: "toolResult",
toolCallId: toolCall.id,
toolName: toolCall.name,
content: [{ type: "text", text: "42" }],
isError: false,
timestamp: Date.now(),
};
let finalResponse: AssistantMessage;
const messagesForFinal = [userMessage, assistantResponse, toolResult];
try {
finalResponse = await completeSimple(
model,
{
systemPrompt: "You are a helpful assistant.",
messages: messagesForFinal,
tools: [testTool],
},
{
apiKey,
reasoning: supportsReasoning ? "high" : undefined,
onPayload: (payload) => {
lastPayload = payload;
},
},
);
} catch (error) {
const msg = error instanceof Error ? error.message : String(error);
console.log(` Final request failed: ${msg}`);
dumpFailurePayload({
label: `${pair.label}-final`,
error: msg,
payload: lastPayload,
messages: messagesForFinal,
});
return null;
}
if (finalResponse.stopReason === "error") {
console.log(` Final request error: ${finalResponse.errorMessage}`);
dumpFailurePayload({
label: `${pair.label}-final`,
error: finalResponse.errorMessage || "Unknown error",
payload: lastPayload,
messages: messagesForFinal,
});
return null;
}
return {
messages: [userMessage, assistantResponse, toolResult, finalResponse],
api: model.api,
};
}
describe.skipIf(!hasAnyApiKey())("Cross-Provider Handoff", () => {
let contexts: Record<string, CachedContext>;
let availablePairs: ProviderModelPair[];
beforeAll(async () => {
contexts = {};
availablePairs = [];
console.log("\n=== Generating Fixtures ===\n");
for (const pair of PROVIDER_MODEL_PAIRS) {
const apiKey = await getApiKey(pair.provider);
if (!apiKey) {
console.log(`[${pair.label}] Skipping - no auth for ${pair.provider}`);
continue;
}
console.log(`[${pair.label}] Generating fixture...`);
const result = await generateContext(pair, apiKey);
if (!result || result.messages.length < 4) {
console.log(`[${pair.label}] Failed to generate fixture, skipping`);
continue;
}
contexts[pair.label] = {
label: pair.label,
provider: pair.provider,
model: pair.model,
api: result.api,
messages: result.messages,
generatedAt: new Date().toISOString(),
};
availablePairs.push(pair);
console.log(`[${pair.label}] Generated ${result.messages.length} messages`);
}
console.log(`\n=== ${availablePairs.length}/${PROVIDER_MODEL_PAIRS.length} contexts available ===\n`);
}, 300000);
it.skipIf(!hasAnyApiKey())("should have at least 2 fixtures to test handoffs", () => {
expect(Object.keys(contexts).length).toBeGreaterThanOrEqual(2);
});
it.skipIf(!hasAnyApiKey())(
"should handle cross-provider handoffs for each target",
async () => {
const contextLabels = Object.keys(contexts);
if (contextLabels.length < 2) {
console.log("Not enough fixtures for handoff test, skipping");
return;
}
console.log("\n=== Testing Cross-Provider Handoffs ===\n");
const results: { target: string; success: boolean; error?: string }[] = [];
for (const targetPair of availablePairs) {
const apiKey = await getApiKey(targetPair.provider);
if (!apiKey) {
console.log(`[Target: ${targetPair.label}] Skipping - no auth`);
continue;
}
// Collect messages from ALL OTHER contexts
const otherMessages: Message[] = [];
for (const [label, ctx] of Object.entries(contexts)) {
if (label === targetPair.label) continue;
otherMessages.push(...ctx.messages);
}
if (otherMessages.length === 0) {
console.log(`[Target: ${targetPair.label}] Skipping - no other contexts`);
continue;
}
const allMessages: Message[] = [
...otherMessages,
{
role: "user",
content:
"Great, thanks for all that help! Now just say 'Hello, handoff successful!' to confirm you received everything.",
timestamp: Date.now(),
},
];
const baseModel = (getModel as (p: string, m: string) => Model<Api> | undefined)(
targetPair.provider,
targetPair.model,
);
if (!baseModel) {
console.log(`[Target: ${targetPair.label}] Model not found`);
continue;
}
const model: Model<Api> = targetPair.apiOverride
? { ...baseModel, api: targetPair.apiOverride }
: baseModel;
const supportsReasoning = model.reasoning === true;
console.log(
`[Target: ${targetPair.label}] Testing with ${otherMessages.length} messages from other providers...`,
);
let lastPayload: unknown;
try {
const response = await completeSimple(
model,
{
systemPrompt: "You are a helpful assistant.",
messages: allMessages,
tools: [testTool],
},
{
apiKey,
reasoning: supportsReasoning ? "high" : undefined,
onPayload: (payload) => {
lastPayload = payload;
},
},
);
if (response.stopReason === "error") {
console.log(`[Target: ${targetPair.label}] FAILED: ${response.errorMessage}`);
dumpFailurePayload({
label: targetPair.label,
error: response.errorMessage || "Unknown error",
payload: lastPayload,
messages: allMessages,
});
results.push({ target: targetPair.label, success: false, error: response.errorMessage });
} else {
const text = response.content
.filter((c) => c.type === "text")
.map((c) => c.text)
.join(" ");
const preview = text.slice(0, 100).replace(/\n/g, " ");
console.log(`[Target: ${targetPair.label}] SUCCESS: ${preview}...`);
results.push({ target: targetPair.label, success: true });
}
} catch (error) {
const msg = error instanceof Error ? error.message : String(error);
console.log(`[Target: ${targetPair.label}] EXCEPTION: ${msg}`);
dumpFailurePayload({
label: targetPair.label,
error: msg,
payload: lastPayload,
messages: allMessages,
});
results.push({ target: targetPair.label, success: false, error: msg });
}
}
console.log("\n=== Results Summary ===\n");
const successes = results.filter((r) => r.success);
const failures = results.filter((r) => !r.success);
console.log(`Passed: ${successes.length}/${results.length}`);
if (failures.length > 0) {
console.log("\nFailures:");
for (const f of failures) {
console.log(` - ${f.target}: ${f.error}`);
}
}
expect(failures.length).toBe(0);
},
600000,
);
});
Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.5 KiB

@@ -1,742 +0,0 @@
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { complete } from "../src/stream.js";
import type { Api, AssistantMessage, Context, Model, StreamOptions, UserMessage } from "../src/types.js";
type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;
import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.js";
import { hasBedrockCredentials } from "./bedrock-utils.js";
import { resolveApiKey } from "./oauth.js";
// Resolve OAuth tokens at module level (async, runs before tests)
const oauthTokens = await Promise.all([
resolveApiKey("anthropic"),
resolveApiKey("github-copilot"),
resolveApiKey("google-gemini-cli"),
resolveApiKey("google-antigravity"),
resolveApiKey("openai-codex"),
]);
const [anthropicOAuthToken, githubCopilotToken, geminiCliToken, antigravityToken, openaiCodexToken] = oauthTokens;
async function testEmptyMessage<TApi extends Api>(llm: Model<TApi>, options: StreamOptionsWithExtras = {}) {
// Test with completely empty content array
const emptyMessage: UserMessage = {
role: "user",
content: [],
timestamp: Date.now(),
};
const context: Context = {
messages: [emptyMessage],
};
const response = await complete(llm, context, options);
// Should either handle gracefully or return an error
expect(response).toBeDefined();
expect(response.role).toBe("assistant");
// Should handle empty string gracefully
if (response.stopReason === "error") {
expect(response.errorMessage).toBeDefined();
} else {
expect(response.content).toBeDefined();
}
}
async function testEmptyStringMessage<TApi extends Api>(llm: Model<TApi>, options: StreamOptionsWithExtras = {}) {
// Test with empty string content
const context: Context = {
messages: [
{
role: "user",
content: "",
timestamp: Date.now(),
},
],
};
const response = await complete(llm, context, options);
expect(response).toBeDefined();
expect(response.role).toBe("assistant");
// Should handle empty string gracefully
if (response.stopReason === "error") {
expect(response.errorMessage).toBeDefined();
} else {
expect(response.content).toBeDefined();
}
}
async function testWhitespaceOnlyMessage<TApi extends Api>(llm: Model<TApi>, options: StreamOptionsWithExtras = {}) {
// Test with whitespace-only content
const context: Context = {
messages: [
{
role: "user",
content: " \n\t ",
timestamp: Date.now(),
},
],
};
const response = await complete(llm, context, options);
expect(response).toBeDefined();
expect(response.role).toBe("assistant");
// Should handle whitespace-only gracefully
if (response.stopReason === "error") {
expect(response.errorMessage).toBeDefined();
} else {
expect(response.content).toBeDefined();
}
}
async function testEmptyAssistantMessage<TApi extends Api>(llm: Model<TApi>, options: StreamOptionsWithExtras = {}) {
// Test with empty assistant message in conversation flow
// User -> Empty Assistant -> User
const emptyAssistant: AssistantMessage = {
role: "assistant",
content: [],
api: llm.api,
provider: llm.provider,
model: llm.id,
usage: {
input: 10,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 10,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "stop",
timestamp: Date.now(),
};
const context: Context = {
messages: [
{
role: "user",
content: "Hello, how are you?",
timestamp: Date.now(),
},
emptyAssistant,
{
role: "user",
content: "Please respond this time.",
timestamp: Date.now(),
},
],
};
const response = await complete(llm, context, options);
expect(response).toBeDefined();
expect(response.role).toBe("assistant");
// Should handle empty assistant message in context gracefully
if (response.stopReason === "error") {
expect(response.errorMessage).toBeDefined();
} else {
expect(response.content).toBeDefined();
expect(response.content.length).toBeGreaterThan(0);
}
}
describe("AI Providers Empty Message Tests", () => {
describe.skipIf(!process.env.GEMINI_API_KEY)("Google Provider Empty Messages", () => {
const llm = getModel("google", "gemini-2.5-flash");
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
await testWhitespaceOnlyMessage(llm);
});
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
await testEmptyAssistantMessage(llm);
});
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions Provider Empty Messages", () => {
const llm = getModel("openai", "gpt-4o-mini");
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
await testWhitespaceOnlyMessage(llm);
});
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
await testEmptyAssistantMessage(llm);
});
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses Provider Empty Messages", () => {
const llm = getModel("openai", "gpt-5-mini");
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
await testWhitespaceOnlyMessage(llm);
});
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
await testEmptyAssistantMessage(llm);
});
});
describe.skipIf(!hasAzureOpenAICredentials())("Azure OpenAI Responses Provider Empty Messages", () => {
const llm = getModel("azure-openai-responses", "gpt-4o-mini");
const azureDeploymentName = resolveAzureDeploymentName(llm.id);
const azureOptions = azureDeploymentName ? { azureDeploymentName } : {};
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
await testEmptyMessage(llm, azureOptions);
});
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
await testEmptyStringMessage(llm, azureOptions);
});
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
await testWhitespaceOnlyMessage(llm, azureOptions);
});
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
await testEmptyAssistantMessage(llm, azureOptions);
});
});
describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic Provider Empty Messages", () => {
const llm = getModel("anthropic", "claude-3-5-haiku-20241022");
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
await testWhitespaceOnlyMessage(llm);
});
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
await testEmptyAssistantMessage(llm);
});
});
describe.skipIf(!process.env.XAI_API_KEY)("xAI Provider Empty Messages", () => {
const llm = getModel("xai", "grok-3");
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
await testWhitespaceOnlyMessage(llm);
});
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
await testEmptyAssistantMessage(llm);
});
});
describe.skipIf(!process.env.GROQ_API_KEY)("Groq Provider Empty Messages", () => {
const llm = getModel("groq", "openai/gpt-oss-20b");
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
await testWhitespaceOnlyMessage(llm);
});
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
await testEmptyAssistantMessage(llm);
});
});
describe.skipIf(!process.env.CEREBRAS_API_KEY)("Cerebras Provider Empty Messages", () => {
const llm = getModel("cerebras", "gpt-oss-120b");
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
await testWhitespaceOnlyMessage(llm);
});
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
await testEmptyAssistantMessage(llm);
});
});
describe.skipIf(!process.env.HF_TOKEN)("Hugging Face Provider Empty Messages", () => {
const llm = getModel("huggingface", "moonshotai/Kimi-K2.5");
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
await testWhitespaceOnlyMessage(llm);
});
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
await testEmptyAssistantMessage(llm);
});
});
describe.skipIf(!process.env.ZAI_API_KEY)("zAI Provider Empty Messages", () => {
const llm = getModel("zai", "glm-4.5-air");
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
await testWhitespaceOnlyMessage(llm);
});
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
await testEmptyAssistantMessage(llm);
});
});
describe.skipIf(!process.env.MISTRAL_API_KEY)("Mistral Provider Empty Messages", () => {
const llm = getModel("mistral", "devstral-medium-latest");
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
await testWhitespaceOnlyMessage(llm);
});
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
await testEmptyAssistantMessage(llm);
});
});
describe.skipIf(!process.env.MINIMAX_API_KEY)("MiniMax Provider Empty Messages", () => {
const llm = getModel("minimax", "MiniMax-M2.7");
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
await testWhitespaceOnlyMessage(llm);
});
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
await testEmptyAssistantMessage(llm);
});
});
describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider Empty Messages", () => {
const llm = getModel("kimi-coding", "kimi-k2-thinking");
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
await testWhitespaceOnlyMessage(llm);
});
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
await testEmptyAssistantMessage(llm);
});
});
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider Empty Messages", () => {
const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash");
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
await testWhitespaceOnlyMessage(llm);
});
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
await testEmptyAssistantMessage(llm);
});
});
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider Empty Messages", () => {
const llm = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0");
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
await testWhitespaceOnlyMessage(llm);
});
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
await testEmptyAssistantMessage(llm);
});
});
// =========================================================================
// OAuth-based providers (credentials from ~/.pi/agent/oauth.json)
// =========================================================================
describe("Anthropic OAuth Provider Empty Messages", () => {
const llm = getModel("anthropic", "claude-3-5-haiku-20241022");
it.skipIf(!anthropicOAuthToken)("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
await testEmptyMessage(llm, { apiKey: anthropicOAuthToken });
});
it.skipIf(!anthropicOAuthToken)("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
await testEmptyStringMessage(llm, { apiKey: anthropicOAuthToken });
});
it.skipIf(!anthropicOAuthToken)(
"should handle whitespace-only content",
{ retry: 3, timeout: 30000 },
async () => {
await testWhitespaceOnlyMessage(llm, { apiKey: anthropicOAuthToken });
},
);
it.skipIf(!anthropicOAuthToken)(
"should handle empty assistant message in conversation",
{ retry: 3, timeout: 30000 },
async () => {
await testEmptyAssistantMessage(llm, { apiKey: anthropicOAuthToken });
},
);
});
describe("GitHub Copilot Provider Empty Messages", () => {
it.skipIf(!githubCopilotToken)(
"gpt-4o - should handle empty content array",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "gpt-4o");
await testEmptyMessage(llm, { apiKey: githubCopilotToken });
},
);
it.skipIf(!githubCopilotToken)(
"gpt-4o - should handle empty string content",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "gpt-4o");
await testEmptyStringMessage(llm, { apiKey: githubCopilotToken });
},
);
it.skipIf(!githubCopilotToken)(
"gpt-4o - should handle whitespace-only content",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "gpt-4o");
await testWhitespaceOnlyMessage(llm, { apiKey: githubCopilotToken });
},
);
it.skipIf(!githubCopilotToken)(
"gpt-4o - should handle empty assistant message in conversation",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "gpt-4o");
await testEmptyAssistantMessage(llm, { apiKey: githubCopilotToken });
},
);
it.skipIf(!githubCopilotToken)(
"claude-sonnet-4 - should handle empty content array",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "claude-sonnet-4");
await testEmptyMessage(llm, { apiKey: githubCopilotToken });
},
);
it.skipIf(!githubCopilotToken)(
"claude-sonnet-4 - should handle empty string content",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "claude-sonnet-4");
await testEmptyStringMessage(llm, { apiKey: githubCopilotToken });
},
);
it.skipIf(!githubCopilotToken)(
"claude-sonnet-4 - should handle whitespace-only content",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "claude-sonnet-4");
await testWhitespaceOnlyMessage(llm, { apiKey: githubCopilotToken });
},
);
it.skipIf(!githubCopilotToken)(
"claude-sonnet-4 - should handle empty assistant message in conversation",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "claude-sonnet-4");
await testEmptyAssistantMessage(llm, { apiKey: githubCopilotToken });
},
);
});
describe("Google Gemini CLI Provider Empty Messages", () => {
it.skipIf(!geminiCliToken)(
"gemini-2.5-flash - should handle empty content array",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
await testEmptyMessage(llm, { apiKey: geminiCliToken });
},
);
it.skipIf(!geminiCliToken)(
"gemini-2.5-flash - should handle empty string content",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
await testEmptyStringMessage(llm, { apiKey: geminiCliToken });
},
);
it.skipIf(!geminiCliToken)(
"gemini-2.5-flash - should handle whitespace-only content",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
await testWhitespaceOnlyMessage(llm, { apiKey: geminiCliToken });
},
);
it.skipIf(!geminiCliToken)(
"gemini-2.5-flash - should handle empty assistant message in conversation",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
await testEmptyAssistantMessage(llm, { apiKey: geminiCliToken });
},
);
});
describe("Google Antigravity Provider Empty Messages", () => {
it.skipIf(!antigravityToken)(
"gemini-3-flash - should handle empty content array",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "gemini-3-flash");
await testEmptyMessage(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"gemini-3-flash - should handle empty string content",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "gemini-3-flash");
await testEmptyStringMessage(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"gemini-3-flash - should handle whitespace-only content",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "gemini-3-flash");
await testWhitespaceOnlyMessage(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"gemini-3-flash - should handle empty assistant message in conversation",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "gemini-3-flash");
await testEmptyAssistantMessage(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"claude-sonnet-4-5 - should handle empty content array",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
await testEmptyMessage(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"claude-sonnet-4-5 - should handle empty string content",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
await testEmptyStringMessage(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"claude-sonnet-4-5 - should handle whitespace-only content",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
await testWhitespaceOnlyMessage(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"claude-sonnet-4-5 - should handle empty assistant message in conversation",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
await testEmptyAssistantMessage(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"gpt-oss-120b-medium - should handle empty content array",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "gpt-oss-120b-medium");
await testEmptyMessage(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"gpt-oss-120b-medium - should handle empty string content",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "gpt-oss-120b-medium");
await testEmptyStringMessage(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"gpt-oss-120b-medium - should handle whitespace-only content",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "gpt-oss-120b-medium");
await testWhitespaceOnlyMessage(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"gpt-oss-120b-medium - should handle empty assistant message in conversation",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "gpt-oss-120b-medium");
await testEmptyAssistantMessage(llm, { apiKey: antigravityToken });
},
);
});
describe("OpenAI Codex Provider Empty Messages", () => {
it.skipIf(!openaiCodexToken)(
"gpt-5.2-codex - should handle empty content array",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("openai-codex", "gpt-5.2-codex");
await testEmptyMessage(llm, { apiKey: openaiCodexToken });
},
);
it.skipIf(!openaiCodexToken)(
"gpt-5.2-codex - should handle empty string content",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("openai-codex", "gpt-5.2-codex");
await testEmptyStringMessage(llm, { apiKey: openaiCodexToken });
},
);
it.skipIf(!openaiCodexToken)(
"gpt-5.2-codex - should handle whitespace-only content",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("openai-codex", "gpt-5.2-codex");
await testWhitespaceOnlyMessage(llm, { apiKey: openaiCodexToken });
},
);
it.skipIf(!openaiCodexToken)(
"gpt-5.2-codex - should handle empty assistant message in conversation",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("openai-codex", "gpt-5.2-codex");
await testEmptyAssistantMessage(llm, { apiKey: openaiCodexToken });
},
);
});
});
@@ -1,597 +0,0 @@
import { afterEach, describe, expect, it } from "vitest";
import {
complete,
fauxAssistantMessage,
fauxText,
fauxThinking,
fauxToolCall,
registerFauxProvider,
stream,
Type,
} from "../src/index.js";
import type { AssistantMessageEvent, Context } from "../src/types.js";
async function collectEvents(streamResult: ReturnType<typeof stream>): Promise<AssistantMessageEvent[]> {
const events: AssistantMessageEvent[] = [];
for await (const event of streamResult) {
events.push(event);
}
return events;
}
const registrations: Array<{ unregister: () => void }> = [];
afterEach(() => {
for (const registration of registrations.splice(0)) {
registration.unregister();
}
});
describe("faux provider", () => {
it("registers a custom provider and estimates usage", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
registration.setResponses([fauxAssistantMessage("hello world")]);
const context: Context = {
systemPrompt: "Be concise.",
messages: [{ role: "user", content: "hi there", timestamp: Date.now() }],
};
const response = await complete(registration.getModel(), context);
expect(response.content).toEqual([{ type: "text", text: "hello world" }]);
expect(response.usage.input).toBeGreaterThan(0);
expect(response.usage.output).toBeGreaterThan(0);
expect(response.usage.totalTokens).toBe(response.usage.input + response.usage.output);
expect(registration.state.callCount).toBe(1);
});
it("supports helper blocks for text, thinking, and tool calls", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
registration.setResponses([
fauxAssistantMessage([fauxThinking("think"), fauxToolCall("echo", { text: "hi" }), fauxText("done")], {
stopReason: "toolUse",
}),
]);
const response = await complete(registration.getModel(), {
messages: [{ role: "user", content: "hi", timestamp: Date.now() }],
});
expect(response.content).toEqual([
{ type: "thinking", thinking: "think" },
{ type: "toolCall", id: expect.any(String), name: "echo", arguments: { text: "hi" } },
{ type: "text", text: "done" },
]);
expect(response.stopReason).toBe("toolUse");
});
it("supports multiple models with per-model reasoning and model-aware factories", async () => {
const registration = registerFauxProvider({
models: [
{ id: "faux-fast", name: "Faux Fast", reasoning: false },
{ id: "faux-thinker", name: "Faux Thinker", reasoning: true },
],
});
registrations.push(registration);
registration.setResponses([
(_context, _options, _state, model) => fauxAssistantMessage(`${model.id}:${String(model.reasoning)}`),
(_context, _options, _state, model) => fauxAssistantMessage(`${model.id}:${String(model.reasoning)}`),
]);
expect(registration.models.map((model) => model.id)).toEqual(["faux-fast", "faux-thinker"]);
expect(registration.getModel()).toBe(registration.models[0]);
expect(registration.getModel("faux-fast")?.reasoning).toBe(false);
expect(registration.getModel("faux-thinker")?.reasoning).toBe(true);
const fast = await complete(registration.getModel("faux-fast")!, {
messages: [{ role: "user", content: "hi", timestamp: Date.now() }],
});
const thinker = await complete(registration.getModel("faux-thinker")!, {
messages: [{ role: "user", content: "hi", timestamp: Date.now() }],
});
expect(fast.content).toEqual([{ type: "text", text: "faux-fast:false" }]);
expect(thinker.content).toEqual([{ type: "text", text: "faux-thinker:true" }]);
});
it("rewrites api, provider, and model on returned messages", async () => {
const registration = registerFauxProvider({
api: "faux:test",
provider: "faux-provider",
models: [{ id: "faux-model" }],
});
registrations.push(registration);
registration.setResponses([fauxAssistantMessage("hello")]);
const response = await complete(registration.getModel(), {
messages: [{ role: "user", content: "hi", timestamp: Date.now() }],
});
expect(response.api).toBe("faux:test");
expect(response.provider).toBe("faux-provider");
expect(response.model).toBe("faux-model");
});
it("consumes queued responses in order and errors when exhausted", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
registration.setResponses([fauxAssistantMessage("first"), fauxAssistantMessage("second")]);
const context: Context = {
messages: [{ role: "user", content: "hi", timestamp: Date.now() }],
};
const first = await complete(registration.getModel(), context);
const second = await complete(registration.getModel(), context);
const exhausted = await complete(registration.getModel(), context);
expect(first.content).toEqual([{ type: "text", text: "first" }]);
expect(second.content).toEqual([{ type: "text", text: "second" }]);
expect(exhausted.stopReason).toBe("error");
expect(exhausted.errorMessage).toBe("No more faux responses queued");
expect(registration.getPendingResponseCount()).toBe(0);
expect(registration.state.callCount).toBe(3);
});
it("can replace and append queued responses", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
registration.setResponses([fauxAssistantMessage("first")]);
const context: Context = {
messages: [{ role: "user", content: "hi", timestamp: Date.now() }],
};
expect((await complete(registration.getModel(), context)).content).toEqual([{ type: "text", text: "first" }]);
expect(registration.getPendingResponseCount()).toBe(0);
registration.setResponses([fauxAssistantMessage("second")]);
expect(registration.getPendingResponseCount()).toBe(1);
expect((await complete(registration.getModel(), context)).content).toEqual([{ type: "text", text: "second" }]);
registration.appendResponses([fauxAssistantMessage("third"), fauxAssistantMessage("fourth")]);
expect(registration.getPendingResponseCount()).toBe(2);
expect((await complete(registration.getModel(), context)).content).toEqual([{ type: "text", text: "third" }]);
expect((await complete(registration.getModel(), context)).content).toEqual([{ type: "text", text: "fourth" }]);
expect(registration.getPendingResponseCount()).toBe(0);
});
it("supports async response factories", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
registration.setResponses([
async (context, _options, state) => fauxAssistantMessage(`${context.messages.length}:${state.callCount}`),
]);
const response = await complete(registration.getModel(), {
messages: [{ role: "user", content: "hi", timestamp: Date.now() }],
});
expect(response.content).toEqual([{ type: "text", text: "1:1" }]);
});
it("emits an error when a response factory throws", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
registration.setResponses([
() => {
throw new Error("boom");
},
]);
const events = await collectEvents(
stream(registration.getModel(), { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }),
);
expect(events).toHaveLength(1);
expect(events[0].type).toBe("error");
if (events[0].type === "error") {
expect(events[0].error.stopReason).toBe("error");
expect(events[0].error.errorMessage).toBe("boom");
}
});
it("estimates prompt and output tokens from serialized context", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
registration.setResponses([fauxAssistantMessage("done")]);
const tool = {
name: "echo",
description: "Echo back text",
parameters: Type.Object({ text: Type.String() }),
};
const context: Context = {
systemPrompt: "sys",
messages: [
{
role: "user",
content: [
{ type: "text", text: "hello" },
{ type: "image", mimeType: "image/png", data: "abcd" },
],
timestamp: 1,
},
fauxAssistantMessage("prior"),
{
role: "toolResult",
toolCallId: "tool-1",
toolName: "echo",
content: [{ type: "text", text: "tool out" }],
isError: false,
timestamp: 2,
},
],
tools: [tool],
};
const response = await complete(registration.getModel(), context);
const promptText = [
"system:sys",
"user:hello\n[image:image/png:4]",
"assistant:prior",
"toolResult:echo\ntool out",
`tools:${JSON.stringify([tool])}`,
].join("\n\n");
const expectedPromptTokens = Math.ceil(promptText.length / 4);
const expectedOutputTokens = Math.ceil("done".length / 4);
expect(response.usage.input).toBe(expectedPromptTokens);
expect(response.usage.output).toBe(expectedOutputTokens);
expect(response.usage.cacheRead).toBe(0);
expect(response.usage.cacheWrite).toBe(0);
expect(response.usage.totalTokens).toBe(expectedPromptTokens + expectedOutputTokens);
});
it("does not share cache across sessions or requests without sessionId", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
registration.setResponses([
fauxAssistantMessage("first"),
fauxAssistantMessage("second"),
fauxAssistantMessage("third"),
]);
const context: Context = {
messages: [{ role: "user", content: "hello", timestamp: Date.now() }],
};
const first = await complete(registration.getModel(), context, {
sessionId: "session-1",
cacheRetention: "short",
});
expect(first.usage.cacheWrite).toBeGreaterThan(0);
context.messages.push(first);
context.messages.push({ role: "user", content: "follow up", timestamp: Date.now() + 1 });
const second = await complete(registration.getModel(), context, {
sessionId: "session-2",
cacheRetention: "short",
});
expect(second.usage.cacheRead).toBe(0);
expect(second.usage.cacheWrite).toBeGreaterThan(0);
const third = await complete(registration.getModel(), context);
expect(third.usage.cacheRead).toBe(0);
expect(third.usage.cacheWrite).toBe(0);
});
it("simulates prompt caching per sessionId", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
registration.setResponses([fauxAssistantMessage("first"), fauxAssistantMessage("second")]);
const context: Context = {
systemPrompt: "Be concise.",
messages: [{ role: "user", content: "hello", timestamp: Date.now() }],
};
const first = await complete(registration.getModel(), context, {
sessionId: "session-1",
cacheRetention: "short",
});
expect(first.usage.cacheRead).toBe(0);
expect(first.usage.cacheWrite).toBeGreaterThan(0);
context.messages.push(first);
context.messages.push({ role: "user", content: "follow up", timestamp: Date.now() + 1 });
const second = await complete(registration.getModel(), context, {
sessionId: "session-1",
cacheRetention: "short",
});
expect(second.usage.cacheRead).toBeGreaterThan(0);
expect(second.usage.input + second.usage.cacheRead).toBeGreaterThan(second.usage.input);
});
it("does not simulate caching when cacheRetention is none", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
registration.setResponses([fauxAssistantMessage("first"), fauxAssistantMessage("second")]);
const context: Context = {
messages: [{ role: "user", content: "hello", timestamp: Date.now() }],
};
await complete(registration.getModel(), context, { sessionId: "session-1", cacheRetention: "none" });
context.messages.push(fauxAssistantMessage("first"));
context.messages.push({ role: "user", content: "follow up", timestamp: Date.now() + 1 });
const second = await complete(registration.getModel(), context, {
sessionId: "session-1",
cacheRetention: "none",
});
expect(second.usage.cacheRead).toBe(0);
expect(second.usage.cacheWrite).toBe(0);
});
it("streams thinking, text, and partial tool call deltas", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
registration.setResponses([
fauxAssistantMessage(
[
fauxThinking("thinking text"),
fauxText("answer text"),
fauxToolCall("echo", { text: "hi", count: 12 }, { id: "tool-1" }),
],
{ stopReason: "toolUse" },
),
]);
const events: string[] = [];
const toolCallDeltas: string[] = [];
const s = stream(registration.getModel(), { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] });
for await (const event of s) {
events.push(event.type);
if (event.type === "toolcall_delta") {
toolCallDeltas.push(event.delta);
}
}
expect(events).toContain("thinking_start");
expect(events).toContain("thinking_delta");
expect(events).toContain("text_start");
expect(events).toContain("text_delta");
expect(events).toContain("toolcall_start");
expect(events).toContain("toolcall_delta");
expect(events).toContain("toolcall_end");
expect(toolCallDeltas.length).toBeGreaterThan(1);
expect(JSON.parse(toolCallDeltas.join(""))).toEqual({ text: "hi", count: 12 });
});
it("streams an exact event order for fixed-size chunks", async () => {
const registration = registerFauxProvider({ tokenSize: { min: 1, max: 1 } });
registrations.push(registration);
registration.setResponses([
fauxAssistantMessage([fauxThinking("go"), fauxText("ok"), fauxToolCall("echo", {}, { id: "tool-1" })], {
stopReason: "toolUse",
}),
]);
const events = await collectEvents(
stream(registration.getModel(), { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }),
);
expect(events.map((event) => event.type)).toEqual([
"start",
"thinking_start",
"thinking_delta",
"thinking_end",
"text_start",
"text_delta",
"text_end",
"toolcall_start",
"toolcall_delta",
"toolcall_end",
"done",
]);
});
it("streams multiple tool calls in one message", async () => {
const registration = registerFauxProvider();
registrations.push(registration);
registration.setResponses([
fauxAssistantMessage(
[
fauxToolCall("echo", { text: "one" }, { id: "tool-1" }),
fauxToolCall("echo", { text: "two" }, { id: "tool-2" }),
],
{ stopReason: "toolUse" },
),
]);
const events = await collectEvents(
stream(registration.getModel(), { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }),
);
expect(events.filter((event) => event.type === "toolcall_start")).toHaveLength(2);
expect(events.filter((event) => event.type === "toolcall_end")).toHaveLength(2);
});
it("streams an explicit assistant error message as a terminal error", async () => {
const registration = registerFauxProvider({ tokenSize: { min: 2, max: 2 } });
registrations.push(registration);
registration.setResponses([
{
...fauxAssistantMessage("partial"),
stopReason: "error",
errorMessage: "upstream failed",
},
]);
const events = await collectEvents(
stream(registration.getModel(), { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }),
);
expect(events.map((event) => event.type)).toEqual(["start", "text_start", "text_delta", "text_end", "error"]);
const terminal = events[events.length - 1];
expect(terminal.type).toBe("error");
if (terminal.type === "error") {
expect(terminal.reason).toBe("error");
expect(terminal.error.stopReason).toBe("error");
expect(terminal.error.errorMessage).toBe("upstream failed");
}
});
it("streams an explicit assistant aborted message as a terminal error", async () => {
const registration = registerFauxProvider({ tokenSize: { min: 2, max: 2 } });
registrations.push(registration);
registration.setResponses([
{
...fauxAssistantMessage("partial"),
stopReason: "aborted",
errorMessage: "Request was aborted",
},
]);
const events = await collectEvents(
stream(registration.getModel(), { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }),
);
expect(events.map((event) => event.type)).toEqual(["start", "text_start", "text_delta", "text_end", "error"]);
const terminal = events[events.length - 1];
expect(terminal.type).toBe("error");
if (terminal.type === "error") {
expect(terminal.reason).toBe("aborted");
expect(terminal.error.stopReason).toBe("aborted");
expect(terminal.error.errorMessage).toBe("Request was aborted");
}
});
it("supports aborting before the first chunk", async () => {
const registration = registerFauxProvider({ tokensPerSecond: 50, tokenSize: { min: 3, max: 3 } });
registrations.push(registration);
registration.setResponses([fauxAssistantMessage("abcdefghijklmnopqrstuvwxyz")]);
const controller = new AbortController();
controller.abort();
const events = await collectEvents(
stream(
registration.getModel(),
{ messages: [{ role: "user", content: "hi", timestamp: Date.now() }] },
{ signal: controller.signal },
),
);
expect(events).toHaveLength(1);
expect(events[0].type).toBe("error");
if (events[0].type === "error") {
expect(events[0].reason).toBe("aborted");
expect(events[0].error.stopReason).toBe("aborted");
}
});
it("supports aborting mid-text stream when paced", async () => {
const registration = registerFauxProvider({ tokensPerSecond: 100, tokenSize: { min: 3, max: 3 } });
registrations.push(registration);
registration.setResponses([fauxAssistantMessage("abcdefghijklmnopqrstuvwxyz")]);
const controller = new AbortController();
const events: string[] = [];
let textDeltaCount = 0;
const s = stream(
registration.getModel(),
{ messages: [{ role: "user", content: "hi", timestamp: Date.now() }] },
{ signal: controller.signal },
);
for await (const event of s) {
events.push(event.type);
if (event.type === "text_delta") {
textDeltaCount++;
controller.abort();
}
}
expect(textDeltaCount).toBe(1);
expect(events).toContain("text_start");
expect(events).toContain("text_delta");
expect(events).toContain("error");
expect(events).not.toContain("text_end");
});
it("supports aborting mid-thinking stream when paced", async () => {
const registration = registerFauxProvider({ tokensPerSecond: 100, tokenSize: { min: 3, max: 3 } });
registrations.push(registration);
registration.setResponses([
{
...fauxAssistantMessage("ignored"),
content: [{ type: "thinking", thinking: "abcdefghijklmnopqrstuvwxyz" }],
},
]);
const controller = new AbortController();
const events: string[] = [];
let thinkingDeltaCount = 0;
const s = stream(
registration.getModel(),
{ messages: [{ role: "user", content: "hi", timestamp: Date.now() }] },
{ signal: controller.signal },
);
for await (const event of s) {
events.push(event.type);
if (event.type === "thinking_delta") {
thinkingDeltaCount++;
controller.abort();
}
}
expect(thinkingDeltaCount).toBe(1);
expect(events).toContain("thinking_start");
expect(events).toContain("thinking_delta");
expect(events).toContain("error");
expect(events).not.toContain("thinking_end");
});
it("supports aborting mid-toolcall stream when paced", async () => {
const registration = registerFauxProvider({ tokensPerSecond: 100, tokenSize: { min: 3, max: 3 } });
registrations.push(registration);
registration.setResponses([
{
...fauxAssistantMessage("done"),
content: [
{
type: "toolCall",
id: "tool-1",
name: "echo",
arguments: { text: "abcdefghijklmnopqrstuvwxyz", count: 123456789 },
},
],
stopReason: "toolUse",
},
]);
const controller = new AbortController();
const events: string[] = [];
let toolCallDeltaCount = 0;
const s = stream(
registration.getModel(),
{ messages: [{ role: "user", content: "hi", timestamp: Date.now() }] },
{ signal: controller.signal },
);
for await (const event of s) {
events.push(event.type);
if (event.type === "toolcall_delta") {
toolCallDeltaCount++;
controller.abort();
}
}
expect(toolCallDeltaCount).toBe(1);
expect(events).toContain("toolcall_start");
expect(events).toContain("toolcall_delta");
expect(events).toContain("error");
expect(events).not.toContain("toolcall_end");
});
it("unregisters the provider", async () => {
const registration = registerFauxProvider();
registration.setResponses([fauxAssistantMessage("hello")]);
registration.unregister();
await expect(
complete(registration.getModel(), { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }),
).rejects.toThrow(`No API provider registered for api: ${registration.api}`);
});
});
@@ -1,103 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import { getModel } from "../src/models.js";
import type { Context } from "../src/types.js";
const mockState = vi.hoisted(() => ({
constructorOpts: undefined as Record<string, unknown> | undefined,
streamParams: undefined as Record<string, unknown> | undefined,
}));
vi.mock("@anthropic-ai/sdk", () => {
const fakeStream = {
async *[Symbol.asyncIterator]() {
yield {
type: "message_start",
message: {
usage: { input_tokens: 10, output_tokens: 0 },
},
};
yield {
type: "message_delta",
delta: { stop_reason: "end_turn" },
usage: { output_tokens: 5 },
};
},
finalMessage: async () => ({
usage: { input_tokens: 10, output_tokens: 5, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 },
}),
};
class FakeAnthropic {
constructor(opts: Record<string, unknown>) {
mockState.constructorOpts = opts;
}
messages = {
stream: (params: Record<string, unknown>) => {
mockState.streamParams = params;
return fakeStream;
},
};
}
return { default: FakeAnthropic };
});
describe("Copilot Claude via Anthropic Messages", () => {
const context: Context = {
systemPrompt: "You are a helpful assistant.",
messages: [{ role: "user", content: "Hello", timestamp: Date.now() }],
};
it("uses Bearer auth, Copilot headers, and valid Anthropic Messages payload", async () => {
const model = getModel("github-copilot", "claude-sonnet-4");
expect(model.api).toBe("anthropic-messages");
const { streamAnthropic } = await import("../src/providers/anthropic.js");
const s = streamAnthropic(model, context, { apiKey: "tid_copilot_session_test_token" });
for await (const event of s) {
if (event.type === "error") break;
}
const opts = mockState.constructorOpts!;
expect(opts).toBeDefined();
// Auth: apiKey null, authToken for Bearer
expect(opts.apiKey).toBeNull();
expect(opts.authToken).toBe("tid_copilot_session_test_token");
const headers = opts.defaultHeaders as Record<string, string>;
// Copilot static headers from model.headers
expect(headers["User-Agent"]).toContain("GitHubCopilotChat");
expect(headers["Copilot-Integration-Id"]).toBe("vscode-chat");
// Dynamic headers
expect(headers["X-Initiator"]).toBe("user");
expect(headers["Openai-Intent"]).toBe("conversation-edits");
// No fine-grained-tool-streaming (Copilot doesn't support it)
const beta = headers["anthropic-beta"] ?? "";
expect(beta).not.toContain("fine-grained-tool-streaming");
// Payload is valid Anthropic Messages format
const params = mockState.streamParams!;
expect(params.model).toBe("claude-sonnet-4");
expect(params.stream).toBe(true);
expect(params.max_tokens).toBeGreaterThan(0);
expect(Array.isArray(params.messages)).toBe(true);
});
it("includes interleaved-thinking beta when reasoning is enabled", async () => {
const model = getModel("github-copilot", "claude-sonnet-4");
const { streamAnthropic } = await import("../src/providers/anthropic.js");
const s = streamAnthropic(model, context, {
apiKey: "tid_copilot_session_test_token",
interleavedThinking: true,
});
for await (const event of s) {
if (event.type === "error") break;
}
const headers = mockState.constructorOpts!.defaultHeaders as Record<string, string>;
expect(headers["anthropic-beta"]).toContain("interleaved-thinking-2025-05-14");
});
});
@@ -1,196 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { loginGitHubCopilot } from "../src/utils/oauth/github-copilot.js";
function jsonResponse(body: unknown, status: number = 200): Response {
return new Response(JSON.stringify(body), {
status,
headers: {
"Content-Type": "application/json",
},
});
}
function getUrl(input: unknown): string {
if (typeof input === "string") {
return input;
}
if (input instanceof URL) {
return input.toString();
}
if (input instanceof Request) {
return input.url;
}
throw new Error(`Unsupported fetch input: ${String(input)}`);
}
describe("GitHub Copilot OAuth device flow", () => {
afterEach(() => {
vi.unstubAllGlobals();
vi.useRealTimers();
});
it("waits before the first poll and increases the safety margin after slow_down", async () => {
vi.useFakeTimers();
const startTime = new Date("2026-03-09T00:00:00Z");
vi.setSystemTime(startTime);
const accessTokenPollTimes: number[] = [];
const accessTokenResponses = [
jsonResponse({ error: "authorization_pending", error_description: "pending" }),
jsonResponse({ error: "slow_down", error_description: "slow down", interval: 10 }),
jsonResponse({ access_token: "ghu_refresh_token" }),
];
const fetchMock = vi.fn(async (input: unknown, init?: RequestInit): Promise<Response> => {
const url = getUrl(input);
if (url.endsWith("/login/device/code")) {
expect(init?.method).toBe("POST");
expect(init?.headers).toMatchObject({
Accept: "application/json",
"Content-Type": "application/x-www-form-urlencoded",
});
expect(String(init?.body)).toContain("client_id=");
expect(String(init?.body)).toContain("scope=read%3Auser");
return jsonResponse({
device_code: "device-code",
user_code: "ABCD-EFGH",
verification_uri: "https://github.com/login/device",
interval: 5,
expires_in: 900,
});
}
if (url.endsWith("/login/oauth/access_token")) {
accessTokenPollTimes.push(Date.now());
expect(init?.method).toBe("POST");
expect(init?.headers).toMatchObject({
Accept: "application/json",
"Content-Type": "application/x-www-form-urlencoded",
});
expect(String(init?.body)).toContain("client_id=");
expect(String(init?.body)).toContain("device_code=device-code");
expect(String(init?.body)).toContain("grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Adevice_code");
const response = accessTokenResponses.shift();
if (!response) {
throw new Error("Unexpected extra access token poll");
}
return response;
}
if (url.includes("/copilot_internal/v2/token")) {
return jsonResponse({
token: "tid=test;exp=9999999999;proxy-ep=proxy.individual.githubcopilot.com;",
expires_at: 9999999999,
});
}
if (url.includes("/models/") && url.endsWith("/policy")) {
return new Response("", { status: 200 });
}
throw new Error(`Unexpected fetch URL: ${url}`);
});
vi.stubGlobal("fetch", fetchMock);
const loginPromise = loginGitHubCopilot({
onAuth: () => {},
onPrompt: async () => "",
onProgress: () => {},
});
await vi.advanceTimersByTimeAsync(0);
expect(accessTokenPollTimes).toHaveLength(0);
await vi.advanceTimersByTimeAsync(5999);
expect(accessTokenPollTimes).toHaveLength(0);
await vi.advanceTimersByTimeAsync(1);
expect(accessTokenPollTimes).toHaveLength(1);
await vi.advanceTimersByTimeAsync(5999);
expect(accessTokenPollTimes).toHaveLength(1);
await vi.advanceTimersByTimeAsync(1);
expect(accessTokenPollTimes).toHaveLength(2);
await vi.advanceTimersByTimeAsync(13999);
expect(accessTokenPollTimes).toHaveLength(2);
await vi.advanceTimersByTimeAsync(1);
await loginPromise;
expect(accessTokenPollTimes).toEqual([
startTime.getTime() + 6000,
startTime.getTime() + 12000,
startTime.getTime() + 26000,
]);
});
it("uses the remaining lifetime for a final poll before timing out after repeated slow_down responses", async () => {
vi.useFakeTimers();
const startTime = new Date("2026-03-09T00:00:00Z");
vi.setSystemTime(startTime);
const accessTokenPollTimes: number[] = [];
const accessTokenResponses = [
jsonResponse({ error: "slow_down", error_description: "slow down", interval: 10 }),
jsonResponse({ error: "slow_down", error_description: "still too fast", interval: 15 }),
jsonResponse({ error: "authorization_pending", error_description: "pending" }),
];
const fetchMock = vi.fn(async (input: unknown): Promise<Response> => {
const url = getUrl(input);
if (url.endsWith("/login/device/code")) {
return jsonResponse({
device_code: "device-code",
user_code: "ABCD-EFGH",
verification_uri: "https://github.com/login/device",
interval: 5,
expires_in: 25,
});
}
if (url.endsWith("/login/oauth/access_token")) {
accessTokenPollTimes.push(Date.now());
const response = accessTokenResponses.shift();
if (!response) {
throw new Error("Unexpected extra access token poll");
}
return response;
}
throw new Error(`Unexpected fetch URL: ${url}`);
});
vi.stubGlobal("fetch", fetchMock);
const loginPromise = loginGitHubCopilot({
onAuth: () => {},
onPrompt: async () => "",
});
const rejection = expect(loginPromise).rejects.toThrow(
/Device flow timed out after one or more slow_down responses/,
);
await vi.advanceTimersByTimeAsync(6000);
expect(accessTokenPollTimes).toEqual([startTime.getTime() + 6000]);
await vi.advanceTimersByTimeAsync(14000);
expect(accessTokenPollTimes).toEqual([startTime.getTime() + 6000, startTime.getTime() + 20000]);
await vi.advanceTimersByTimeAsync(4999);
expect(accessTokenPollTimes).toEqual([startTime.getTime() + 6000, startTime.getTime() + 20000]);
await vi.advanceTimersByTimeAsync(1);
await rejection;
expect(accessTokenPollTimes).toEqual([
startTime.getTime() + 6000,
startTime.getTime() + 20000,
startTime.getTime() + 25000,
]);
});
});
@@ -1,103 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { streamGoogleGeminiCli } from "../src/providers/google-gemini-cli.js";
import type { Context, Model } from "../src/types.js";
const originalFetch = global.fetch;
const apiKey = JSON.stringify({ token: "token", projectId: "project" });
const createSseResponse = () => {
const sse = `${[
`data: ${JSON.stringify({
response: {
candidates: [
{
content: { role: "model", parts: [{ text: "Hello" }] },
finishReason: "STOP",
},
],
},
})}`,
].join("\n\n")}\n\n`;
const encoder = new TextEncoder();
const stream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(encoder.encode(sse));
controller.close();
},
});
return new Response(stream, {
status: 200,
headers: { "content-type": "text/event-stream" },
});
};
afterEach(() => {
global.fetch = originalFetch;
vi.restoreAllMocks();
});
describe("google-gemini-cli Claude thinking header", () => {
const context: Context = {
messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }],
};
it("adds anthropic-beta for Claude thinking models", async () => {
const fetchMock = vi.fn(async (_input: string | URL, init?: RequestInit) => {
const headers = new Headers(init?.headers);
expect(headers.get("anthropic-beta")).toBe("interleaved-thinking-2025-05-14");
return createSseResponse();
});
global.fetch = fetchMock as typeof fetch;
const model: Model<"google-gemini-cli"> = {
id: "claude-opus-4-5-thinking",
name: "Claude Opus 4.5 Thinking",
api: "google-gemini-cli",
provider: "google-antigravity",
baseUrl: "https://cloudcode-pa.googleapis.com",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 128000,
maxTokens: 8192,
};
const stream = streamGoogleGeminiCli(model, context, { apiKey });
for await (const _event of stream) {
// exhaust stream
}
await stream.result();
});
it("does not add anthropic-beta for Gemini models", async () => {
const fetchMock = vi.fn(async (_input: string | URL, init?: RequestInit) => {
const headers = new Headers(init?.headers);
expect(headers.has("anthropic-beta")).toBe(false);
return createSseResponse();
});
global.fetch = fetchMock as typeof fetch;
const model: Model<"google-gemini-cli"> = {
id: "gemini-2.5-flash",
name: "Gemini 2.5 Flash",
api: "google-gemini-cli",
provider: "google-gemini-cli",
baseUrl: "https://cloudcode-pa.googleapis.com",
reasoning: false,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 128000,
maxTokens: 8192,
};
const stream = streamGoogleGeminiCli(model, context, { apiKey });
for await (const _event of stream) {
// exhaust stream
}
await stream.result();
});
});
@@ -1,108 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { streamGoogleGeminiCli } from "../src/providers/google-gemini-cli.js";
import type { Context, Model } from "../src/types.js";
const originalFetch = global.fetch;
afterEach(() => {
global.fetch = originalFetch;
vi.restoreAllMocks();
});
describe("google-gemini-cli empty stream retry", () => {
it("retries empty SSE responses without duplicate start", async () => {
const emptyStream = new ReadableStream<Uint8Array>({
start(controller) {
controller.close();
},
});
const sse = `${[
`data: ${JSON.stringify({
response: {
candidates: [
{
content: { role: "model", parts: [{ text: "Hello" }] },
finishReason: "STOP",
},
],
usageMetadata: {
promptTokenCount: 1,
candidatesTokenCount: 1,
totalTokenCount: 2,
},
},
})}`,
].join("\n\n")}\n\n`;
const encoder = new TextEncoder();
const dataStream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(encoder.encode(sse));
controller.close();
},
});
let callCount = 0;
const fetchMock = vi.fn(async () => {
callCount += 1;
if (callCount === 1) {
return new Response(emptyStream, {
status: 200,
headers: { "content-type": "text/event-stream" },
});
}
return new Response(dataStream, {
status: 200,
headers: { "content-type": "text/event-stream" },
});
});
global.fetch = fetchMock as typeof fetch;
const model: Model<"google-gemini-cli"> = {
id: "gemini-2.5-flash",
name: "Gemini 2.5 Flash",
api: "google-gemini-cli",
provider: "google-gemini-cli",
baseUrl: "https://cloudcode-pa.googleapis.com",
reasoning: false,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 128000,
maxTokens: 8192,
};
const context: Context = {
messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }],
};
const stream = streamGoogleGeminiCli(model, context, {
apiKey: JSON.stringify({ token: "token", projectId: "project" }),
});
let startCount = 0;
let doneCount = 0;
let text = "";
for await (const event of stream) {
if (event.type === "start") {
startCount += 1;
}
if (event.type === "done") {
doneCount += 1;
}
if (event.type === "text_delta") {
text += event.delta;
}
}
const result = await stream.result();
expect(text).toBe("Hello");
expect(result.stopReason).toBe("stop");
expect(startCount).toBe(1);
expect(doneCount).toBe(1);
expect(fetchMock).toHaveBeenCalledTimes(2);
});
});
@@ -1,53 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { extractRetryDelay } from "../src/providers/google-gemini-cli.js";
describe("extractRetryDelay header parsing", () => {
afterEach(() => {
vi.useRealTimers();
});
it("prefers Retry-After seconds header", () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2025-01-01T00:00:00Z"));
const response = new Response("", { headers: { "Retry-After": "5" } });
const delay = extractRetryDelay("Please retry in 1s", response);
expect(delay).toBe(6000);
});
it("parses Retry-After HTTP date header", () => {
vi.useFakeTimers();
const now = new Date("2025-01-01T00:00:00Z");
vi.setSystemTime(now);
const retryAt = new Date(now.getTime() + 12000).toUTCString();
const response = new Response("", { headers: { "Retry-After": retryAt } });
const delay = extractRetryDelay("", response);
expect(delay).toBe(13000);
});
it("parses x-ratelimit-reset header", () => {
vi.useFakeTimers();
const now = new Date("2025-01-01T00:00:00Z");
vi.setSystemTime(now);
const resetAtMs = now.getTime() + 20000;
const resetSeconds = Math.floor(resetAtMs / 1000).toString();
const response = new Response("", { headers: { "x-ratelimit-reset": resetSeconds } });
const delay = extractRetryDelay("", response);
expect(delay).toBe(21000);
});
it("parses x-ratelimit-reset-after header", () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2025-01-01T00:00:00Z"));
const response = new Response("", { headers: { "x-ratelimit-reset-after": "30" } });
const delay = extractRetryDelay("", response);
expect(delay).toBe(31000);
});
});
@@ -1,172 +0,0 @@
import { describe, expect, it } from "vitest";
import { convertMessages } from "../src/providers/google-shared.js";
import type { Context, Model } from "../src/types.js";
const SKIP_THOUGHT_SIGNATURE = "skip_thought_signature_validator";
function makeGemini3Model(id = "gemini-3-pro-preview"): Model<"google-generative-ai"> {
return {
id,
name: "Gemini 3 Pro Preview",
api: "google-generative-ai",
provider: "google",
baseUrl: "https://generativelanguage.googleapis.com",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 128000,
maxTokens: 8192,
};
}
describe("google-shared convertMessages — Gemini 3 unsigned tool calls", () => {
it("uses skip_thought_signature_validator for unsigned tool calls on Gemini 3", () => {
const model = makeGemini3Model();
const now = Date.now();
const context: Context = {
messages: [
{ role: "user", content: "Hi", timestamp: now },
{
role: "assistant",
content: [
{
type: "toolCall",
id: "call_1",
name: "bash",
arguments: { command: "ls -la" },
// No thoughtSignature: simulates Claude via Antigravity.
},
],
api: "google-gemini-cli",
provider: "google-antigravity",
model: "claude-sonnet-4-20250514",
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "stop",
timestamp: now,
},
],
};
const contents = convertMessages(model, context);
const modelTurn = contents.find((c) => c.role === "model");
expect(modelTurn).toBeTruthy();
// Should be a structured functionCall, NOT text fallback
const fcPart = modelTurn?.parts?.find((p) => p.functionCall !== undefined);
expect(fcPart).toBeTruthy();
expect(fcPart?.functionCall?.name).toBe("bash");
expect(fcPart?.functionCall?.args).toEqual({ command: "ls -la" });
expect(fcPart?.thoughtSignature).toBe(SKIP_THOUGHT_SIGNATURE);
// No text fallback should exist
const textParts = modelTurn?.parts?.filter((p) => p.text !== undefined) ?? [];
const historicalText = textParts.filter((p) => p.text?.includes("Historical context"));
expect(historicalText).toHaveLength(0);
});
it("preserves valid thoughtSignature when present (same provider/model)", () => {
const model = makeGemini3Model();
const now = Date.now();
// Valid base64 signature (16 bytes = 24 chars base64)
const validSig = "AAAAAAAAAAAAAAAAAAAAAA==";
const context: Context = {
messages: [
{ role: "user", content: "Hi", timestamp: now },
{
role: "assistant",
content: [
{
type: "toolCall",
id: "call_1",
name: "bash",
arguments: { command: "echo hi" },
thoughtSignature: validSig,
},
],
api: "google-generative-ai",
provider: "google",
model: "gemini-3-pro-preview",
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "stop",
timestamp: now,
},
],
};
const contents = convertMessages(model, context);
const modelTurn = contents.find((c) => c.role === "model");
const fcPart = modelTurn?.parts?.find((p) => p.functionCall !== undefined);
expect(fcPart).toBeTruthy();
expect(fcPart?.thoughtSignature).toBe(validSig);
});
it("does not add sentinel for non-Gemini-3 models", () => {
const model: Model<"google-generative-ai"> = {
id: "gemini-2.5-flash",
name: "Gemini 2.5 Flash",
api: "google-generative-ai",
provider: "google",
baseUrl: "https://generativelanguage.googleapis.com",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 128000,
maxTokens: 8192,
};
const now = Date.now();
const context: Context = {
messages: [
{ role: "user", content: "Hi", timestamp: now },
{
role: "assistant",
content: [
{
type: "toolCall",
id: "call_1",
name: "bash",
arguments: { command: "ls" },
// No thoughtSignature
},
],
api: "google-gemini-cli",
provider: "google-antigravity",
model: "claude-sonnet-4-20250514",
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "stop",
timestamp: now,
},
],
};
const contents = convertMessages(model, context);
const modelTurn = contents.find((c) => c.role === "model");
const fcPart = modelTurn?.parts?.find((p) => p.functionCall !== undefined);
expect(fcPart).toBeTruthy();
// No sentinel, no thoughtSignature at all
expect(fcPart?.thoughtSignature).toBeUndefined();
});
});
@@ -1,124 +0,0 @@
import { describe, expect, it } from "vitest";
import { convertMessages } from "../src/providers/google-shared.js";
import type { Context, Model } from "../src/types.js";
function makeModel<TApi extends "google-generative-ai" | "google-gemini-cli">(
api: TApi,
provider: Model<TApi>["provider"],
id: string,
): Model<TApi> {
return {
id,
name: id,
api,
provider,
baseUrl: "https://example.com",
reasoning: true,
input: ["text", "image"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 128000,
maxTokens: 8192,
};
}
function makeContext(model: { api: string; provider: string; id: string }): Context {
const now = Date.now();
return {
messages: [
{ role: "user", content: "read the files", timestamp: now },
{
role: "assistant",
content: [
{ type: "toolCall", id: "call_a", name: "read", arguments: { path: "a.txt" } },
{ type: "toolCall", id: "call_img", name: "read", arguments: { path: "image.png" } },
{ type: "toolCall", id: "call_b", name: "read", arguments: { path: "b.txt" } },
],
api: model.api,
provider: model.provider,
model: model.id,
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "toolUse",
timestamp: now,
},
{
role: "toolResult",
toolCallId: "call_a",
toolName: "read",
content: [{ type: "text", text: "alpha text" }],
isError: false,
timestamp: now,
},
{
role: "toolResult",
toolCallId: "call_img",
toolName: "read",
content: [{ type: "image", data: "abc", mimeType: "image/png" }],
isError: false,
timestamp: now,
},
{
role: "toolResult",
toolCallId: "call_b",
toolName: "read",
content: [{ type: "text", text: "beta text" }],
isError: false,
timestamp: now,
},
],
};
}
describe("google-shared image tool result routing", () => {
it("keeps separate synthetic image turn for Gemini 2.x Google API models", () => {
const model = makeModel("google-generative-ai", "google", "gemini-2.5-flash");
const contents = convertMessages(model, makeContext(model));
expect(contents).toHaveLength(5);
expect(contents[2].parts?.every((part) => part.functionResponse)).toBe(true);
expect(contents[3].parts?.[0]?.text).toBe("Tool result image:");
expect(contents[3].parts?.[1]?.inlineData).toBeTruthy();
expect(contents[4].parts?.[0]?.functionResponse).toBeTruthy();
});
it("nests image tool results for Gemini 3 Google API models", () => {
const model = makeModel("google-generative-ai", "google", "gemini-3-pro-preview");
const contents = convertMessages(model, makeContext(model));
expect(contents).toHaveLength(3);
const toolResultTurn = contents[2];
expect(toolResultTurn.parts).toHaveLength(3);
const imageResponse = toolResultTurn.parts?.[1]?.functionResponse;
expect(imageResponse).toBeTruthy();
expect(imageResponse?.parts).toHaveLength(1);
expect(imageResponse?.parts?.[0]?.inlineData).toBeTruthy();
});
it("nests image tool results for non-Gemini models on Antigravity / Cloud Code Assist", () => {
const model = makeModel("google-gemini-cli", "google-antigravity", "claude-sonnet-4-6");
const contents = convertMessages(model, makeContext(model));
expect(contents).toHaveLength(3);
const toolResultTurn = contents[2];
expect(toolResultTurn.parts).toHaveLength(3);
const imageResponse = toolResultTurn.parts?.[1]?.functionResponse;
expect(imageResponse).toBeTruthy();
expect(imageResponse?.parts).toHaveLength(1);
expect(imageResponse?.parts?.[0]?.inlineData).toBeTruthy();
});
it("keeps separate synthetic image turn for Gemini 2.x Cloud Code Assist models", () => {
const model = makeModel("google-gemini-cli", "google-gemini-cli", "gemini-2.5-flash");
const contents = convertMessages(model, makeContext(model));
expect(contents).toHaveLength(5);
expect(contents[3].parts?.[0]?.text).toBe("Tool result image:");
expect(contents[3].parts?.[1]?.inlineData).toBeTruthy();
});
});
@@ -1,183 +0,0 @@
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { streamSimple } from "../src/stream.js";
import type { Api, Context, Model, SimpleStreamOptions } from "../src/types.js";
import { resolveApiKey } from "./oauth.js";
type SimpleOptionsWithExtras = SimpleStreamOptions & Record<string, unknown>;
interface RunResult {
thinkingEventCount: number;
thinkingCharCount: number;
text: string;
outputTokens: number;
contentTypes: string[];
}
interface DisableExpectations {
requestOptions?: SimpleOptionsWithExtras;
minPongs?: number;
maxOutputTokens?: number;
}
const oauthTokens = await Promise.all([resolveApiKey("google-gemini-cli"), resolveApiKey("google-antigravity")]);
const [geminiCliToken, antigravityToken] = oauthTokens;
function makeContext(): Context {
return {
systemPrompt: "You are a precise assistant. Follow the requested output format exactly.",
messages: [
{
role: "user",
content:
"Before replying, carefully solve 36863 * 5279 internally. Then reply with the word pong repeated exactly 40 times, separated by single spaces. Do not add any other text.",
timestamp: Date.now(),
},
],
};
}
function countPongs(text: string): number {
return text.match(/\bpong\b/gi)?.length ?? 0;
}
async function runWithoutReasoning<TApi extends Api>(
model: Model<TApi>,
options: SimpleOptionsWithExtras = {},
): Promise<RunResult> {
const s = streamSimple(model, makeContext(), {
maxTokens: 160,
temperature: 0,
...options,
});
let thinkingEventCount = 0;
let thinkingCharCount = 0;
for await (const event of s) {
if (event.type === "thinking_start" || event.type === "thinking_end") {
thinkingEventCount += 1;
}
if (event.type === "thinking_delta") {
thinkingEventCount += 1;
thinkingCharCount += event.delta.length;
}
}
const response = await s.result();
expect(response.stopReason, response.errorMessage).toBe("stop");
const text = response.content
.filter((block) => block.type === "text")
.map((block) => block.text)
.join("")
.trim();
return {
thinkingEventCount,
thinkingCharCount,
text,
outputTokens: response.usage.output,
contentTypes: response.content.map((block) => block.type),
};
}
async function expectThinkingDisabledE2E<TApi extends Api>(model: Model<TApi>, expectations: DisableExpectations = {}) {
const result = await runWithoutReasoning(model, expectations.requestOptions);
expect(result.thinkingEventCount).toBe(0);
expect(result.thinkingCharCount).toBe(0);
expect(result.contentTypes).not.toContain("thinking");
expect(countPongs(result.text)).toBeGreaterThanOrEqual(expectations.minPongs ?? 35);
if (expectations.maxOutputTokens !== undefined) {
expect(result.outputTokens).toBeLessThan(expectations.maxOutputTokens);
}
}
describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic thinking disable E2E", () => {
it("disables thinking for budget-based reasoning models", { retry: 2, timeout: 30000 }, async () => {
await expectThinkingDisabledE2E(getModel("anthropic", "claude-sonnet-4-5"), {
requestOptions: { maxTokens: 320, temperature: 0 },
});
});
it("disables thinking for adaptive reasoning models", { retry: 2, timeout: 30000 }, async () => {
await expectThinkingDisabledE2E(getModel("anthropic", "claude-sonnet-4-6"), {
requestOptions: { maxTokens: 320, temperature: 0 },
});
});
});
describe.skipIf(!process.env.GEMINI_API_KEY)("Google thinking disable E2E", () => {
it("disables thinking for Gemini 2.5", { retry: 2, timeout: 30000 }, async () => {
await expectThinkingDisabledE2E(getModel("google", "gemini-2.5-flash"));
});
it("disables thinking for Gemini 3.x", { retry: 2, timeout: 30000 }, async () => {
await expectThinkingDisabledE2E(getModel("google", "gemini-3-flash-preview"));
});
it("does not error when thinking is off for Gemini 3.1 Pro", { retry: 2, timeout: 30000 }, async () => {
await expectThinkingDisabledE2E(getModel("google", "gemini-3.1-pro-preview"), {
requestOptions: { maxTokens: 512 },
minPongs: 20,
});
});
});
describe("Google Vertex thinking disable E2E", () => {
const vertexProject = process.env.GOOGLE_CLOUD_PROJECT || process.env.GCLOUD_PROJECT;
const vertexLocation = process.env.GOOGLE_CLOUD_LOCATION;
const vertexApiKey = process.env.GOOGLE_CLOUD_API_KEY;
const vertexOptions = vertexApiKey
? ({ apiKey: vertexApiKey } satisfies SimpleOptionsWithExtras)
: vertexProject && vertexLocation
? ({ project: vertexProject, location: vertexLocation } satisfies SimpleOptionsWithExtras)
: undefined;
it.skipIf(!vertexOptions)("disables thinking for Gemini 2.5", { retry: 2, timeout: 30000 }, async () => {
await expectThinkingDisabledE2E(getModel("google-vertex", "gemini-2.5-flash"), {
requestOptions: vertexOptions,
});
});
it.skipIf(!vertexOptions)("disables thinking for Gemini 3.x", { retry: 2, timeout: 30000 }, async () => {
await expectThinkingDisabledE2E(getModel("google-vertex", "gemini-3-flash-preview"), {
requestOptions: vertexOptions,
});
});
});
describe("Google Gemini CLI thinking disable E2E", () => {
it.skipIf(!geminiCliToken)("disables thinking for Gemini 2.5", { retry: 2, timeout: 30000 }, async () => {
await expectThinkingDisabledE2E(getModel("google-gemini-cli", "gemini-2.5-flash"), {
requestOptions: { apiKey: geminiCliToken! },
maxOutputTokens: 100,
});
});
});
describe("Google Antigravity thinking disable E2E", () => {
it.skipIf(!antigravityToken)("disables thinking for Gemini 3.x", { retry: 2, timeout: 30000 }, async () => {
await expectThinkingDisabledE2E(getModel("google-antigravity", "gemini-3-flash"), {
requestOptions: { apiKey: antigravityToken! },
maxOutputTokens: 100,
});
});
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI thinking disable E2E", () => {
it("disables thinking for Responses reasoning models", { retry: 2, timeout: 30000 }, async () => {
await expectThinkingDisabledE2E(getModel("openai", "gpt-5.4-mini"), {
requestOptions: { temperature: undefined },
});
});
});
describe.skipIf(!process.env.OPENROUTER_API_KEY)("OpenRouter thinking disable E2E", () => {
it("disables thinking for Qwen 3.5 reasoning models", { retry: 2, timeout: 30000 }, async () => {
await expectThinkingDisabledE2E(getModel("openrouter", "qwen/qwen3.5-plus-02-15"), {
maxOutputTokens: 100,
});
});
});
@@ -1,38 +0,0 @@
import { describe, expect, it } from "vitest";
import { isThinkingPart, retainThoughtSignature } from "../src/providers/google-shared.js";
describe("Google thinking detection (thoughtSignature)", () => {
it("treats part.thought === true as thinking", () => {
expect(isThinkingPart({ thought: true, thoughtSignature: undefined })).toBe(true);
expect(isThinkingPart({ thought: true, thoughtSignature: "opaque-signature" })).toBe(true);
});
it("does not treat thoughtSignature alone as thinking", () => {
// Per Google docs, thoughtSignature is for context replay and can appear on any part type.
// Only thought === true indicates thinking content.
// See: https://ai.google.dev/gemini-api/docs/thought-signatures
expect(isThinkingPart({ thought: undefined, thoughtSignature: "opaque-signature" })).toBe(false);
expect(isThinkingPart({ thought: false, thoughtSignature: "opaque-signature" })).toBe(false);
});
it("does not treat empty/missing signatures as thinking if thought is not set", () => {
expect(isThinkingPart({ thought: undefined, thoughtSignature: undefined })).toBe(false);
expect(isThinkingPart({ thought: false, thoughtSignature: "" })).toBe(false);
});
it("preserves the existing signature when subsequent deltas omit thoughtSignature", () => {
const first = retainThoughtSignature(undefined, "sig-1");
expect(first).toBe("sig-1");
const second = retainThoughtSignature(first, undefined);
expect(second).toBe("sig-1");
const third = retainThoughtSignature(second, "");
expect(third).toBe("sig-1");
});
it("updates the signature when a new non-empty signature arrives", () => {
const updated = retainThoughtSignature("sig-1", "sig-2");
expect(updated).toBe("sig-2");
});
});
@@ -1,105 +0,0 @@
import { Type } from "@sinclair/typebox";
import { afterEach, describe, expect, it, vi } from "vitest";
import { streamGoogleGeminiCli } from "../src/providers/google-gemini-cli.js";
import type { Context, Model, ToolCall } from "../src/types.js";
const emptySchema = Type.Object({});
const originalFetch = global.fetch;
afterEach(() => {
global.fetch = originalFetch;
vi.restoreAllMocks();
});
describe("google providers tool call missing args", () => {
it("defaults arguments to empty object when provider omits args field", async () => {
// Simulate a tool call response where args is missing (no-arg tool)
const sse = `${[
`data: ${JSON.stringify({
response: {
candidates: [
{
content: {
role: "model",
parts: [
{
functionCall: {
name: "get_status",
// args intentionally omitted
},
},
],
},
finishReason: "STOP",
},
],
usageMetadata: {
promptTokenCount: 10,
candidatesTokenCount: 5,
totalTokenCount: 15,
},
},
})}`,
].join("\n\n")}\n\n`;
const encoder = new TextEncoder();
const dataStream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(encoder.encode(sse));
controller.close();
},
});
const fetchMock = vi.fn(async () => {
return new Response(dataStream, {
status: 200,
headers: { "content-type": "text/event-stream" },
});
});
global.fetch = fetchMock as typeof fetch;
const model: Model<"google-gemini-cli"> = {
id: "gemini-2.5-flash",
name: "Gemini 2.5 Flash",
api: "google-gemini-cli",
provider: "google-gemini-cli",
baseUrl: "https://cloudcode-pa.googleapis.com",
reasoning: false,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 128000,
maxTokens: 8192,
};
const context: Context = {
messages: [{ role: "user", content: "Check status", timestamp: Date.now() }],
tools: [
{
name: "get_status",
description: "Get current status",
parameters: emptySchema,
},
],
};
const stream = streamGoogleGeminiCli(model, context, {
apiKey: JSON.stringify({ token: "token", projectId: "project" }),
});
for await (const _ of stream) {
// consume stream
}
const result = await stream.result();
expect(result.stopReason).toBe("toolUse");
expect(result.content).toHaveLength(1);
const toolCall = result.content[0] as ToolCall;
expect(toolCall.type).toBe("toolCall");
expect(toolCall.name).toBe("get_status");
expect(toolCall.arguments).toEqual({});
});
});
@@ -1,144 +0,0 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const googleGenAiMock = vi.hoisted(() => ({
constructorCalls: [] as Array<Record<string, unknown>>,
}));
vi.mock("@google/genai", () => {
class GoogleGenAI {
models = {
generateContentStream: async function* () {
yield {
responseId: "vertex-response-id",
candidates: [
{
content: { parts: [{ text: "ok" }] },
finishReason: "STOP",
},
],
usageMetadata: {
promptTokenCount: 1,
candidatesTokenCount: 1,
totalTokenCount: 2,
},
};
},
};
constructor(config: Record<string, unknown>) {
googleGenAiMock.constructorCalls.push(config);
}
}
return {
GoogleGenAI,
ThinkingLevel: {
THINKING_LEVEL_UNSPECIFIED: "THINKING_LEVEL_UNSPECIFIED",
MINIMAL: "MINIMAL",
LOW: "LOW",
MEDIUM: "MEDIUM",
HIGH: "HIGH",
},
};
});
import { getModel } from "../src/models.js";
import { streamGoogleVertex } from "../src/providers/google-vertex.js";
import type { Context } from "../src/types.js";
const model = getModel("google-vertex", "gemini-3-flash-preview");
const context: Context = {
messages: [{ role: "user", content: "hello", timestamp: Date.now() }],
};
const originalGoogleCloudApiKey = process.env.GOOGLE_CLOUD_API_KEY;
beforeEach(() => {
googleGenAiMock.constructorCalls.length = 0;
delete process.env.GOOGLE_CLOUD_API_KEY;
});
afterEach(() => {
if (originalGoogleCloudApiKey === undefined) {
delete process.env.GOOGLE_CLOUD_API_KEY;
} else {
process.env.GOOGLE_CLOUD_API_KEY = originalGoogleCloudApiKey;
}
});
describe("google-vertex api key resolution", () => {
it("falls back to ADC when options.apiKey is a placeholder marker", async () => {
const stream = streamGoogleVertex(model, context, {
apiKey: "<authenticated>",
project: "test-project",
location: "us-central1",
});
await stream.result();
expect(googleGenAiMock.constructorCalls).toHaveLength(1);
expect(googleGenAiMock.constructorCalls[0]).toMatchObject({
vertexai: true,
project: "test-project",
location: "us-central1",
apiVersion: "v1",
});
expect(googleGenAiMock.constructorCalls[0]).not.toHaveProperty("apiKey");
});
it("falls back to ADC when options.apiKey is the gcp-vertex-credentials marker", async () => {
const stream = streamGoogleVertex(model, context, {
apiKey: "gcp-vertex-credentials",
project: "test-project",
location: "us-central1",
});
await stream.result();
expect(googleGenAiMock.constructorCalls).toHaveLength(1);
expect(googleGenAiMock.constructorCalls[0]).toMatchObject({
vertexai: true,
project: "test-project",
location: "us-central1",
apiVersion: "v1",
});
expect(googleGenAiMock.constructorCalls[0]).not.toHaveProperty("apiKey");
});
it("falls back to ADC when GOOGLE_CLOUD_API_KEY is a placeholder marker", async () => {
process.env.GOOGLE_CLOUD_API_KEY = "<authenticated>";
const stream = streamGoogleVertex(model, context, {
project: "test-project",
location: "us-central1",
});
await stream.result();
expect(googleGenAiMock.constructorCalls).toHaveLength(1);
expect(googleGenAiMock.constructorCalls[0]).toMatchObject({
vertexai: true,
project: "test-project",
location: "us-central1",
apiVersion: "v1",
});
expect(googleGenAiMock.constructorCalls[0]).not.toHaveProperty("apiKey");
});
it("still uses the API key client for real API keys", async () => {
const stream = streamGoogleVertex(model, context, {
apiKey: "AIzaSyExampleRealisticLookingApiKey123456",
});
await stream.result();
expect(googleGenAiMock.constructorCalls).toHaveLength(1);
expect(googleGenAiMock.constructorCalls[0]).toMatchObject({
vertexai: true,
apiKey: "AIzaSyExampleRealisticLookingApiKey123456",
apiVersion: "v1",
});
expect(googleGenAiMock.constructorCalls[0]).not.toHaveProperty("project");
expect(googleGenAiMock.constructorCalls[0]).not.toHaveProperty("location");
});
});
@@ -1,481 +0,0 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { Type } from "@sinclair/typebox";
import { describe, expect, it } from "vitest";
import type { Api, Context, Model, Tool, ToolResultMessage } from "../src/index.js";
import { complete, getModel } from "../src/index.js";
import type { StreamOptions } from "../src/types.js";
type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;
import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.js";
import { hasBedrockCredentials } from "./bedrock-utils.js";
import { resolveApiKey } from "./oauth.js";
// Resolve OAuth tokens at module level (async, runs before tests)
const oauthTokens = await Promise.all([
resolveApiKey("anthropic"),
resolveApiKey("github-copilot"),
resolveApiKey("google-gemini-cli"),
resolveApiKey("google-antigravity"),
resolveApiKey("openai-codex"),
]);
const [anthropicOAuthToken, githubCopilotToken, geminiCliToken, antigravityToken, openaiCodexToken] = oauthTokens;
/**
* Test that tool results containing only images work correctly across all providers.
* This verifies that:
* 1. Tool results can contain image content blocks
* 2. Providers correctly pass images from tool results to the LLM
* 3. The LLM can see and describe images returned by tools
*/
async function handleToolWithImageResult<TApi extends Api>(model: Model<TApi>, options?: StreamOptionsWithExtras) {
// Check if the model supports images
if (!model.input.includes("image")) {
console.log(`Skipping tool image result test - model ${model.id} doesn't support images`);
return;
}
// Read the test image
const imagePath = join(__dirname, "data", "red-circle.png");
const imageBuffer = readFileSync(imagePath);
const base64Image = imageBuffer.toString("base64");
// Define a tool that returns only an image (no text)
const getImageSchema = Type.Object({});
const getImageTool: Tool<typeof getImageSchema> = {
name: "get_circle",
description: "Returns a circle image for visualization",
parameters: getImageSchema,
};
const context: Context = {
systemPrompt: "You are a helpful assistant that uses tools when asked.",
messages: [
{
role: "user",
content: "Call the get_circle tool to get an image, and describe what you see, shapes, colors, etc.",
timestamp: Date.now(),
},
],
tools: [getImageTool],
};
// First request - LLM should call the tool
const firstResponse = await complete(model, context, options);
expect(firstResponse.stopReason).toBe("toolUse");
// Find the tool call
const toolCall = firstResponse.content.find((b) => b.type === "toolCall");
expect(toolCall).toBeTruthy();
if (!toolCall || toolCall.type !== "toolCall") {
throw new Error("Expected tool call");
}
expect(toolCall.name).toBe("get_circle");
// Add the tool call to context
context.messages.push(firstResponse);
// Create tool result with ONLY an image (no text)
const toolResult: ToolResultMessage = {
role: "toolResult",
toolCallId: toolCall.id,
toolName: toolCall.name,
content: [
{
type: "image",
data: base64Image,
mimeType: "image/png",
},
],
isError: false,
timestamp: Date.now(),
};
context.messages.push(toolResult);
// Second request - LLM should describe the image from the tool result
const secondResponse = await complete(model, context, options);
expect(secondResponse.stopReason).toBe("stop");
expect(secondResponse.errorMessage).toBeFalsy();
// Verify the LLM can see and describe the image
const textContent = secondResponse.content.find((b) => b.type === "text");
expect(textContent).toBeTruthy();
if (textContent && textContent.type === "text") {
const lowerContent = textContent.text.toLowerCase();
// Should mention red and circle since that's what the image shows
expect(lowerContent).toContain("red");
expect(lowerContent).toContain("circle");
}
}
/**
* Test that tool results containing both text and images work correctly across all providers.
* This verifies that:
* 1. Tool results can contain mixed content blocks (text + images)
* 2. Providers correctly pass both text and images from tool results to the LLM
* 3. The LLM can see both the text and images in tool results
*/
async function handleToolWithTextAndImageResult<TApi extends Api>(
model: Model<TApi>,
options?: StreamOptionsWithExtras,
) {
// Check if the model supports images
if (!model.input.includes("image")) {
console.log(`Skipping tool text+image result test - model ${model.id} doesn't support images`);
return;
}
// Read the test image
const imagePath = join(__dirname, "data", "red-circle.png");
const imageBuffer = readFileSync(imagePath);
const base64Image = imageBuffer.toString("base64");
// Define a tool that returns both text and an image
const getImageSchema = Type.Object({});
const getImageTool: Tool<typeof getImageSchema> = {
name: "get_circle_with_description",
description: "Returns a circle image with a text description",
parameters: getImageSchema,
};
const context: Context = {
systemPrompt: "You are a helpful assistant that uses tools when asked.",
messages: [
{
role: "user",
content:
"Use the get_circle_with_description tool and tell me what you learned. Also say what color the shape is.",
timestamp: Date.now(),
},
],
tools: [getImageTool],
};
// First request - LLM should call the tool
const firstResponse = await complete(model, context, options);
expect(firstResponse.stopReason).toBe("toolUse");
// Find the tool call
const toolCall = firstResponse.content.find((b) => b.type === "toolCall");
expect(toolCall).toBeTruthy();
if (!toolCall || toolCall.type !== "toolCall") {
throw new Error("Expected tool call");
}
expect(toolCall.name).toBe("get_circle_with_description");
// Add the tool call to context
context.messages.push(firstResponse);
// Create tool result with BOTH text and image
const toolResult: ToolResultMessage = {
role: "toolResult",
toolCallId: toolCall.id,
toolName: toolCall.name,
content: [
{
type: "text",
text: "This is a geometric shape with specific properties: it has a diameter of 100 pixels.",
},
{
type: "image",
data: base64Image,
mimeType: "image/png",
},
],
isError: false,
timestamp: Date.now(),
};
context.messages.push(toolResult);
// Second request - LLM should describe both the text and image from the tool result
const secondResponse = await complete(model, context, options);
expect(secondResponse.stopReason).toBe("stop");
expect(secondResponse.errorMessage).toBeFalsy();
// Verify the LLM can see both text and image
const textContent = secondResponse.content.find((b) => b.type === "text");
expect(textContent).toBeTruthy();
if (textContent && textContent.type === "text") {
const lowerContent = textContent.text.toLowerCase();
// Should mention details from the text (diameter/pixels)
expect(lowerContent.match(/diameter|100|pixel/)).toBeTruthy();
// Should also mention the visual properties (red and circle)
expect(lowerContent).toContain("red");
expect(lowerContent).toContain("circle");
}
}
describe("Tool Results with Images", () => {
describe.skipIf(!process.env.GEMINI_API_KEY)("Google Provider (gemini-2.5-flash)", () => {
const llm = getModel("google", "gemini-2.5-flash");
it("should handle tool result with only image", { retry: 3, timeout: 30000 }, async () => {
await handleToolWithImageResult(llm);
});
it("should handle tool result with text and image", { retry: 3, timeout: 30000 }, async () => {
await handleToolWithTextAndImageResult(llm);
});
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions Provider (gpt-4o-mini)", () => {
const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini");
void _compat;
const llm: Model<"openai-completions"> = {
...baseModel,
api: "openai-completions",
};
it("should handle tool result with only image", { retry: 3, timeout: 30000 }, async () => {
await handleToolWithImageResult(llm);
});
it("should handle tool result with text and image", { retry: 3, timeout: 30000 }, async () => {
await handleToolWithTextAndImageResult(llm);
});
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses Provider (gpt-5-mini)", () => {
const llm = getModel("openai", "gpt-5-mini");
it("should handle tool result with only image", { retry: 3, timeout: 30000 }, async () => {
await handleToolWithImageResult(llm);
});
it("should handle tool result with text and image", { retry: 3, timeout: 30000 }, async () => {
await handleToolWithTextAndImageResult(llm);
});
});
describe.skipIf(!hasAzureOpenAICredentials())("Azure OpenAI Responses Provider (gpt-4o-mini)", () => {
const llm = getModel("azure-openai-responses", "gpt-4o-mini");
const azureDeploymentName = resolveAzureDeploymentName(llm.id);
const azureOptions = azureDeploymentName ? { azureDeploymentName } : {};
it("should handle tool result with only image", { retry: 3, timeout: 30000 }, async () => {
await handleToolWithImageResult(llm, azureOptions);
});
it("should handle tool result with text and image", { retry: 3, timeout: 30000 }, async () => {
await handleToolWithTextAndImageResult(llm, azureOptions);
});
});
describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic Provider (claude-haiku-4-5)", () => {
const model = getModel("anthropic", "claude-haiku-4-5");
it("should handle tool result with only image", { retry: 3, timeout: 30000 }, async () => {
await handleToolWithImageResult(model);
});
it("should handle tool result with text and image", { retry: 3, timeout: 30000 }, async () => {
await handleToolWithTextAndImageResult(model);
});
});
describe.skipIf(!process.env.OPENROUTER_API_KEY)("OpenRouter Provider (glm-4.5v)", () => {
const llm = getModel("openrouter", "z-ai/glm-4.5v");
it("should handle tool result with only image", { retry: 3, timeout: 30000 }, async () => {
await handleToolWithImageResult(llm);
});
it("should handle tool result with text and image", { retry: 3, timeout: 30000 }, async () => {
await handleToolWithTextAndImageResult(llm);
});
});
describe.skipIf(!process.env.MISTRAL_API_KEY)("Mistral Provider (pixtral-12b)", () => {
const llm = getModel("mistral", "pixtral-12b");
it("should handle tool result with only image", { retry: 5, timeout: 30000 }, async () => {
await handleToolWithImageResult(llm);
});
it("should handle tool result with text and image", { retry: 5, timeout: 30000 }, async () => {
await handleToolWithTextAndImageResult(llm);
});
});
describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider (kimi-for-coding)", () => {
const llm = getModel("kimi-coding", "kimi-for-coding");
it("should handle tool result with only image", { retry: 3, timeout: 30000 }, async () => {
await handleToolWithImageResult(llm);
});
it("should handle tool result with text and image", { retry: 3, timeout: 30000 }, async () => {
await handleToolWithTextAndImageResult(llm);
});
});
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider (google/gemini-2.5-flash)", () => {
const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash");
it("should handle tool result with only image", { retry: 3, timeout: 30000 }, async () => {
await handleToolWithImageResult(llm);
});
it("should handle tool result with text and image", { retry: 3, timeout: 30000 }, async () => {
await handleToolWithTextAndImageResult(llm);
});
});
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider (claude-sonnet-4-5)", () => {
const llm = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0");
it("should handle tool result with only image", { retry: 3, timeout: 30000 }, async () => {
await handleToolWithImageResult(llm);
});
it("should handle tool result with text and image", { retry: 3, timeout: 30000 }, async () => {
await handleToolWithTextAndImageResult(llm);
});
});
// =========================================================================
// OAuth-based providers (credentials from ~/.pi/agent/oauth.json)
// =========================================================================
describe("Anthropic OAuth Provider (claude-sonnet-4-5)", () => {
const model = getModel("anthropic", "claude-sonnet-4-5");
it.skipIf(!anthropicOAuthToken)(
"should handle tool result with only image",
{ retry: 3, timeout: 30000 },
async () => {
await handleToolWithImageResult(model, { apiKey: anthropicOAuthToken });
},
);
it.skipIf(!anthropicOAuthToken)(
"should handle tool result with text and image",
{ retry: 3, timeout: 30000 },
async () => {
await handleToolWithTextAndImageResult(model, { apiKey: anthropicOAuthToken });
},
);
});
describe("GitHub Copilot Provider", () => {
it.skipIf(!githubCopilotToken)(
"gpt-4o - should handle tool result with only image",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "gpt-4o");
await handleToolWithImageResult(llm, { apiKey: githubCopilotToken });
},
);
it.skipIf(!githubCopilotToken)(
"gpt-4o - should handle tool result with text and image",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "gpt-4o");
await handleToolWithTextAndImageResult(llm, { apiKey: githubCopilotToken });
},
);
it.skipIf(!githubCopilotToken)(
"claude-sonnet-4 - should handle tool result with only image",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "claude-sonnet-4");
await handleToolWithImageResult(llm, { apiKey: githubCopilotToken });
},
);
it.skipIf(!githubCopilotToken)(
"claude-sonnet-4 - should handle tool result with text and image",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "claude-sonnet-4");
await handleToolWithTextAndImageResult(llm, { apiKey: githubCopilotToken });
},
);
});
describe("Google Gemini CLI Provider", () => {
it.skipIf(!geminiCliToken)(
"gemini-2.5-flash - should handle tool result with only image",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
await handleToolWithImageResult(llm, { apiKey: geminiCliToken });
},
);
it.skipIf(!geminiCliToken)(
"gemini-2.5-flash - should handle tool result with text and image",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
await handleToolWithTextAndImageResult(llm, { apiKey: geminiCliToken });
},
);
});
describe("Google Antigravity Provider", () => {
it.skipIf(!antigravityToken)(
"gemini-3-flash - should handle tool result with only image",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "gemini-3-flash");
await handleToolWithImageResult(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"gemini-3-flash - should handle tool result with text and image",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "gemini-3-flash");
await handleToolWithTextAndImageResult(llm, { apiKey: antigravityToken });
},
);
/** These two don't work, the model simply won't call the tool, works in pi
it.skipIf(!antigravityToken)(
"claude-sonnet-4-5 - should handle tool result with only image",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
await handleToolWithImageResult(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"claude-sonnet-4-5 - should handle tool result with text and image",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
await handleToolWithTextAndImageResult(llm, { apiKey: antigravityToken });
},
);**/
// Note: gpt-oss-120b-medium does not support images, so not tested here
});
describe("OpenAI Codex Provider", () => {
it.skipIf(!openaiCodexToken)(
"gpt-5.2-codex - should handle tool result with only image",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("openai-codex", "gpt-5.2-codex");
await handleToolWithImageResult(llm, { apiKey: openaiCodexToken });
},
);
it.skipIf(!openaiCodexToken)(
"gpt-5.2-codex - should handle tool result with text and image",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("openai-codex", "gpt-5.2-codex");
await handleToolWithTextAndImageResult(llm, { apiKey: openaiCodexToken });
},
);
});
});
@@ -1,145 +0,0 @@
import { Type } from "@sinclair/typebox";
import { describe, expect, it } from "vitest";
import { getEnvApiKey } from "../src/env-api-keys.js";
import { getModel } from "../src/models.js";
import { completeSimple } from "../src/stream.js";
import type { Api, Context, Model, StopReason, Tool, ToolCall, ToolResultMessage } from "../src/types.js";
import { StringEnum } from "../src/utils/typebox-helpers.js";
import { hasBedrockCredentials } from "./bedrock-utils.js";
const calculatorSchema = Type.Object({
a: Type.Number({ description: "First number" }),
b: Type.Number({ description: "Second number" }),
operation: StringEnum(["add", "subtract", "multiply", "divide"], {
description: "The operation to perform.",
}),
});
const calculatorTool: Tool<typeof calculatorSchema> = {
name: "calculator",
description: "Perform basic arithmetic operations",
parameters: calculatorSchema,
};
type CalculatorOperation = "add" | "subtract" | "multiply" | "divide";
type CalculatorArguments = {
a: number;
b: number;
operation: CalculatorOperation;
};
function asCalculatorArguments(args: ToolCall["arguments"]): CalculatorArguments {
if (typeof args !== "object" || args === null) {
throw new Error("Tool arguments must be an object");
}
const value = args as Record<string, unknown>;
const operation = value.operation;
if (
typeof value.a !== "number" ||
typeof value.b !== "number" ||
(operation !== "add" && operation !== "subtract" && operation !== "multiply" && operation !== "divide")
) {
throw new Error("Invalid calculator arguments");
}
return { a: value.a, b: value.b, operation };
}
function evaluateCalculatorCall(toolCall: ToolCall): number {
const { a, b, operation } = asCalculatorArguments(toolCall.arguments);
switch (operation) {
case "add":
return a + b;
case "subtract":
return a - b;
case "multiply":
return a * b;
case "divide":
return a / b;
}
}
async function assertSecondToolCallWithInterleavedThinking<TApi extends Api>(
llm: Model<TApi>,
reasoning: "high" | "xhigh",
) {
const context: Context = {
systemPrompt: [
"You are a helpful assistant that must use tools for arithmetic.",
"Always think before every tool call, not just the first one.",
"Do not answer with plain text when a tool call is required.",
].join(" "),
messages: [
{
role: "user",
content: [
"Use calculator to calculate 328 * 29.",
"You must call the calculator tool exactly once.",
"Provide the final answer based on the best guess given the tool result, even if it seems unreliable.",
"Start by thinking about the steps you will take to solve the problem.",
].join(" "),
timestamp: Date.now(),
},
],
tools: [calculatorTool],
};
const firstResponse = await completeSimple(llm, context, { reasoning });
expect(firstResponse.stopReason, `Error: ${firstResponse.errorMessage}`).toBe("toolUse" satisfies StopReason);
expect(firstResponse.content.some((block) => block.type === "thinking")).toBe(true);
expect(firstResponse.content.some((block) => block.type === "toolCall")).toBe(true);
const firstToolCall = firstResponse.content.find((block) => block.type === "toolCall");
expect(firstToolCall?.type).toBe("toolCall");
if (!firstToolCall || firstToolCall.type !== "toolCall") {
throw new Error("Expected first response to include a tool call");
}
context.messages.push(firstResponse);
const correctAnswer = evaluateCalculatorCall(firstToolCall);
const firstToolResult: ToolResultMessage = {
role: "toolResult",
toolCallId: firstToolCall.id,
toolName: firstToolCall.name,
content: [{ type: "text", text: `The answer is ${correctAnswer} or ${correctAnswer * 2}.` }],
isError: false,
timestamp: Date.now(),
};
context.messages.push(firstToolResult);
const secondResponse = await completeSimple(llm, context, { reasoning });
expect(secondResponse.stopReason, `Error: ${secondResponse.errorMessage}`).toBe("stop" satisfies StopReason);
expect(secondResponse.content.some((block) => block.type === "thinking")).toBe(true);
expect(secondResponse.content.some((block) => block.type === "text")).toBe(true);
}
const hasAnthropicCredentials = !!getEnvApiKey("anthropic");
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock interleaved thinking", () => {
it("should do interleaved thinking on Claude Opus 4.5", { retry: 3 }, async () => {
const llm = getModel("amazon-bedrock", "global.anthropic.claude-opus-4-5-20251101-v1:0");
await assertSecondToolCallWithInterleavedThinking(llm, "high");
});
it("should do interleaved thinking on Claude Opus 4.6", { retry: 3 }, async () => {
const llm = getModel("amazon-bedrock", "global.anthropic.claude-opus-4-6-v1");
await assertSecondToolCallWithInterleavedThinking(llm, "high");
});
});
describe.skipIf(!hasAnthropicCredentials)("Anthropic interleaved thinking", () => {
it("should do interleaved thinking on Claude Opus 4.5", { retry: 3 }, async () => {
const llm = getModel("anthropic", "claude-opus-4-5");
await assertSecondToolCallWithInterleavedThinking(llm, "high");
});
it("should do interleaved thinking on Claude Opus 4.6", { retry: 3 }, async () => {
const llm = getModel("anthropic", "claude-opus-4-6");
await assertSecondToolCallWithInterleavedThinking(llm, "high");
});
});
@@ -1,102 +0,0 @@
import { spawnSync } from "node:child_process";
import { createRequire } from "node:module";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
const require = createRequire(import.meta.url);
const tsxLoader = require.resolve("tsx/esm");
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const aiEntryUrl = new URL("../src/index.ts", import.meta.url).href;
const SDK_SPECIFIERS = [
"@anthropic-ai/sdk",
"openai",
"@google/genai",
"@mistralai/mistralai",
"@aws-sdk/client-bedrock-runtime",
] as const;
type ProbeResult = {
loadedSpecifiers: string[];
};
function runProbe(action: string): ProbeResult {
const script = `
import { registerHooks } from "node:module";
const targets = new Set(${JSON.stringify(SDK_SPECIFIERS)});
const loaded = [];
registerHooks({
resolve(specifier, context, nextResolve) {
if (targets.has(specifier)) {
loaded.push(specifier);
}
return nextResolve(specifier, context);
},
});
const mod = await import(${JSON.stringify(aiEntryUrl)});
${action}
console.log(JSON.stringify({ loadedSpecifiers: [...new Set(loaded)] }));
`;
const result = spawnSync(process.execPath, ["--import", tsxLoader, "--input-type=module", "--eval", script], {
cwd: packageRoot,
encoding: "utf8",
});
if (result.status !== 0) {
throw new Error(`Probe failed (exit ${result.status})\nSTDOUT:\n${result.stdout}\nSTDERR:\n${result.stderr}`);
}
const stdoutLines = result.stdout
.split(/\r?\n/)
.map((line) => line.trim())
.filter((line) => line.length > 0);
const lastLine = stdoutLines.at(-1);
if (!lastLine) {
throw new Error(`Probe produced no output\nSTDERR:\n${result.stderr}`);
}
return JSON.parse(lastLine) as ProbeResult;
}
describe("lazy provider module loading", () => {
it("does not load provider SDKs when importing the root barrel", () => {
const result = runProbe("");
expect(result.loadedSpecifiers).toEqual([]);
});
it("loads only the Anthropic SDK when calling the root lazy wrapper", () => {
const result = runProbe(`
const model = {
id: "claude-sonnet-4-20250514",
name: "Claude Sonnet 4",
api: "anthropic-messages",
provider: "anthropic",
baseUrl: "https://api.anthropic.com",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 200000,
maxTokens: 8192,
};
const context = { messages: [{ role: "user", content: "hi" }] };
await mod.streamSimpleAnthropic(model, context).result();
`);
expect(result.loadedSpecifiers).toEqual(["@anthropic-ai/sdk"]);
});
it("loads only the Anthropic SDK when dispatching through streamSimple", () => {
const result = runProbe(`
const model = mod.getModel("anthropic", "claude-sonnet-4-20250514");
const context = { messages: [{ role: "user", content: "hi" }] };
await mod.streamSimple(model, context).result();
`);
expect(result.loadedSpecifiers).toEqual(["@anthropic-ai/sdk"]);
});
});
@@ -1,89 +0,0 @@
/**
* Test helper for resolving API keys from ~/.pi/agent/auth.json
*
* Supports both API key and OAuth credentials.
* OAuth tokens are automatically refreshed if expired and saved back to auth.json.
*/
import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
import { homedir } from "os";
import { dirname, join } from "path";
import { getOAuthApiKey } from "../src/utils/oauth/index.js";
import type { OAuthCredentials, OAuthProvider } from "../src/utils/oauth/types.js";
const AUTH_PATH = join(homedir(), ".pi", "agent", "auth.json");
type ApiKeyCredential = {
type: "api_key";
key: string;
};
type OAuthCredentialEntry = {
type: "oauth";
} & OAuthCredentials;
type AuthCredential = ApiKeyCredential | OAuthCredentialEntry;
type AuthStorage = Record<string, AuthCredential>;
function loadAuthStorage(): AuthStorage {
if (!existsSync(AUTH_PATH)) {
return {};
}
try {
const content = readFileSync(AUTH_PATH, "utf-8");
return JSON.parse(content);
} catch {
return {};
}
}
function saveAuthStorage(storage: AuthStorage): void {
const configDir = dirname(AUTH_PATH);
if (!existsSync(configDir)) {
mkdirSync(configDir, { recursive: true, mode: 0o700 });
}
writeFileSync(AUTH_PATH, JSON.stringify(storage, null, 2), "utf-8");
chmodSync(AUTH_PATH, 0o600);
}
/**
* Resolve API key for a provider from ~/.pi/agent/auth.json
*
* For API key credentials, returns the key directly.
* For OAuth credentials, returns the access token (refreshing if expired and saving back).
*
* For google-gemini-cli and google-antigravity, returns JSON-encoded { token, projectId }
*/
export async function resolveApiKey(provider: string): Promise<string | undefined> {
const storage = loadAuthStorage();
const entry = storage[provider];
if (!entry) return undefined;
if (entry.type === "api_key") {
return entry.key;
}
if (entry.type === "oauth") {
// Build OAuthCredentials record for getOAuthApiKey
const oauthCredentials: Record<string, OAuthCredentials> = {};
for (const [key, value] of Object.entries(storage)) {
if (value.type === "oauth") {
const { type: _, ...creds } = value;
oauthCredentials[key] = creds;
}
}
const result = await getOAuthApiKey(provider as OAuthProvider, oauthCredentials);
if (!result) return undefined;
// Save refreshed credentials back to auth.json
storage[provider] = { type: "oauth", ...result.newCredentials };
saveAuthStorage(storage);
return result.apiKey;
}
return undefined;
}
@@ -1,36 +0,0 @@
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { complete } from "../src/stream.js";
import type { Context } from "../src/types.js";
import { resolveApiKey } from "./oauth.js";
const codexToken = await resolveApiKey("openai-codex");
describe("openai-codex cache affinity e2e", () => {
it.skipIf(!codexToken)("handles SSE requests with aligned cache-affinity identifiers", async () => {
const model = getModel("openai-codex", "gpt-5.3-codex");
const sessionId = "0195d6e4-4cf9-7f44-a2d8-f8f7f49ee9d3";
const context: Context = {
systemPrompt: "You are a helpful assistant. Reply exactly as requested.",
messages: [
{
role: "user",
content: "Reply with exactly: cache affinity e2e success",
timestamp: Date.now(),
},
],
};
const response = await complete(model, context, {
apiKey: codexToken,
sessionId,
transport: "sse",
});
expect(response.stopReason, response.errorMessage).not.toBe("error");
expect(response.errorMessage).toBeUndefined();
expect(response.content.map((block) => (block.type === "text" ? block.text : "")).join("")).toContain(
"cache affinity e2e success",
);
});
});
@@ -1,597 +0,0 @@
import { mkdtempSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { streamOpenAICodexResponses } from "../src/providers/openai-codex-responses.js";
import type { Context, Model } from "../src/types.js";
const originalFetch = global.fetch;
const originalAgentDir = process.env.PI_CODING_AGENT_DIR;
afterEach(() => {
global.fetch = originalFetch;
if (originalAgentDir === undefined) {
delete process.env.PI_CODING_AGENT_DIR;
} else {
process.env.PI_CODING_AGENT_DIR = originalAgentDir;
}
vi.restoreAllMocks();
});
function mockToken(): string {
const payload = Buffer.from(
JSON.stringify({ "https://api.openai.com/auth": { chatgpt_account_id: "acc_test" } }),
"utf8",
).toString("base64");
return `aaa.${payload}.bbb`;
}
function buildSSEPayload({
status,
includeDone = false,
}: {
status: "completed" | "incomplete";
includeDone?: boolean;
}): string {
const terminalType = status === "incomplete" ? "response.incomplete" : "response.completed";
const events = [
`data: ${JSON.stringify({
type: "response.output_item.added",
item: { type: "message", id: "msg_1", role: "assistant", status: "in_progress", content: [] },
})}`,
`data: ${JSON.stringify({ type: "response.content_part.added", part: { type: "output_text", text: "" } })}`,
`data: ${JSON.stringify({ type: "response.output_text.delta", delta: "Hello" })}`,
`data: ${JSON.stringify({
type: "response.output_item.done",
item: {
type: "message",
id: "msg_1",
role: "assistant",
status: "completed",
content: [{ type: "output_text", text: "Hello" }],
},
})}`,
`data: ${JSON.stringify({
type: terminalType,
response: {
status,
incomplete_details: status === "incomplete" ? { reason: "max_output_tokens" } : null,
usage: {
input_tokens: 5,
output_tokens: 3,
total_tokens: 8,
input_tokens_details: { cached_tokens: 0 },
},
},
})}`,
];
if (includeDone) {
events.push("data: [DONE]");
}
return `${events.join("\n\n")}\n\n`;
}
describe("openai-codex streaming", () => {
it("streams SSE responses into AssistantMessageEventStream", async () => {
const tempDir = mkdtempSync(join(tmpdir(), "pi-codex-stream-"));
process.env.PI_CODING_AGENT_DIR = tempDir;
const payload = Buffer.from(
JSON.stringify({ "https://api.openai.com/auth": { chatgpt_account_id: "acc_test" } }),
"utf8",
).toString("base64");
const token = `aaa.${payload}.bbb`;
const sse = `${[
`data: ${JSON.stringify({
type: "response.output_item.added",
item: { type: "message", id: "msg_1", role: "assistant", status: "in_progress", content: [] },
})}`,
`data: ${JSON.stringify({ type: "response.content_part.added", part: { type: "output_text", text: "" } })}`,
`data: ${JSON.stringify({ type: "response.output_text.delta", delta: "Hello" })}`,
`data: ${JSON.stringify({
type: "response.output_item.done",
item: {
type: "message",
id: "msg_1",
role: "assistant",
status: "completed",
content: [{ type: "output_text", text: "Hello" }],
},
})}`,
`data: ${JSON.stringify({
type: "response.completed",
response: {
status: "completed",
usage: {
input_tokens: 5,
output_tokens: 3,
total_tokens: 8,
input_tokens_details: { cached_tokens: 0 },
},
},
})}`,
].join("\n\n")}\n\n`;
const encoder = new TextEncoder();
const stream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(encoder.encode(sse));
controller.close();
},
});
const fetchMock = vi.fn(async (input: string | URL, init?: RequestInit) => {
const url = typeof input === "string" ? input : input.toString();
if (url === "https://api.github.com/repos/openai/codex/releases/latest") {
return new Response(JSON.stringify({ tag_name: "rust-v0.0.0" }), { status: 200 });
}
if (url.startsWith("https://raw.githubusercontent.com/openai/codex/")) {
return new Response("PROMPT", { status: 200, headers: { etag: '"etag"' } });
}
if (url === "https://chatgpt.com/backend-api/codex/responses") {
const headers = init?.headers instanceof Headers ? init.headers : undefined;
expect(headers?.get("Authorization")).toBe(`Bearer ${token}`);
expect(headers?.get("chatgpt-account-id")).toBe("acc_test");
expect(headers?.get("OpenAI-Beta")).toBe("responses=experimental");
expect(headers?.get("originator")).toBe("pi");
expect(headers?.get("accept")).toBe("text/event-stream");
expect(headers?.has("x-api-key")).toBe(false);
return new Response(stream, {
status: 200,
headers: { "content-type": "text/event-stream" },
});
}
return new Response("not found", { status: 404 });
});
global.fetch = fetchMock as typeof fetch;
const model: Model<"openai-codex-responses"> = {
id: "gpt-5.1-codex",
name: "GPT-5.1 Codex",
api: "openai-codex-responses",
provider: "openai-codex",
baseUrl: "https://chatgpt.com/backend-api",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 400000,
maxTokens: 128000,
};
const context: Context = {
systemPrompt: "You are a helpful assistant.",
messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }],
};
const streamResult = streamOpenAICodexResponses(model, context, { apiKey: token });
let sawTextDelta = false;
let sawDone = false;
for await (const event of streamResult) {
if (event.type === "text_delta") {
sawTextDelta = true;
}
if (event.type === "done") {
sawDone = true;
expect(event.message.content.find((c) => c.type === "text")?.text).toBe("Hello");
}
}
expect(sawTextDelta).toBe(true);
expect(sawDone).toBe(true);
});
it("completes after response.completed even when the SSE body stays open", async () => {
const tempDir = mkdtempSync(join(tmpdir(), "pi-codex-stream-"));
process.env.PI_CODING_AGENT_DIR = tempDir;
const token = mockToken();
const encoder = new TextEncoder();
const sse = buildSSEPayload({ status: "completed", includeDone: true });
const stream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(encoder.encode(sse));
},
});
global.fetch = vi.fn(async (input: string | URL) => {
const url = typeof input === "string" ? input : input.toString();
if (url === "https://api.github.com/repos/openai/codex/releases/latest") {
return new Response(JSON.stringify({ tag_name: "rust-v0.0.0" }), { status: 200 });
}
if (url.startsWith("https://raw.githubusercontent.com/openai/codex/")) {
return new Response("PROMPT", { status: 200, headers: { etag: '"etag"' } });
}
if (url === "https://chatgpt.com/backend-api/codex/responses") {
return new Response(stream, {
status: 200,
headers: { "content-type": "text/event-stream" },
});
}
return new Response("not found", { status: 404 });
}) as typeof fetch;
const model: Model<"openai-codex-responses"> = {
id: "gpt-5.1-codex",
name: "GPT-5.1 Codex",
api: "openai-codex-responses",
provider: "openai-codex",
baseUrl: "https://chatgpt.com/backend-api",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 400000,
maxTokens: 128000,
};
const context: Context = {
systemPrompt: "You are a helpful assistant.",
messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }],
};
const result = await Promise.race([
streamOpenAICodexResponses(model, context, { apiKey: token }).result(),
new Promise<never>((_, reject) => {
setTimeout(() => reject(new Error("Timed out waiting for completed SSE stream")), 1000);
}),
]);
expect(result.content.find((c) => c.type === "text")?.text).toBe("Hello");
expect(result.stopReason).toBe("stop");
});
it("maps response.incomplete to stopReason length even when the SSE body stays open", async () => {
const tempDir = mkdtempSync(join(tmpdir(), "pi-codex-stream-"));
process.env.PI_CODING_AGENT_DIR = tempDir;
const token = mockToken();
const encoder = new TextEncoder();
const sse = buildSSEPayload({ status: "incomplete" });
const stream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(encoder.encode(sse));
},
});
global.fetch = vi.fn(async (input: string | URL) => {
const url = typeof input === "string" ? input : input.toString();
if (url === "https://api.github.com/repos/openai/codex/releases/latest") {
return new Response(JSON.stringify({ tag_name: "rust-v0.0.0" }), { status: 200 });
}
if (url.startsWith("https://raw.githubusercontent.com/openai/codex/")) {
return new Response("PROMPT", { status: 200, headers: { etag: '"etag"' } });
}
if (url === "https://chatgpt.com/backend-api/codex/responses") {
return new Response(stream, {
status: 200,
headers: { "content-type": "text/event-stream" },
});
}
return new Response("not found", { status: 404 });
}) as typeof fetch;
const model: Model<"openai-codex-responses"> = {
id: "gpt-5.1-codex",
name: "GPT-5.1 Codex",
api: "openai-codex-responses",
provider: "openai-codex",
baseUrl: "https://chatgpt.com/backend-api",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 400000,
maxTokens: 128000,
};
const context: Context = {
systemPrompt: "You are a helpful assistant.",
messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }],
};
const result = await Promise.race([
streamOpenAICodexResponses(model, context, { apiKey: token }).result(),
new Promise<never>((_, reject) => {
setTimeout(() => reject(new Error("Timed out waiting for incomplete SSE stream")), 1000);
}),
]);
expect(result.content.find((c) => c.type === "text")?.text).toBe("Hello");
expect(result.stopReason).toBe("length");
});
it("sets session_id/x-client-request-id headers and prompt_cache_key when sessionId is provided", async () => {
const tempDir = mkdtempSync(join(tmpdir(), "pi-codex-stream-"));
process.env.PI_CODING_AGENT_DIR = tempDir;
const payload = Buffer.from(
JSON.stringify({ "https://api.openai.com/auth": { chatgpt_account_id: "acc_test" } }),
"utf8",
).toString("base64");
const token = `aaa.${payload}.bbb`;
const sse = `${[
`data: ${JSON.stringify({
type: "response.output_item.added",
item: { type: "message", id: "msg_1", role: "assistant", status: "in_progress", content: [] },
})}`,
`data: ${JSON.stringify({ type: "response.content_part.added", part: { type: "output_text", text: "" } })}`,
`data: ${JSON.stringify({ type: "response.output_text.delta", delta: "Hello" })}`,
`data: ${JSON.stringify({
type: "response.output_item.done",
item: {
type: "message",
id: "msg_1",
role: "assistant",
status: "completed",
content: [{ type: "output_text", text: "Hello" }],
},
})}`,
`data: ${JSON.stringify({
type: "response.completed",
response: {
status: "completed",
usage: {
input_tokens: 5,
output_tokens: 3,
total_tokens: 8,
input_tokens_details: { cached_tokens: 0 },
},
},
})}`,
].join("\n\n")}\n\n`;
const encoder = new TextEncoder();
const stream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(encoder.encode(sse));
controller.close();
},
});
const sessionId = "test-session-123";
const fetchMock = vi.fn(async (input: string | URL, init?: RequestInit) => {
const url = typeof input === "string" ? input : input.toString();
if (url === "https://api.github.com/repos/openai/codex/releases/latest") {
return new Response(JSON.stringify({ tag_name: "rust-v0.0.0" }), { status: 200 });
}
if (url.startsWith("https://raw.githubusercontent.com/openai/codex/")) {
return new Response("PROMPT", { status: 200, headers: { etag: '"etag"' } });
}
if (url === "https://chatgpt.com/backend-api/codex/responses") {
const headers = init?.headers instanceof Headers ? init.headers : undefined;
// Verify sessionId is set in headers
expect(headers?.get("session_id")).toBe(sessionId);
expect(headers?.get("x-client-request-id")).toBe(sessionId);
// Verify sessionId is set in request body as prompt_cache_key
const body = typeof init?.body === "string" ? (JSON.parse(init.body) as Record<string, unknown>) : null;
expect(body?.prompt_cache_key).toBe(sessionId);
return new Response(stream, {
status: 200,
headers: { "content-type": "text/event-stream" },
});
}
return new Response("not found", { status: 404 });
});
global.fetch = fetchMock as typeof fetch;
const model: Model<"openai-codex-responses"> = {
id: "gpt-5.1-codex",
name: "GPT-5.1 Codex",
api: "openai-codex-responses",
provider: "openai-codex",
baseUrl: "https://chatgpt.com/backend-api",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 400000,
maxTokens: 128000,
};
const context: Context = {
systemPrompt: "You are a helpful assistant.",
messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }],
};
const streamResult = streamOpenAICodexResponses(model, context, { apiKey: token, sessionId });
await streamResult.result();
});
it.each(["gpt-5.3-codex", "gpt-5.4"])("clamps %s minimal reasoning effort to low", async (modelId) => {
const tempDir = mkdtempSync(join(tmpdir(), "pi-codex-stream-"));
process.env.PI_CODING_AGENT_DIR = tempDir;
const payload = Buffer.from(
JSON.stringify({ "https://api.openai.com/auth": { chatgpt_account_id: "acc_test" } }),
"utf8",
).toString("base64");
const token = `aaa.${payload}.bbb`;
const sse = `${[
`data: ${JSON.stringify({
type: "response.output_item.added",
item: { type: "message", id: "msg_1", role: "assistant", status: "in_progress", content: [] },
})}`,
`data: ${JSON.stringify({ type: "response.content_part.added", part: { type: "output_text", text: "" } })}`,
`data: ${JSON.stringify({ type: "response.output_text.delta", delta: "Hello" })}`,
`data: ${JSON.stringify({
type: "response.output_item.done",
item: {
type: "message",
id: "msg_1",
role: "assistant",
status: "completed",
content: [{ type: "output_text", text: "Hello" }],
},
})}`,
`data: ${JSON.stringify({
type: "response.completed",
response: {
status: "completed",
usage: {
input_tokens: 5,
output_tokens: 3,
total_tokens: 8,
input_tokens_details: { cached_tokens: 0 },
},
},
})}`,
].join("\n\n")}\n\n`;
const encoder = new TextEncoder();
const stream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(encoder.encode(sse));
controller.close();
},
});
const fetchMock = vi.fn(async (input: string | URL, init?: RequestInit) => {
const url = typeof input === "string" ? input : input.toString();
if (url === "https://api.github.com/repos/openai/codex/releases/latest") {
return new Response(JSON.stringify({ tag_name: "rust-v0.0.0" }), { status: 200 });
}
if (url.startsWith("https://raw.githubusercontent.com/openai/codex/")) {
return new Response("PROMPT", { status: 200, headers: { etag: '"etag"' } });
}
if (url === "https://chatgpt.com/backend-api/codex/responses") {
const body = typeof init?.body === "string" ? (JSON.parse(init.body) as Record<string, unknown>) : null;
expect(body?.reasoning).toEqual({ effort: "low", summary: "auto" });
return new Response(stream, {
status: 200,
headers: { "content-type": "text/event-stream" },
});
}
return new Response("not found", { status: 404 });
});
global.fetch = fetchMock as typeof fetch;
const model: Model<"openai-codex-responses"> = {
id: modelId,
name: modelId,
api: "openai-codex-responses",
provider: "openai-codex",
baseUrl: "https://chatgpt.com/backend-api",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 400000,
maxTokens: 128000,
};
const context: Context = {
systemPrompt: "You are a helpful assistant.",
messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }],
};
const streamResult = streamOpenAICodexResponses(model, context, {
apiKey: token,
reasoningEffort: "minimal",
});
await streamResult.result();
});
it("does not set session_id/x-client-request-id headers when sessionId is not provided", async () => {
const tempDir = mkdtempSync(join(tmpdir(), "pi-codex-stream-"));
process.env.PI_CODING_AGENT_DIR = tempDir;
const payload = Buffer.from(
JSON.stringify({ "https://api.openai.com/auth": { chatgpt_account_id: "acc_test" } }),
"utf8",
).toString("base64");
const token = `aaa.${payload}.bbb`;
const sse = `${[
`data: ${JSON.stringify({
type: "response.output_item.added",
item: { type: "message", id: "msg_1", role: "assistant", status: "in_progress", content: [] },
})}`,
`data: ${JSON.stringify({ type: "response.content_part.added", part: { type: "output_text", text: "" } })}`,
`data: ${JSON.stringify({ type: "response.output_text.delta", delta: "Hello" })}`,
`data: ${JSON.stringify({
type: "response.output_item.done",
item: {
type: "message",
id: "msg_1",
role: "assistant",
status: "completed",
content: [{ type: "output_text", text: "Hello" }],
},
})}`,
`data: ${JSON.stringify({
type: "response.completed",
response: {
status: "completed",
usage: {
input_tokens: 5,
output_tokens: 3,
total_tokens: 8,
input_tokens_details: { cached_tokens: 0 },
},
},
})}`,
].join("\n\n")}\n\n`;
const encoder = new TextEncoder();
const stream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(encoder.encode(sse));
controller.close();
},
});
const fetchMock = vi.fn(async (input: string | URL, init?: RequestInit) => {
const url = typeof input === "string" ? input : input.toString();
if (url === "https://api.github.com/repos/openai/codex/releases/latest") {
return new Response(JSON.stringify({ tag_name: "rust-v0.0.0" }), { status: 200 });
}
if (url.startsWith("https://raw.githubusercontent.com/openai/codex/")) {
return new Response("PROMPT", { status: 200, headers: { etag: '"etag"' } });
}
if (url === "https://chatgpt.com/backend-api/codex/responses") {
const headers = init?.headers instanceof Headers ? init.headers : undefined;
// Verify headers are not set when sessionId is not provided
expect(headers?.has("session_id")).toBe(false);
expect(headers?.has("x-client-request-id")).toBe(false);
return new Response(stream, {
status: 200,
headers: { "content-type": "text/event-stream" },
});
}
return new Response("not found", { status: 404 });
});
global.fetch = fetchMock as typeof fetch;
const model: Model<"openai-codex-responses"> = {
id: "gpt-5.1-codex",
name: "GPT-5.1 Codex",
api: "openai-codex-responses",
provider: "openai-codex",
baseUrl: "https://chatgpt.com/backend-api",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 400000,
maxTokens: 128000,
};
const context: Context = {
systemPrompt: "You are a helpful assistant.",
messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }],
};
// No sessionId provided
const streamResult = streamOpenAICodexResponses(model, context, { apiKey: token });
await streamResult.result();
});
});
@@ -1,549 +0,0 @@
import { Type } from "@sinclair/typebox";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { getModel } from "../src/models.js";
import { streamSimple } from "../src/stream.js";
import type { Tool } from "../src/types.js";
const mockState = vi.hoisted(() => ({
lastParams: undefined as unknown,
chunks: undefined as
| Array<null | {
id?: string;
choices?: Array<{ delta: Record<string, unknown>; finish_reason: string | null; usage?: unknown }>;
usage?: {
prompt_tokens: number;
completion_tokens: number;
prompt_tokens_details: { cached_tokens: number; cache_write_tokens?: number };
completion_tokens_details: { reasoning_tokens: number };
};
}>
| undefined,
}));
vi.mock("openai", () => {
class FakeOpenAI {
chat = {
completions: {
create: async (params: unknown) => {
mockState.lastParams = params;
return {
async *[Symbol.asyncIterator]() {
const chunks = mockState.chunks ?? [
{
choices: [{ delta: {}, finish_reason: "stop" }],
usage: {
prompt_tokens: 1,
completion_tokens: 1,
prompt_tokens_details: { cached_tokens: 0 },
completion_tokens_details: { reasoning_tokens: 0 },
},
},
];
for (const chunk of chunks) {
yield chunk;
}
},
};
},
},
};
}
return { default: FakeOpenAI };
});
describe("openai-completions tool_choice", () => {
beforeEach(() => {
mockState.lastParams = undefined;
mockState.chunks = undefined;
});
it("forwards toolChoice from simple options to payload", async () => {
const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!;
const model = { ...baseModel, api: "openai-completions" } as const;
const tools: Tool[] = [
{
name: "ping",
description: "Ping tool",
parameters: Type.Object({
ok: Type.Boolean(),
}),
},
];
let payload: unknown;
await streamSimple(
model,
{
messages: [
{
role: "user",
content: "Call ping with ok=true",
timestamp: Date.now(),
},
],
tools,
},
{
apiKey: "test",
toolChoice: "required",
onPayload: (params: unknown) => {
payload = params;
},
} as unknown as Parameters<typeof streamSimple>[2],
).result();
const params = (payload ?? mockState.lastParams) as { tool_choice?: string; tools?: unknown[] };
expect(params.tool_choice).toBe("required");
expect(Array.isArray(params.tools)).toBe(true);
expect(params.tools?.length ?? 0).toBeGreaterThan(0);
});
it("omits strict when compat disables strict mode", async () => {
const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!;
const model = {
...baseModel,
api: "openai-completions",
compat: { supportsStrictMode: false },
} as const;
const tools: Tool[] = [
{
name: "ping",
description: "Ping tool",
parameters: Type.Object({
ok: Type.Boolean(),
}),
},
];
let payload: unknown;
await streamSimple(
model,
{
messages: [
{
role: "user",
content: "Call ping with ok=true",
timestamp: Date.now(),
},
],
tools,
},
{
apiKey: "test",
onPayload: (params: unknown) => {
payload = params;
},
} as unknown as Parameters<typeof streamSimple>[2],
).result();
const params = (payload ?? mockState.lastParams) as { tools?: Array<{ function?: Record<string, unknown> }> };
const tool = params.tools?.[0]?.function;
expect(tool).toBeTruthy();
expect(tool?.strict).toBeUndefined();
expect("strict" in (tool ?? {})).toBe(false);
});
it("maps groq qwen3 reasoning levels to default reasoning_effort", async () => {
const model = getModel("groq", "qwen/qwen3-32b")!;
let payload: unknown;
await streamSimple(
model,
{
messages: [
{
role: "user",
content: "Hi",
timestamp: Date.now(),
},
],
},
{
apiKey: "test",
reasoning: "medium",
onPayload: (params: unknown) => {
payload = params;
},
},
).result();
const params = (payload ?? mockState.lastParams) as { reasoning_effort?: string };
expect(params.reasoning_effort).toBe("default");
});
it("keeps normal reasoning_effort for groq models without compat mapping", async () => {
const model = getModel("groq", "openai/gpt-oss-20b")!;
let payload: unknown;
await streamSimple(
model,
{
messages: [
{
role: "user",
content: "Hi",
timestamp: Date.now(),
},
],
},
{
apiKey: "test",
reasoning: "medium",
onPayload: (params: unknown) => {
payload = params;
},
},
).result();
const params = (payload ?? mockState.lastParams) as { reasoning_effort?: string };
expect(params.reasoning_effort).toBe("medium");
});
it("enables tool_stream for supported z.ai models with tools", async () => {
const model = getModel("zai", "glm-5")!;
const tools: Tool[] = [
{
name: "ping",
description: "Ping tool",
parameters: Type.Object({
ok: Type.Boolean(),
}),
},
];
let payload: unknown;
await streamSimple(
model,
{
messages: [
{
role: "user",
content: "Call ping with ok=true",
timestamp: Date.now(),
},
],
tools,
},
{
apiKey: "test",
onPayload: (params: unknown) => {
payload = params;
},
},
).result();
const params = (payload ?? mockState.lastParams) as { tool_stream?: boolean };
expect(params.tool_stream).toBe(true);
});
it("stores z.ai tool_stream support in model compat metadata", () => {
expect(getModel("zai", "glm-5")?.compat?.zaiToolStream).toBe(true);
expect(getModel("zai", "glm-4.7")?.compat?.zaiToolStream).toBe(true);
expect(getModel("zai", "glm-4.7-flash")?.compat?.zaiToolStream).toBe(true);
expect(getModel("zai", "glm-4.6v")?.compat?.zaiToolStream).toBe(true);
expect(getModel("zai", "glm-4.5-air")?.compat?.zaiToolStream).toBeUndefined();
});
it("omits tool_stream for unsupported z.ai models", async () => {
const model = getModel("zai", "glm-4.5-air")!;
const tools: Tool[] = [
{
name: "ping",
description: "Ping tool",
parameters: Type.Object({
ok: Type.Boolean(),
}),
},
];
let payload: unknown;
await streamSimple(
model,
{
messages: [
{
role: "user",
content: "Call ping with ok=true",
timestamp: Date.now(),
},
],
tools,
},
{
apiKey: "test",
onPayload: (params: unknown) => {
payload = params;
},
},
).result();
const params = (payload ?? mockState.lastParams) as { tool_stream?: boolean };
expect(params.tool_stream).toBeUndefined();
});
it("respects explicit z.ai tool_stream compat override", async () => {
const baseModel = getModel("zai", "glm-4.5-air")!;
const model = {
...baseModel,
compat: {
...baseModel.compat,
zaiToolStream: true,
},
} as const;
const tools: Tool[] = [
{
name: "ping",
description: "Ping tool",
parameters: Type.Object({
ok: Type.Boolean(),
}),
},
];
let payload: unknown;
await streamSimple(
model,
{
messages: [
{
role: "user",
content: "Call ping with ok=true",
timestamp: Date.now(),
},
],
tools,
},
{
apiKey: "test",
onPayload: (params: unknown) => {
payload = params;
},
},
).result();
const params = (payload ?? mockState.lastParams) as { tool_stream?: boolean };
expect(params.tool_stream).toBe(true);
});
it("omits tool_stream when no tools are provided", async () => {
const model = getModel("zai", "glm-5")!;
let payload: unknown;
await streamSimple(
model,
{
messages: [
{
role: "user",
content: "Hi",
timestamp: Date.now(),
},
],
},
{
apiKey: "test",
onPayload: (params: unknown) => {
payload = params;
},
},
).result();
const params = (payload ?? mockState.lastParams) as { tool_stream?: boolean };
expect(params.tool_stream).toBeUndefined();
});
it("maps non-standard provider finish_reason values to stopReason error", async () => {
mockState.chunks = [
{
choices: [{ delta: { content: "partial" }, finish_reason: null }],
},
{
choices: [{ delta: {}, finish_reason: "network_error" }],
usage: {
prompt_tokens: 1,
completion_tokens: 1,
prompt_tokens_details: { cached_tokens: 0 },
completion_tokens_details: { reasoning_tokens: 0 },
},
},
];
const model = getModel("zai", "glm-5")!;
const response = await streamSimple(
model,
{
messages: [
{
role: "user",
content: "Hi",
timestamp: Date.now(),
},
],
},
{ apiKey: "test" },
).result();
expect(response.stopReason).toBe("error");
expect(response.errorMessage).toBe("Provider finish_reason: network_error");
});
it("ignores null stream chunks from openai-compatible providers", async () => {
mockState.chunks = [
null,
{
id: "chatcmpl-test",
choices: [{ delta: { content: "OK" }, finish_reason: null }],
},
{
id: "chatcmpl-test",
choices: [{ delta: {}, finish_reason: "stop" }],
usage: {
prompt_tokens: 3,
completion_tokens: 1,
prompt_tokens_details: { cached_tokens: 0 },
completion_tokens_details: { reasoning_tokens: 0 },
},
},
];
const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!;
const model = { ...baseModel, api: "openai-completions" } as const;
const response = await streamSimple(
model,
{
messages: [
{
role: "user",
content: "Reply with exactly OK",
timestamp: Date.now(),
},
],
},
{ apiKey: "test" },
).result();
expect(response.stopReason).toBe("stop");
expect(response.errorMessage).toBeUndefined();
expect(response.responseId).toBe("chatcmpl-test");
expect(response.usage.totalTokens).toBe(4);
expect(response.content).toEqual([{ type: "text", text: "OK" }]);
});
it("preserves prompt_tokens_details.cache_write_tokens from chunk usage", async () => {
mockState.chunks = [
{
id: "chatcmpl-cache-write",
choices: [{ delta: { content: "OK" }, finish_reason: null }],
},
{
id: "chatcmpl-cache-write",
choices: [{ delta: {}, finish_reason: "stop" }],
usage: {
prompt_tokens: 100,
completion_tokens: 5,
prompt_tokens_details: { cached_tokens: 50, cache_write_tokens: 30 },
completion_tokens_details: { reasoning_tokens: 0 },
},
},
];
const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!;
const model = { ...baseModel, api: "openai-completions" } as const;
const response = await streamSimple(
model,
{
messages: [
{
role: "user",
content: "Reply with exactly OK",
timestamp: Date.now(),
},
],
},
{ apiKey: "test" },
).result();
expect(response.usage.input).toBe(50);
expect(response.usage.cacheRead).toBe(20);
expect(response.usage.cacheWrite).toBe(30);
expect(response.usage.totalTokens).toBe(105);
});
it("preserves prompt_tokens_details.cache_write_tokens from choice usage fallback", async () => {
mockState.chunks = [
{
id: "chatcmpl-cache-write-choice",
choices: [{ delta: { content: "OK" }, finish_reason: null }],
},
{
id: "chatcmpl-cache-write-choice",
choices: [
{
delta: {},
finish_reason: "stop",
usage: {
prompt_tokens: 100,
completion_tokens: 5,
prompt_tokens_details: { cached_tokens: 50, cache_write_tokens: 30 },
completion_tokens_details: { reasoning_tokens: 0 },
},
},
],
},
];
const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!;
const model = { ...baseModel, api: "openai-completions" } as const;
const response = await streamSimple(
model,
{
messages: [
{
role: "user",
content: "Reply with exactly OK",
timestamp: Date.now(),
},
],
},
{ apiKey: "test" },
).result();
expect(response.usage.input).toBe(50);
expect(response.usage.cacheRead).toBe(20);
expect(response.usage.cacheWrite).toBe(30);
expect(response.usage.totalTokens).toBe(105);
});
it("uses OpenRouter reasoning object instead of reasoning_effort", async () => {
const model = getModel("openrouter", "deepseek/deepseek-r1")!;
let payload: unknown;
await streamSimple(
model,
{
messages: [
{
role: "user",
content: "Hi",
timestamp: Date.now(),
},
],
},
{
apiKey: "test",
reasoning: "high",
onPayload: (params: unknown) => {
payload = params;
},
},
).result();
const params = (payload ?? mockState.lastParams) as {
reasoning?: { effort?: string };
reasoning_effort?: string;
};
expect(params.reasoning).toEqual({ effort: "high" });
expect(params.reasoning_effort).toBeUndefined();
});
});
@@ -1,99 +0,0 @@
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { convertMessages } from "../src/providers/openai-completions.js";
import type {
AssistantMessage,
Context,
Model,
OpenAICompletionsCompat,
ToolResultMessage,
Usage,
} from "../src/types.js";
const emptyUsage: Usage = {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
};
const compat: Required<OpenAICompletionsCompat> = {
supportsStore: true,
supportsDeveloperRole: true,
supportsReasoningEffort: true,
reasoningEffortMap: {},
supportsUsageInStreaming: true,
maxTokensField: "max_completion_tokens",
requiresToolResultName: false,
requiresAssistantAfterToolResult: false,
requiresThinkingAsText: false,
thinkingFormat: "openai",
openRouterRouting: {},
vercelGatewayRouting: {},
zaiToolStream: false,
supportsStrictMode: true,
};
function buildToolResult(toolCallId: string, timestamp: number): ToolResultMessage {
return {
role: "toolResult",
toolCallId,
toolName: "read",
content: [
{ type: "text", text: "Read image file [image/png]" },
{ type: "image", data: "ZmFrZQ==", mimeType: "image/png" },
],
isError: false,
timestamp,
};
}
describe("openai-completions convertMessages", () => {
it("batches tool-result images after consecutive tool results", () => {
const baseModel = getModel("openai", "gpt-4o-mini");
const model: Model<"openai-completions"> = {
...baseModel,
api: "openai-completions",
input: ["text", "image"],
};
const now = Date.now();
const assistantMessage: AssistantMessage = {
role: "assistant",
content: [
{ type: "toolCall", id: "tool-1", name: "read", arguments: { path: "img-1.png" } },
{ type: "toolCall", id: "tool-2", name: "read", arguments: { path: "img-2.png" } },
],
api: model.api,
provider: model.provider,
model: model.id,
usage: emptyUsage,
stopReason: "toolUse",
timestamp: now,
};
const context: Context = {
messages: [
{ role: "user", content: "Read the images", timestamp: now - 2 },
assistantMessage,
buildToolResult("tool-1", now + 1),
buildToolResult("tool-2", now + 2),
],
};
const messages = convertMessages(model, context, compat);
const roles = messages.map((message) => message.role);
expect(roles).toEqual(["user", "assistant", "tool", "tool", "user"]);
const imageMessage = messages[messages.length - 1];
expect(imageMessage.role).toBe("user");
expect(Array.isArray(imageMessage.content)).toBe(true);
const imageParts = (imageMessage.content as Array<{ type?: string }>).filter(
(part) => part?.type === "image_url",
);
expect(imageParts.length).toBe(2);
});
});
@@ -1,32 +0,0 @@
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { complete } from "../src/stream.js";
import type { Context } from "../src/types.js";
describe.skipIf(!process.env.OPENAI_API_KEY)("openai responses cache affinity e2e", () => {
it("handles direct OpenAI Responses requests with aligned cache-affinity identifiers", { retry: 2 }, async () => {
const model = getModel("openai", "gpt-5.4");
const sessionId = "0195d6e4-4cf9-7f44-a2d8-f8f7f49ee9d3";
const context: Context = {
systemPrompt: "You are a helpful assistant. Reply exactly as requested.",
messages: [
{
role: "user",
content: "Reply with exactly: openai cache affinity e2e success",
timestamp: Date.now(),
},
],
};
const response = await complete(model, context, {
apiKey: process.env.OPENAI_API_KEY!,
sessionId,
});
expect(response.stopReason, response.errorMessage).not.toBe("error");
expect(response.errorMessage).toBeUndefined();
expect(response.content.map((block) => (block.type === "text" ? block.text : "")).join("")).toContain(
"openai cache affinity e2e success",
);
});
});
@@ -1,115 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { getModel } from "../src/models.js";
import { streamOpenAIResponses } from "../src/providers/openai-responses.js";
type CapturedHeaders = Headers | string[][] | Record<string, string | readonly string[]> | undefined;
function getHeader(headers: CapturedHeaders, name: string): string | null {
if (!headers) return null;
if (headers instanceof Headers) return headers.get(name);
const lowerName = name.toLowerCase();
if (Array.isArray(headers)) {
const match = headers.find(([key]) => key?.toLowerCase() === lowerName);
return match?.[1] ?? null;
}
for (const [key, value] of Object.entries(headers)) {
if (key.toLowerCase() === lowerName) return typeof value === "string" ? value : value.join(", ");
}
return null;
}
async function captureOpenAIResponseHeaders(
options: Parameters<typeof streamOpenAIResponses>[2],
): Promise<{ sessionId: string | null; clientRequestId: string | null }> {
const captured = { sessionId: null as string | null, clientRequestId: null as string | null };
vi.spyOn(globalThis, "fetch").mockImplementation(async (_input, init) => {
captured.sessionId = getHeader(init?.headers, "session_id");
captured.clientRequestId = getHeader(init?.headers, "x-client-request-id");
return new Response("data: [DONE]\n\n", {
status: 200,
headers: { "content-type": "text/event-stream" },
});
});
const stream = streamOpenAIResponses(
getModel("openai", "gpt-5.4"),
{
systemPrompt: "sys",
messages: [{ role: "user", content: "hi", timestamp: Date.now() }],
},
{ apiKey: "test-key", ...options },
);
for await (const event of stream) {
if (event.type === "done" || event.type === "error") break;
}
return captured;
}
describe("openai-responses provider defaults", () => {
afterEach(() => {
vi.restoreAllMocks();
});
it("omits reasoning when no reasoning is requested", async () => {
const model = getModel("github-copilot", "gpt-5-mini");
let capturedPayload: unknown;
vi.spyOn(globalThis, "fetch").mockResolvedValue(
new Response("data: [DONE]\n\n", {
status: 200,
headers: { "content-type": "text/event-stream" },
}),
);
const stream = streamOpenAIResponses(
model,
{
systemPrompt: "sys",
messages: [{ role: "user", content: "hi", timestamp: Date.now() }],
},
{
apiKey: "test-key",
onPayload: (payload) => {
capturedPayload = payload;
},
},
);
for await (const event of stream) {
if (event.type === "done" || event.type === "error") break;
}
expect(capturedPayload).not.toBeNull();
expect(capturedPayload).not.toMatchObject({
reasoning: expect.anything(),
});
});
it("sets cache-affinity headers for official OpenAI Responses requests with a sessionId", async () => {
const captured = await captureOpenAIResponseHeaders({ sessionId: "session-123" });
expect(captured).toEqual({ sessionId: "session-123", clientRequestId: "session-123" });
});
it("lets explicit headers override the default OpenAI cache-affinity headers", async () => {
const captured = await captureOpenAIResponseHeaders({
sessionId: "session-123",
headers: {
session_id: "override-session",
"x-client-request-id": "override-request",
},
});
expect(captured).toEqual({ sessionId: "override-session", clientRequestId: "override-request" });
});
it("omits OpenAI cache-affinity headers when cacheRetention is none", async () => {
const captured = await captureOpenAIResponseHeaders({ cacheRetention: "none", sessionId: "session-123" });
expect(captured).toEqual({ sessionId: null, clientRequestId: null });
});
});
@@ -1,66 +0,0 @@
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { convertResponsesMessages } from "../src/providers/openai-responses-shared.js";
import type { AssistantMessage, Context, ToolResultMessage, Usage } from "../src/types.js";
import { shortHash } from "../src/utils/hash.js";
const COPILOT_RAW_TOOL_CALL_ID =
"call_4VnzVawQXPB9MgYib7CiQFEY|I9b95oN1wD/cHXKTw3PpRkL6KkCtzTJhUxMouMWYwHeTo2j3htzfSk7YPx2vifiIM4g3A8XXyOj8q4Bt6SLUG7gqY1E3ELkrkVQNHglRfUmWj84lqxJY+Puieb3VKyX0FB+83TUzn91cDMF/4gzt990IzqVrc+nIb9RRscRD070Du16q1glydVjWR0SBJsE6TbY/esOjFpqplogQqrajm1eI++f3eLi73R6q7hVusY0QbeFySVxABCjhN0lXB04caBe1rzHjYzul6MAXj7uq+0r17VLq+yrtyYhN12wkmFqHeqTyEei6EFPbMy24Nc+IbJlkP0OCg02W+gOnyBFcbi2ctvJFSOhSjt1CqBdqCnnhwUqXjbWiT0wh3DmLScRgTHmGkaI+oAcQQjfic65nxj+TnEkReA==";
const usage: Usage = {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
};
describe("OpenAI Responses foreign tool call ID normalization", () => {
it("hashes foreign Copilot tool item IDs into a bounded Codex-safe fc_<hash> shape", () => {
const model = getModel("openai-codex", "gpt-5.3-codex");
const assistant: AssistantMessage = {
role: "assistant",
content: [
{
type: "toolCall",
id: COPILOT_RAW_TOOL_CALL_ID,
name: "edit",
arguments: { path: "src/styles/app.css" },
},
],
api: "openai-responses",
provider: "github-copilot",
model: "gpt-5.3-codex",
usage,
stopReason: "toolUse",
timestamp: Date.now() - 2000,
};
const toolResult: ToolResultMessage = {
role: "toolResult",
toolCallId: COPILOT_RAW_TOOL_CALL_ID,
toolName: "edit",
content: [{ type: "text", text: "ok" }],
isError: false,
timestamp: Date.now() - 1000,
};
const context: Context = {
systemPrompt: "You are concise.",
messages: [{ role: "user", content: "Use the tool.", timestamp: Date.now() - 3000 }, assistant, toolResult],
};
const input = convertResponsesMessages(model, context, new Set(["openai", "openai-codex", "opencode"]));
const functionCall = input.find((item) => item.type === "function_call");
expect(functionCall).toBeDefined();
expect(functionCall?.type).toBe("function_call");
if (!functionCall || functionCall.type !== "function_call") {
throw new Error("Expected function_call item");
}
const expectedItemId = `fc_${shortHash(COPILOT_RAW_TOOL_CALL_ID.split("|")[1]!)}`;
expect(functionCall.id).toBe(expectedItemId);
expect(functionCall.id?.length ?? 0).toBeLessThanOrEqual(64);
expect(functionCall.id).toMatch(/^fc_[A-Za-z0-9]+$/);
});
});
@@ -1,101 +0,0 @@
import type { ResponseStreamEvent } from "openai/resources/responses/responses.js";
import { describe, expect, it, vi } from "vitest";
import { processResponsesStream } from "../src/providers/openai-responses-shared.js";
import type { AssistantMessage, AssistantMessageEvent, Model } from "../src/types.js";
import { AssistantMessageEventStream } from "../src/utils/event-stream.js";
function createOutput(model: Model<"openai-responses">): AssistantMessage {
return {
role: "assistant",
content: [],
api: model.api,
provider: model.provider,
model: model.id,
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "stop",
timestamp: Date.now(),
};
}
async function* createFunctionCallEvents(argumentsJson: string): AsyncIterable<ResponseStreamEvent> {
yield {
type: "response.output_item.added",
item: {
type: "function_call",
id: "fc_test",
call_id: "call_test",
name: "edit",
arguments: "",
},
} as ResponseStreamEvent;
yield {
type: "response.function_call_arguments.delta",
delta: '{"path":"README.md"',
} as ResponseStreamEvent;
yield {
type: "response.function_call_arguments.delta",
delta: ',"content":"updated"}',
} as ResponseStreamEvent;
yield {
type: "response.function_call_arguments.done",
arguments: argumentsJson,
} as ResponseStreamEvent;
yield {
type: "response.output_item.done",
item: {
type: "function_call",
id: "fc_test",
call_id: "call_test",
name: "edit",
arguments: argumentsJson,
},
} as ResponseStreamEvent;
}
describe("openai responses partialJson cleanup", () => {
it("removes partialJson from persisted tool-call blocks at output_item.done", async () => {
const model: Model<"openai-responses"> = {
id: "gpt-5-mini",
name: "GPT-5 Mini",
api: "openai-responses",
provider: "openai",
baseUrl: "https://api.openai.com/v1",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 400000,
maxTokens: 128000,
};
const output = createOutput(model);
const stream = new AssistantMessageEventStream();
const pushSpy = vi.spyOn(stream, "push");
const argumentsJson = '{"path":"README.md","content":"updated"}';
await processResponsesStream(createFunctionCallEvents(argumentsJson), output, stream, model);
expect(output.content).toHaveLength(1);
const persistedToolCall = output.content[0];
expect(persistedToolCall?.type).toBe("toolCall");
if (!persistedToolCall || persistedToolCall.type !== "toolCall") {
throw new Error("Expected toolCall block");
}
expect(persistedToolCall.arguments).toEqual({ path: "README.md", content: "updated" });
expect("partialJson" in persistedToolCall).toBe(false);
const emittedEvents = pushSpy.mock.calls.map(([event]) => event as AssistantMessageEvent);
const toolCallEnd = emittedEvents.find((event) => event.type === "toolcall_end");
expect(toolCallEnd).toBeDefined();
if (!toolCallEnd || toolCallEnd.type !== "toolcall_end") {
throw new Error("Expected toolcall_end event");
}
expect(toolCallEnd.toolCall).toBe(persistedToolCall);
expect("partialJson" in toolCallEnd.toolCall).toBe(false);
});
});
@@ -1,292 +0,0 @@
import { Type } from "@sinclair/typebox";
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { complete, getEnvApiKey } from "../src/stream.js";
import type { AssistantMessage, Context, Message, Tool, ToolCall } from "../src/types.js";
const testToolSchema = Type.Object({
value: Type.Number({ description: "A number to double" }),
});
const testTool: Tool<typeof testToolSchema> = {
name: "double_number",
description: "Doubles a number and returns the result",
parameters: testToolSchema,
};
describe.skipIf(!process.env.OPENAI_API_KEY || !process.env.ANTHROPIC_API_KEY)(
"OpenAI Responses reasoning replay e2e",
() => {
it("skips reasoning-only history after an aborted turn", { retry: 2 }, async () => {
const model = getModel("openai", "gpt-5-mini");
const apiKey = getEnvApiKey("openai");
if (!apiKey) {
throw new Error("Missing OPENAI_API_KEY");
}
const userMessage: Message = {
role: "user",
content: "Use the double_number tool to double 21.",
timestamp: Date.now(),
};
const assistantResponse = await complete(
model,
{
systemPrompt: "You are a helpful assistant. Use the tool.",
messages: [userMessage],
tools: [testTool],
},
{
apiKey,
reasoningEffort: "high",
},
);
const thinkingBlock = assistantResponse.content.find(
(block) => block.type === "thinking" && block.thinkingSignature,
);
if (!thinkingBlock || thinkingBlock.type !== "thinking") {
throw new Error("Missing thinking signature from OpenAI Responses");
}
const corruptedAssistant: AssistantMessage = {
...assistantResponse,
content: [thinkingBlock],
stopReason: "aborted",
};
const followUp: Message = {
role: "user",
content: "Say hello to confirm you can continue.",
timestamp: Date.now(),
};
const context: Context = {
systemPrompt: "You are a helpful assistant.",
messages: [userMessage, corruptedAssistant, followUp],
tools: [testTool],
};
const response = await complete(model, context, {
apiKey,
reasoningEffort: "high",
});
// The key assertion: no 400 error from orphaned reasoning item
expect(response.stopReason, `Error: ${response.errorMessage}`).not.toBe("error");
expect(response.errorMessage).toBeFalsy();
// Model should respond (text or tool call)
expect(response.content.length).toBeGreaterThan(0);
});
it("handles same-provider different-model handoff with tool calls", { retry: 2 }, async () => {
// This tests the scenario where:
// 1. Model A (gpt-5-mini) generates reasoning + function_call
// 2. User switches to Model B (gpt-5.2-codex) - same provider, different model
// 3. transform-messages: isSameModel=false, thinking converted to text
// 4. But tool call ID still has OpenAI pairing history (fc_xxx paired with rs_xxx)
// 5. Without fix: OpenAI returns 400 "function_call without required reasoning item"
// 6. With fix: tool calls/results converted to text, conversation continues
const modelA = getModel("openai", "gpt-5-mini");
const modelB = getModel("openai", "gpt-5.2-codex");
const apiKey = getEnvApiKey("openai");
if (!apiKey) {
throw new Error("Missing OPENAI_API_KEY");
}
const userMessage: Message = {
role: "user",
content: "Use the double_number tool to double 21.",
timestamp: Date.now(),
};
// Get a real response from Model A with reasoning + tool call
const assistantResponse = await complete(
modelA,
{
systemPrompt: "You are a helpful assistant. Always use the tool when asked.",
messages: [userMessage],
tools: [testTool],
},
{
apiKey,
reasoningEffort: "high",
},
);
const toolCallBlock = assistantResponse.content.find((block) => block.type === "toolCall") as
| ToolCall
| undefined;
if (!toolCallBlock) {
throw new Error("Missing tool call from OpenAI Responses - model did not use the tool");
}
// Provide a tool result
const toolResult: Message = {
role: "toolResult",
toolCallId: toolCallBlock.id,
toolName: toolCallBlock.name,
content: [{ type: "text", text: "42" }],
isError: false,
timestamp: Date.now(),
};
const followUp: Message = {
role: "user",
content: "What was the result? Answer with just the number.",
timestamp: Date.now(),
};
// Now continue with Model B (different model, same provider)
const context: Context = {
systemPrompt: "You are a helpful assistant. Answer concisely.",
messages: [userMessage, assistantResponse, toolResult, followUp],
tools: [testTool],
};
let capturedPayload: any = null;
const response = await complete(modelB, context, {
apiKey,
reasoningEffort: "high",
onPayload: (payload) => {
capturedPayload = payload;
},
});
// The key assertion: no 400 error from orphaned function_call
expect(response.stopReason, `Error: ${response.errorMessage}`).not.toBe("error");
expect(response.errorMessage).toBeFalsy();
expect(response.content.length).toBeGreaterThan(0);
// Log what was sent for debugging
const input = capturedPayload?.input as any[];
const functionCalls = input?.filter((item: any) => item.type === "function_call") || [];
const reasoningItems = input?.filter((item: any) => item.type === "reasoning") || [];
console.log("Payload sent to API:");
console.log("- function_calls:", functionCalls.length);
console.log("- reasoning items:", reasoningItems.length);
console.log("- full input:", JSON.stringify(input, null, 2));
// Verify the model understood the context
const responseText = response.content
.filter((b) => b.type === "text")
.map((b) => (b as any).text)
.join("");
expect(responseText).toContain("42");
});
it("handles cross-provider handoff from Anthropic to OpenAI Codex", { retry: 2 }, async () => {
// This tests cross-provider handoff:
// 1. Anthropic model generates thinking + function_call (toolu_xxx ID)
// 2. User switches to OpenAI Codex
// 3. transform-messages: isSameModel=false, thinking converted to text
// 4. Tool call ID is Anthropic format (toolu_xxx), no OpenAI pairing history
// 5. Should work because foreign IDs have no pairing expectation
const anthropicModel = getModel("anthropic", "claude-sonnet-4-5");
const codexModel = getModel("openai", "gpt-5.2-codex");
const anthropicApiKey = getEnvApiKey("anthropic");
const openaiApiKey = getEnvApiKey("openai");
if (!anthropicApiKey || !openaiApiKey) {
throw new Error("Missing API keys");
}
const userMessage: Message = {
role: "user",
content: "Use the double_number tool to double 21.",
timestamp: Date.now(),
};
// Get a real response from Anthropic with thinking + tool call
const assistantResponse = await complete(
anthropicModel,
{
systemPrompt: "You are a helpful assistant. Always use the tool when asked.",
messages: [userMessage],
tools: [testTool],
},
{
apiKey: anthropicApiKey,
thinkingEnabled: true,
thinkingBudgetTokens: 5000,
},
);
const toolCallBlock = assistantResponse.content.find((block) => block.type === "toolCall") as
| ToolCall
| undefined;
if (!toolCallBlock) {
throw new Error("Missing tool call from Anthropic - model did not use the tool");
}
console.log("Anthropic tool call ID:", toolCallBlock.id);
// Provide a tool result
const toolResult: Message = {
role: "toolResult",
toolCallId: toolCallBlock.id,
toolName: toolCallBlock.name,
content: [{ type: "text", text: "42" }],
isError: false,
timestamp: Date.now(),
};
const followUp: Message = {
role: "user",
content: "What was the result? Answer with just the number.",
timestamp: Date.now(),
};
// Now continue with Codex (different provider)
const context: Context = {
systemPrompt: "You are a helpful assistant. Answer concisely.",
messages: [userMessage, assistantResponse, toolResult, followUp],
tools: [testTool],
};
let capturedPayload: any = null;
const response = await complete(codexModel, context, {
apiKey: openaiApiKey,
reasoningEffort: "high",
onPayload: (payload) => {
capturedPayload = payload;
},
});
// Log what was sent
const input = capturedPayload?.input as any[];
const functionCalls = input?.filter((item: any) => item.type === "function_call") || [];
const reasoningItems = input?.filter((item: any) => item.type === "reasoning") || [];
console.log("Payload sent to Codex:");
console.log("- function_calls:", functionCalls.length);
console.log("- reasoning items:", reasoningItems.length);
if (functionCalls.length > 0) {
console.log(
"- function_call IDs:",
functionCalls.map((fc: any) => fc.id),
);
}
// The key assertion: no 400 error
expect(response.stopReason, `Error: ${response.errorMessage}`).not.toBe("error");
expect(response.errorMessage).toBeFalsy();
expect(response.content.length).toBeGreaterThan(0);
// Verify the model understood the context
const responseText = response.content
.filter((b) => b.type === "text")
.map((b) => (b as any).text)
.join("");
expect(responseText).toContain("42");
});
},
);
@@ -1,200 +0,0 @@
import { readFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { Type } from "@sinclair/typebox";
import type { ResponseFunctionCallOutputItemList } from "openai/resources/responses/responses.js";
import { describe, expect, it } from "vitest";
import type { Api, Context, Model, StreamOptions, Tool, ToolResultMessage } from "../src/index.js";
import { complete, getModel } from "../src/index.js";
import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.js";
import { resolveApiKey } from "./oauth.js";
type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const oauthTokens = await Promise.all([resolveApiKey("github-copilot"), resolveApiKey("openai-codex")]);
const [githubCopilotToken, openaiCodexToken] = oauthTokens;
const getImageSchema = Type.Object({});
const getImageTool: Tool<typeof getImageSchema> = {
name: "get_circle_with_description",
description: "Returns a red circle image with a short text description.",
parameters: getImageSchema,
};
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null;
}
function isResponsePayload(value: unknown): value is { input: unknown[] } {
return isRecord(value) && Array.isArray(value.input);
}
function isFunctionCallOutputItem(
value: unknown,
): value is { type: "function_call_output"; output: string | ResponseFunctionCallOutputItemList } {
return isRecord(value) && value.type === "function_call_output" && "output" in value;
}
function isInputTextItem(value: unknown): value is { type: "input_text"; text: string } {
return isRecord(value) && value.type === "input_text" && typeof value.text === "string";
}
function isInputImageItem(value: unknown): value is { type: "input_image"; image_url: string } {
return isRecord(value) && value.type === "input_image" && typeof value.image_url === "string";
}
async function verifyToolResultImagesStayInFunctionCallOutput<TApi extends Api>(
model: Model<TApi>,
options?: StreamOptionsWithExtras,
) {
if (!model.input.includes("image")) {
console.log(`Skipping responses tool-result image test. Model ${model.id} does not support images.`);
return;
}
const imagePath = join(__dirname, "data", "red-circle.png");
const base64Image = readFileSync(imagePath).toString("base64");
const toolText = "A red circle with a diameter of 100 pixels.";
const context: Context = {
systemPrompt: "You are a helpful assistant that always uses the provided tool when asked.",
messages: [
{
role: "user",
content:
"Call get_circle_with_description, then describe both the tool text and the image. Mention the color and shape.",
timestamp: Date.now(),
},
],
tools: [getImageTool],
};
const firstResponse = await complete(model, context, options);
expect(firstResponse.stopReason, `Error: ${firstResponse.errorMessage}`).toBe("toolUse");
const toolCall = firstResponse.content.find((block) => block.type === "toolCall");
expect(toolCall).toBeTruthy();
if (!toolCall || toolCall.type !== "toolCall") {
throw new Error("Expected tool call");
}
context.messages.push(firstResponse);
context.messages.push({
role: "toolResult",
toolCallId: toolCall.id,
toolName: toolCall.name,
content: [
{ type: "text", text: toolText },
{ type: "image", data: base64Image, mimeType: "image/png" },
],
isError: false,
timestamp: Date.now(),
} satisfies ToolResultMessage);
let capturedPayload: unknown;
const secondResponse = await complete(model, context, {
...options,
onPayload: (payload) => {
capturedPayload = payload;
},
});
expect(secondResponse.stopReason, `Error: ${secondResponse.errorMessage}`).toBe("stop");
expect(secondResponse.errorMessage).toBeFalsy();
expect(isResponsePayload(capturedPayload)).toBe(true);
if (!isResponsePayload(capturedPayload)) {
throw new Error("Expected payload with input array");
}
const functionCallOutputIndex = capturedPayload.input.findIndex((item) => isFunctionCallOutputItem(item));
expect(functionCallOutputIndex).toBeGreaterThanOrEqual(0);
const functionCallOutput = capturedPayload.input[functionCallOutputIndex];
if (!isFunctionCallOutputItem(functionCallOutput)) {
throw new Error("Expected function_call_output item");
}
expect(Array.isArray(functionCallOutput.output)).toBe(true);
if (!Array.isArray(functionCallOutput.output)) {
throw new Error("Expected function_call_output output to be a content array");
}
const outputItems = functionCallOutput.output;
const textItem = outputItems.find((item) => isInputTextItem(item));
const imageItem = outputItems.find((item) => isInputImageItem(item));
expect(textItem).toBeTruthy();
expect(imageItem).toBeTruthy();
if (!textItem || !imageItem) {
throw new Error("Expected both input_text and input_image in function_call_output");
}
expect(textItem.text).toContain(toolText);
expect(imageItem.image_url.startsWith("data:image/png;base64,")).toBe(true);
const laterUserMessages = capturedPayload.input
.slice(functionCallOutputIndex + 1)
.filter((item) => isRecord(item) && item.role === "user");
expect(laterUserMessages).toHaveLength(0);
const responseText = secondResponse.content
.filter((block) => block.type === "text")
.map((block) => block.text)
.join(" ")
.toLowerCase();
expect(responseText).toContain("red");
expect(responseText).toContain("circle");
}
describe("Responses API tool result images", () => {
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses Provider (gpt-5-mini)", () => {
const model = getModel("openai", "gpt-5-mini");
it("should send tool result images in function_call_output", { retry: 3, timeout: 30000 }, async () => {
await verifyToolResultImagesStayInFunctionCallOutput(model, { reasoningEffort: "low" });
});
});
describe.skipIf(!hasAzureOpenAICredentials())("Azure OpenAI Responses Provider (gpt-4o-mini)", () => {
const model = getModel("azure-openai-responses", "gpt-4o-mini");
const azureDeploymentName = resolveAzureDeploymentName(model.id);
const azureOptions = azureDeploymentName ? { azureDeploymentName } : {};
it("should send tool result images in function_call_output", { retry: 3, timeout: 30000 }, async () => {
await verifyToolResultImagesStayInFunctionCallOutput(model, azureOptions);
});
});
describe("GitHub Copilot Responses Provider (gpt-5-mini)", () => {
const model = getModel("github-copilot", "gpt-5-mini");
it.skipIf(!githubCopilotToken)(
"should send tool result images in function_call_output",
{ retry: 3, timeout: 30000 },
async () => {
await verifyToolResultImagesStayInFunctionCallOutput(model, {
apiKey: githubCopilotToken,
reasoningEffort: "low",
});
},
);
});
describe("OpenAI Codex Responses Provider (gpt-5.2-codex)", () => {
const model = getModel("openai-codex", "gpt-5.2-codex");
it.skipIf(!openaiCodexToken)(
"should send tool result images in function_call_output",
{ retry: 3, timeout: 30000 },
async () => {
await verifyToolResultImagesStayInFunctionCallOutput(model, {
apiKey: openaiCodexToken,
reasoningEffort: "low",
});
},
);
});
});
@@ -1,78 +0,0 @@
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { completeSimple } from "../src/stream.js";
function createLongSystemPrompt(): string {
const nonce = `${Date.now()}-${Math.random()}`;
return `You are a concise assistant.\nCache nonce: ${nonce}\n\n${Array(80)
.fill(
"Prompt-caching probe content. Keep this exact text stable across requests so the provider can reuse prefix tokens and report cache read and cache write usage.",
)
.join("\n\n")}`;
}
describe.skipIf(!process.env.OPENROUTER_API_KEY)("OpenRouter cache_write repro E2E", () => {
it(
"regression: preserves cache_write_tokens on openai-completions stream path",
{ retry: 2, timeout: 90000 },
async () => {
const model = getModel("openrouter", "google/gemini-2.5-flash");
const context = {
systemPrompt: createLongSystemPrompt(),
messages: [
{
role: "user" as const,
content: "Reply with exactly: OK",
timestamp: Date.now(),
},
],
};
const options = {
apiKey: process.env.OPENROUTER_API_KEY!,
maxTokens: 32,
temperature: 0,
onPayload: (payload: unknown) => {
const params = payload as {
messages?: Array<{
role?: string;
content?: string | Array<{ type?: string; text?: string; cache_control?: { type: string } }>;
}>;
};
const messages = params.messages;
if (!Array.isArray(messages)) return payload;
for (let i = messages.length - 1; i >= 0; i--) {
const msg = messages[i];
if (msg.role !== "user") continue;
if (typeof msg.content === "string") {
msg.content = [{ type: "text", text: msg.content, cache_control: { type: "ephemeral" } }];
break;
}
if (!Array.isArray(msg.content)) continue;
for (let j = msg.content.length - 1; j >= 0; j--) {
const part = msg.content[j];
if (part.type === "text") {
part.cache_control = { type: "ephemeral" };
break;
}
}
break;
}
return payload;
},
};
const first = await completeSimple(model, context, options);
expect(first.stopReason, first.errorMessage).toBe("stop");
const second = await completeSimple(model, context, options);
expect(second.stopReason, second.errorMessage).toBe("stop");
// Regression expectation: cache_write_tokens from provider usage must be preserved.
// With the cache_control marker above, at least one of the two calls should create cache.
const hasCacheWrite = first.usage.cacheWrite > 0 || second.usage.cacheWrite > 0;
expect(hasCacheWrite).toBe(true);
},
);
});
@@ -1,64 +0,0 @@
import { describe, expect, it } from "vitest";
import type { AssistantMessage } from "../src/types.js";
import { isContextOverflow } from "../src/utils/overflow.js";
function createErrorMessage(errorMessage: string): AssistantMessage {
return {
role: "assistant",
content: [],
api: "openai-completions",
provider: "ollama",
model: "qwen3.5:35b",
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
total: 0,
},
},
stopReason: "error",
errorMessage,
timestamp: Date.now(),
};
}
describe("isContextOverflow", () => {
it("detects explicit Ollama prompt-too-long errors", () => {
const message = createErrorMessage("400 `prompt too long; exceeded max context length by 100918 tokens`");
expect(isContextOverflow(message, 32768)).toBe(true);
});
it("does not treat generic non-overflow Ollama errors as overflow", () => {
const message = createErrorMessage("500 `model runner crashed unexpectedly`");
expect(isContextOverflow(message, 32768)).toBe(false);
});
it("does not treat Bedrock throttling 'Too many tokens' as overflow", () => {
// Bedrock returns this for HTTP 429 rate limiting, NOT context overflow.
// formatBedrockError uses a human-readable prefix for ThrottlingException.
const message = createErrorMessage("Throttling error: Too many tokens, please wait before trying again.");
expect(isContextOverflow(message, 200000)).toBe(false);
});
it("does not treat Bedrock service unavailable as overflow", () => {
const message = createErrorMessage("Service unavailable: The service is temporarily unavailable.");
expect(isContextOverflow(message, 200000)).toBe(false);
});
it("does not treat generic rate limit errors as overflow", () => {
const message = createErrorMessage("Rate limit exceeded, please retry after 30 seconds.");
expect(isContextOverflow(message, 200000)).toBe(false);
});
it("does not treat HTTP 429 style errors as overflow", () => {
const message = createErrorMessage("Too many requests. Please slow down.");
expect(isContextOverflow(message, 200000)).toBe(false);
});
});
@@ -1,145 +0,0 @@
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { complete } from "../src/stream.js";
import type { Api, Context, Model, StreamOptions } from "../src/types.js";
import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.js";
import { resolveApiKey } from "./oauth.js";
type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;
const oauthTokens = await Promise.all([
resolveApiKey("github-copilot"),
resolveApiKey("google-gemini-cli"),
resolveApiKey("google-antigravity"),
resolveApiKey("openai-codex"),
]);
const [githubCopilotToken, geminiCliToken, antigravityToken, openaiCodexToken] = oauthTokens;
async function expectResponseId<TApi extends Api>(model: Model<TApi>, options: StreamOptionsWithExtras = {}) {
const context: Context = {
systemPrompt: "You are a helpful assistant. Be concise.",
messages: [{ role: "user", content: "Reply with exactly: response id test", timestamp: Date.now() }],
};
const response = await complete(model, context, options);
expect(response.stopReason, response.errorMessage).not.toBe("error");
expect(response.responseId).toBeTruthy();
expect(typeof response.responseId).toBe("string");
}
describe("responseId E2E Tests", () => {
describe.skipIf(!process.env.GEMINI_API_KEY)("Google Provider", () => {
const llm = getModel("google", "gemini-2.5-flash");
it("should expose responseId", { retry: 3, timeout: 30000 }, async () => {
await expectResponseId(llm);
});
});
describe("Google Vertex Provider", () => {
const vertexProject = process.env.GOOGLE_CLOUD_PROJECT || process.env.GCLOUD_PROJECT;
const vertexLocation = process.env.GOOGLE_CLOUD_LOCATION;
const vertexApiKey = process.env.GOOGLE_CLOUD_API_KEY;
const isVertexConfigured = Boolean(vertexProject && vertexLocation);
const vertexOptions = { project: vertexProject, location: vertexLocation } as const;
const llm = getModel("google-vertex", "gemini-3-flash-preview");
it.skipIf(!isVertexConfigured)("should expose responseId with ADC", { retry: 3, timeout: 30000 }, async () => {
await expectResponseId(llm, vertexOptions);
});
it.skipIf(!vertexApiKey)("should expose responseId with API key", { retry: 3, timeout: 30000 }, async () => {
await expectResponseId(llm, { apiKey: vertexApiKey! });
});
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions Provider", () => {
const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini");
void _compat;
const llm: Model<"openai-completions"> = {
...baseModel,
api: "openai-completions",
};
it("should expose responseId", { retry: 3, timeout: 30000 }, async () => {
await expectResponseId(llm);
});
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses Provider", () => {
const llm = getModel("openai", "gpt-5-mini");
it("should expose responseId", { retry: 3, timeout: 30000 }, async () => {
await expectResponseId(llm);
});
});
describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic Provider", () => {
const llm = getModel("anthropic", "claude-sonnet-4-5");
it("should expose responseId", { retry: 3, timeout: 30000 }, async () => {
await expectResponseId(llm);
});
});
describe.skipIf(!hasAzureOpenAICredentials())("Azure OpenAI Responses Provider", () => {
const llm = getModel("azure-openai-responses", "gpt-4o-mini");
const azureDeploymentName = resolveAzureDeploymentName(llm.id);
const azureOptions = azureDeploymentName ? { azureDeploymentName } : {};
it("should expose responseId", { retry: 3, timeout: 30000 }, async () => {
await expectResponseId(llm, azureOptions);
});
});
describe.skipIf(!process.env.MISTRAL_API_KEY)("Mistral Provider", () => {
const llm = getModel("mistral", "devstral-medium-latest");
it("should expose responseId", { retry: 3, timeout: 30000 }, async () => {
await expectResponseId(llm);
});
});
describe("GitHub Copilot Provider", () => {
it.skipIf(!githubCopilotToken)("OpenAI path should expose responseId", { retry: 3, timeout: 30000 }, async () => {
const llm = getModel("github-copilot", "gpt-5.3-codex");
await expectResponseId(llm, { apiKey: githubCopilotToken });
});
it.skipIf(!githubCopilotToken)(
"Anthropic path should expose responseId",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "claude-sonnet-4");
await expectResponseId(llm, { apiKey: githubCopilotToken });
},
);
});
describe("Google Gemini CLI Provider", () => {
it.skipIf(!geminiCliToken)("should expose responseId", { retry: 3, timeout: 30000 }, async () => {
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
await expectResponseId(llm, { apiKey: geminiCliToken });
});
});
describe("Google Antigravity Provider", () => {
it.skipIf(!antigravityToken)("Gemini path should expose responseId", { retry: 3, timeout: 30000 }, async () => {
const llm = getModel("google-antigravity", "gemini-3.1-pro-high");
await expectResponseId(llm, { apiKey: antigravityToken });
});
it.skipIf(!antigravityToken)("Claude path should expose responseId", { retry: 3, timeout: 30000 }, async () => {
const llm = getModel("google-antigravity", "claude-sonnet-4-6");
await expectResponseId(llm, { apiKey: antigravityToken });
});
});
describe("OpenAI Codex Provider", () => {
it.skipIf(!openaiCodexToken)("should expose responseId", { retry: 3, timeout: 30000 }, async () => {
const llm = getModel("openai-codex", "gpt-5.2-codex");
await expectResponseId(llm, { apiKey: openaiCodexToken });
});
});
});
File diff suppressed because it is too large. Load diff
@@ -1,34 +0,0 @@
import { describe, expect, it } from "vitest";
import { getModel, supportsXhigh } from "../src/models.js";
describe("supportsXhigh", () => {
it("returns true for Anthropic Opus 4.6 on anthropic-messages API", () => {
const model = getModel("anthropic", "claude-opus-4-6");
expect(model).toBeDefined();
expect(supportsXhigh(model!)).toBe(true);
});
it("returns true for Anthropic Opus 4.7 on anthropic-messages API", () => {
const model = getModel("anthropic", "claude-opus-4-7");
expect(model).toBeDefined();
expect(supportsXhigh(model!)).toBe(true);
});
it("returns false for non-Opus Anthropic models", () => {
const model = getModel("anthropic", "claude-sonnet-4-5");
expect(model).toBeDefined();
expect(supportsXhigh(model!)).toBe(false);
});
it("returns true for GPT-5.4 models", () => {
const model = getModel("openai-codex", "gpt-5.4");
expect(model).toBeDefined();
expect(supportsXhigh(model!)).toBe(true);
});
it("returns true for OpenRouter Opus 4.6 (openai-completions API)", () => {
const model = getModel("openrouter", "anthropic/claude-opus-4.6");
expect(model).toBeDefined();
expect(supportsXhigh(model!)).toBe(true);
});
});
@@ -1,304 +0,0 @@
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { stream } from "../src/stream.js";
import type { Api, Context, Model, StreamOptions } from "../src/types.js";
type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;
import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.js";
import { hasBedrockCredentials } from "./bedrock-utils.js";
import { resolveApiKey } from "./oauth.js";
// Resolve OAuth tokens at module level (async, runs before tests)
const oauthTokens = await Promise.all([
resolveApiKey("anthropic"),
resolveApiKey("github-copilot"),
resolveApiKey("google-gemini-cli"),
resolveApiKey("google-antigravity"),
resolveApiKey("openai-codex"),
]);
const [anthropicOAuthToken, githubCopilotToken, geminiCliToken, antigravityToken, openaiCodexToken] = oauthTokens;
async function testTokensOnAbort<TApi extends Api>(llm: Model<TApi>, options: StreamOptionsWithExtras = {}) {
const context: Context = {
messages: [
{
role: "user",
content: "Write a long poem with 20 stanzas about the beauty of nature.",
timestamp: Date.now(),
},
],
systemPrompt: "You are a helpful assistant.",
};
const controller = new AbortController();
const response = stream(llm, context, { ...options, signal: controller.signal });
let abortFired = false;
let text = "";
for await (const event of response) {
if (!abortFired && (event.type === "text_delta" || event.type === "thinking_delta")) {
text += event.delta;
if (text.length >= 1000) {
abortFired = true;
controller.abort();
}
}
}
const msg = await response.result();
expect(msg.stopReason).toBe("aborted");
// OpenAI providers, OpenAI Codex, Gemini CLI, zai, Amazon Bedrock, and the GPT-OSS model on Antigravity only send usage in the final chunk,
// so when aborted they have no token stats. Anthropic and Google send usage information early in the stream.
// MiniMax and Kimi report input tokens but not output tokens differently on aborted requests.
if (
llm.api === "openai-completions" ||
llm.api === "mistral-conversations" ||
llm.api === "openai-responses" ||
llm.api === "azure-openai-responses" ||
llm.api === "openai-codex-responses" ||
llm.provider === "google-gemini-cli" ||
llm.provider === "zai" ||
llm.provider === "amazon-bedrock" ||
llm.provider === "vercel-ai-gateway" ||
(llm.provider === "google-antigravity" && llm.id.includes("gpt-oss"))
) {
expect(msg.usage.input).toBe(0);
expect(msg.usage.output).toBe(0);
} else if (llm.provider === "minimax") {
// MiniMax M2.7 does not report token usage for aborted requests.
expect(msg.usage.input).toBe(0);
expect(msg.usage.output).toBe(0);
} else if (llm.provider === "kimi-coding") {
// Kimi reports input tokens early but output tokens only in the final chunk.
expect(msg.usage.input).toBeGreaterThan(0);
expect(msg.usage.output).toBe(0);
} else {
expect(msg.usage.input).toBeGreaterThan(0);
expect(msg.usage.output).toBeGreaterThan(0);
// Some providers (Antigravity, Copilot) have zero cost rates
if (llm.cost.input > 0) {
expect(msg.usage.cost.input).toBeGreaterThan(0);
expect(msg.usage.cost.total).toBeGreaterThan(0);
}
}
}
describe("Token Statistics on Abort", () => {
describe.skipIf(!process.env.GEMINI_API_KEY)("Google Provider", () => {
const llm = getModel("google", "gemini-2.5-flash");
it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
await testTokensOnAbort(llm, { thinking: { enabled: true } });
});
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions Provider", () => {
const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!;
void _compat;
const llm: Model<"openai-completions"> = {
...baseModel,
api: "openai-completions",
};
it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
await testTokensOnAbort(llm);
});
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses Provider", () => {
const llm = getModel("openai", "gpt-5.4-mini");
it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
await testTokensOnAbort(llm, { reasoningEffort: "low" });
});
});
describe.skipIf(!hasAzureOpenAICredentials())("Azure OpenAI Responses Provider", () => {
const llm = getModel("azure-openai-responses", "gpt-4o-mini");
const azureDeploymentName = resolveAzureDeploymentName(llm.id);
const azureOptions = azureDeploymentName ? { azureDeploymentName } : {};
it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
await testTokensOnAbort(llm, azureOptions);
});
});
describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic Provider", () => {
const llm = getModel("anthropic", "claude-sonnet-4-6");
it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
await testTokensOnAbort(llm);
});
});
describe.skipIf(!process.env.XAI_API_KEY)("xAI Provider", () => {
const llm = getModel("xai", "grok-3-fast");
it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
await testTokensOnAbort(llm);
});
});
describe.skipIf(!process.env.GROQ_API_KEY)("Groq Provider", () => {
const llm = getModel("groq", "openai/gpt-oss-20b");
it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
await testTokensOnAbort(llm);
});
});
describe.skipIf(!process.env.CEREBRAS_API_KEY)("Cerebras Provider", () => {
const llm = getModel("cerebras", "qwen-3-235b-a22b-instruct-2507");
it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
await testTokensOnAbort(llm);
});
});
describe.skipIf(!process.env.HF_TOKEN)("Hugging Face Provider", () => {
const llm = getModel("huggingface", "moonshotai/Kimi-K2.5");
it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
await testTokensOnAbort(llm);
});
});
describe.skipIf(!process.env.ZAI_API_KEY)("zAI Provider", () => {
const llm = getModel("zai", "glm-4.5-flash");
it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
await testTokensOnAbort(llm);
});
});
describe.skipIf(!process.env.MISTRAL_API_KEY)("Mistral Provider", () => {
const llm = getModel("mistral", "devstral-medium-latest");
it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
await testTokensOnAbort(llm);
});
});
describe.skipIf(!process.env.MINIMAX_API_KEY)("MiniMax Provider", () => {
const llm = getModel("minimax", "MiniMax-M2.7");
it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
await testTokensOnAbort(llm);
});
});
describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider", () => {
const llm = getModel("kimi-coding", "kimi-for-coding");
it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
await testTokensOnAbort(llm);
});
});
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider", () => {
const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash");
it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
await testTokensOnAbort(llm);
});
});
// =========================================================================
// OAuth-based providers (credentials from ~/.pi/agent/oauth.json)
// =========================================================================
describe("Anthropic OAuth Provider", () => {
const llm = getModel("anthropic", "claude-sonnet-4-6");
it.skipIf(!anthropicOAuthToken)(
"should include token stats when aborted mid-stream",
{ retry: 3, timeout: 30000 },
async () => {
await testTokensOnAbort(llm, { apiKey: anthropicOAuthToken });
},
);
});
describe("GitHub Copilot Provider", () => {
it.skipIf(!githubCopilotToken)(
"gpt-4o - should include token stats when aborted mid-stream",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "gpt-4o");
await testTokensOnAbort(llm, { apiKey: githubCopilotToken });
},
);
it.skipIf(!githubCopilotToken)(
"claude-sonnet-4 - should include token stats when aborted mid-stream",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "claude-sonnet-4");
await testTokensOnAbort(llm, { apiKey: githubCopilotToken });
},
);
});
describe("Google Gemini CLI Provider", () => {
it.skipIf(!geminiCliToken)(
"gemini-2.5-flash - should include token stats when aborted mid-stream",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
await testTokensOnAbort(llm, { apiKey: geminiCliToken });
},
);
});
describe("Google Antigravity Provider", () => {
it.skipIf(!antigravityToken)(
"gemini-3-flash - should include token stats when aborted mid-stream",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "gemini-3-flash");
await testTokensOnAbort(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"claude-sonnet-4-6 - should include token stats when aborted mid-stream",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "claude-sonnet-4-6");
await testTokensOnAbort(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"gpt-oss-120b-medium - should include token stats when aborted mid-stream",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "gpt-oss-120b-medium");
await testTokensOnAbort(llm, { apiKey: antigravityToken });
},
);
});
describe("OpenAI Codex Provider", () => {
it.skipIf(!openaiCodexToken)(
"gpt-5.2-codex - should include token stats when aborted mid-stream",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("openai-codex", "gpt-5.2-codex");
await testTokensOnAbort(llm, { apiKey: openaiCodexToken });
},
);
});
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider", () => {
const llm = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0");
it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
await testTokensOnAbort(llm);
});
});
});
@@ -1,291 +0,0 @@
/**
* Tool Call ID Normalization Tests
*
* Tests that tool call IDs from OpenAI Responses API (github-copilot, openai-codex, opencode)
* are properly normalized when sent to other providers.
*
* OpenAI Responses API generates IDs in format: {call_id}|{id}
* where {id} can be 400+ chars with special characters (+, /, =).
*
* Regression test for: https://github.com/badlogic/pi-mono/issues/1022
*/
import { Type } from "@sinclair/typebox";
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { completeSimple, getEnvApiKey } from "../src/stream.js";
import type { AssistantMessage, Message, Tool, ToolResultMessage } from "../src/types.js";
import { resolveApiKey } from "./oauth.js";
// Resolve API keys
const copilotToken = await resolveApiKey("github-copilot");
const openrouterKey = getEnvApiKey("openrouter");
const codexToken = await resolveApiKey("openai-codex");
// Simple echo tool for testing
const echoToolSchema = Type.Object({
message: Type.String({ description: "Message to echo back" }),
});
const echoTool: Tool<typeof echoToolSchema> = {
name: "echo",
description: "Echoes the message back",
parameters: echoToolSchema,
};
/**
* Test 1: Live cross-provider handoff
*
* 1. Use github-copilot gpt-5.2-codex to generate a tool call
* 2. Switch to openrouter openai/gpt-5.2-codex and complete
* 3. Switch to openai-codex gpt-5.2-codex and complete
*
* Both should succeed without "call_id too long" errors.
*/
describe("Tool Call ID Normalization - Live Handoff", () => {
it.skipIf(!copilotToken || !openrouterKey)(
"github-copilot -> openrouter should normalize pipe-separated IDs",
async () => {
const copilotModel = getModel("github-copilot", "gpt-5.2-codex");
const openrouterModel = getModel("openrouter", "openai/gpt-5.2-codex");
// Step 1: Generate tool call with github-copilot
const userMessage: Message = {
role: "user",
content: "Use the echo tool to echo 'hello world'",
timestamp: Date.now(),
};
const assistantResponse = await completeSimple(
copilotModel,
{
systemPrompt: "You are a helpful assistant. Use the echo tool when asked.",
messages: [userMessage],
tools: [echoTool],
},
{ apiKey: copilotToken },
);
expect(assistantResponse.stopReason, `Copilot error: ${assistantResponse.errorMessage}`).toBe("toolUse");
const toolCall = assistantResponse.content.find((c) => c.type === "toolCall");
expect(toolCall).toBeDefined();
expect(toolCall!.type).toBe("toolCall");
// Verify it's a pipe-separated ID (OpenAI Responses format)
if (toolCall?.type === "toolCall") {
expect(toolCall.id).toContain("|");
console.log(`Tool call ID from github-copilot: ${toolCall.id.slice(0, 80)}...`);
}
// Create tool result
const toolResult: ToolResultMessage = {
role: "toolResult",
toolCallId: (toolCall as any).id,
toolName: "echo",
content: [{ type: "text", text: "hello world" }],
isError: false,
timestamp: Date.now(),
};
// Step 2: Complete with openrouter (uses openai-completions API)
const openrouterResponse = await completeSimple(
openrouterModel,
{
systemPrompt: "You are a helpful assistant.",
messages: [
userMessage,
assistantResponse,
toolResult,
{ role: "user", content: "Say hi", timestamp: Date.now() },
],
tools: [echoTool],
},
{ apiKey: openrouterKey },
);
// Should NOT fail with "call_id too long" error
expect(openrouterResponse.stopReason, `OpenRouter error: ${openrouterResponse.errorMessage}`).not.toBe(
"error",
);
expect(openrouterResponse.errorMessage).toBeUndefined();
},
60000,
);
it.skipIf(!copilotToken || !codexToken)(
"github-copilot -> openai-codex should normalize pipe-separated IDs",
async () => {
const copilotModel = getModel("github-copilot", "gpt-5.2-codex");
const codexModel = getModel("openai-codex", "gpt-5.2-codex");
// Step 1: Generate tool call with github-copilot
const userMessage: Message = {
role: "user",
content: "Use the echo tool to echo 'test message'",
timestamp: Date.now(),
};
const assistantResponse = await completeSimple(
copilotModel,
{
systemPrompt: "You are a helpful assistant. Use the echo tool when asked.",
messages: [userMessage],
tools: [echoTool],
},
{ apiKey: copilotToken },
);
expect(assistantResponse.stopReason, `Copilot error: ${assistantResponse.errorMessage}`).toBe("toolUse");
const toolCall = assistantResponse.content.find((c) => c.type === "toolCall");
expect(toolCall).toBeDefined();
// Create tool result
const toolResult: ToolResultMessage = {
role: "toolResult",
toolCallId: (toolCall as any).id,
toolName: "echo",
content: [{ type: "text", text: "test message" }],
isError: false,
timestamp: Date.now(),
};
// Step 2: Complete with openai-codex (uses openai-codex-responses API)
const codexResponse = await completeSimple(
codexModel,
{
systemPrompt: "You are a helpful assistant.",
messages: [
userMessage,
assistantResponse,
toolResult,
{ role: "user", content: "Say hi", timestamp: Date.now() },
],
tools: [echoTool],
},
{ apiKey: codexToken },
);
// Should NOT fail with ID validation error
expect(codexResponse.stopReason, `Codex error: ${codexResponse.errorMessage}`).not.toBe("error");
expect(codexResponse.errorMessage).toBeUndefined();
},
60000,
);
});
/**
* Test 2: Prefilled context with exact failing IDs from issue #1022
*
* Uses the exact tool call ID format that caused the error:
* "call_xxx|very_long_base64_with_special_chars+/="
*/
describe("Tool Call ID Normalization - Prefilled Context", () => {
// Exact tool call ID from issue #1022 JSONL
const FAILING_TOOL_CALL_ID =
"call_pAYbIr76hXIjncD9UE4eGfnS|t5nnb2qYMFWGSsr13fhCd1CaCu3t3qONEPuOudu4HSVEtA8YJSL6FAZUxvoOoD792VIJWl91g87EdqsCWp9krVsdBysQoDaf9lMCLb8BS4EYi4gQd5kBQBYLlgD71PYwvf+TbMD9J9/5OMD42oxSRj8H+vRf78/l2Xla33LWz4nOgsddBlbvabICRs8GHt5C9PK5keFtzyi3lsyVKNlfduK3iphsZqs4MLv4zyGJnvZo/+QzShyk5xnMSQX/f98+aEoNflEApCdEOXipipgeiNWnpFSHbcwmMkZoJhURNu+JEz3xCh1mrXeYoN5o+trLL3IXJacSsLYXDrYTipZZbJFRPAucgbnjYBC+/ZzJOfkwCs+Gkw7EoZR7ZQgJ8ma+9586n4tT4cI8DEhBSZsWMjrCt8dxKg==";
// Build prefilled context with the failing ID
function buildPrefilledMessages(): Message[] {
const userMessage: Message = {
role: "user",
content: "Use the echo tool to echo 'hello'",
timestamp: Date.now() - 2000,
};
const assistantMessage: AssistantMessage = {
role: "assistant",
content: [
{
type: "toolCall",
id: FAILING_TOOL_CALL_ID,
name: "echo",
arguments: { message: "hello" },
},
],
api: "openai-responses",
provider: "github-copilot",
model: "gpt-5.2-codex",
usage: {
input: 100,
output: 50,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 150,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "toolUse",
timestamp: Date.now() - 1500,
};
const toolResult: ToolResultMessage = {
role: "toolResult",
toolCallId: FAILING_TOOL_CALL_ID,
toolName: "echo",
content: [{ type: "text", text: "hello" }],
isError: false,
timestamp: Date.now() - 1000,
};
const followUpUser: Message = {
role: "user",
content: "Say hi",
timestamp: Date.now(),
};
return [userMessage, assistantMessage, toolResult, followUpUser];
}
it.skipIf(!openrouterKey)(
"openrouter should handle prefilled context with long pipe-separated IDs",
async () => {
const model = getModel("openrouter", "openai/gpt-5.2-codex");
const messages = buildPrefilledMessages();
const response = await completeSimple(
model,
{
systemPrompt: "You are a helpful assistant.",
messages,
tools: [echoTool],
},
{ apiKey: openrouterKey },
);
// Should NOT fail with "call_id too long" error
expect(response.stopReason, `OpenRouter error: ${response.errorMessage}`).not.toBe("error");
if (response.errorMessage) {
expect(response.errorMessage).not.toContain("call_id");
expect(response.errorMessage).not.toContain("too long");
}
},
30000,
);
it.skipIf(!codexToken)(
"openai-codex should handle prefilled context with long pipe-separated IDs",
async () => {
const model = getModel("openai-codex", "gpt-5.2-codex");
const messages = buildPrefilledMessages();
const response = await completeSimple(
model,
{
systemPrompt: "You are a helpful assistant.",
messages,
tools: [echoTool],
},
{ apiKey: codexToken },
);
// Should NOT fail with ID validation error
expect(response.stopReason, `Codex error: ${response.errorMessage}`).not.toBe("error");
if (response.errorMessage) {
expect(response.errorMessage).not.toContain("id");
expect(response.errorMessage).not.toContain("additional characters");
}
},
30000,
);
});
@@ -1,313 +0,0 @@
import { Type } from "@sinclair/typebox";
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { complete } from "../src/stream.js";
import type { Api, Context, Model, StreamOptions, Tool } from "../src/types.js";
type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;
import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.js";
import { hasBedrockCredentials } from "./bedrock-utils.js";
import { resolveApiKey } from "./oauth.js";
// Resolve OAuth tokens at module level (async, runs before tests)
const oauthTokens = await Promise.all([
resolveApiKey("anthropic"),
resolveApiKey("github-copilot"),
resolveApiKey("google-gemini-cli"),
resolveApiKey("google-antigravity"),
resolveApiKey("openai-codex"),
]);
const [anthropicOAuthToken, githubCopilotToken, geminiCliToken, antigravityToken, openaiCodexToken] = oauthTokens;
// Simple calculate tool
const calculateSchema = Type.Object({
expression: Type.String({ description: "The mathematical expression to evaluate" }),
});
const calculateTool: Tool = {
name: "calculate",
description: "Evaluate mathematical expressions",
parameters: calculateSchema,
};
async function testToolCallWithoutResult<TApi extends Api>(model: Model<TApi>, options: StreamOptionsWithExtras = {}) {
// Step 1: Create context with the calculate tool
const context: Context = {
systemPrompt: "You are a helpful assistant. Use the calculate tool when asked to perform calculations.",
messages: [],
tools: [calculateTool],
};
// Step 2: Ask the LLM to make a tool call
context.messages.push({
role: "user",
content: "Please calculate 25 * 18 using the calculate tool.",
timestamp: Date.now(),
});
// Step 3: Get the assistant's response (should contain a tool call)
const firstResponse = await complete(model, context, options);
context.messages.push(firstResponse);
console.log("First response:", JSON.stringify(firstResponse, null, 2));
// Verify the response contains a tool call
const hasToolCall = firstResponse.content.some((block) => block.type === "toolCall");
expect(hasToolCall).toBe(true);
if (!hasToolCall) {
throw new Error("Expected assistant to make a tool call, but none was found");
}
// Step 4: Send a user message WITHOUT providing tool result
// This simulates the scenario where a tool call was aborted/cancelled
context.messages.push({
role: "user",
content: "Never mind, just tell me what is 2+2?",
timestamp: Date.now(),
});
// Step 5: The fix should filter out the orphaned tool call, and the request should succeed
const secondResponse = await complete(model, context, options);
console.log("Second response:", JSON.stringify(secondResponse, null, 2));
// The request should succeed (not error) - that's the main thing we're testing
expect(secondResponse.stopReason).not.toBe("error");
// Should have some content in the response
expect(secondResponse.content.length).toBeGreaterThan(0);
// The LLM may choose to answer directly or make a new tool call - either is fine
// The important thing is it didn't fail with the orphaned tool call error
const textContent = secondResponse.content
.filter((block) => block.type === "text")
.map((block) => (block.type === "text" ? block.text : ""))
.join(" ");
const toolCalls = secondResponse.content.filter((block) => block.type === "toolCall").length;
expect(toolCalls || textContent.length).toBeGreaterThan(0);
console.log("Answer:", textContent);
// Verify the stop reason is either "stop" or "toolUse" (new tool call)
expect(["stop", "toolUse"]).toContain(secondResponse.stopReason);
}
describe("Tool Call Without Result Tests", () => {
// =========================================================================
// API Key-based providers
// =========================================================================
describe.skipIf(!process.env.GEMINI_API_KEY)("Google Provider", () => {
const model = getModel("google", "gemini-2.5-flash");
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
await testToolCallWithoutResult(model);
});
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions Provider", () => {
const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!;
void _compat;
const model: Model<"openai-completions"> = {
...baseModel,
api: "openai-completions",
};
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
await testToolCallWithoutResult(model);
});
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses Provider", () => {
const model = getModel("openai", "gpt-5-mini");
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
await testToolCallWithoutResult(model);
});
});
describe.skipIf(!hasAzureOpenAICredentials())("Azure OpenAI Responses Provider", () => {
const model = getModel("azure-openai-responses", "gpt-4o-mini");
const azureDeploymentName = resolveAzureDeploymentName(model.id);
const azureOptions = azureDeploymentName ? { azureDeploymentName } : {};
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
await testToolCallWithoutResult(model, azureOptions);
});
});
describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic Provider", () => {
const model = getModel("anthropic", "claude-3-5-haiku-20241022");
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
await testToolCallWithoutResult(model);
});
});
describe.skipIf(!process.env.XAI_API_KEY)("xAI Provider", () => {
const model = getModel("xai", "grok-3-fast");
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
await testToolCallWithoutResult(model);
});
});
describe.skipIf(!process.env.GROQ_API_KEY)("Groq Provider", () => {
const model = getModel("groq", "openai/gpt-oss-20b");
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
await testToolCallWithoutResult(model);
});
});
describe.skipIf(!process.env.CEREBRAS_API_KEY)("Cerebras Provider", () => {
const model = getModel("cerebras", "gpt-oss-120b");
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
await testToolCallWithoutResult(model);
});
});
describe.skipIf(!process.env.HF_TOKEN)("Hugging Face Provider", () => {
const model = getModel("huggingface", "moonshotai/Kimi-K2.5");
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
await testToolCallWithoutResult(model);
});
});
describe.skipIf(!process.env.ZAI_API_KEY)("zAI Provider", () => {
const model = getModel("zai", "glm-4.5-flash");
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
await testToolCallWithoutResult(model);
});
});
describe.skipIf(!process.env.MISTRAL_API_KEY)("Mistral Provider", () => {
const model = getModel("mistral", "devstral-medium-latest");
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
await testToolCallWithoutResult(model);
});
});
describe.skipIf(!process.env.MINIMAX_API_KEY)("MiniMax Provider", () => {
const model = getModel("minimax", "MiniMax-M2.7");
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
await testToolCallWithoutResult(model);
});
});
describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider", () => {
const model = getModel("kimi-coding", "kimi-k2-thinking");
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
await testToolCallWithoutResult(model);
});
});
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider", () => {
const model = getModel("vercel-ai-gateway", "google/gemini-2.5-flash");
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
await testToolCallWithoutResult(model);
});
});
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider", () => {
const model = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0");
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
await testToolCallWithoutResult(model);
});
});
// =========================================================================
// OAuth-based providers (credentials from ~/.pi/agent/oauth.json)
// =========================================================================
describe("Anthropic OAuth Provider", () => {
const model = getModel("anthropic", "claude-3-5-haiku-20241022");
it.skipIf(!anthropicOAuthToken)(
"should filter out tool calls without corresponding tool results",
{ retry: 3, timeout: 30000 },
async () => {
await testToolCallWithoutResult(model, { apiKey: anthropicOAuthToken });
},
);
});
describe("GitHub Copilot Provider", () => {
it.skipIf(!githubCopilotToken)(
"gpt-4o - should filter out tool calls without corresponding tool results",
{ retry: 3, timeout: 30000 },
async () => {
const model = getModel("github-copilot", "gpt-4o");
await testToolCallWithoutResult(model, { apiKey: githubCopilotToken });
},
);
it.skipIf(!githubCopilotToken)(
"claude-sonnet-4 - should filter out tool calls without corresponding tool results",
{ retry: 3, timeout: 30000 },
async () => {
const model = getModel("github-copilot", "claude-sonnet-4");
await testToolCallWithoutResult(model, { apiKey: githubCopilotToken });
},
);
});
describe("Google Gemini CLI Provider", () => {
it.skipIf(!geminiCliToken)(
"gemini-2.5-flash - should filter out tool calls without corresponding tool results",
{ retry: 3, timeout: 30000 },
async () => {
const model = getModel("google-gemini-cli", "gemini-2.5-flash");
await testToolCallWithoutResult(model, { apiKey: geminiCliToken });
},
);
});
describe("Google Antigravity Provider", () => {
it.skipIf(!antigravityToken)(
"gemini-3-flash - should filter out tool calls without corresponding tool results",
{ retry: 3, timeout: 30000 },
async () => {
const model = getModel("google-antigravity", "gemini-3-flash");
await testToolCallWithoutResult(model, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"claude-sonnet-4-5 - should filter out tool calls without corresponding tool results",
{ retry: 3, timeout: 30000 },
async () => {
const model = getModel("google-antigravity", "claude-sonnet-4-5");
await testToolCallWithoutResult(model, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"gpt-oss-120b-medium - should filter out tool calls without corresponding tool results",
{ retry: 3, timeout: 30000 },
async () => {
const model = getModel("google-antigravity", "gpt-oss-120b-medium");
await testToolCallWithoutResult(model, { apiKey: antigravityToken });
},
);
});
describe("OpenAI Codex Provider", () => {
it.skipIf(!openaiCodexToken)(
"gpt-5.2-codex - should filter out tool calls without corresponding tool results",
{ retry: 3, timeout: 30000 },
async () => {
const model = getModel("openai-codex", "gpt-5.2-codex");
await testToolCallWithoutResult(model, { apiKey: openaiCodexToken });
},
);
});
});
@@ -1,695 +0,0 @@
/**
* Test totalTokens field across all providers.
*
* totalTokens represents the total number of tokens processed by the LLM,
* including input (with cache) and output (with thinking). This is the
* base for calculating context size for the next request.
*
* - OpenAI Completions: Uses native total_tokens field
* - OpenAI Responses: Uses native total_tokens field
* - Google: Uses native totalTokenCount field
* - Anthropic: Computed as input + output + cacheRead + cacheWrite
* - Other OpenAI-compatible providers: Uses native total_tokens field
*/
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { complete } from "../src/stream.js";
import type { Api, Context, Model, StreamOptions, Usage } from "../src/types.js";
type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;
import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.js";
import { hasBedrockCredentials } from "./bedrock-utils.js";
import { resolveApiKey } from "./oauth.js";
// Resolve OAuth tokens at module level (async, runs before tests)
const oauthTokens = await Promise.all([
resolveApiKey("anthropic"),
resolveApiKey("github-copilot"),
resolveApiKey("google-gemini-cli"),
resolveApiKey("google-antigravity"),
resolveApiKey("openai-codex"),
]);
const [anthropicOAuthToken, githubCopilotToken, geminiCliToken, antigravityToken, openaiCodexToken] = oauthTokens;
// Generate a long system prompt to trigger caching (>2k bytes for most providers)
const LONG_SYSTEM_PROMPT = `You are a helpful assistant. Be concise in your responses.
Here is some additional context that makes this system prompt long enough to trigger caching:
${Array(50)
.fill(
"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris.",
)
.join("\n\n")}
Remember: Always be helpful and concise.`;
async function testTotalTokensWithCache<TApi extends Api>(
llm: Model<TApi>,
options: StreamOptionsWithExtras = {},
): Promise<{ first: Usage; second: Usage }> {
// First request - no cache
const context1: Context = {
systemPrompt: LONG_SYSTEM_PROMPT,
messages: [
{
role: "user",
content: "What is 2 + 2? Reply with just the number.",
timestamp: Date.now(),
},
],
};
const response1 = await complete(llm, context1, options);
expect(response1.stopReason).toBe("stop");
// Second request - should trigger cache read (same system prompt, add conversation)
const context2: Context = {
systemPrompt: LONG_SYSTEM_PROMPT,
messages: [
...context1.messages,
response1, // Include previous assistant response
{
role: "user",
content: "What is 3 + 3? Reply with just the number.",
timestamp: Date.now(),
},
],
};
const response2 = await complete(llm, context2, options);
expect(response2.stopReason).toBe("stop");
return { first: response1.usage, second: response2.usage };
}
function logUsage(label: string, usage: Usage) {
const computed = usage.input + usage.output + usage.cacheRead + usage.cacheWrite;
console.log(` ${label}:`);
console.log(
` input: ${usage.input}, output: ${usage.output}, cacheRead: ${usage.cacheRead}, cacheWrite: ${usage.cacheWrite}`,
);
console.log(` totalTokens: ${usage.totalTokens}, computed: ${computed}`);
}
function assertTotalTokensEqualsComponents(usage: Usage) {
const computed = usage.input + usage.output + usage.cacheRead + usage.cacheWrite;
expect(usage.totalTokens).toBe(computed);
}
describe("totalTokens field", () => {
// =========================================================================
// Anthropic
// =========================================================================
describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic (API Key)", () => {
it(
"claude-3-5-haiku - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("anthropic", "claude-3-5-haiku-20241022");
console.log(`\nAnthropic / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.ANTHROPIC_API_KEY });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
// Anthropic should have cache activity
const hasCache = second.cacheRead > 0 || second.cacheWrite > 0 || first.cacheWrite > 0;
expect(hasCache).toBe(true);
},
);
});
describe("Anthropic (OAuth)", () => {
it.skipIf(!anthropicOAuthToken)(
"claude-sonnet-4 - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("anthropic", "claude-sonnet-4-20250514");
console.log(`\nAnthropic OAuth / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: anthropicOAuthToken });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
// Anthropic should have cache activity
const hasCache = second.cacheRead > 0 || second.cacheWrite > 0 || first.cacheWrite > 0;
expect(hasCache).toBe(true);
},
);
});
// =========================================================================
// OpenAI
// =========================================================================
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions", () => {
it(
"gpt-4o-mini - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!;
void _compat;
const llm: Model<"openai-completions"> = {
...baseModel,
api: "openai-completions",
};
console.log(`\nOpenAI Completions / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm);
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses", () => {
it("gpt-4o - should return totalTokens equal to sum of components", { retry: 3, timeout: 60000 }, async () => {
const llm = getModel("openai", "gpt-4o");
console.log(`\nOpenAI Responses / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm);
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
});
});
describe.skipIf(!hasAzureOpenAICredentials())("Azure OpenAI Responses", () => {
it(
"gpt-4o-mini - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("azure-openai-responses", "gpt-4o-mini");
const azureDeploymentName = resolveAzureDeploymentName(llm.id);
const azureOptions = azureDeploymentName ? { azureDeploymentName } : {};
console.log(`\nAzure OpenAI Responses / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, azureOptions);
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
});
// =========================================================================
// Google
// =========================================================================
describe.skipIf(!process.env.GEMINI_API_KEY)("Google", () => {
it(
"gemini-2.0-flash - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("google", "gemini-2.0-flash");
console.log(`\nGoogle / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm);
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
});
// =========================================================================
// xAI
// =========================================================================
describe.skipIf(!process.env.XAI_API_KEY)("xAI", () => {
it(
"grok-3-fast - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("xai", "grok-3-fast");
console.log(`\nxAI / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.XAI_API_KEY });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
});
// =========================================================================
// Groq
// =========================================================================
describe.skipIf(!process.env.GROQ_API_KEY)("Groq", () => {
it(
"openai/gpt-oss-120b - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("groq", "openai/gpt-oss-120b");
console.log(`\nGroq / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.GROQ_API_KEY });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
});
// =========================================================================
// Cerebras
// =========================================================================
describe.skipIf(!process.env.CEREBRAS_API_KEY)("Cerebras", () => {
it(
"gpt-oss-120b - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("cerebras", "gpt-oss-120b");
console.log(`\nCerebras / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.CEREBRAS_API_KEY });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
});
// =========================================================================
// Hugging Face
// =========================================================================
describe.skipIf(!process.env.HF_TOKEN)("Hugging Face", () => {
it("Kimi-K2.5 - should return totalTokens equal to sum of components", { retry: 3, timeout: 60000 }, async () => {
const llm = getModel("huggingface", "moonshotai/Kimi-K2.5");
console.log(`\nHugging Face / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.HF_TOKEN });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
});
});
// =========================================================================
// z.ai
// =========================================================================
describe.skipIf(!process.env.ZAI_API_KEY)("z.ai", () => {
it(
"glm-4.5-flash - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("zai", "glm-4.5-flash");
console.log(`\nz.ai / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.ZAI_API_KEY });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
});
// =========================================================================
// Mistral
// =========================================================================
describe.skipIf(!process.env.MISTRAL_API_KEY)("Mistral", () => {
it(
"devstral-medium-latest - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("mistral", "devstral-medium-latest");
console.log(`\nMistral / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.MISTRAL_API_KEY });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
});
// =========================================================================
// MiniMax
// =========================================================================
describe.skipIf(!process.env.MINIMAX_API_KEY)("MiniMax", () => {
it(
"MiniMax-M2.7 - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("minimax", "MiniMax-M2.7");
console.log(`\nMiniMax / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.MINIMAX_API_KEY });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
});
// =========================================================================
// Kimi For Coding
// =========================================================================
describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding", () => {
it(
"kimi-k2-thinking - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("kimi-coding", "kimi-k2-thinking");
console.log(`\nKimi For Coding / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.KIMI_API_KEY });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
});
// =========================================================================
// Vercel AI Gateway
// =========================================================================
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway", () => {
it(
"google/gemini-2.5-flash - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash");
console.log(`\nVercel AI Gateway / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.AI_GATEWAY_API_KEY });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
});
// =========================================================================
// OpenRouter - Multiple backend providers
// =========================================================================
describe.skipIf(!process.env.OPENROUTER_API_KEY)("OpenRouter", () => {
it(
"anthropic/claude-sonnet-4 - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("openrouter", "anthropic/claude-sonnet-4");
console.log(`\nOpenRouter / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.OPENROUTER_API_KEY });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
it(
"deepseek/deepseek-chat - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("openrouter", "deepseek/deepseek-chat");
console.log(`\nOpenRouter / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.OPENROUTER_API_KEY });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
it(
"mistralai/mistral-small-3.2-24b-instruct - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("openrouter", "mistralai/mistral-small-3.2-24b-instruct");
console.log(`\nOpenRouter / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.OPENROUTER_API_KEY });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
it(
"google/gemini-2.0-flash-001 - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("openrouter", "google/gemini-2.0-flash-001");
console.log(`\nOpenRouter / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.OPENROUTER_API_KEY });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
it(
"meta-llama/llama-4-maverick - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("openrouter", "meta-llama/llama-4-maverick");
console.log(`\nOpenRouter / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.OPENROUTER_API_KEY });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
});
// =========================================================================
// GitHub Copilot (OAuth)
// =========================================================================
describe("GitHub Copilot (OAuth)", () => {
it.skipIf(!githubCopilotToken)(
"gpt-4o - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("github-copilot", "gpt-4o");
console.log(`\nGitHub Copilot / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: githubCopilotToken });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
it.skipIf(!githubCopilotToken)(
"claude-sonnet-4 - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("github-copilot", "claude-sonnet-4");
console.log(`\nGitHub Copilot / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: githubCopilotToken });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
});
// =========================================================================
// Google Gemini CLI (OAuth)
// =========================================================================
describe("Google Gemini CLI (OAuth)", () => {
it.skipIf(!geminiCliToken)(
"gemini-2.5-flash - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
console.log(`\nGoogle Gemini CLI / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: geminiCliToken });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
});
// =========================================================================
// Google Antigravity (OAuth)
// =========================================================================
describe("Google Antigravity (OAuth)", () => {
it.skipIf(!antigravityToken)(
"gemini-3-flash - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("google-antigravity", "gemini-3-flash");
console.log(`\nGoogle Antigravity / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: antigravityToken });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
it.skipIf(!antigravityToken)(
"claude-sonnet-4-5 - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
console.log(`\nGoogle Antigravity / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: antigravityToken });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
it.skipIf(!antigravityToken)(
"gpt-oss-120b-medium - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("google-antigravity", "gpt-oss-120b-medium");
console.log(`\nGoogle Antigravity / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: antigravityToken });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
});
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock", () => {
it(
"claude-sonnet-4-5 - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0");
console.log(`\nAmazon Bedrock / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm);
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
});
// =========================================================================
// OpenAI Codex (OAuth)
// =========================================================================
describe("OpenAI Codex (OAuth)", () => {
it.skipIf(!openaiCodexToken)(
"gpt-5.2-codex - should return totalTokens equal to sum of components",
{ retry: 3, timeout: 60000 },
async () => {
const llm = getModel("openai-codex", "gpt-5.2-codex");
console.log(`\nOpenAI Codex / ${llm.id}:`);
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: openaiCodexToken });
logUsage("First request", first);
logUsage("Second request", second);
assertTotalTokensEqualsComponents(first);
assertTotalTokensEqualsComponents(second);
},
);
});
});
@@ -1,115 +0,0 @@
import { describe, expect, it } from "vitest";
import { transformMessages } from "../src/providers/transform-messages.js";
import type { AssistantMessage, Message, Model, ToolCall } from "../src/types.js";
// Normalize function matching what anthropic.ts uses
function anthropicNormalizeToolCallId(
id: string,
_model: Model<"anthropic-messages">,
_source: AssistantMessage,
): string {
return id.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 64);
}
function makeCopilotClaudeModel(): Model<"anthropic-messages"> {
return {
id: "claude-sonnet-4",
name: "Claude Sonnet 4",
api: "anthropic-messages",
provider: "github-copilot",
baseUrl: "https://api.individual.githubcopilot.com",
reasoning: true,
input: ["text", "image"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 128000,
maxTokens: 16000,
};
}
describe("OpenAI to Anthropic session migration for Copilot Claude", () => {
it("converts thinking blocks to plain text when source model differs", () => {
const model = makeCopilotClaudeModel();
const messages: Message[] = [
{ role: "user", content: "hello", timestamp: Date.now() },
{
role: "assistant",
content: [
{
type: "thinking",
thinking: "Let me think about this...",
thinkingSignature: "reasoning_content",
},
{ type: "text", text: "Hi there!" },
],
api: "openai-completions",
provider: "github-copilot",
model: "gpt-4o",
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "stop",
timestamp: Date.now(),
},
];
const result = transformMessages(messages, model, anthropicNormalizeToolCallId);
const assistantMsg = result.find((m) => m.role === "assistant") as AssistantMessage;
// Thinking block should be converted to text since models differ
const textBlocks = assistantMsg.content.filter((b) => b.type === "text");
const thinkingBlocks = assistantMsg.content.filter((b) => b.type === "thinking");
expect(thinkingBlocks).toHaveLength(0);
expect(textBlocks.length).toBeGreaterThanOrEqual(2);
});
it("removes thoughtSignature from tool calls when migrating between models", () => {
const model = makeCopilotClaudeModel();
const messages: Message[] = [
{ role: "user", content: "run a command", timestamp: Date.now() },
{
role: "assistant",
content: [
{
type: "toolCall",
id: "call_123",
name: "bash",
arguments: { command: "ls" },
thoughtSignature: JSON.stringify({ type: "reasoning.encrypted", id: "call_123", data: "encrypted" }),
},
],
api: "openai-responses",
provider: "github-copilot",
model: "gpt-5",
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "toolUse",
timestamp: Date.now(),
},
{
role: "toolResult",
toolCallId: "call_123",
toolName: "bash",
content: [{ type: "text", text: "output" }],
isError: false,
timestamp: Date.now(),
},
];
const result = transformMessages(messages, model, anthropicNormalizeToolCallId);
const assistantMsg = result.find((m) => m.role === "assistant") as AssistantMessage;
const toolCall = assistantMsg.content.find((b) => b.type === "toolCall") as ToolCall;
expect(toolCall.thoughtSignature).toBeUndefined();
});
});
@@ -1,754 +0,0 @@
import { Type } from "@sinclair/typebox";
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { complete } from "../src/stream.js";
import type { Api, Context, Model, StreamOptions, ToolResultMessage } from "../src/types.js";
type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;
import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.js";
import { hasBedrockCredentials } from "./bedrock-utils.js";
import { resolveApiKey } from "./oauth.js";
// Empty schema for test tools - must be proper OBJECT type for Cloud Code Assist
const emptySchema = Type.Object({});
// Resolve OAuth tokens at module level (async, runs before tests)
const oauthTokens = await Promise.all([
resolveApiKey("anthropic"),
resolveApiKey("github-copilot"),
resolveApiKey("google-gemini-cli"),
resolveApiKey("google-antigravity"),
resolveApiKey("openai-codex"),
]);
const [anthropicOAuthToken, githubCopilotToken, geminiCliToken, antigravityToken, openaiCodexToken] = oauthTokens;
/**
* Test for Unicode surrogate pair handling in tool results.
*
* Issue: When tool results contain emoji or other characters outside the Basic Multilingual Plane,
* they may be incorrectly serialized as unpaired surrogates, causing "no low surrogate in string"
* errors when sent to the API provider.
*
* Example error from Anthropic:
* "The request body is not valid JSON: no low surrogate in string: line 1 column 197667"
*/
async function testEmojiInToolResults<TApi extends Api>(llm: Model<TApi>, options: StreamOptionsWithExtras = {}) {
const toolCallId = llm.provider === "mistral" ? "testtool1" : "test_1";
// Simulate a tool that returns emoji
const context: Context = {
systemPrompt: "You are a helpful assistant.",
messages: [
{
role: "user",
content: "Use the test tool",
timestamp: Date.now(),
},
{
role: "assistant",
content: [
{
type: "toolCall",
id: toolCallId,
name: "test_tool",
arguments: {},
},
],
api: llm.api,
provider: llm.provider,
model: llm.id,
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "toolUse",
timestamp: Date.now(),
},
],
tools: [
{
name: "test_tool",
description: "A test tool",
parameters: emptySchema,
},
],
};
// Add tool result with various problematic Unicode characters
const toolResult: ToolResultMessage = {
role: "toolResult",
toolCallId: toolCallId,
toolName: "test_tool",
content: [
{
type: "text",
text: `Test with emoji 🙈 and other characters:
- Monkey emoji: 🙈
- Thumbs up: 👍
- Heart: ❤️
- Thinking face: 🤔
- Rocket: 🚀
- Mixed text: Mario Zechner wann? Wo? Bin grad äußersr eventuninformiert 🙈
- Japanese: こんにちは
- Chinese: 你好
- Mathematical symbols: ∑∫∂√
- Special quotes: "curly" 'quotes'`,
},
],
isError: false,
timestamp: Date.now(),
};
context.messages.push(toolResult);
// Add follow-up user message
context.messages.push({
role: "user",
content: "Summarize the tool result briefly.",
timestamp: Date.now(),
});
// This should not throw a surrogate pair error
const response = await complete(llm, context, options);
expect(response.stopReason).not.toBe("error");
expect(response.errorMessage).toBeFalsy();
expect(response.content.length).toBeGreaterThan(0);
}
async function testRealWorldLinkedInData<TApi extends Api>(llm: Model<TApi>, options: StreamOptionsWithExtras = {}) {
const toolCallId = llm.provider === "mistral" ? "linkedin1" : "linkedin_1";
const context: Context = {
systemPrompt: "You are a helpful assistant.",
messages: [
{
role: "user",
content: "Use the linkedin tool to get comments",
timestamp: Date.now(),
},
{
role: "assistant",
content: [
{
type: "toolCall",
id: toolCallId,
name: "linkedin_skill",
arguments: {},
},
],
api: llm.api,
provider: llm.provider,
model: llm.id,
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "toolUse",
timestamp: Date.now(),
},
],
tools: [
{
name: "linkedin_skill",
description: "Get LinkedIn comments",
parameters: emptySchema,
},
],
};
// Real-world tool result from LinkedIn with emoji
const toolResult: ToolResultMessage = {
role: "toolResult",
toolCallId: toolCallId,
toolName: "linkedin_skill",
content: [
{
type: "text",
text: `Post: Hab einen "Generative KI für Nicht-Techniker" Workshop gebaut.
Unanswered Comments: 2
=> {
"comments": [
{
"author": "Matthias Neumayer's graphic link",
"text": "Leider nehmen das viel zu wenige Leute ernst"
},
{
"author": "Matthias Neumayer's graphic link",
"text": "Mario Zechner wann? Wo? Bin grad äußersr eventuninformiert 🙈"
}
]
}`,
},
],
isError: false,
timestamp: Date.now(),
};
context.messages.push(toolResult);
context.messages.push({
role: "user",
content: "How many comments are there?",
timestamp: Date.now(),
});
// This should not throw a surrogate pair error
const response = await complete(llm, context, options);
expect(response.stopReason).not.toBe("error");
expect(response.errorMessage).toBeFalsy();
expect(response.content.some((b) => b.type === "text")).toBe(true);
}
async function testUnpairedHighSurrogate<TApi extends Api>(llm: Model<TApi>, options: StreamOptionsWithExtras = {}) {
const toolCallId = llm.provider === "mistral" ? "testtool2" : "test_2";
const context: Context = {
systemPrompt: "You are a helpful assistant.",
messages: [
{
role: "user",
content: "Use the test tool",
timestamp: Date.now(),
},
{
role: "assistant",
content: [
{
type: "toolCall",
id: toolCallId,
name: "test_tool",
arguments: {},
},
],
api: llm.api,
provider: llm.provider,
model: llm.id,
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "toolUse",
timestamp: Date.now(),
},
],
tools: [
{
name: "test_tool",
description: "A test tool",
parameters: emptySchema,
},
],
};
// Construct a string with an intentionally unpaired high surrogate
// This simulates what might happen if text processing corrupts emoji
const unpairedSurrogate = String.fromCharCode(0xd83d); // High surrogate without low surrogate
const toolResult: ToolResultMessage = {
role: "toolResult",
toolCallId: toolCallId,
toolName: "test_tool",
content: [{ type: "text", text: `Text with unpaired surrogate: ${unpairedSurrogate} <- should be sanitized` }],
isError: false,
timestamp: Date.now(),
};
context.messages.push(toolResult);
context.messages.push({
role: "user",
content: "What did the tool return?",
timestamp: Date.now(),
});
// This should not throw a surrogate pair error
// The unpaired surrogate should be sanitized before sending to API
const response = await complete(llm, context, options);
expect(response.stopReason).not.toBe("error");
expect(response.errorMessage).toBeFalsy();
expect(response.content.length).toBeGreaterThan(0);
}
describe("AI Providers Unicode Surrogate Pair Tests", () => {
describe.skipIf(!process.env.GEMINI_API_KEY)("Google Provider Unicode Handling", () => {
const llm = getModel("google", "gemini-2.5-flash");
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
await testEmojiInToolResults(llm);
});
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
await testRealWorldLinkedInData(llm);
});
it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => {
await testUnpairedHighSurrogate(llm);
});
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions Provider Unicode Handling", () => {
const llm = getModel("openai", "gpt-4o-mini");
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
await testEmojiInToolResults(llm);
});
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
await testRealWorldLinkedInData(llm);
});
it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => {
await testUnpairedHighSurrogate(llm);
});
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses Provider Unicode Handling", () => {
const llm = getModel("openai", "gpt-5-mini");
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
await testEmojiInToolResults(llm);
});
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
await testRealWorldLinkedInData(llm);
});
it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => {
await testUnpairedHighSurrogate(llm);
});
});
describe.skipIf(!hasAzureOpenAICredentials())("Azure OpenAI Responses Provider Unicode Handling", () => {
const llm = getModel("azure-openai-responses", "gpt-4o-mini");
const azureDeploymentName = resolveAzureDeploymentName(llm.id);
const azureOptions = azureDeploymentName ? { azureDeploymentName } : {};
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
await testEmojiInToolResults(llm, azureOptions);
});
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
await testRealWorldLinkedInData(llm, azureOptions);
});
it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => {
await testUnpairedHighSurrogate(llm, azureOptions);
});
});
describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic Provider Unicode Handling", () => {
const llm = getModel("anthropic", "claude-3-5-haiku-20241022");
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
await testEmojiInToolResults(llm);
});
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
await testRealWorldLinkedInData(llm);
});
it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => {
await testUnpairedHighSurrogate(llm);
});
});
// =========================================================================
// OAuth-based providers (credentials from ~/.pi/agent/oauth.json)
// =========================================================================
describe("Anthropic OAuth Provider Unicode Handling", () => {
const llm = getModel("anthropic", "claude-3-5-haiku-20241022");
it.skipIf(!anthropicOAuthToken)("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
await testEmojiInToolResults(llm, { apiKey: anthropicOAuthToken });
});
it.skipIf(!anthropicOAuthToken)(
"should handle real-world LinkedIn comment data with emoji",
{ retry: 3, timeout: 30000 },
async () => {
await testRealWorldLinkedInData(llm, { apiKey: anthropicOAuthToken });
},
);
it.skipIf(!anthropicOAuthToken)(
"should handle unpaired high surrogate (0xD83D) in tool results",
{ retry: 3, timeout: 30000 },
async () => {
await testUnpairedHighSurrogate(llm, { apiKey: anthropicOAuthToken });
},
);
});
describe("GitHub Copilot Provider Unicode Handling", () => {
it.skipIf(!githubCopilotToken)(
"gpt-4o - should handle emoji in tool results",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "gpt-4o");
await testEmojiInToolResults(llm, { apiKey: githubCopilotToken });
},
);
it.skipIf(!githubCopilotToken)(
"gpt-4o - should handle real-world LinkedIn comment data with emoji",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "gpt-4o");
await testRealWorldLinkedInData(llm, { apiKey: githubCopilotToken });
},
);
it.skipIf(!githubCopilotToken)(
"gpt-4o - should handle unpaired high surrogate (0xD83D) in tool results",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "gpt-4o");
await testUnpairedHighSurrogate(llm, { apiKey: githubCopilotToken });
},
);
it.skipIf(!githubCopilotToken)(
"claude-sonnet-4 - should handle emoji in tool results",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "claude-sonnet-4");
await testEmojiInToolResults(llm, { apiKey: githubCopilotToken });
},
);
it.skipIf(!githubCopilotToken)(
"claude-sonnet-4 - should handle real-world LinkedIn comment data with emoji",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "claude-sonnet-4");
await testRealWorldLinkedInData(llm, { apiKey: githubCopilotToken });
},
);
it.skipIf(!githubCopilotToken)(
"claude-sonnet-4 - should handle unpaired high surrogate (0xD83D) in tool results",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("github-copilot", "claude-sonnet-4");
await testUnpairedHighSurrogate(llm, { apiKey: githubCopilotToken });
},
);
});
describe("Google Gemini CLI Provider Unicode Handling", () => {
it.skipIf(!geminiCliToken)(
"gemini-2.5-flash - should handle emoji in tool results",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
await testEmojiInToolResults(llm, { apiKey: geminiCliToken });
},
);
it.skipIf(!geminiCliToken)(
"gemini-2.5-flash - should handle real-world LinkedIn comment data with emoji",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
await testRealWorldLinkedInData(llm, { apiKey: geminiCliToken });
},
);
it.skipIf(!geminiCliToken)(
"gemini-2.5-flash - should handle unpaired high surrogate (0xD83D) in tool results",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-gemini-cli", "gemini-2.5-flash");
await testUnpairedHighSurrogate(llm, { apiKey: geminiCliToken });
},
);
});
describe("Google Antigravity Provider Unicode Handling", () => {
it.skipIf(!antigravityToken)(
"gemini-3-flash - should handle emoji in tool results",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "gemini-3-flash");
await testEmojiInToolResults(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"gemini-3-flash - should handle real-world LinkedIn comment data with emoji",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "gemini-3-flash");
await testRealWorldLinkedInData(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"gemini-3-flash - should handle unpaired high surrogate (0xD83D) in tool results",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "gemini-3-flash");
await testUnpairedHighSurrogate(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"claude-sonnet-4-5 - should handle emoji in tool results",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
await testEmojiInToolResults(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"claude-sonnet-4-5 - should handle real-world LinkedIn comment data with emoji",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
await testRealWorldLinkedInData(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"claude-sonnet-4-5 - should handle unpaired high surrogate (0xD83D) in tool results",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "claude-sonnet-4-5");
await testUnpairedHighSurrogate(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"gpt-oss-120b-medium - should handle emoji in tool results",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "gpt-oss-120b-medium");
await testEmojiInToolResults(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"gpt-oss-120b-medium - should handle real-world LinkedIn comment data with emoji",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "gpt-oss-120b-medium");
await testRealWorldLinkedInData(llm, { apiKey: antigravityToken });
},
);
it.skipIf(!antigravityToken)(
"gpt-oss-120b-medium - should handle unpaired high surrogate (0xD83D) in tool results",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("google-antigravity", "gpt-oss-120b-medium");
await testUnpairedHighSurrogate(llm, { apiKey: antigravityToken });
},
);
});
describe.skipIf(!process.env.XAI_API_KEY)("xAI Provider Unicode Handling", () => {
const llm = getModel("xai", "grok-3");
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
await testEmojiInToolResults(llm);
});
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
await testRealWorldLinkedInData(llm);
});
it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => {
await testUnpairedHighSurrogate(llm);
});
});
describe.skipIf(!process.env.GROQ_API_KEY)("Groq Provider Unicode Handling", () => {
const llm = getModel("groq", "openai/gpt-oss-20b");
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
await testEmojiInToolResults(llm);
});
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
await testRealWorldLinkedInData(llm);
});
it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => {
await testUnpairedHighSurrogate(llm);
});
});
describe.skipIf(!process.env.CEREBRAS_API_KEY)("Cerebras Provider Unicode Handling", () => {
const llm = getModel("cerebras", "gpt-oss-120b");
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
await testEmojiInToolResults(llm);
});
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
await testRealWorldLinkedInData(llm);
});
it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => {
await testUnpairedHighSurrogate(llm);
});
});
describe.skipIf(!process.env.HF_TOKEN)("Hugging Face Provider Unicode Handling", () => {
const llm = getModel("huggingface", "moonshotai/Kimi-K2.5");
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
await testEmojiInToolResults(llm);
});
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
await testRealWorldLinkedInData(llm);
});
it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => {
await testUnpairedHighSurrogate(llm);
});
});
describe.skipIf(!process.env.ZAI_API_KEY)("zAI Provider Unicode Handling", () => {
const llm = getModel("zai", "glm-4.5-air");
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
await testEmojiInToolResults(llm);
});
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
await testRealWorldLinkedInData(llm);
});
it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => {
await testUnpairedHighSurrogate(llm);
});
});
describe.skipIf(!process.env.MISTRAL_API_KEY)("Mistral Provider Unicode Handling", () => {
const llm = getModel("mistral", "devstral-medium-latest");
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
await testEmojiInToolResults(llm);
});
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
await testRealWorldLinkedInData(llm);
});
it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => {
await testUnpairedHighSurrogate(llm);
});
});
describe.skipIf(!process.env.MINIMAX_API_KEY)("MiniMax Provider Unicode Handling", () => {
const llm = getModel("minimax", "MiniMax-M2.7");
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
await testEmojiInToolResults(llm);
});
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
await testRealWorldLinkedInData(llm);
});
it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => {
await testUnpairedHighSurrogate(llm);
});
});
describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider Unicode Handling", () => {
const llm = getModel("kimi-coding", "kimi-k2-thinking");
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
await testEmojiInToolResults(llm);
});
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
await testRealWorldLinkedInData(llm);
});
it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => {
await testUnpairedHighSurrogate(llm);
});
});
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider Unicode Handling", () => {
const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash");
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
await testEmojiInToolResults(llm);
});
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
await testRealWorldLinkedInData(llm);
});
it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => {
await testUnpairedHighSurrogate(llm);
});
});
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider Unicode Handling", () => {
const llm = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0");
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
await testEmojiInToolResults(llm);
});
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
await testRealWorldLinkedInData(llm);
});
it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => {
await testUnpairedHighSurrogate(llm);
});
});
describe("OpenAI Codex Provider Unicode Handling", () => {
it.skipIf(!openaiCodexToken)(
"gpt-5.2-codex - should handle emoji in tool results",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("openai-codex", "gpt-5.2-codex");
await testEmojiInToolResults(llm, { apiKey: openaiCodexToken });
},
);
it.skipIf(!openaiCodexToken)(
"gpt-5.2-codex - should handle real-world LinkedIn comment data with emoji",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("openai-codex", "gpt-5.2-codex");
await testRealWorldLinkedInData(llm, { apiKey: openaiCodexToken });
},
);
it.skipIf(!openaiCodexToken)(
"gpt-5.2-codex - should handle unpaired high surrogate (0xD83D) in tool results",
{ retry: 3, timeout: 30000 },
async () => {
const llm = getModel("openai-codex", "gpt-5.2-codex");
await testUnpairedHighSurrogate(llm, { apiKey: openaiCodexToken });
},
);
});
});
@@ -1,39 +0,0 @@
import { Type } from "@sinclair/typebox";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { ToolCall } from "../src/types.js";
import { validateToolArguments } from "../src/utils/validation.js";
afterEach(() => {
vi.restoreAllMocks();
});
describe("validateToolArguments", () => {
it("falls back to raw arguments without writing to stderr when runtime code generation is blocked", () => {
const originalFunction = globalThis.Function;
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
const tool = {
name: "echo",
description: "Echo tool",
parameters: Type.Object({
count: Type.Number(),
}),
};
const toolCall: ToolCall = {
type: "toolCall",
id: "tool-1",
name: "echo",
arguments: { count: "42" as unknown as number },
};
globalThis.Function = (() => {
throw new EvalError("Code generation from strings disallowed for this context");
}) as unknown as FunctionConstructor;
try {
expect(validateToolArguments(tool, toolCall)).toEqual(toolCall.arguments);
expect(errorSpy).not.toHaveBeenCalled();
} finally {
globalThis.Function = originalFunction;
}
});
});
@@ -1,71 +0,0 @@
import { describe, expect, it } from "vitest";
import { getModel } from "../src/models.js";
import { stream } from "../src/stream.js";
import type { Context, Model } from "../src/types.js";
function makeContext(): Context {
return {
messages: [
{
role: "user",
content: `What is ${(Math.random() * 100) | 0} + ${(Math.random() * 100) | 0}? Think step by step.`,
timestamp: Date.now(),
},
],
};
}
describe.skipIf(!process.env.OPENAI_API_KEY)("xhigh reasoning", () => {
describe("codex-max (supports xhigh)", () => {
// Note: codex models only support the responses API, not chat completions
it("should work with openai-responses", async () => {
const model = getModel("openai", "gpt-5.1-codex-max");
const s = stream(model, makeContext(), { reasoningEffort: "xhigh" });
let hasThinking = false;
for await (const event of s) {
if (event.type === "thinking_start" || event.type === "thinking_delta") {
hasThinking = true;
}
}
const response = await s.result();
expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("stop");
expect(response.content.some((b) => b.type === "text")).toBe(true);
expect(hasThinking || response.content.some((b) => b.type === "thinking")).toBe(true);
});
});
describe("gpt-5-mini (does not support xhigh)", () => {
it("should error with openai-responses when using xhigh", async () => {
const model = getModel("openai", "gpt-5-mini");
const s = stream(model, makeContext(), { reasoningEffort: "xhigh" });
for await (const _ of s) {
// drain events
}
const response = await s.result();
expect(response.stopReason).toBe("error");
expect(response.errorMessage).toContain("xhigh");
});
it("should error with openai-completions when using xhigh", async () => {
const { compat: _compat, ...baseModel } = getModel("openai", "gpt-5-mini");
void _compat;
const model: Model<"openai-completions"> = {
...baseModel,
api: "openai-completions",
};
const s = stream(model, makeContext(), { reasoningEffort: "xhigh" });
for await (const _ of s) {
// drain events
}
const response = await s.result();
expect(response.stopReason).toBe("error");
expect(response.errorMessage).toContain("xhigh");
});
});
});
@@ -1,25 +0,0 @@
import { describe, expect, it } from "vitest";
import { MODELS } from "../src/models.generated.js";
import { complete } from "../src/stream.js";
import type { Model } from "../src/types.js";
describe.skipIf(!process.env.OPENCODE_API_KEY)("OpenCode Models Smoke Test", () => {
const providers = [
{ key: "opencode", label: "OpenCode Zen" },
{ key: "opencode-go", label: "OpenCode Go" },
] as const;
providers.forEach(({ key, label }) => {
const providerModels = Object.values(MODELS[key]);
providerModels.forEach((model) => {
it(`${label}: ${model.id}`, async () => {
const response = await complete(model as Model<any>, {
messages: [{ role: "user", content: "Say hello.", timestamp: Date.now() }],
});
expect(response.content).toBeTruthy();
expect(response.stopReason).toBe("stop");
}, 60000);
});
});
});