ZacharyZhang-NYandClaude Opus 5.5
0970b84336
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
316 lines
19 KiB
TypeScript
316 lines
19 KiB
TypeScript
import { randomUUID } from "node:crypto";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { describe, expect, test } from "vitest";
|
|
import type { JsonRecord, ProviderEventStreamMessage } from "../../src/main/services/provider-http";
|
|
import type { AgentTurnMessage } from "../../src/main/services/provider-tool-turn";
|
|
import { accumulateResponsesStream, parseResponsesOutput, responsesToolDefs, toResponsesInput } from "../../src/main/services/provider-tool-turns/responses-format";
|
|
import { accumulateChatStream, parseChatChoice, toChatMessages } from "../../src/main/services/provider-tool-turns/chat-format";
|
|
import { parseAnthropicContent, toAnthropicMessages } from "../../src/main/services/provider-tool-turns/anthropic-format";
|
|
import { accumulateGeminiStream, antigravityRequestBody } from "../../src/main/services/provider-tool-turns/gemini-format";
|
|
import { bedrockToolConfig, parseBedrockOutput, toBedrockMessages } from "../../src/main/services/provider-tool-turns/bedrock-format";
|
|
import { parseGeminiResponse, vertexRequestBody } from "../../src/main/services/provider-tool-turns/gemini-format";
|
|
import { signAwsRequest } from "../../src/main/services/provider-tool-turns/aws-sigv4";
|
|
import { selectToolTurn, supportsToolTurn } from "../../src/main/services/provider-tool-turns/router";
|
|
import { resolveTurnCredential } from "../../src/main/services/provider-tool-turns/credential-resolver";
|
|
import { DatabaseConnection } from "../../src/main/database/connection";
|
|
import { OrganizationService } from "../../src/main/services/organization-service";
|
|
import { ProviderService } from "../../src/main/services/provider-service";
|
|
import { providerAdapters } from "../../src/main/services/provider-adapters";
|
|
import { AuthBrokerService } from "../../src/main/services/auth-broker-service";
|
|
import { loadConfiguredProvider } from "../../src/main/services/provider-internal";
|
|
import { IntegrationKeyStore } from "../../src/main/services/integration-key-store";
|
|
|
|
const CONVO: AgentTurnMessage[] = [
|
|
{ role: "user", text: "do it" },
|
|
{ role: "assistant", text: "calling", toolCalls: [{ id: "c1", name: "hire_agent", arguments: { name: "Sam" } }] },
|
|
{ role: "tool", results: [{ toolCallId: "c1", toolName: "hire_agent", output: "ok" }] },
|
|
];
|
|
|
|
async function* stream(items: JsonRecord[]): AsyncIterable<ProviderEventStreamMessage> {
|
|
for (const data of items) yield { eventName: null, data, rawData: JSON.stringify(data) };
|
|
}
|
|
|
|
const testVault = {
|
|
encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
|
|
decryptSecret: (ciphertext: Buffer) => ciphertext.toString("utf8"),
|
|
};
|
|
|
|
describe("openai responses encoding", () => {
|
|
test("encodes the conversation and tool calls", () => {
|
|
expect(toResponsesInput(CONVO)).toEqual([
|
|
{ role: "user", content: [{ type: "input_text", text: "do it" }] },
|
|
{ type: "message", role: "assistant", content: [{ type: "output_text", text: "calling", annotations: [] }] },
|
|
{ type: "function_call", call_id: "c1", name: "hire_agent", arguments: JSON.stringify({ name: "Sam" }) },
|
|
{ type: "function_call_output", call_id: "c1", output: "ok" },
|
|
]);
|
|
});
|
|
|
|
test("parses output text and function calls", () => {
|
|
const data = { status: "completed", output: [
|
|
{ type: "message", content: [{ type: "output_text", text: "done" }] },
|
|
{ type: "function_call", call_id: "c2", name: "create_task", arguments: '{"title":"x"}' },
|
|
] };
|
|
expect(parseResponsesOutput(data)).toMatchObject({ text: "done", toolCalls: [{ id: "c2", name: "create_task", arguments: { title: "x" } }] });
|
|
});
|
|
|
|
test("accumulates the SSE completion event", async () => {
|
|
const result = await accumulateResponsesStream(stream([
|
|
{ type: "response.output_text.delta", delta: "partial" },
|
|
{ type: "response.completed", response: { output: [{ type: "function_call", call_id: "c3", name: "write_deliverable", arguments: '{"path":"a"}' }] } },
|
|
]));
|
|
expect(result.toolCalls).toEqual([{ id: "c3", name: "write_deliverable", arguments: { path: "a" } }]);
|
|
});
|
|
|
|
test("accumulates output_item.done when the completion output is empty (Codex/ChatGPT backend)", async () => {
|
|
// The Codex backend streams each tool call as an output_item.done event and
|
|
// then sends a response.completed whose response.output is [].
|
|
const result = await accumulateResponsesStream(stream([
|
|
{ type: "response.output_item.added", item: { id: "fc1", type: "function_call", status: "in_progress", arguments: "", call_id: "call_1", name: "hire_agent" } },
|
|
{ type: "response.function_call_arguments.done", arguments: '{"name":"Dr. Li"}', item_id: "fc1" },
|
|
{ type: "response.output_item.done", item: { id: "fc1", type: "function_call", status: "completed", arguments: '{"name":"Dr. Li","department":"Engineering"}', call_id: "call_1", name: "hire_agent" } },
|
|
{ type: "response.output_item.done", item: { id: "m1", type: "message", content: [{ type: "output_text", text: "Staffing the team." }] } },
|
|
{ type: "response.completed", response: { output: [], status: "completed" } },
|
|
]));
|
|
expect(result.toolCalls).toEqual([{ id: "call_1", name: "hire_agent", arguments: { name: "Dr. Li", department: "Engineering" } }]);
|
|
expect(result.text).toBe("Staffing the team.");
|
|
});
|
|
});
|
|
|
|
describe("openai chat completions encoding", () => {
|
|
test("encodes system, tool calls and tool results", () => {
|
|
expect(toChatMessages("sys", CONVO)).toEqual([
|
|
{ role: "system", content: "sys" },
|
|
{ role: "user", content: "do it" },
|
|
{ role: "assistant", content: "calling", tool_calls: [{ id: "c1", type: "function", function: { name: "hire_agent", arguments: JSON.stringify({ name: "Sam" }) } }] },
|
|
{ role: "tool", tool_call_id: "c1", content: "ok" },
|
|
]);
|
|
});
|
|
|
|
test("parses a tool call choice", () => {
|
|
const data = { choices: [{ message: { content: "hi", tool_calls: [{ id: "c9", function: { name: "f", arguments: '{"a":1}' } }] }, finish_reason: "tool_calls" }] };
|
|
expect(parseChatChoice(data)).toMatchObject({ text: "hi", toolCalls: [{ id: "c9", name: "f", arguments: { a: 1 } }] });
|
|
});
|
|
|
|
test("accumulates streamed content deltas and indexed tool-call fragments", async () => {
|
|
// The Kimi coding endpoint streams reasoning_content first (which must be
|
|
// ignored), then content deltas, then tool calls split across chunks: the
|
|
// opening fragment carries index/id/name, later fragments append arguments.
|
|
const result = await accumulateChatStream(stream([
|
|
{ id: "cmpl-1", choices: [{ delta: { role: "assistant", reasoning_content: "thinking..." } }] },
|
|
{ choices: [{ delta: { content: "Staffing " } }] },
|
|
{ choices: [{ delta: { content: "the team." } }] },
|
|
{ choices: [{ delta: { tool_calls: [{ index: 0, id: "c1", type: "function", function: { name: "hire_agent", arguments: "" } }] } }] },
|
|
{ choices: [{ delta: { tool_calls: [{ index: 0, function: { arguments: '{"name":' } }, { index: 1, id: "c2", type: "function", function: { name: "create_task", arguments: '{"title":"x"}' } }] } }] },
|
|
{ choices: [{ delta: { tool_calls: [{ index: 0, function: { arguments: '"Sam"}' } }] }, finish_reason: "tool_calls" }] },
|
|
{ choices: [], usage: { prompt_tokens: 10, completion_tokens: 5 } },
|
|
]));
|
|
expect(result.text).toBe("Staffing the team.");
|
|
expect(result.toolCalls).toEqual([
|
|
{ id: "c1", name: "hire_agent", arguments: { name: "Sam" } },
|
|
{ id: "c2", name: "create_task", arguments: { title: "x" } },
|
|
]);
|
|
});
|
|
|
|
test("fails the turn on a streamed error event and on an empty stream", async () => {
|
|
await expect(accumulateChatStream(stream([
|
|
{ error: { message: "The usage limit has been reached" } },
|
|
]))).rejects.toThrow("The usage limit has been reached");
|
|
await expect(accumulateChatStream(stream([]))).rejects.toThrow("Chat completions stream ended without any events.");
|
|
});
|
|
});
|
|
|
|
describe("anthropic encoding", () => {
|
|
test("encodes tool_use and tool_result", () => {
|
|
expect(toAnthropicMessages(CONVO)).toEqual([
|
|
{ role: "user", content: [{ type: "text", text: "do it" }] },
|
|
{ role: "assistant", content: [{ type: "text", text: "calling" }, { type: "tool_use", id: "c1", name: "hire_agent", input: { name: "Sam" } }] },
|
|
{ role: "user", content: [{ type: "tool_result", tool_use_id: "c1", content: "ok" }] },
|
|
]);
|
|
});
|
|
|
|
test("parses text and tool_use blocks", () => {
|
|
const data = { stop_reason: "tool_use", content: [{ type: "text", text: "ok" }, { type: "tool_use", id: "tu", name: "f", input: { x: 1 } }] };
|
|
expect(parseAnthropicContent(data)).toMatchObject({ text: "ok", toolCalls: [{ id: "tu", name: "f", arguments: { x: 1 } }] });
|
|
});
|
|
});
|
|
|
|
describe("antigravity (gemini) encoding", () => {
|
|
test("wraps the request with the Antigravity system instruction and tool declarations", () => {
|
|
const body = antigravityRequestBody({ model: "claude-sonnet-4-5", projectId: "proj", instructions: "be helpful", messages: CONVO, tools: [{ name: "f", description: "d", parameters: { type: "object" } }], requestId: "agent-1" });
|
|
expect(body.project).toBe("proj");
|
|
expect(body.requestType).toBe("agent");
|
|
expect(body.userAgent).toBe("antigravity");
|
|
const request = body.request as JsonRecord;
|
|
const system = request.systemInstruction as { role: string; parts: Array<{ text: string }> };
|
|
expect(system.role).toBe("user");
|
|
expect(system.parts[0].text).toContain("You are Antigravity");
|
|
expect(system.parts[system.parts.length - 1].text).toBe("be helpful");
|
|
// claude-* models use the legacy `parameters` field
|
|
expect(request.tools).toEqual([{ functionDeclarations: [{ name: "f", description: "d", parameters: { type: "object" } }] }]);
|
|
});
|
|
|
|
test("gemini models use parametersJsonSchema and omit the tool-call id", () => {
|
|
const body = antigravityRequestBody({ model: "gemini-3.1-pro-high", projectId: "proj", instructions: null, messages: CONVO, tools: [{ name: "f", description: "d", parameters: { type: "object" } }], requestId: "agent-2" });
|
|
const request = body.request as JsonRecord;
|
|
expect(request.tools).toEqual([{ functionDeclarations: [{ name: "f", description: "d", parametersJsonSchema: { type: "object" } }] }]);
|
|
const contents = request.contents as Array<{ role: string; parts: Array<JsonRecord> }>;
|
|
const assistant = contents.find((c) => c.role === "model");
|
|
const fnCall = assistant?.parts.find((p) => "functionCall" in p)?.functionCall as JsonRecord;
|
|
expect(fnCall.id).toBeUndefined();
|
|
});
|
|
|
|
test("accumulates streamed candidates", async () => {
|
|
const result = await accumulateGeminiStream(stream([
|
|
{ response: { candidates: [{ content: { parts: [{ text: "hello" }, { functionCall: { name: "f", args: { x: 1 }, id: "g1" } }] }, finishReason: "STOP" }] } },
|
|
]));
|
|
expect(result.text).toBe("hello");
|
|
expect(result.toolCalls).toEqual([{ id: "g1", name: "f", arguments: { x: 1 } }]);
|
|
});
|
|
});
|
|
|
|
describe("bedrock converse encoding", () => {
|
|
test("encodes toolUse/toolResult and the tool config", () => {
|
|
expect(toBedrockMessages(CONVO)).toEqual([
|
|
{ role: "user", content: [{ text: "do it" }] },
|
|
{ role: "assistant", content: [{ text: "calling" }, { toolUse: { toolUseId: "c1", name: "hire_agent", input: { name: "Sam" } } }] },
|
|
{ role: "user", content: [{ toolResult: { toolUseId: "c1", content: [{ text: "ok" }] } }] },
|
|
]);
|
|
expect(bedrockToolConfig([{ name: "f", description: "d", parameters: { type: "object" } }])).toEqual({
|
|
tools: [{ toolSpec: { name: "f", description: "d", inputSchema: { json: { type: "object" } } } }],
|
|
toolChoice: { auto: {} },
|
|
});
|
|
});
|
|
|
|
test("parses the converse output", () => {
|
|
const data = { stopReason: "tool_use", output: { message: { content: [{ text: "ok" }, { toolUse: { toolUseId: "t1", name: "f", input: { x: 1 } } }] } } };
|
|
expect(parseBedrockOutput(data)).toMatchObject({ text: "ok", toolCalls: [{ id: "t1", name: "f", arguments: { x: 1 } }] });
|
|
});
|
|
});
|
|
|
|
describe("vertex (gemini) encoding", () => {
|
|
test("builds a generateContent body with tools and parses the response", () => {
|
|
const body = vertexRequestBody({ model: "gemini-2.5-pro", instructions: "sys", messages: CONVO, tools: [{ name: "f", description: "d", parameters: { type: "object" } }] });
|
|
expect((body.systemInstruction as JsonRecord).parts).toEqual([{ text: "sys" }]);
|
|
expect(body.tools).toEqual([{ functionDeclarations: [{ name: "f", description: "d", parametersJsonSchema: { type: "object" } }] }]);
|
|
const parsed = parseGeminiResponse({ candidates: [{ content: { parts: [{ text: "hi" }, { functionCall: { name: "f", args: { x: 1 }, id: "v1" } }] }, finishReason: "STOP" }] });
|
|
expect(parsed).toMatchObject({ text: "hi", toolCalls: [{ id: "v1", name: "f", arguments: { x: 1 } }] });
|
|
});
|
|
});
|
|
|
|
describe("aws sigv4", () => {
|
|
test("encodes the model path and emits a SigV4 authorization header", () => {
|
|
const signed = signAwsRequest({
|
|
method: "POST",
|
|
host: "bedrock-runtime.us-east-1.amazonaws.com",
|
|
rawPath: "/model/anthropic.claude-sonnet-4-5-v1:0/converse",
|
|
body: "{}",
|
|
signer: { region: "us-east-1", service: "bedrock", accessKeyId: "AKIA", secretAccessKey: "secret" },
|
|
});
|
|
expect(signed.url).toContain("/model/anthropic.claude-sonnet-4-5-v1%3A0/converse");
|
|
expect(signed.headers.Authorization).toMatch(/^AWS4-HMAC-SHA256 Credential=AKIA\/\d{8}\/us-east-1\/bedrock\/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=[0-9a-f]{64}$/);
|
|
});
|
|
});
|
|
|
|
describe("tool-turn router", () => {
|
|
test("supports every agent-capable provider (incl. cloud and custom) and rejects unknown ones", () => {
|
|
for (const key of ["openai", "anthropic", "xai", "groq", "mistral", "cerebras", "openrouter", "kimi", "kimi-cn", "gpt-account", "github-copilot", "google-antigravity", "claude-pro", "kimi-code", "custom-openai", "custom-anthropic", "azure-openai", "aws-bedrock", "vertex-ai"]) {
|
|
expect(supportsToolTurn(key)).toBe(true);
|
|
expect(typeof selectToolTurn(key)).toBe("function");
|
|
}
|
|
expect(supportsToolTurn("not-a-provider")).toBe(false);
|
|
expect(() => selectToolTurn("not-a-provider")).toThrow();
|
|
});
|
|
});
|
|
|
|
describe("credential resolver", () => {
|
|
test("returns the api key for api-key providers and the access token for fresh OAuth", () => {
|
|
const connection = new DatabaseConnection(join(tmpdir(), `ac-cred-${randomUUID()}.db`));
|
|
const organization = new OrganizationService(connection.db);
|
|
const providers = new ProviderService(connection.db, providerAdapters, testVault);
|
|
const authBroker = new AuthBrokerService(connection.db, testVault);
|
|
const integrationKeys = new IntegrationKeyStore(connection.db, testVault);
|
|
let orgId = "";
|
|
let openaiId = "";
|
|
let gptAccountId = "";
|
|
const future = Date.now() + 60 * 60 * 1000;
|
|
connection.transaction(() => {
|
|
orgId = organization.createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" });
|
|
openaiId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'openai'").get(orgId) as { id: string }).id;
|
|
gptAccountId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'gpt-account'").get(orgId) as { id: string }).id;
|
|
providers.saveCredential({ orgId, providerId: openaiId, secret: "sk-live", model: "gpt-5.4" });
|
|
providers.saveCredential({ orgId, providerId: gptAccountId, secret: JSON.stringify({ access: "tok", refresh: "r", expires: future, accountId: "acct_1" }), model: "gpt-5.4" });
|
|
});
|
|
|
|
return (async () => {
|
|
const apiKeyCred = await resolveTurnCredential({ authBroker, integrationKeys, orgId, provider: loadConfiguredProvider(connection.db, orgId, openaiId) });
|
|
expect(apiKeyCred).toEqual({ apiKey: "sk-live" });
|
|
const oauthCred = await resolveTurnCredential({ authBroker, integrationKeys, orgId, provider: loadConfiguredProvider(connection.db, orgId, gptAccountId) });
|
|
expect(oauthCred.accessToken).toBe("tok");
|
|
expect(oauthCred.accountId).toBe("acct_1");
|
|
connection.close();
|
|
})();
|
|
});
|
|
|
|
test("an expired Antigravity credential without a saved client secret points to Settings", async () => {
|
|
const connection = new DatabaseConnection(join(tmpdir(), `ac-cred-${randomUUID()}.db`));
|
|
const organization = new OrganizationService(connection.db);
|
|
const providers = new ProviderService(connection.db, providerAdapters, testVault);
|
|
const authBroker = new AuthBrokerService(connection.db, testVault);
|
|
const integrationKeys = new IntegrationKeyStore(connection.db, testVault);
|
|
let orgId = "";
|
|
let antigravityId = "";
|
|
connection.transaction(() => {
|
|
orgId = organization.createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" });
|
|
antigravityId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'google-antigravity'").get(orgId) as { id: string }).id;
|
|
providers.saveCredential({ orgId, providerId: antigravityId, secret: JSON.stringify({ access: "old", refresh: "r", expires: 0, projectId: "p" }), model: "gemini-3-pro" });
|
|
});
|
|
await expect(resolveTurnCredential({ authBroker, integrationKeys, orgId, provider: loadConfiguredProvider(connection.db, orgId, antigravityId) }))
|
|
.rejects.toThrow("Settings > Integrations");
|
|
connection.close();
|
|
});
|
|
});
|
|
|
|
describe("responses tool definitions", () => {
|
|
const tools = [{ name: "hire_agent", description: "Add a report.", parameters: { type: "object", properties: {} } }];
|
|
|
|
test("omit strict when null (so the Codex endpoint keeps the tools), keep it when boolean", () => {
|
|
const codex = responsesToolDefs(tools, null)[0] as Record<string, unknown>;
|
|
expect("strict" in codex).toBe(false); // strict:null is malformed and makes the endpoint drop the tools
|
|
expect(responsesToolDefs(tools)[0]).toMatchObject({ type: "function", name: "hire_agent", strict: false });
|
|
expect(responsesToolDefs(tools, true)[0]).toMatchObject({ strict: true });
|
|
});
|
|
});
|
|
|
|
describe("mid-run follow-up text rides on the tool-result turn", () => {
|
|
const convo: AgentTurnMessage[] = [
|
|
{ role: "user", text: "go" },
|
|
{ role: "assistant", text: "", toolCalls: [{ id: "c1", name: "f", arguments: {} }] },
|
|
{ role: "tool", results: [{ toolCallId: "c1", toolName: "f", output: "ok" }], followUp: "New: stop" },
|
|
];
|
|
|
|
test("each wire format keeps the result and appends the text in order", () => {
|
|
expect(toResponsesInput(convo).slice(-2)).toEqual([
|
|
{ type: "function_call_output", call_id: "c1", output: "ok" },
|
|
{ role: "user", content: [{ type: "input_text", text: "New: stop" }] },
|
|
]);
|
|
expect(toChatMessages(null, convo).slice(-2)).toEqual([
|
|
{ role: "tool", tool_call_id: "c1", content: "ok" },
|
|
{ role: "user", content: "New: stop" },
|
|
]);
|
|
expect(toAnthropicMessages(convo).at(-1)).toEqual({
|
|
role: "user",
|
|
content: [{ type: "tool_result", tool_use_id: "c1", content: "ok" }, { type: "text", text: "New: stop" }],
|
|
});
|
|
expect(toBedrockMessages(convo).at(-1)).toEqual({
|
|
role: "user",
|
|
content: [{ toolResult: { toolUseId: "c1", content: [{ text: "ok" }] } }, { text: "New: stop" }],
|
|
});
|
|
const gemini = vertexRequestBody({ model: "gemini-3-pro", instructions: null, messages: convo, tools: [] }) as { contents: Array<{ role: string; parts: unknown[] }> };
|
|
expect(gemini.contents.at(-1)).toEqual({
|
|
role: "user",
|
|
parts: [{ functionResponse: { name: "f", response: { output: "ok" } } }, { text: "New: stop" }],
|
|
});
|
|
});
|
|
});
|