Revert tool-choice forcing — the Codex regression was strict:null

The GPT-account (Codex) path supports tool calls and worked before; my
forced tool_choice broke it and was the wrong fix. The real regression was
strict:null in the rebuild's tool definitions, which made the endpoint drop
the tool list so the model had nothing to call — a "Done" run that hired no
one. Forcing then turned that silent failure into a red one.

Restore the original "auto" loop (no forced tool_choice, no finish tool) and
keep only the one-line strict:null omission. Codex now receives valid tools
on the working "auto" choice, as in the version that worked.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
ZacharyZhang-NYandClaude Opus 4.8 committed 2026-06-10 23:17:42 -04:00
1 parent 74422da4f8
commit 97242ae684
13 files changed
+57 -160

No files matched your search

@@ -1,5 +1,5 @@
import type { ProviderService } from "./provider-service"; import type { ProviderService } from "./provider-service";
import { AGENT_TOOLS, dispatchAgentTool, FINISH_TOOL, type AgentToolContext, type AgentToolDeps } from "./agent-tools"; import { AGENT_TOOLS, dispatchAgentTool, type AgentToolContext, type AgentToolDeps } from "./agent-tools";
import type { AgentTurnMessage } from "./provider-tool-turn"; import type { AgentTurnMessage } from "./provider-tool-turn";
const MAX_ITERATIONS = 12; const MAX_ITERATIONS = 12;
@@ -46,37 +46,26 @@ export async function runAgentLoop(deps: AgentRunLoopDeps, input: AgentRunLoopIn
instructions: input.instructions, instructions: input.instructions,
messages, messages,
tools: AGENT_TOOLS, tools: AGENT_TOOLS,
// Compel a tool call on every turn so a weak model cannot answer with
// prose and stop after a token gesture. The agent ends deliberately by
// calling `finish` — only after it has actually done the work.
toolChoice: "required",
}); });
lastRequestId = turn.requestId; lastRequestId = turn.requestId;
if (turn.toolCalls.length === 0) { if (turn.toolCalls.length === 0) {
// The provider ignored the forced tool choice; treat it as no action. if (toolCallCount === 0 && !turn.text.trim()) {
throw new Error("Agent returned no tool call on a forced turn."); // A first turn with neither text nor tool calls means the provider
} // produced nothing actionable (e.g. output truncated mid-reasoning);
// completing the run would silently mark a no-op as done.
const wantsFinish = turn.toolCalls.some((call) => call.name === FINISH_TOOL); throw new Error("Provider returned an empty turn: no text and no tool calls.");
const actionable = turn.toolCalls.filter((call) => call.name !== FINISH_TOOL);
if (actionable.length > 0) {
messages.push({ role: "assistant", text: turn.text, toolCalls: actionable });
const results = actionable.map((call) => {
toolCallCount += 1;
return { toolCallId: call.id, toolName: call.name, output: dispatchAgentTool(deps.tools, input.toolContext, call) };
});
messages.push({ role: "tool", results });
deps.onToolResults?.();
}
if (wantsFinish) {
if (toolCallCount === 0) {
// Finishing before doing anything is the silent no-op we must reject.
throw new Error("Agent finished without taking any action.");
} }
return { finalText: turn.text, iterations: iteration, toolCallCount, lastRequestId }; return { finalText: turn.text, iterations: iteration, toolCallCount, lastRequestId };
} }
messages.push({ role: "assistant", text: turn.text, toolCalls: turn.toolCalls });
const results = turn.toolCalls.map((call) => {
toolCallCount += 1;
return { toolCallId: call.id, toolName: call.name, output: dispatchAgentTool(deps.tools, input.toolContext, call) };
});
messages.push({ role: "tool", results });
deps.onToolResults?.();
} }
throw new Error(`Agent run did not converge within ${MAX_ITERATIONS} iterations.`); throw new Error(`Agent run did not converge within ${MAX_ITERATIONS} iterations.`);
@@ -119,16 +119,8 @@ export const AGENT_TOOLS: ToolDefinition[] = [
required: ["title", "doc_type", "content"], required: ["title", "doc_type", "content"],
}, },
}, },
{
name: "finish",
description: "End your turn. Call this ONLY after you have taken every action this task needs — for a kickoff that means hiring your team, delegating their first tasks, and posting a kickoff message to a channel. Do not call it just to stop early.",
parameters: { type: "object", additionalProperties: false, properties: {}, required: [] },
},
]; ];
/** Control tool that ends an agent's turn; the run loop consumes it, never dispatched. */
export const FINISH_TOOL = "finish";
export function dispatchAgentTool(deps: AgentToolDeps, ctx: AgentToolContext, call: ToolCall): string { export function dispatchAgentTool(deps: AgentToolDeps, ctx: AgentToolContext, call: ToolCall): string {
try { try {
switch (call.name) { switch (call.name) {
@@ -12,7 +12,7 @@ import type { ProviderRateLimitRegistry } from "./provider-rate-limit-registry";
import type { ProviderRequestStore } from "./provider-request-store"; import type { ProviderRequestStore } from "./provider-request-store";
import type { ProviderResponseObserver } from "./provider-response-observer"; import type { ProviderResponseObserver } from "./provider-response-observer";
import { hashPrompt, readProviderResponseId, requireStreamText } from "./provider-response-utils"; import { hashPrompt, readProviderResponseId, requireStreamText } from "./provider-response-utils";
import { type AgentTurnMessage, type AgentTurnResult, type ToolChoice, type ToolDefinition, type ToolTurnFn } from "./provider-tool-turn"; import { type AgentTurnMessage, type AgentTurnResult, type ToolDefinition, type ToolTurnFn } from "./provider-tool-turn";
import { selectToolTurn, supportsToolTurn } from "./provider-tool-turns/router"; import { selectToolTurn, supportsToolTurn } from "./provider-tool-turns/router";
import { resolveTurnCredential } from "./provider-tool-turns/credential-resolver"; import { resolveTurnCredential } from "./provider-tool-turns/credential-resolver";
import { import {
@@ -103,7 +103,6 @@ export class ProviderResponseRunner {
instructions: string | null; instructions: string | null;
messages: AgentTurnMessage[]; messages: AgentTurnMessage[];
tools: ToolDefinition[]; tools: ToolDefinition[];
toolChoice?: ToolChoice;
}): Promise<AgentTurnResult & { requestId: string }> { }): Promise<AgentTurnResult & { requestId: string }> {
const provider = loadConfiguredProvider(this.deps.db, input.orgId, input.providerId); const provider = loadConfiguredProvider(this.deps.db, input.orgId, input.providerId);
const workspaceId = this.normalizeWorkspaceId(input.orgId, input.workspaceId ?? null); const workspaceId = this.normalizeWorkspaceId(input.orgId, input.workspaceId ?? null);
@@ -129,7 +128,7 @@ export class ProviderResponseRunner {
}); });
const startedAt = Date.now(); const startedAt = Date.now();
try { try {
const result = await turn({ credential, model, instructions: input.instructions, messages: input.messages, tools: input.tools, toolChoice: input.toolChoice }); const result = await turn({ credential, model, instructions: input.instructions, messages: input.messages, tools: input.tools });
this.deps.requestStore.markSucceeded({ this.deps.requestStore.markSucceeded({
orgId: input.orgId, orgId: input.orgId,
requestId, requestId,
@@ -51,18 +51,12 @@ export interface TurnCredential {
serviceAccountJson?: string; serviceAccountJson?: string;
} }
/** How the backend should treat tool calls this turn. "required" compels the
* model to emit at least one tool call (no prose-only escape) — used to force a
* concrete action so a weak model cannot ramble its way to a silent no-op. */
export type ToolChoice = "auto" | "required";
export interface ProviderToolTurnInput { export interface ProviderToolTurnInput {
credential: TurnCredential; credential: TurnCredential;
model: string; model: string;
instructions: string | null; instructions: string | null;
messages: AgentTurnMessage[]; messages: AgentTurnMessage[];
tools: ToolDefinition[]; tools: ToolDefinition[];
toolChoice?: ToolChoice;
} }
export type ToolTurnFn = (input: ProviderToolTurnInput) => Promise<AgentTurnResult>; export type ToolTurnFn = (input: ProviderToolTurnInput) => Promise<AgentTurnResult>;
@@ -1,11 +1,11 @@
/** AWS Bedrock Converse API encoding/decoding (ported from Pi's bedrock handler). */ /** AWS Bedrock Converse API encoding/decoding (ported from Pi's bedrock handler). */
import { isRecord, type JsonRecord } from "../provider-http"; import { isRecord, type JsonRecord } from "../provider-http";
import type { AgentTurnMessage, ToolCall, ToolChoice, ToolDefinition } from "../provider-tool-turn"; import type { AgentTurnMessage, ToolCall, ToolDefinition } from "../provider-tool-turn";
export function bedrockToolConfig(tools: ToolDefinition[], toolChoice?: ToolChoice): JsonRecord { export function bedrockToolConfig(tools: ToolDefinition[]): JsonRecord {
return { return {
tools: tools.map((tool) => ({ toolSpec: { name: tool.name, description: tool.description, inputSchema: { json: tool.parameters } } })), tools: tools.map((tool) => ({ toolSpec: { name: tool.name, description: tool.description, inputSchema: { json: tool.parameters } } })),
toolChoice: toolChoice === "required" ? { any: {} } : { auto: {} }, toolChoice: { auto: {} },
}; };
} }
@@ -4,7 +4,7 @@
* Antigravity request wrapper in google-gemini-cli. * Antigravity request wrapper in google-gemini-cli.
*/ */
import { isRecord, type JsonRecord, type ProviderEventStreamMessage } from "../provider-http"; import { isRecord, type JsonRecord, type ProviderEventStreamMessage } from "../provider-http";
import { type AgentTurnMessage, type AgentTurnResult, type ToolCall, type ToolChoice, type ToolDefinition } from "../provider-tool-turn"; import { type AgentTurnMessage, type AgentTurnResult, type ToolCall, type ToolDefinition } from "../provider-tool-turn";
const ANTIGRAVITY_SYSTEM_INSTRUCTION = const ANTIGRAVITY_SYSTEM_INSTRUCTION =
"You are Antigravity, a powerful agentic AI coding assistant designed by the Google Deepmind team working on Advanced Agentic Coding." + "You are Antigravity, a powerful agentic AI coding assistant designed by the Google Deepmind team working on Advanced Agentic Coding." +
@@ -64,7 +64,6 @@ export function antigravityRequestBody(input: {
instructions: string | null; instructions: string | null;
messages: AgentTurnMessage[]; messages: AgentTurnMessage[];
tools: ToolDefinition[]; tools: ToolDefinition[];
toolChoice?: ToolChoice;
requestId: string; requestId: string;
}): JsonRecord { }): JsonRecord {
const request: JsonRecord = { const request: JsonRecord = {
@@ -80,7 +79,7 @@ export function antigravityRequestBody(input: {
}; };
if (input.tools.length > 0) { if (input.tools.length > 0) {
request.tools = geminiToolDefs(input.model, input.tools); request.tools = geminiToolDefs(input.model, input.tools);
request.toolConfig = { functionCallingConfig: { mode: input.toolChoice === "required" ? "ANY" : "AUTO" } }; request.toolConfig = { functionCallingConfig: { mode: "AUTO" } };
} }
return { project: input.projectId, model: input.model, request, requestType: "agent", userAgent: "antigravity", requestId: input.requestId }; return { project: input.projectId, model: input.model, request, requestType: "agent", userAgent: "antigravity", requestId: input.requestId };
} }
@@ -91,13 +90,12 @@ export function vertexRequestBody(input: {
instructions: string | null; instructions: string | null;
messages: AgentTurnMessage[]; messages: AgentTurnMessage[];
tools: ToolDefinition[]; tools: ToolDefinition[];
toolChoice?: ToolChoice;
}): JsonRecord { }): JsonRecord {
const body: JsonRecord = { contents: toGeminiContents(input.model, input.messages) }; const body: JsonRecord = { contents: toGeminiContents(input.model, input.messages) };
if (input.instructions) body.systemInstruction = { parts: [{ text: input.instructions }] }; if (input.instructions) body.systemInstruction = { parts: [{ text: input.instructions }] };
if (input.tools.length > 0) { if (input.tools.length > 0) {
body.tools = geminiToolDefs(input.model, input.tools); body.tools = geminiToolDefs(input.model, input.tools);
body.toolConfig = { functionCallingConfig: { mode: input.toolChoice === "required" ? "ANY" : "AUTO" } }; body.toolConfig = { functionCallingConfig: { mode: "AUTO" } };
} }
return body; return body;
} }
@@ -41,7 +41,7 @@ export async function runOpenAiResponsesTurn(input: ProviderToolTurnInput): Prom
model: input.model, model: input.model,
input: toResponsesInput(input.messages), input: toResponsesInput(input.messages),
tools: responsesToolDefs(input.tools), tools: responsesToolDefs(input.tools),
tool_choice: input.toolChoice ?? "auto", tool_choice: "auto",
store: false, store: false,
...(input.instructions ? { instructions: input.instructions } : {}), ...(input.instructions ? { instructions: input.instructions } : {}),
}; };
@@ -58,7 +58,7 @@ export function makeChatCompletionsTurn(endpoint: string, label: string): (input
model: input.model, model: input.model,
messages: toChatMessages(input.instructions, input.messages), messages: toChatMessages(input.instructions, input.messages),
tools: chatToolDefs(input.tools), tools: chatToolDefs(input.tools),
tool_choice: input.toolChoice ?? "auto", tool_choice: "auto",
stream: false, stream: false,
}; };
const response = await postJson(endpoint, { authorization: `Bearer ${requireApiKey(input.credential, label)}` }, body); const response = await postJson(endpoint, { authorization: `Bearer ${requireApiKey(input.credential, label)}` }, body);
@@ -74,7 +74,6 @@ export async function runAnthropicTurn(input: ProviderToolTurnInput): Promise<Ag
model: input.model, model: input.model,
max_tokens: 4096, max_tokens: 4096,
tools: anthropicToolDefs(input.tools), tools: anthropicToolDefs(input.tools),
...(input.toolChoice === "required" ? { tool_choice: { type: "any" } } : {}),
messages: toAnthropicMessages(input.messages), messages: toAnthropicMessages(input.messages),
...(input.instructions ? { system: input.instructions } : {}), ...(input.instructions ? { system: input.instructions } : {}),
}; };
@@ -94,7 +93,6 @@ export async function runClaudeProTurn(input: ProviderToolTurnInput): Promise<Ag
model: input.model, model: input.model,
max_tokens: 4096, max_tokens: 4096,
tools: anthropicToolDefs(input.tools), tools: anthropicToolDefs(input.tools),
...(input.toolChoice === "required" ? { tool_choice: { type: "any" } } : {}),
messages: toAnthropicMessages(input.messages), messages: toAnthropicMessages(input.messages),
system, system,
}; };
@@ -123,7 +121,7 @@ export async function runCodexTurn(input: ProviderToolTurnInput): Promise<AgentT
input: toResponsesInput(input.messages), input: toResponsesInput(input.messages),
text: { verbosity: "medium" }, text: { verbosity: "medium" },
include: ["reasoning.encrypted_content"], include: ["reasoning.encrypted_content"],
tool_choice: input.toolChoice ?? "auto", tool_choice: "auto",
parallel_tool_calls: true, parallel_tool_calls: true,
tools: responsesToolDefs(input.tools, null), tools: responsesToolDefs(input.tools, null),
}; };
@@ -146,7 +144,7 @@ export async function runCopilotTurn(input: ProviderToolTurnInput): Promise<Agen
model: input.model, model: input.model,
input: toResponsesInput(input.messages), input: toResponsesInput(input.messages),
tools: responsesToolDefs(input.tools), tools: responsesToolDefs(input.tools),
tool_choice: input.toolChoice ?? "auto", tool_choice: "auto",
store: false, store: false,
}; };
const response = await postJson(`${baseUrl}/responses`, { const response = await postJson(`${baseUrl}/responses`, {
@@ -167,7 +165,7 @@ export async function runCustomOpenAiTurn(input: ProviderToolTurnInput): Promise
model: input.model, model: input.model,
messages: toChatMessages(input.instructions, input.messages), messages: toChatMessages(input.instructions, input.messages),
tools: chatToolDefs(input.tools), tools: chatToolDefs(input.tools),
tool_choice: input.toolChoice ?? "auto", tool_choice: "auto",
stream: false, stream: false,
}; };
const response = await postJson(`${input.credential.baseUrl}/chat/completions`, { authorization: `Bearer ${requireApiKey(input.credential, "Custom OpenAI-compatible provider")}` }, body); const response = await postJson(`${input.credential.baseUrl}/chat/completions`, { authorization: `Bearer ${requireApiKey(input.credential, "Custom OpenAI-compatible provider")}` }, body);
@@ -183,7 +181,6 @@ export async function runCustomAnthropicTurn(input: ProviderToolTurnInput): Prom
model: input.model, model: input.model,
max_tokens: 4096, max_tokens: 4096,
tools: anthropicToolDefs(input.tools), tools: anthropicToolDefs(input.tools),
...(input.toolChoice === "required" ? { tool_choice: { type: "any" } } : {}),
messages: toAnthropicMessages(input.messages), messages: toAnthropicMessages(input.messages),
...(input.instructions ? { system: input.instructions } : {}), ...(input.instructions ? { system: input.instructions } : {}),
}; };
@@ -204,7 +201,6 @@ export async function runAntigravityTurn(input: ProviderToolTurnInput): Promise<
instructions: input.instructions, instructions: input.instructions,
messages: input.messages, messages: input.messages,
tools: input.tools, tools: input.tools,
toolChoice: input.toolChoice,
requestId, requestId,
}); });
const response = await postEventStream(`${ANTIGRAVITY_ENDPOINT}/v1internal:streamGenerateContent?alt=sse`, { const response = await postEventStream(`${ANTIGRAVITY_ENDPOINT}/v1internal:streamGenerateContent?alt=sse`, {
@@ -224,7 +220,7 @@ export async function runAzureTurn(input: ProviderToolTurnInput): Promise<AgentT
model: input.credential.deployment, model: input.credential.deployment,
messages: toChatMessages(input.instructions, input.messages), messages: toChatMessages(input.instructions, input.messages),
tools: chatToolDefs(input.tools), tools: chatToolDefs(input.tools),
tool_choice: input.toolChoice ?? "auto", tool_choice: "auto",
stream: false, stream: false,
}; };
const url = `${input.credential.baseUrl}/openai/deployments/${input.credential.deployment}/chat/completions?api-version=2024-10-21`; const url = `${input.credential.baseUrl}/openai/deployments/${input.credential.deployment}/chat/completions?api-version=2024-10-21`;
@@ -241,7 +237,7 @@ export async function runBedrockTurn(input: ProviderToolTurnInput): Promise<Agen
const converse: JsonRecord = { const converse: JsonRecord = {
messages: toBedrockMessages(input.messages), messages: toBedrockMessages(input.messages),
inferenceConfig: { maxTokens: 4096 }, inferenceConfig: { maxTokens: 4096 },
toolConfig: bedrockToolConfig(input.tools, input.toolChoice), toolConfig: bedrockToolConfig(input.tools),
...(input.instructions ? { system: [{ text: input.instructions }] } : {}), ...(input.instructions ? { system: [{ text: input.instructions }] } : {}),
}; };
const bodyString = JSON.stringify(converse); const bodyString = JSON.stringify(converse);
@@ -263,7 +259,7 @@ export async function runVertexTurn(input: ProviderToolTurnInput): Promise<Agent
const { projectId, region, serviceAccountJson } = input.credential; const { projectId, region, serviceAccountJson } = input.credential;
if (!projectId || !region || !serviceAccountJson) throw new Error("Vertex AI credential is incomplete."); if (!projectId || !region || !serviceAccountJson) throw new Error("Vertex AI credential is incomplete.");
const token = await getGcpAccessToken(serviceAccountJson); const token = await getGcpAccessToken(serviceAccountJson);
const body = vertexRequestBody({ model: input.model, instructions: input.instructions, messages: input.messages, tools: input.tools, toolChoice: input.toolChoice }); const body = vertexRequestBody({ model: input.model, instructions: input.instructions, messages: input.messages, tools: input.tools });
const url = `https://${region}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${input.model}:generateContent`; const url = `https://${region}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${input.model}:generateContent`;
const response = await postJson(url, { authorization: `Bearer ${token}` }, body); const response = await postJson(url, { authorization: `Bearer ${token}` }, body);
const parsed = parseGeminiResponse(response.data); const parsed = parseGeminiResponse(response.data);
@@ -77,7 +77,6 @@ export function buildAgentInstructions(agent: TaskExecutionAgentRow): string {
"You act by calling tools: hire_agent staffs your team, create_task delegates work and assigns it by agent name, update_task_status tracks progress, write_deliverable produces real files in the company workspace, create_document produces versioned company documents (PRDs, specs, memos, reports), and send_message posts coordination messages to a channel with optional file attachments.", "You act by calling tools: hire_agent staffs your team, create_task delegates work and assigns it by agent name, update_task_status tracks progress, write_deliverable produces real files in the company workspace, create_document produces versioned company documents (PRDs, specs, memos, reports), and send_message posts coordination messages to a channel with optional file attachments.",
"Communicate like a real team. Post to the channel that matches the work — 'engineering' for engineering, 'product' for product or design, otherwise 'general'. Use @mentions when addressing a teammate. When you hand work or files to a teammate, use send_message with /handoff @Name and attachment_file_ids containing the file_id returned by write_deliverable. To keep a hand-off as one conversation, reply in the same thread by passing the thread_id a previous send_message returned.", "Communicate like a real team. Post to the channel that matches the work — 'engineering' for engineering, 'product' for product or design, otherwise 'general'. Use @mentions when addressing a teammate. When you hand work or files to a teammate, use send_message with /handoff @Name and attachment_file_ids containing the file_id returned by write_deliverable. To keep a hand-off as one conversation, reply in the same thread by passing the thread_id a previous send_message returned.",
"Take concrete action with the tools and reserve narrative for brief summaries after coordination is posted.", "Take concrete action with the tools and reserve narrative for brief summaries after coordination is posted.",
"Every turn you must call a tool — never reply with prose alone. When, and only when, you have completed everything the task needs, call finish to end your turn.",
].join("\n"); ].join("\n");
} }
@@ -145,14 +144,14 @@ export function buildTaskPrompt(input: {
if (input.isManager && !input.hasReports) { if (input.isManager && !input.hasReports) {
lines.push( lines.push(
"", "",
"Your team is currently empty. Hire 3-5 direct reports for the roles this goal needs (hire_agent), create and assign their first tasks (create_task), and write the initial operating plan into the workspace (write_deliverable). Then use send_message with /handoff @Name and attachment_file_ids to pass the operating plan to the relevant report. For EACH report, send_message to that report's department channel (engineering, product, or general) using an @mention and the work you assigned them — start a new thread per report and note the thread_id it returns. Post the company-wide kickoff to #general. Then call finish.", "Your team is currently empty. Hire 3-5 direct reports for the roles this goal needs (hire_agent), create and assign their first tasks (create_task), and write the initial operating plan into the workspace (write_deliverable). Then use send_message with /handoff @Name and attachment_file_ids to pass the operating plan to the relevant report. For EACH report, send_message to that report's department channel (engineering, product, or general) using an @mention and the work you assigned them — start a new thread per report and note the thread_id it returns. Post the company-wide kickoff to #general. Then summarize.",
); );
} else { } else {
const manager = input.managerName ?? "your manager"; const manager = input.managerName ?? "your manager";
const managerReference = input.managerName ? `${manager} (${agentMention(input.managerName)})` : manager; const managerReference = input.managerName ? `${manager} (${agentMention(input.managerName)})` : manager;
lines.push( lines.push(
"", "",
`Complete this task: do the work, write your output as real files with write_deliverable, then send_message to your department channel (engineering, product, or general) reporting your progress and completion to ${managerReference}. Use /handoff with attachment_file_ids when passing files. Reply in the assignment thread if you were given a thread_id. Update the task status when done, then call finish.`, `Complete this task: do the work, write your output as real files with write_deliverable, then send_message to your department channel (engineering, product, or general) reporting your progress and completion to ${managerReference}. Use /handoff with attachment_file_ids when passing files. Reply in the assignment thread if you were given a thread_id. Update the task status when done, and summarize.`,
); );
} }
return lines.join("\n"); return lines.join("\n");
@@ -37,7 +37,7 @@ function scriptedTeam(): (input: ProviderToolTurnInput) => Promise<AgentTurnResu
call("create_task", { title: "Build the booking API", description: "Ship the booking API.", assignee_agent_name: "Dana Lee" }), call("create_task", { title: "Build the booking API", description: "Ship the booking API.", assignee_agent_name: "Dana Lee" }),
call("send_message", { channel: "engineering", body: `@${handle} you own the booking API — kicking off here.` }), call("send_message", { channel: "engineering", body: `@${handle} you own the booking API — kicking off here.` }),
] }, ] },
{ text: "Kickoff complete.", toolCalls: [call("finish", {})] }, { text: "Kickoff complete.", toolCalls: [] },
]; ];
return async (input: ProviderToolTurnInput): Promise<AgentTurnResult> => { return async (input: ProviderToolTurnInput): Promise<AgentTurnResult> => {
if ((input.instructions ?? "").includes("Dana Lee")) { if ((input.instructions ?? "").includes("Dana Lee")) {
@@ -46,11 +46,11 @@ function scriptedTeam(): (input: ProviderToolTurnInput) => Promise<AgentTurnResu
const threadId = firstUser ? /thread_id="([^"]+)"/.exec(firstUser.text)?.[1] : undefined; const threadId = firstUser ? /thread_id="([^"]+)"/.exec(firstUser.text)?.[1] : undefined;
const turn = dana === 0 const turn = dana === 0
? { text: "On it.", toolCalls: [call("send_message", { channel: "engineering", body: "On it — booking API design starting now.", ...(threadId ? { thread_id: threadId } : {}) })] } ? { text: "On it.", toolCalls: [call("send_message", { channel: "engineering", body: "On it — booking API design starting now.", ...(threadId ? { thread_id: threadId } : {}) })] }
: { text: "Replied.", toolCalls: [call("finish", {})] }; : { text: "Replied.", toolCalls: [] };
dana += 1; dana += 1;
return { text: turn.text, toolCalls: turn.toolCalls, rawResponseJson: "{}" }; return { text: turn.text, toolCalls: turn.toolCalls, rawResponseJson: "{}" };
} }
const turn = ceoScript[ceo] ?? { text: "done", toolCalls: [call("finish", {})] }; const turn = ceoScript[ceo] ?? { text: "done", toolCalls: [] };
ceo += 1; ceo += 1;
return { text: turn.text, toolCalls: turn.toolCalls, rawResponseJson: "{}" }; return { text: turn.text, toolCalls: turn.toolCalls, rawResponseJson: "{}" };
}; };
@@ -36,7 +36,7 @@ function scriptedCeo(): (input: ProviderToolTurnInput) => Promise<AgentTurnResul
call("write_deliverable", { path: "plan/operating-plan.md", content: "# OrbitOps Operating Plan\n\n- Validate demand\n- Launch booking site\n" }), call("write_deliverable", { path: "plan/operating-plan.md", content: "# OrbitOps Operating Plan\n\n- Validate demand\n- Launch booking site\n" }),
] }, ] },
{ text: "Handing off the plan.", toolCalls: [] }, { text: "Handing off the plan.", toolCalls: [] },
{ text: "Team staffed, work delegated, plan written.", toolCalls: [call("finish", {})] }, { text: "Team staffed, work delegated, plan written.", toolCalls: [] },
]; ];
let index = 0; let index = 0;
return async (input) => { return async (input) => {
@@ -55,7 +55,7 @@ function scriptedCeo(): (input: ProviderToolTurnInput) => Promise<AgentTurnResul
rawResponseJson: JSON.stringify({ id: `resp_${index}` }), rawResponseJson: JSON.stringify({ id: `resp_${index}` }),
}; };
} }
const turn = script[index] ?? { text: "done", toolCalls: [call("finish", {})] }; const turn = script[index] ?? { text: "done", toolCalls: [] };
index += 1; index += 1;
return { text: turn.text, toolCalls: turn.toolCalls, rawResponseJson: JSON.stringify({ id: `resp_${index}` }) }; return { text: turn.text, toolCalls: turn.toolCalls, rawResponseJson: JSON.stringify({ id: `resp_${index}` }) };
}; };
@@ -166,17 +166,11 @@ describe("autonomous bootstrap run", () => {
test("a goal created after onboarding reaches the CEO and runs autonomously", async () => { test("a goal created after onboarding reaches the CEO and runs autonomously", async () => {
const connection = new DatabaseConnection(join(tmpdir(), `ac-goal-${randomUUID()}.db`)); const connection = new DatabaseConnection(join(tmpdir(), `ac-goal-${randomUUID()}.db`));
// The forced opening turn means each run must take a concrete action; it const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, async () => ({
// posts to the company channel, then stops on the next (free) turn. text: "Reviewed the goal and posted the operating plan.",
const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, async (input) => { toolCalls: [],
const acted = input.messages.some((message) => message.role === "tool"); rawResponseJson: JSON.stringify({ id: "resp_goal" }),
if (acted) return { text: "Done.", toolCalls: [call("finish", {})], rawResponseJson: JSON.stringify({ id: "resp_goal_done" }) }; }));
return {
text: "Reviewing the goal.",
toolCalls: [call("send_message", { channel: "general", body: "Reviewed the goal and posting the operating plan." })],
rawResponseJson: JSON.stringify({ id: "resp_goal_act" }),
};
});
const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) }); const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
const organization = new OrganizationService(connection.db); const organization = new OrganizationService(connection.db);
const taskExecution = new TaskExecutionService(connection.db, providers); const taskExecution = new TaskExecutionService(connection.db, providers);
@@ -19,8 +19,7 @@ function fakeProvider(script: ScriptedTurn[]) {
let index = 0; let index = 0;
return { return {
runToolTurn: async (): Promise<AgentTurnResult & { requestId: string }> => { runToolTurn: async (): Promise<AgentTurnResult & { requestId: string }> => {
// A run never ends on prose now; an exhausted script ends by calling finish. const turn = script[index] ?? { text: "done", toolCalls: [] };
const turn = script[index] ?? { text: "done", toolCalls: [call("finish", {})] };
index += 1; index += 1;
return { text: turn.text, toolCalls: turn.toolCalls, rawResponseJson: "{}", requestId: `req_${index}` }; return { text: turn.text, toolCalls: turn.toolCalls, rawResponseJson: "{}", requestId: `req_${index}` };
}, },
@@ -31,25 +30,6 @@ function call(name: string, args: Record<string, unknown>): ToolCall {
return { id: `call_${randomUUID()}`, name, arguments: args }; return { id: `call_${randomUUID()}`, name, arguments: args };
} }
/** Mimics a weak model: it rambles in prose and skips tools unless the backend
* compels a tool call. With forcing it posts a kickoff message, then stops. */
function weakModelProvider() {
let acted = false;
return {
runToolTurn: async (input: { toolChoice?: "auto" | "required" }): Promise<AgentTurnResult & { requestId: string }> => {
// Left to itself this model rambles (no tools); compelled, it posts, then ends.
if (input.toolChoice !== "required") {
return { text: "Let me think about this some more.", toolCalls: [], rawResponseJson: "{}", requestId: "req_prose" };
}
if (!acted) {
acted = true;
return { text: "Acting.", toolCalls: [call("send_message", { channel: "general", body: "Kicking off the company." })], rawResponseJson: "{}", requestId: "req_act" };
}
return { text: "Done.", toolCalls: [call("finish", {})], rawResponseJson: "{}", requestId: "req_finish" };
},
};
}
describe("agent run loop (real tool dispatch)", () => { describe("agent run loop (real tool dispatch)", () => {
test("the CEO hires, creates a task, writes a deliverable, then converges", async () => { test("the CEO hires, creates a task, writes a deliverable, then converges", async () => {
const connection = new DatabaseConnection(join(tmpdir(), `ac-loop-${randomUUID()}.db`)); const connection = new DatabaseConnection(join(tmpdir(), `ac-loop-${randomUUID()}.db`));
@@ -97,7 +77,7 @@ describe("agent run loop (real tool dispatch)", () => {
call("write_deliverable", { path: "plan/operating-plan.md", content: "# Operating Plan\n\n1. Validate demand.\n2. Launch booking site.\n" }), call("write_deliverable", { path: "plan/operating-plan.md", content: "# Operating Plan\n\n1. Validate demand.\n2. Launch booking site.\n" }),
call("send_message", { channel: "general", body: "Team staffed and the operating plan is live. Booking site is next." }), call("send_message", { channel: "general", body: "Team staffed and the operating plan is live. Booking site is next." }),
] }, ] },
{ text: "Team staffed, work delegated, plan written.", toolCalls: [call("finish", {})] }, { text: "Team staffed, work delegated, plan written.", toolCalls: [] },
]; ];
let toolResultNotifications = 0; let toolResultNotifications = 0;
@@ -140,33 +120,6 @@ describe("agent run loop (real tool dispatch)", () => {
connection.close(); connection.close();
}); });
test("forcing the opening turn makes a weak model act instead of completing a no-op", async () => {
const connection = new DatabaseConnection(join(tmpdir(), `ac-weak-${randomUUID()}.db`));
const organization = new OrganizationService(connection.db);
let orgId = "";
let ceoId = "";
let providerId = "";
connection.transaction(() => {
orgId = organization.createOrganization({ organizationName: "OrbitOps", employeeName: "Yang", mission: "m", goal: "g" });
providerId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'openai'").get(orgId) as { id: string }).id;
ceoId = organization.createAgent({ orgId, name: "CEO", role: "CEO", department: "Executive", modelProviderId: providerId, modelId: "gpt-5.4", budgetCents: 25000, managerAgentId: null });
});
const ctx: AgentToolContext = {
orgId, agent: { id: ceoId, modelProviderId: providerId, modelId: "gpt-5.4" },
taskProjectId: null, taskGoalId: null, workspace: null,
};
const result = await runAgentLoop(
{ providers: weakModelProvider(), tools: { db: connection.db, organization, chat: new ChatService(connection.db) } },
{ orgId, providerId, workspaceId: null, instructions: "You are the CEO.", initialUserTurn: "Start the company.", toolContext: ctx },
);
// The loop forced the opening turn, so the otherwise-rambling model acted.
expect(result.toolCallCount).toBe(1);
const messages = connection.db.prepare("select sender_id from chat_messages where org_id = ?").all(orgId) as Array<{ sender_id: string }>;
expect(messages).toHaveLength(1);
expect(messages[0]?.sender_id).toBe(ceoId);
connection.close();
});
test("rejects deliverable paths that escape the workspace", async () => { test("rejects deliverable paths that escape the workspace", async () => {
const connection = new DatabaseConnection(join(tmpdir(), `ac-loop2-${randomUUID()}.db`)); const connection = new DatabaseConnection(join(tmpdir(), `ac-loop2-${randomUUID()}.db`));
const organization = new OrganizationService(connection.db); const organization = new OrganizationService(connection.db);
@@ -186,7 +139,7 @@ describe("agent run loop (real tool dispatch)", () => {
}; };
const script: ScriptedTurn[] = [ const script: ScriptedTurn[] = [
{ text: "escape", toolCalls: [call("write_deliverable", { path: "../escape.txt", content: "x" })] }, { text: "escape", toolCalls: [call("write_deliverable", { path: "../escape.txt", content: "x" })] },
{ text: "done", toolCalls: [call("finish", {})] }, { text: "done", toolCalls: [] },
]; ];
const result = await runAgentLoop( const result = await runAgentLoop(
{ providers: fakeProvider(script), tools: { db: connection.db, organization, chat: new ChatService(connection.db) } }, { providers: fakeProvider(script), tools: { db: connection.db, organization, chat: new ChatService(connection.db) } },
@@ -198,7 +151,7 @@ describe("agent run loop (real tool dispatch)", () => {
connection.close(); connection.close();
}); });
test("a first turn that takes no action (prose only, no tool calls) fails the run", async () => { test("a first turn with no text and no tool calls fails the run instead of completing a no-op", async () => {
const connection = new DatabaseConnection(join(tmpdir(), `ac-loop3-${randomUUID()}.db`)); const connection = new DatabaseConnection(join(tmpdir(), `ac-loop3-${randomUUID()}.db`));
const organization = new OrganizationService(connection.db); const organization = new OrganizationService(connection.db);
let orgId = ""; let orgId = "";
@@ -213,12 +166,10 @@ describe("agent run loop (real tool dispatch)", () => {
orgId, agent: { id: ceoId, modelProviderId: providerId, modelId: "gpt-5.4" }, orgId, agent: { id: ceoId, modelProviderId: providerId, modelId: "gpt-5.4" },
taskProjectId: null, taskGoalId: null, workspace: null, taskProjectId: null, taskGoalId: null, workspace: null,
}; };
// A weak model that answers the kickoff with prose and zero tool calls must
// not be recorded as a success — the forced first turn produced no action.
await expect(runAgentLoop( await expect(runAgentLoop(
{ providers: fakeProvider([{ text: "Here is my plan: first I will think about it.", toolCalls: [] }]), tools: { db: connection.db, organization, chat: new ChatService(connection.db) } }, { providers: fakeProvider([{ text: "", toolCalls: [] }]), tools: { db: connection.db, organization, chat: new ChatService(connection.db) } },
{ orgId, providerId, workspaceId: null, instructions: "i", initialUserTurn: "u", toolContext: ctx }, { orgId, providerId, workspaceId: null, instructions: "i", initialUserTurn: "u", toolContext: ctx },
)).rejects.toThrow(/no tool call on a forced turn/); )).rejects.toThrow(/empty turn/);
connection.close(); connection.close();
}); });
}); });
@@ -210,22 +210,13 @@ describe("credential resolver", () => {
}); });
}); });
describe("tool definitions and forced tool choice", () => { describe("responses tool definitions", () => {
const tools = [{ name: "hire_agent", description: "Add a report.", parameters: { type: "object", properties: {} } }]; const tools = [{ name: "hire_agent", description: "Add a report.", parameters: { type: "object", properties: {} } }];
test("Responses tool defs omit strict when null, keep it when boolean", () => { test("omit strict when null (so the Codex endpoint keeps the tools), keep it when boolean", () => {
const codex = responsesToolDefs(tools, null)[0] as Record<string, unknown>; const codex = responsesToolDefs(tools, null)[0] as Record<string, unknown>;
expect("strict" in codex).toBe(false); // null would make the endpoint drop the tools expect("strict" in codex).toBe(false); // strict:null is malformed and makes the endpoint drop the tools
expect(responsesToolDefs(tools)[0]).toMatchObject({ type: "function", name: "hire_agent", strict: false }); expect(responsesToolDefs(tools)[0]).toMatchObject({ type: "function", name: "hire_agent", strict: false });
expect(responsesToolDefs(tools, true)[0]).toMatchObject({ strict: true }); expect(responsesToolDefs(tools, true)[0]).toMatchObject({ strict: true });
}); });
test("Bedrock and Gemini encode a forced tool choice", () => {
expect(bedrockToolConfig(tools, "required").toolChoice).toEqual({ any: {} });
expect(bedrockToolConfig(tools).toolChoice).toEqual({ auto: {} });
const vertex = vertexRequestBody({ model: "m", instructions: null, messages: [], tools, toolChoice: "required" });
expect(vertex.toolConfig).toEqual({ functionCallingConfig: { mode: "ANY" } });
const anti = antigravityRequestBody({ model: "m", projectId: "p", instructions: null, messages: [], tools, requestId: "r" });
expect((anti.request as Record<string, unknown>).toolConfig).toEqual({ functionCallingConfig: { mode: "AUTO" } });
});
}); });
@@ -37,11 +37,7 @@ describe("task execution service", () => {
const fakeToolTurn = async (input: ProviderToolTurnInput): Promise<AgentTurnResult> => { const fakeToolTurn = async (input: ProviderToolTurnInput): Promise<AgentTurnResult> => {
const first = input.messages[0]; const first = input.messages[0];
if (first?.role === "user") capturedPrompt = first.text; if (first?.role === "user") capturedPrompt = first.text;
const acted = input.messages.some((message) => message.role === "tool"); return { text: "executed:gpt-5.4", toolCalls: [], rawResponseJson: JSON.stringify({ id: "resp-task-1" }) };
const toolCall = acted
? { id: `call_${randomUUID()}`, name: "finish", arguments: {} }
: { id: `call_${randomUUID()}`, name: "create_document", arguments: { title: "Run note", doc_type: "memo", content: "Executed." } };
return { text: "executed:gpt-5.4", toolCalls: [toolCall], rawResponseJson: JSON.stringify({ id: "resp-task-1" }) };
}; };
const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn); const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
const skills = new SkillService(connection.db, join(tmpdir(), `agentcompany-task-skills-${randomUUID()}`)); const skills = new SkillService(connection.db, join(tmpdir(), `agentcompany-task-skills-${randomUUID()}`));
@@ -145,13 +141,11 @@ describe("task execution service", () => {
const organization = new OrganizationService(connection.db); const organization = new OrganizationService(connection.db);
const approvals = new ApprovalService(connection.db); const approvals = new ApprovalService(connection.db);
const snapshot = new SnapshotService(connection.db); const snapshot = new SnapshotService(connection.db);
const fakeToolTurn = async (input: ProviderToolTurnInput): Promise<AgentTurnResult> => { const fakeToolTurn = async (): Promise<AgentTurnResult> => ({
const acted = input.messages.some((message) => message.role === "tool"); text: "executed:gpt-5.4",
const toolCall = acted toolCalls: [],
? { id: `call_${randomUUID()}`, name: "finish", arguments: {} } rawResponseJson: JSON.stringify({ id: "resp-task-approved" }),
: { id: `call_${randomUUID()}`, name: "create_document", arguments: { title: "Run note", doc_type: "memo", content: "Executed." } }; });
return { text: "executed:gpt-5.4", toolCalls: [toolCall], rawResponseJson: JSON.stringify({ id: "resp-task-approved" }) };
};
const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn); const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
const execution = new TaskExecutionService(connection.db, providers); const execution = new TaskExecutionService(connection.db, providers);
let orgId = ""; let orgId = "";