Force agents to act so weak models can't complete a no-op
The whole premise is cheap models + skills + a team — so the control plane, not the model, must guarantee action. Every adapter hardcoded tool_choice:"auto", letting a weak model answer the kickoff with prose and "succeed" having hired no one, delegated nothing, and said nothing. Now the agent loop compels a tool call on the opening turn (tool_choice plumbed through every backend: OpenAI/Responses/Chat, Anthropic, Codex, Copilot, Azure, Custom, Gemini ANY mode, Bedrock any). If the forced turn still yields no tool call, the run fails loudly instead of recording a deceptive success — surfacing the problem (let it crash) rather than a silent dead company. After it has acted, the model chooses freely. Tested: a stub that rambles on "auto" but acts when forced now posts a kickoff message; a prose-only first turn fails the run. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
fa01aa682f
commit
ef9ee463c9
9 files changed
+107
-35
No files matched your search
@@ -46,15 +46,19 @@ export async function runAgentLoop(deps: AgentRunLoopDeps, input: AgentRunLoopIn
|
|||||||
instructions: input.instructions,
|
instructions: input.instructions,
|
||||||
messages,
|
messages,
|
||||||
tools: AGENT_TOOLS,
|
tools: AGENT_TOOLS,
|
||||||
|
// Compel a concrete action on the opening turn so even a weak model
|
||||||
|
// cannot answer with prose and quietly finish a no-op; once it has acted,
|
||||||
|
// let it choose freely (continue working, or stop when done).
|
||||||
|
toolChoice: iteration === 1 ? "required" : "auto",
|
||||||
});
|
});
|
||||||
lastRequestId = turn.requestId;
|
lastRequestId = turn.requestId;
|
||||||
|
|
||||||
if (turn.toolCalls.length === 0) {
|
if (turn.toolCalls.length === 0) {
|
||||||
if (toolCallCount === 0 && !turn.text.trim()) {
|
if (iteration === 1) {
|
||||||
// A first turn with neither text nor tool calls means the provider
|
// The opening turn was forced to call a tool. None means the agent took
|
||||||
// produced nothing actionable (e.g. output truncated mid-reasoning);
|
// no action at all — fail loudly instead of recording a deceptive
|
||||||
// completing the run would silently mark a no-op as done.
|
// "success" that hired no one, delegated nothing, and said nothing.
|
||||||
throw new Error("Provider returned an empty turn: no text and no tool calls.");
|
throw new Error("Agent took no action on its first turn — the model returned no tool calls.");
|
||||||
}
|
}
|
||||||
return { finalText: turn.text, iterations: iteration, toolCallCount, lastRequestId };
|
return { finalText: turn.text, iterations: iteration, toolCallCount, lastRequestId };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import type { ProviderRateLimitRegistry } from "./provider-rate-limit-registry";
|
|||||||
import type { ProviderRequestStore } from "./provider-request-store";
|
import type { ProviderRequestStore } from "./provider-request-store";
|
||||||
import type { ProviderResponseObserver } from "./provider-response-observer";
|
import type { ProviderResponseObserver } from "./provider-response-observer";
|
||||||
import { hashPrompt, readProviderResponseId, requireStreamText } from "./provider-response-utils";
|
import { hashPrompt, readProviderResponseId, requireStreamText } from "./provider-response-utils";
|
||||||
import { type AgentTurnMessage, type AgentTurnResult, type ToolDefinition, type ToolTurnFn } from "./provider-tool-turn";
|
import { type AgentTurnMessage, type AgentTurnResult, type ToolChoice, type ToolDefinition, type ToolTurnFn } from "./provider-tool-turn";
|
||||||
import { selectToolTurn, supportsToolTurn } from "./provider-tool-turns/router";
|
import { selectToolTurn, supportsToolTurn } from "./provider-tool-turns/router";
|
||||||
import { resolveTurnCredential } from "./provider-tool-turns/credential-resolver";
|
import { resolveTurnCredential } from "./provider-tool-turns/credential-resolver";
|
||||||
import {
|
import {
|
||||||
@@ -103,6 +103,7 @@ export class ProviderResponseRunner {
|
|||||||
instructions: string | null;
|
instructions: string | null;
|
||||||
messages: AgentTurnMessage[];
|
messages: AgentTurnMessage[];
|
||||||
tools: ToolDefinition[];
|
tools: ToolDefinition[];
|
||||||
|
toolChoice?: ToolChoice;
|
||||||
}): Promise<AgentTurnResult & { requestId: string }> {
|
}): Promise<AgentTurnResult & { requestId: string }> {
|
||||||
const provider = loadConfiguredProvider(this.deps.db, input.orgId, input.providerId);
|
const provider = loadConfiguredProvider(this.deps.db, input.orgId, input.providerId);
|
||||||
const workspaceId = this.normalizeWorkspaceId(input.orgId, input.workspaceId ?? null);
|
const workspaceId = this.normalizeWorkspaceId(input.orgId, input.workspaceId ?? null);
|
||||||
@@ -128,7 +129,7 @@ export class ProviderResponseRunner {
|
|||||||
});
|
});
|
||||||
const startedAt = Date.now();
|
const startedAt = Date.now();
|
||||||
try {
|
try {
|
||||||
const result = await turn({ credential, model, instructions: input.instructions, messages: input.messages, tools: input.tools });
|
const result = await turn({ credential, model, instructions: input.instructions, messages: input.messages, tools: input.tools, toolChoice: input.toolChoice });
|
||||||
this.deps.requestStore.markSucceeded({
|
this.deps.requestStore.markSucceeded({
|
||||||
orgId: input.orgId,
|
orgId: input.orgId,
|
||||||
requestId,
|
requestId,
|
||||||
|
|||||||
@@ -51,12 +51,18 @@ export interface TurnCredential {
|
|||||||
serviceAccountJson?: string;
|
serviceAccountJson?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** How the backend should treat tool calls this turn. "required" compels the
|
||||||
|
* model to emit at least one tool call (no prose-only escape) — used to force a
|
||||||
|
* concrete action so a weak model cannot ramble its way to a silent no-op. */
|
||||||
|
export type ToolChoice = "auto" | "required";
|
||||||
|
|
||||||
export interface ProviderToolTurnInput {
|
export interface ProviderToolTurnInput {
|
||||||
credential: TurnCredential;
|
credential: TurnCredential;
|
||||||
model: string;
|
model: string;
|
||||||
instructions: string | null;
|
instructions: string | null;
|
||||||
messages: AgentTurnMessage[];
|
messages: AgentTurnMessage[];
|
||||||
tools: ToolDefinition[];
|
tools: ToolDefinition[];
|
||||||
|
toolChoice?: ToolChoice;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type ToolTurnFn = (input: ProviderToolTurnInput) => Promise<AgentTurnResult>;
|
export type ToolTurnFn = (input: ProviderToolTurnInput) => Promise<AgentTurnResult>;
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
/** AWS Bedrock Converse API encoding/decoding (ported from Pi's bedrock handler). */
|
/** AWS Bedrock Converse API encoding/decoding (ported from Pi's bedrock handler). */
|
||||||
import { isRecord, type JsonRecord } from "../provider-http";
|
import { isRecord, type JsonRecord } from "../provider-http";
|
||||||
import type { AgentTurnMessage, ToolCall, ToolDefinition } from "../provider-tool-turn";
|
import type { AgentTurnMessage, ToolCall, ToolChoice, ToolDefinition } from "../provider-tool-turn";
|
||||||
|
|
||||||
export function bedrockToolConfig(tools: ToolDefinition[]): JsonRecord {
|
export function bedrockToolConfig(tools: ToolDefinition[], toolChoice?: ToolChoice): JsonRecord {
|
||||||
return {
|
return {
|
||||||
tools: tools.map((tool) => ({ toolSpec: { name: tool.name, description: tool.description, inputSchema: { json: tool.parameters } } })),
|
tools: tools.map((tool) => ({ toolSpec: { name: tool.name, description: tool.description, inputSchema: { json: tool.parameters } } })),
|
||||||
toolChoice: { auto: {} },
|
toolChoice: toolChoice === "required" ? { any: {} } : { auto: {} },
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
* Antigravity request wrapper in google-gemini-cli.
|
* Antigravity request wrapper in google-gemini-cli.
|
||||||
*/
|
*/
|
||||||
import { isRecord, type JsonRecord, type ProviderEventStreamMessage } from "../provider-http";
|
import { isRecord, type JsonRecord, type ProviderEventStreamMessage } from "../provider-http";
|
||||||
import { type AgentTurnMessage, type AgentTurnResult, type ToolCall, type ToolDefinition } from "../provider-tool-turn";
|
import { type AgentTurnMessage, type AgentTurnResult, type ToolCall, type ToolChoice, type ToolDefinition } from "../provider-tool-turn";
|
||||||
|
|
||||||
const ANTIGRAVITY_SYSTEM_INSTRUCTION =
|
const ANTIGRAVITY_SYSTEM_INSTRUCTION =
|
||||||
"You are Antigravity, a powerful agentic AI coding assistant designed by the Google Deepmind team working on Advanced Agentic Coding." +
|
"You are Antigravity, a powerful agentic AI coding assistant designed by the Google Deepmind team working on Advanced Agentic Coding." +
|
||||||
@@ -64,6 +64,7 @@ export function antigravityRequestBody(input: {
|
|||||||
instructions: string | null;
|
instructions: string | null;
|
||||||
messages: AgentTurnMessage[];
|
messages: AgentTurnMessage[];
|
||||||
tools: ToolDefinition[];
|
tools: ToolDefinition[];
|
||||||
|
toolChoice?: ToolChoice;
|
||||||
requestId: string;
|
requestId: string;
|
||||||
}): JsonRecord {
|
}): JsonRecord {
|
||||||
const request: JsonRecord = {
|
const request: JsonRecord = {
|
||||||
@@ -79,7 +80,7 @@ export function antigravityRequestBody(input: {
|
|||||||
};
|
};
|
||||||
if (input.tools.length > 0) {
|
if (input.tools.length > 0) {
|
||||||
request.tools = geminiToolDefs(input.model, input.tools);
|
request.tools = geminiToolDefs(input.model, input.tools);
|
||||||
request.toolConfig = { functionCallingConfig: { mode: "AUTO" } };
|
request.toolConfig = { functionCallingConfig: { mode: input.toolChoice === "required" ? "ANY" : "AUTO" } };
|
||||||
}
|
}
|
||||||
return { project: input.projectId, model: input.model, request, requestType: "agent", userAgent: "antigravity", requestId: input.requestId };
|
return { project: input.projectId, model: input.model, request, requestType: "agent", userAgent: "antigravity", requestId: input.requestId };
|
||||||
}
|
}
|
||||||
@@ -90,12 +91,13 @@ export function vertexRequestBody(input: {
|
|||||||
instructions: string | null;
|
instructions: string | null;
|
||||||
messages: AgentTurnMessage[];
|
messages: AgentTurnMessage[];
|
||||||
tools: ToolDefinition[];
|
tools: ToolDefinition[];
|
||||||
|
toolChoice?: ToolChoice;
|
||||||
}): JsonRecord {
|
}): JsonRecord {
|
||||||
const body: JsonRecord = { contents: toGeminiContents(input.model, input.messages) };
|
const body: JsonRecord = { contents: toGeminiContents(input.model, input.messages) };
|
||||||
if (input.instructions) body.systemInstruction = { parts: [{ text: input.instructions }] };
|
if (input.instructions) body.systemInstruction = { parts: [{ text: input.instructions }] };
|
||||||
if (input.tools.length > 0) {
|
if (input.tools.length > 0) {
|
||||||
body.tools = geminiToolDefs(input.model, input.tools);
|
body.tools = geminiToolDefs(input.model, input.tools);
|
||||||
body.toolConfig = { functionCallingConfig: { mode: "AUTO" } };
|
body.toolConfig = { functionCallingConfig: { mode: input.toolChoice === "required" ? "ANY" : "AUTO" } };
|
||||||
}
|
}
|
||||||
return body;
|
return body;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ export async function runOpenAiResponsesTurn(input: ProviderToolTurnInput): Prom
|
|||||||
model: input.model,
|
model: input.model,
|
||||||
input: toResponsesInput(input.messages),
|
input: toResponsesInput(input.messages),
|
||||||
tools: responsesToolDefs(input.tools),
|
tools: responsesToolDefs(input.tools),
|
||||||
tool_choice: "auto",
|
tool_choice: input.toolChoice ?? "auto",
|
||||||
store: false,
|
store: false,
|
||||||
...(input.instructions ? { instructions: input.instructions } : {}),
|
...(input.instructions ? { instructions: input.instructions } : {}),
|
||||||
};
|
};
|
||||||
@@ -58,7 +58,7 @@ export function makeChatCompletionsTurn(endpoint: string, label: string): (input
|
|||||||
model: input.model,
|
model: input.model,
|
||||||
messages: toChatMessages(input.instructions, input.messages),
|
messages: toChatMessages(input.instructions, input.messages),
|
||||||
tools: chatToolDefs(input.tools),
|
tools: chatToolDefs(input.tools),
|
||||||
tool_choice: "auto",
|
tool_choice: input.toolChoice ?? "auto",
|
||||||
stream: false,
|
stream: false,
|
||||||
};
|
};
|
||||||
const response = await postJson(endpoint, { authorization: `Bearer ${requireApiKey(input.credential, label)}` }, body);
|
const response = await postJson(endpoint, { authorization: `Bearer ${requireApiKey(input.credential, label)}` }, body);
|
||||||
@@ -74,6 +74,7 @@ export async function runAnthropicTurn(input: ProviderToolTurnInput): Promise<Ag
|
|||||||
model: input.model,
|
model: input.model,
|
||||||
max_tokens: 4096,
|
max_tokens: 4096,
|
||||||
tools: anthropicToolDefs(input.tools),
|
tools: anthropicToolDefs(input.tools),
|
||||||
|
...(input.toolChoice === "required" ? { tool_choice: { type: "any" } } : {}),
|
||||||
messages: toAnthropicMessages(input.messages),
|
messages: toAnthropicMessages(input.messages),
|
||||||
...(input.instructions ? { system: input.instructions } : {}),
|
...(input.instructions ? { system: input.instructions } : {}),
|
||||||
};
|
};
|
||||||
@@ -93,6 +94,7 @@ export async function runClaudeProTurn(input: ProviderToolTurnInput): Promise<Ag
|
|||||||
model: input.model,
|
model: input.model,
|
||||||
max_tokens: 4096,
|
max_tokens: 4096,
|
||||||
tools: anthropicToolDefs(input.tools),
|
tools: anthropicToolDefs(input.tools),
|
||||||
|
...(input.toolChoice === "required" ? { tool_choice: { type: "any" } } : {}),
|
||||||
messages: toAnthropicMessages(input.messages),
|
messages: toAnthropicMessages(input.messages),
|
||||||
system,
|
system,
|
||||||
};
|
};
|
||||||
@@ -121,7 +123,7 @@ export async function runCodexTurn(input: ProviderToolTurnInput): Promise<AgentT
|
|||||||
input: toResponsesInput(input.messages),
|
input: toResponsesInput(input.messages),
|
||||||
text: { verbosity: "medium" },
|
text: { verbosity: "medium" },
|
||||||
include: ["reasoning.encrypted_content"],
|
include: ["reasoning.encrypted_content"],
|
||||||
tool_choice: "auto",
|
tool_choice: input.toolChoice ?? "auto",
|
||||||
parallel_tool_calls: true,
|
parallel_tool_calls: true,
|
||||||
tools: responsesToolDefs(input.tools, null),
|
tools: responsesToolDefs(input.tools, null),
|
||||||
};
|
};
|
||||||
@@ -144,7 +146,7 @@ export async function runCopilotTurn(input: ProviderToolTurnInput): Promise<Agen
|
|||||||
model: input.model,
|
model: input.model,
|
||||||
input: toResponsesInput(input.messages),
|
input: toResponsesInput(input.messages),
|
||||||
tools: responsesToolDefs(input.tools),
|
tools: responsesToolDefs(input.tools),
|
||||||
tool_choice: "auto",
|
tool_choice: input.toolChoice ?? "auto",
|
||||||
store: false,
|
store: false,
|
||||||
};
|
};
|
||||||
const response = await postJson(`${baseUrl}/responses`, {
|
const response = await postJson(`${baseUrl}/responses`, {
|
||||||
@@ -165,7 +167,7 @@ export async function runCustomOpenAiTurn(input: ProviderToolTurnInput): Promise
|
|||||||
model: input.model,
|
model: input.model,
|
||||||
messages: toChatMessages(input.instructions, input.messages),
|
messages: toChatMessages(input.instructions, input.messages),
|
||||||
tools: chatToolDefs(input.tools),
|
tools: chatToolDefs(input.tools),
|
||||||
tool_choice: "auto",
|
tool_choice: input.toolChoice ?? "auto",
|
||||||
stream: false,
|
stream: false,
|
||||||
};
|
};
|
||||||
const response = await postJson(`${input.credential.baseUrl}/chat/completions`, { authorization: `Bearer ${requireApiKey(input.credential, "Custom OpenAI-compatible provider")}` }, body);
|
const response = await postJson(`${input.credential.baseUrl}/chat/completions`, { authorization: `Bearer ${requireApiKey(input.credential, "Custom OpenAI-compatible provider")}` }, body);
|
||||||
@@ -181,6 +183,7 @@ export async function runCustomAnthropicTurn(input: ProviderToolTurnInput): Prom
|
|||||||
model: input.model,
|
model: input.model,
|
||||||
max_tokens: 4096,
|
max_tokens: 4096,
|
||||||
tools: anthropicToolDefs(input.tools),
|
tools: anthropicToolDefs(input.tools),
|
||||||
|
...(input.toolChoice === "required" ? { tool_choice: { type: "any" } } : {}),
|
||||||
messages: toAnthropicMessages(input.messages),
|
messages: toAnthropicMessages(input.messages),
|
||||||
...(input.instructions ? { system: input.instructions } : {}),
|
...(input.instructions ? { system: input.instructions } : {}),
|
||||||
};
|
};
|
||||||
@@ -201,6 +204,7 @@ export async function runAntigravityTurn(input: ProviderToolTurnInput): Promise<
|
|||||||
instructions: input.instructions,
|
instructions: input.instructions,
|
||||||
messages: input.messages,
|
messages: input.messages,
|
||||||
tools: input.tools,
|
tools: input.tools,
|
||||||
|
toolChoice: input.toolChoice,
|
||||||
requestId,
|
requestId,
|
||||||
});
|
});
|
||||||
const response = await postEventStream(`${ANTIGRAVITY_ENDPOINT}/v1internal:streamGenerateContent?alt=sse`, {
|
const response = await postEventStream(`${ANTIGRAVITY_ENDPOINT}/v1internal:streamGenerateContent?alt=sse`, {
|
||||||
@@ -220,7 +224,7 @@ export async function runAzureTurn(input: ProviderToolTurnInput): Promise<AgentT
|
|||||||
model: input.credential.deployment,
|
model: input.credential.deployment,
|
||||||
messages: toChatMessages(input.instructions, input.messages),
|
messages: toChatMessages(input.instructions, input.messages),
|
||||||
tools: chatToolDefs(input.tools),
|
tools: chatToolDefs(input.tools),
|
||||||
tool_choice: "auto",
|
tool_choice: input.toolChoice ?? "auto",
|
||||||
stream: false,
|
stream: false,
|
||||||
};
|
};
|
||||||
const url = `${input.credential.baseUrl}/openai/deployments/${input.credential.deployment}/chat/completions?api-version=2024-10-21`;
|
const url = `${input.credential.baseUrl}/openai/deployments/${input.credential.deployment}/chat/completions?api-version=2024-10-21`;
|
||||||
@@ -237,7 +241,7 @@ export async function runBedrockTurn(input: ProviderToolTurnInput): Promise<Agen
|
|||||||
const converse: JsonRecord = {
|
const converse: JsonRecord = {
|
||||||
messages: toBedrockMessages(input.messages),
|
messages: toBedrockMessages(input.messages),
|
||||||
inferenceConfig: { maxTokens: 4096 },
|
inferenceConfig: { maxTokens: 4096 },
|
||||||
toolConfig: bedrockToolConfig(input.tools),
|
toolConfig: bedrockToolConfig(input.tools, input.toolChoice),
|
||||||
...(input.instructions ? { system: [{ text: input.instructions }] } : {}),
|
...(input.instructions ? { system: [{ text: input.instructions }] } : {}),
|
||||||
};
|
};
|
||||||
const bodyString = JSON.stringify(converse);
|
const bodyString = JSON.stringify(converse);
|
||||||
@@ -259,7 +263,7 @@ export async function runVertexTurn(input: ProviderToolTurnInput): Promise<Agent
|
|||||||
const { projectId, region, serviceAccountJson } = input.credential;
|
const { projectId, region, serviceAccountJson } = input.credential;
|
||||||
if (!projectId || !region || !serviceAccountJson) throw new Error("Vertex AI credential is incomplete.");
|
if (!projectId || !region || !serviceAccountJson) throw new Error("Vertex AI credential is incomplete.");
|
||||||
const token = await getGcpAccessToken(serviceAccountJson);
|
const token = await getGcpAccessToken(serviceAccountJson);
|
||||||
const body = vertexRequestBody({ model: input.model, instructions: input.instructions, messages: input.messages, tools: input.tools });
|
const body = vertexRequestBody({ model: input.model, instructions: input.instructions, messages: input.messages, tools: input.tools, toolChoice: input.toolChoice });
|
||||||
const url = `https://${region}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${input.model}:generateContent`;
|
const url = `https://${region}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${input.model}:generateContent`;
|
||||||
const response = await postJson(url, { authorization: `Bearer ${token}` }, body);
|
const response = await postJson(url, { authorization: `Bearer ${token}` }, body);
|
||||||
const parsed = parseGeminiResponse(response.data);
|
const parsed = parseGeminiResponse(response.data);
|
||||||
|
|||||||
@@ -166,11 +166,18 @@ describe("autonomous bootstrap run", () => {
|
|||||||
|
|
||||||
test("a goal created after onboarding reaches the CEO and runs autonomously", async () => {
|
test("a goal created after onboarding reaches the CEO and runs autonomously", async () => {
|
||||||
const connection = new DatabaseConnection(join(tmpdir(), `ac-goal-${randomUUID()}.db`));
|
const connection = new DatabaseConnection(join(tmpdir(), `ac-goal-${randomUUID()}.db`));
|
||||||
const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, async () => ({
|
// The forced opening turn means each run must take a concrete action; it
|
||||||
text: "Reviewed the goal and posted the operating plan.",
|
// posts to the company channel, then stops on the next (free) turn.
|
||||||
toolCalls: [],
|
const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, async (input) => {
|
||||||
rawResponseJson: JSON.stringify({ id: "resp_goal" }),
|
if (input.toolChoice === "required") {
|
||||||
}));
|
return {
|
||||||
|
text: "Reviewing the goal.",
|
||||||
|
toolCalls: [call("send_message", { channel: "general", body: "Reviewed the goal and posting the operating plan." })],
|
||||||
|
rawResponseJson: JSON.stringify({ id: "resp_goal_act" }),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { text: "Done.", toolCalls: [], rawResponseJson: JSON.stringify({ id: "resp_goal_done" }) };
|
||||||
|
});
|
||||||
const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
|
const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
|
||||||
const organization = new OrganizationService(connection.db);
|
const organization = new OrganizationService(connection.db);
|
||||||
const taskExecution = new TaskExecutionService(connection.db, providers);
|
const taskExecution = new TaskExecutionService(connection.db, providers);
|
||||||
|
|||||||
@@ -30,6 +30,21 @@ function call(name: string, args: Record<string, unknown>): ToolCall {
|
|||||||
return { id: `call_${randomUUID()}`, name, arguments: args };
|
return { id: `call_${randomUUID()}`, name, arguments: args };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Mimics a weak model: it rambles in prose and skips tools unless the backend
|
||||||
|
* compels a tool call. With forcing it posts a kickoff message, then stops. */
|
||||||
|
function weakModelProvider() {
|
||||||
|
let forced = false;
|
||||||
|
return {
|
||||||
|
runToolTurn: async (input: { toolChoice?: "auto" | "required" }): Promise<AgentTurnResult & { requestId: string }> => {
|
||||||
|
if (input.toolChoice === "required" && !forced) {
|
||||||
|
forced = true;
|
||||||
|
return { text: "Acting.", toolCalls: [call("send_message", { channel: "general", body: "Kicking off the company." })], rawResponseJson: "{}", requestId: "req_forced" };
|
||||||
|
}
|
||||||
|
return { text: "Let me think about this some more.", toolCalls: [], rawResponseJson: "{}", requestId: "req_prose" };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
describe("agent run loop (real tool dispatch)", () => {
|
describe("agent run loop (real tool dispatch)", () => {
|
||||||
test("the CEO hires, creates a task, writes a deliverable, then converges", async () => {
|
test("the CEO hires, creates a task, writes a deliverable, then converges", async () => {
|
||||||
const connection = new DatabaseConnection(join(tmpdir(), `ac-loop-${randomUUID()}.db`));
|
const connection = new DatabaseConnection(join(tmpdir(), `ac-loop-${randomUUID()}.db`));
|
||||||
@@ -120,6 +135,33 @@ describe("agent run loop (real tool dispatch)", () => {
|
|||||||
connection.close();
|
connection.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("forcing the opening turn makes a weak model act instead of completing a no-op", async () => {
|
||||||
|
const connection = new DatabaseConnection(join(tmpdir(), `ac-weak-${randomUUID()}.db`));
|
||||||
|
const organization = new OrganizationService(connection.db);
|
||||||
|
let orgId = "";
|
||||||
|
let ceoId = "";
|
||||||
|
let providerId = "";
|
||||||
|
connection.transaction(() => {
|
||||||
|
orgId = organization.createOrganization({ organizationName: "OrbitOps", employeeName: "Yang", mission: "m", goal: "g" });
|
||||||
|
providerId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'openai'").get(orgId) as { id: string }).id;
|
||||||
|
ceoId = organization.createAgent({ orgId, name: "CEO", role: "CEO", department: "Executive", modelProviderId: providerId, modelId: "gpt-5.4", budgetCents: 25000, managerAgentId: null });
|
||||||
|
});
|
||||||
|
const ctx: AgentToolContext = {
|
||||||
|
orgId, agent: { id: ceoId, modelProviderId: providerId, modelId: "gpt-5.4" },
|
||||||
|
taskProjectId: null, taskGoalId: null, workspace: null,
|
||||||
|
};
|
||||||
|
const result = await runAgentLoop(
|
||||||
|
{ providers: weakModelProvider(), tools: { db: connection.db, organization, chat: new ChatService(connection.db) } },
|
||||||
|
{ orgId, providerId, workspaceId: null, instructions: "You are the CEO.", initialUserTurn: "Start the company.", toolContext: ctx },
|
||||||
|
);
|
||||||
|
// The loop forced the opening turn, so the otherwise-rambling model acted.
|
||||||
|
expect(result.toolCallCount).toBe(1);
|
||||||
|
const messages = connection.db.prepare("select sender_id from chat_messages where org_id = ?").all(orgId) as Array<{ sender_id: string }>;
|
||||||
|
expect(messages).toHaveLength(1);
|
||||||
|
expect(messages[0]?.sender_id).toBe(ceoId);
|
||||||
|
connection.close();
|
||||||
|
});
|
||||||
|
|
||||||
test("rejects deliverable paths that escape the workspace", async () => {
|
test("rejects deliverable paths that escape the workspace", async () => {
|
||||||
const connection = new DatabaseConnection(join(tmpdir(), `ac-loop2-${randomUUID()}.db`));
|
const connection = new DatabaseConnection(join(tmpdir(), `ac-loop2-${randomUUID()}.db`));
|
||||||
const organization = new OrganizationService(connection.db);
|
const organization = new OrganizationService(connection.db);
|
||||||
@@ -151,7 +193,7 @@ describe("agent run loop (real tool dispatch)", () => {
|
|||||||
connection.close();
|
connection.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
test("a first turn with no text and no tool calls fails the run instead of completing a no-op", async () => {
|
test("a first turn that takes no action (prose only, no tool calls) fails the run", async () => {
|
||||||
const connection = new DatabaseConnection(join(tmpdir(), `ac-loop3-${randomUUID()}.db`));
|
const connection = new DatabaseConnection(join(tmpdir(), `ac-loop3-${randomUUID()}.db`));
|
||||||
const organization = new OrganizationService(connection.db);
|
const organization = new OrganizationService(connection.db);
|
||||||
let orgId = "";
|
let orgId = "";
|
||||||
@@ -166,10 +208,12 @@ describe("agent run loop (real tool dispatch)", () => {
|
|||||||
orgId, agent: { id: ceoId, modelProviderId: providerId, modelId: "gpt-5.4" },
|
orgId, agent: { id: ceoId, modelProviderId: providerId, modelId: "gpt-5.4" },
|
||||||
taskProjectId: null, taskGoalId: null, workspace: null,
|
taskProjectId: null, taskGoalId: null, workspace: null,
|
||||||
};
|
};
|
||||||
|
// A weak model that answers the kickoff with prose and zero tool calls must
|
||||||
|
// not be recorded as a success — the forced first turn produced no action.
|
||||||
await expect(runAgentLoop(
|
await expect(runAgentLoop(
|
||||||
{ providers: fakeProvider([{ text: "", toolCalls: [] }]), tools: { db: connection.db, organization, chat: new ChatService(connection.db) } },
|
{ providers: fakeProvider([{ text: "Here is my plan: first I will think about it.", toolCalls: [] }]), tools: { db: connection.db, organization, chat: new ChatService(connection.db) } },
|
||||||
{ orgId, providerId, workspaceId: null, instructions: "i", initialUserTurn: "u", toolContext: ctx },
|
{ orgId, providerId, workspaceId: null, instructions: "i", initialUserTurn: "u", toolContext: ctx },
|
||||||
)).rejects.toThrow(/empty turn/);
|
)).rejects.toThrow(/no action on its first turn/);
|
||||||
connection.close();
|
connection.close();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -37,7 +37,10 @@ describe("task execution service", () => {
|
|||||||
const fakeToolTurn = async (input: ProviderToolTurnInput): Promise<AgentTurnResult> => {
|
const fakeToolTurn = async (input: ProviderToolTurnInput): Promise<AgentTurnResult> => {
|
||||||
const first = input.messages[0];
|
const first = input.messages[0];
|
||||||
if (first?.role === "user") capturedPrompt = first.text;
|
if (first?.role === "user") capturedPrompt = first.text;
|
||||||
return { text: "executed:gpt-5.4", toolCalls: [], rawResponseJson: JSON.stringify({ id: "resp-task-1" }) };
|
if (input.toolChoice === "required") {
|
||||||
|
return { text: "executed:gpt-5.4", toolCalls: [{ id: `call_${randomUUID()}`, name: "create_document", arguments: { title: "Run note", doc_type: "memo", content: "Executed." } }], rawResponseJson: JSON.stringify({ id: "resp-task-1a" }) };
|
||||||
|
}
|
||||||
|
return { text: "executed:gpt-5.4", toolCalls: [], rawResponseJson: JSON.stringify({ id: "resp-task-1b" }) };
|
||||||
};
|
};
|
||||||
const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
|
const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
|
||||||
const skills = new SkillService(connection.db, join(tmpdir(), `agentcompany-task-skills-${randomUUID()}`));
|
const skills = new SkillService(connection.db, join(tmpdir(), `agentcompany-task-skills-${randomUUID()}`));
|
||||||
@@ -141,11 +144,12 @@ describe("task execution service", () => {
|
|||||||
const organization = new OrganizationService(connection.db);
|
const organization = new OrganizationService(connection.db);
|
||||||
const approvals = new ApprovalService(connection.db);
|
const approvals = new ApprovalService(connection.db);
|
||||||
const snapshot = new SnapshotService(connection.db);
|
const snapshot = new SnapshotService(connection.db);
|
||||||
const fakeToolTurn = async (): Promise<AgentTurnResult> => ({
|
const fakeToolTurn = async (input: ProviderToolTurnInput): Promise<AgentTurnResult> => {
|
||||||
text: "executed:gpt-5.4",
|
if (input.toolChoice === "required") {
|
||||||
toolCalls: [],
|
return { text: "executed:gpt-5.4", toolCalls: [{ id: `call_${randomUUID()}`, name: "create_document", arguments: { title: "Run note", doc_type: "memo", content: "Executed." } }], rawResponseJson: JSON.stringify({ id: "resp-task-approved-a" }) };
|
||||||
rawResponseJson: JSON.stringify({ id: "resp-task-approved" }),
|
}
|
||||||
});
|
return { text: "executed:gpt-5.4", toolCalls: [], rawResponseJson: JSON.stringify({ id: "resp-task-approved-b" }) };
|
||||||
|
};
|
||||||
const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
|
const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, fakeToolTurn);
|
||||||
const execution = new TaskExecutionService(connection.db, providers);
|
const execution = new TaskExecutionService(connection.db, providers);
|
||||||
let orgId = "";
|
let orgId = "";
|
||||||
|
|||||||
Reference in new issue
Block a user