Files
AgentCompany/AgentCompany-New/tests/integration/agent-reply.test.ts
T
ZacharyZhang-NYandClaude Opus 4.8 f7543e506d Compel agents to keep acting until they explicitly finish
Forcing only the opening turn let a weak model take one token action and
stop, so a kickoff hired one agent (or none) and went quiet. Now every turn
is forced (tool_choice required) and the agent ends only by calling the new
`finish` tool — so it must keep hiring, delegating, and messaging until the
work is actually done. Finishing with zero actions, or returning no tool
call at all, fails the run loudly instead of faking success.

Prompts tell each agent it must call a tool every turn and finish only when
done. Tests drive weak-model stubs that act then finish, proving a full team
forms rather than a single gesture.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-10 22:17:49 -04:00

113 lines
5.8 KiB
TypeScript

import { randomUUID } from "node:crypto";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, test } from "vitest";
import { DatabaseConnection } from "../../src/main/database/connection";
import { OnboardingService } from "../../src/main/services/onboarding-service";
import { ProviderService } from "../../src/main/services/provider-service";
import { providerAdapters } from "../../src/main/services/provider-adapters";
import { TaskExecutionService } from "../../src/main/services/task-execution-service";
import { SnapshotService } from "../../src/main/services/snapshot-service";
import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
import { canonicalMentionHandle } from "../../src/main/services/mention-handles";
import type { AgentTurnResult, ProviderToolTurnInput, ToolCall } from "../../src/main/services/provider-tool-turn";
const testVault = {
encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
decryptSecret: (ciphertext: Buffer) => ciphertext.toString("utf8"),
};
function call(name: string, args: Record<string, unknown>): ToolCall {
return { id: `call_${randomUUID()}`, name, arguments: args };
}
/**
* Scripts two distinct agents off the same provider, branching on the
* instructions (which name the acting agent). The CEO staffs one engineer and
* @mentions them in #engineering; the engineer, once woken, replies in the same
* thread — the round trip that makes the company feel alive.
*/
function scriptedTeam(): (input: ProviderToolTurnInput) => Promise<AgentTurnResult> {
const handle = canonicalMentionHandle("Dana Lee");
let ceo = 0;
let dana = 0;
const ceoScript = [
{ text: "Staffing engineering.", toolCalls: [call("hire_agent", { name: "Dana Lee", role: "Engineer", department: "Engineering" })] },
{ text: "Delegating and pinging.", toolCalls: [
call("create_task", { title: "Build the booking API", description: "Ship the booking API.", assignee_agent_name: "Dana Lee" }),
call("send_message", { channel: "engineering", body: `@${handle} you own the booking API — kicking off here.` }),
] },
{ text: "Kickoff complete.", toolCalls: [call("finish", {})] },
];
return async (input: ProviderToolTurnInput): Promise<AgentTurnResult> => {
if ((input.instructions ?? "").includes("Dana Lee")) {
// Reply inside the thread the CEO started (the prompt carries its id).
const firstUser = input.messages.find((message) => message.role === "user");
const threadId = firstUser ? /thread_id="([^"]+)"/.exec(firstUser.text)?.[1] : undefined;
const turn = dana === 0
? { text: "On it.", toolCalls: [call("send_message", { channel: "engineering", body: "On it — booking API design starting now.", ...(threadId ? { thread_id: threadId } : {}) })] }
: { text: "Replied.", toolCalls: [call("finish", {})] };
dana += 1;
return { text: turn.text, toolCalls: turn.toolCalls, rawResponseJson: "{}" };
}
const turn = ceoScript[ceo] ?? { text: "done", toolCalls: [call("finish", {})] };
ceo += 1;
return { text: turn.text, toolCalls: turn.toolCalls, rawResponseJson: "{}" };
};
}
describe("inter-agent reply", () => {
test("an @mention wakes the teammate, who replies in the same thread", async () => {
const connection = new DatabaseConnection(join(tmpdir(), `ac-reply-${randomUUID()}.db`));
const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, scriptedTeam());
const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
const taskExecution = new TaskExecutionService(connection.db, providers);
const snapshot = new SnapshotService(connection.db);
let orgId = "";
connection.transaction(() => {
orgId = onboarding.bootstrap({
companyName: "OrbitOps AI",
companyDescription: "AI-run operations company.",
employeeName: "Yang",
workspacePath: "",
executionMode: "ai_ceo",
autoApproveHires: true,
reviewDeliverables: false,
goalTitle: "Stand up a service business.",
goalDescription: "Win the first customers.",
providerKey: "openai",
providerSecret: "sk-test-key",
providerModel: "gpt-5.4",
}).orgId;
});
// Run the CEO's kickoff: it hires Dana and @mentions her in #engineering.
const ceoRun = claimNextQueuedRun(connection);
await taskExecution.executeRun(ceoRun!.org_id, ceoRun!.id);
const dana = (connection.db.prepare("select id from agents where org_id = ? and name = 'Dana Lee'").get(orgId) as { id: string });
expect(dana).toBeTruthy();
// The @mention woke Dana: a queued chat_mention run was created for her.
const wake = connection.db.prepare(
"select id, state, trigger from task_runs where org_id = ? and agent_id = ? and trigger = 'chat_mention'",
).get(orgId, dana.id) as { id: string; state: string; trigger: string } | undefined;
expect(wake).toMatchObject({ state: "queued", trigger: "chat_mention" });
// Execute Dana's woken run: she replies in the CEO's thread.
await taskExecution.executeRun(orgId, wake!.id);
const engineering = connection.db.prepare("select id from chat_spaces where org_id = ? and name = 'engineering'").get(orgId) as { id: string };
const messages = snapshot.load().chatMessages.filter((m) => m.orgId === orgId && m.spaceId === engineering.id);
const senders = new Set(messages.map((m) => m.senderId));
// Both the CEO and Dana have spoken in #engineering, in one shared thread.
expect(messages.length).toBe(2);
expect(senders.size).toBe(2);
expect(new Set(messages.map((m) => m.threadId)).size).toBe(1);
expect(messages.some((m) => m.senderId === dana.id && m.bodyMd.includes("On it"))).toBe(true);
connection.close();
});
});