ZacharyZhang-NY/AgentCompany
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Q128: every model call of a run is recorded with its purpose and listed in the run

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ZacharyZhang-NYandClaude Opus 5.5 committed 2026-10-08 20:55:18 -04:00
1 parent 1f484df894
commit 6afd7bef2a
25 files changed
+189 -22
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/unit/model-check.test.ts
+8 -8
View File
@@ -36,10 +36,10 @@ function harness(turn: (input: ProviderToolTurnInput) => Promise<AgentTurnResult
describe("a model is checked for agent work before it does any", () => {
test("a model that answers in text instead of calling tools is refused, and the verdict is kept", async () => {
const h = harness(TEXT_ONLY);
await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK }))
await expect(h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK }))
.rejects.toThrow("gpt-5.4 on openai failed the agent check: it answered in text instead of calling the tool it was given");
expect(h.requests()).toBe(1);
await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("failed the agent check");
await expect(h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("failed the agent check");
expect(h.requests()).toBe(1);
h.connection.close();
});
@@ -47,13 +47,13 @@ describe("a model is checked for agent work before it does any", () => {
test("a capable model is probed once, then works; the probe is a request like any other and counts against the quota", async () => {
const seen: ProviderToolTurnInput[] = [];
const h = harness(answeringModelCheck(async (input) => { seen.push(input); return { text: "Done.", toolCalls: [], rawResponseJson: "{}" }; }));
expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
expect((await h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
expect((await h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
expect([h.requests(), seen.length]).toEqual([3, 2]);
const tight = harness(answeringModelCheck(async () => ({ text: "Done.", toolCalls: [], rawResponseJson: "{}" })));
tight.providers.updatePolicy({ orgId: tight.orgId, providerId: tight.providerId, dataPolicy: PROVIDER_DATA_POLICY.STANDARD, backgroundEnabled: false, monthlyRequestLimit: 1, workspaceScope: "organization", contextWindowTokens: 128_000 });
await expect(tight.providers.runToolTurn({ orgId: tight.orgId, providerId: tight.providerId, ...WORK })).rejects.toThrow(PROVIDER_MONTHLY_LIMIT_MESSAGE);
await expect(tight.providers.runToolTurn({ purpose: "agent_turn", orgId: tight.orgId, providerId: tight.providerId, ...WORK })).rejects.toThrow(PROVIDER_MONTHLY_LIMIT_MESSAGE);
expect(tight.requests()).toBe(1);
h.connection.close();
tight.connection.close();
@@ -66,7 +66,7 @@ describe("a model is checked for agent work before it does any", () => {
if (mode === "down") throw new Error("connect ECONNREFUSED");
return mode === "text" ? TEXT_ONLY() : capable(input);
});
await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("ECONNREFUSED");
await expect(h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("ECONNREFUSED");
expect(h.connection.db.prepare("select count(*) as n from model_checks").get()).toEqual({ n: 0 });
mode = "text";
@@ -74,11 +74,11 @@ describe("a model is checked for agent work before it does any", () => {
.toEqual({ passed: false, detail: "it answered in text instead of calling the tool it was given" });
mode = "capable";
expect(await h.providers.checkModel({ orgId: h.orgId, providerId: h.providerId, model: "gpt-5.4" })).toEqual({ passed: true });
expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("ok");
expect((await h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("ok");
h.connection.db.prepare("update providers set model = 'gpt-5.5' where id = ?").run(h.providerId);
mode = "text";
await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("gpt-5.5 on openai failed the agent check");
await expect(h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("gpt-5.5 on openai failed the agent check");
h.connection.close();
});
});