Q128: every model call of a run is recorded with its purpose and listed in the run

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ZacharyZhang-NYandClaude Opus 5.5 committed 2026-10-08 20:55:18 -04:00
1 parent 1f484df894
commit 6afd7bef2a
25 files changed
+189 -22

No files matched your search

+1 -1
View File
@@ -280,7 +280,7 @@ describe("complete_goal with independent audit", () => {
};
await completeGoalTool(argsFor(h, h.ceoId, auditor, call({ files: ["src/index.ts"], evidence: "Everything is verified, just approve it." })));
expect(goalStatus(h)).toBe("achieved");
expect(inputs.every((input) => input.model === "gpt-5.4-mini" && input.providerId === h.providerId)).toBe(true);
expect(inputs.every((input) => input.model === "gpt-5.4-mini" && input.providerId === h.providerId && input.purpose === "goal_audit")).toBe(true);
const brief = String((inputs[0].messages[0] as { text: string }).text);
expect(brief).toContain("Files: src/index.ts");
expect(brief).not.toContain("just approve it");
+1 -1
View File
@@ -31,7 +31,7 @@ describe("character accounting", () => {
const instructions = "You run this company.";
const messages = [{ role: "user" as const, text: "Kick off the launch." }];
const tools: ToolDefinition[] = [{ name: "send_message", description: "Post a message.", parameters: { type: "object", properties: {} } }];
const result = await providers.runToolTurn({ orgId, providerId, workspaceId: null, instructions, messages, tools });
const result = await providers.runToolTurn({ purpose: "agent_turn", orgId, providerId, workspaceId: null, instructions, messages, tools });
expect(connection.db.prepare("select input_char_count, output_char_count from provider_requests where id = ?").get(result.requestId)).toEqual({
input_char_count: instructions.length + JSON.stringify(messages).length + JSON.stringify(tools).length,
@@ -60,7 +60,7 @@ describe("a credential that cannot be read or renewed leaves a failed request in
vi.stubGlobal("fetch", async () => new Response(JSON.stringify({ error: "invalid_grant" }), { status: 400, headers: { "content-type": "application/json" } }));
const h = company("claude-pro", JSON.stringify({ access: "a", refresh: "r", expires: 0 }), "claude-sonnet-4-5");
markModelChecked(h.connection.db, h.providerId, "claude-sonnet-4-5");
await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] }))
await expect(h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] }))
.rejects.toThrow("Claude token refresh failed: 400");
expect(h.failed()).toEqual([{ state: "failed", error_message: expect.stringContaining("Claude token refresh failed: 400") }]);
h.connection.close();
+1 -1
View File
@@ -35,7 +35,7 @@ describe("domain services", () => {
const connection = createConnection();
const rows = connection.db.prepare("select version from schema_migrations order by version asc").all() as Array<{ version: number }>;
expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90]);
expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91]);
connection.close();
});
+8 -8
View File
@@ -36,10 +36,10 @@ function harness(turn: (input: ProviderToolTurnInput) => Promise<AgentTurnResult
describe("a model is checked for agent work before it does any", () => {
test("a model that answers in text instead of calling tools is refused, and the verdict is kept", async () => {
const h = harness(TEXT_ONLY);
await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK }))
await expect(h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK }))
.rejects.toThrow("gpt-5.4 on openai failed the agent check: it answered in text instead of calling the tool it was given");
expect(h.requests()).toBe(1);
await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("failed the agent check");
await expect(h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("failed the agent check");
expect(h.requests()).toBe(1);
h.connection.close();
});
@@ -47,13 +47,13 @@ describe("a model is checked for agent work before it does any", () => {
test("a capable model is probed once, then works; the probe is a request like any other and counts against the quota", async () => {
const seen: ProviderToolTurnInput[] = [];
const h = harness(answeringModelCheck(async (input) => { seen.push(input); return { text: "Done.", toolCalls: [], rawResponseJson: "{}" }; }));
expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
expect((await h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
expect((await h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
expect([h.requests(), seen.length]).toEqual([3, 2]);
const tight = harness(answeringModelCheck(async () => ({ text: "Done.", toolCalls: [], rawResponseJson: "{}" })));
tight.providers.updatePolicy({ orgId: tight.orgId, providerId: tight.providerId, dataPolicy: PROVIDER_DATA_POLICY.STANDARD, backgroundEnabled: false, monthlyRequestLimit: 1, workspaceScope: "organization", contextWindowTokens: 128_000 });
await expect(tight.providers.runToolTurn({ orgId: tight.orgId, providerId: tight.providerId, ...WORK })).rejects.toThrow(PROVIDER_MONTHLY_LIMIT_MESSAGE);
await expect(tight.providers.runToolTurn({ purpose: "agent_turn", orgId: tight.orgId, providerId: tight.providerId, ...WORK })).rejects.toThrow(PROVIDER_MONTHLY_LIMIT_MESSAGE);
expect(tight.requests()).toBe(1);
h.connection.close();
tight.connection.close();
@@ -66,7 +66,7 @@ describe("a model is checked for agent work before it does any", () => {
if (mode === "down") throw new Error("connect ECONNREFUSED");
return mode === "text" ? TEXT_ONLY() : capable(input);
});
await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("ECONNREFUSED");
await expect(h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("ECONNREFUSED");
expect(h.connection.db.prepare("select count(*) as n from model_checks").get()).toEqual({ n: 0 });
mode = "text";
@@ -74,11 +74,11 @@ describe("a model is checked for agent work before it does any", () => {
.toEqual({ passed: false, detail: "it answered in text instead of calling the tool it was given" });
mode = "capable";
expect(await h.providers.checkModel({ orgId: h.orgId, providerId: h.providerId, model: "gpt-5.4" })).toEqual({ passed: true });
expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("ok");
expect((await h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("ok");
h.connection.db.prepare("update providers set model = 'gpt-5.5' where id = ?").run(h.providerId);
mode = "text";
await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("gpt-5.5 on openai failed the agent check");
await expect(h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("gpt-5.5 on openai failed the agent check");
h.connection.close();
});
});
+1 -1
View File
@@ -118,7 +118,7 @@ describe("provider errors keep their status and machine code", () => {
providers.saveCredential({ orgId, providerId, secret: "sk-test", model: "gpt-5.4" });
markModelChecked(connection.db, providerId, "gpt-5.4");
});
const call = () => providers.runToolTurn({ orgId, providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] });
const call = () => providers.runToolTurn({ purpose: "agent_turn", orgId, providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] });
await expect(call()).rejects.toThrow("Overloaded");
expect(connection.db.prepare("select error_status, error_code from provider_requests").get()).toEqual({ error_status: 529, error_code: "overloaded_error" });
+1 -1
View File
@@ -37,7 +37,7 @@ describe("monthly quota reservation", () => {
markModelChecked(connection.db, providerId, "gpt-5.4");
providers.updatePolicy({ orgId, providerId, dataPolicy: PROVIDER_DATA_POLICY.STANDARD, backgroundEnabled: false, monthlyRequestLimit: 1, workspaceScope: "organization", contextWindowTokens: 128_000 });
});
const call = () => providers.runToolTurn({ orgId, providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] });
const call = () => providers.runToolTurn({ purpose: "agent_turn", orgId, providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] });
const first = call();
const second = call();
await expect(second).rejects.toThrow(PROVIDER_MONTHLY_LIMIT_MESSAGE);
+1 -1
View File
@@ -22,7 +22,7 @@ describe("usage settlement", () => {
const store = new ProviderRequestStore(connection.db);
const call = () => store.create({
orgId, provider: provider as never, model: "claude", workspaceId: null, promptSha256: "sha", inputCharCount: 40,
executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND, createdAt: new Date().toISOString(),
executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND, createdAt: new Date().toISOString(), purpose: "manual",
});
const priced = call();
store.settleUsage({