Q128: every model call of a run is recorded with its purpose and listed in the run

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ZacharyZhang-NYandClaude Opus 5.5 committed 2026-10-08 20:55:18 -04:00
1 parent 1f484df894
commit 6afd7bef2a
25 files changed
+189 -22

No files matched your search

+1
View File
@@ -79,6 +79,7 @@ export const platformContracts = {
grantConnector: z.strictObject({ orgId: id, agentId: id, toolkitSlug: z.string(), accountLabel: z.string(), employeeId: id }) satisfies z.ZodType<C.GrantConnectorInput>,
revokeConnector: z.strictObject({ orgId: id, agentId: id, toolkitSlug: z.string() }) satisfies z.ZodType<C.RevokeConnectorInput>,
approvalDrifts: z.strictObject({ orgId: id }),
runRequests: z.strictObject({ orgId: id, runId: id }),
search: z.strictObject({
orgId: id, employeeId: id, query: z.string(), entityTypes: z.array(z.enum(SEARCH_ENTITY_TYPES)).optional(), limit: z.number().optional(),
}) satisfies z.ZodType<C.SearchInput>,
+2
View File
@@ -31,6 +31,7 @@ import { SprintService } from "../services/sprint-service";
import { ComposioConnectionService } from "../services/composio/composio-connection-service";
import { ConnectorGrantsService } from "../services/composio/connector-grants";
import { approvalDrifts } from "../services/approval-drift";
import { requestsForRun } from "../services/provider-request-store";
import { setGoalAuditor } from "../services/goal-audit-inputs";
import { IntegrationKeyStore } from "../services/integration-key-store";
import { importStagedSkills, stageBuiltinSkillsLibrary } from "../services/builtin-skills";
@@ -425,6 +426,7 @@ export function registerIpcHandlers(deps: IpcDeps): void {
}
});
handle("grantConnector", (input) => mutate(() => new ConnectorGrantsService(deps.connection.db).grantFromBoard(input)));
handle("runRequests", (input) => ok(requestsForRun(deps.connection.db, input.orgId, input.runId)));
handle("approvalDrifts", (input) => {
try {
return ok(approvalDrifts(deps.connection.db, input.orgId));
@@ -0,0 +1,8 @@
// Requests from before purposes were recorded stay 'unrecorded' rather than being guessed.
export const requestPurposeSchema = `
alter table provider_requests add column purpose text not null default 'unrecorded';
create index if not exists provider_requests_run on provider_requests(task_run_id);
insert into schema_migrations(version, name, applied_at)
values (91, 'request_purpose', datetime('now'));
`;
+2
View File
@@ -89,6 +89,7 @@ import { goalAuditorSchema } from "./migrations/087-goal-auditor";
import { hostAccessNoteSchema } from "./migrations/088-host-access-note";
import { fileVersionsSchema } from "./migrations/089-file-versions";
import { erasureReceiptsSchema } from "./migrations/090-erasure-receipts";
import { requestPurposeSchema } from "./migrations/091-request-purpose";
interface Migration {
version: number;
@@ -187,6 +188,7 @@ const MIGRATIONS: Migration[] = [
{ version: 88, name: "host_access_note", sql: hostAccessNoteSchema },
{ version: 89, name: "file_versions", sql: fileVersionsSchema },
{ version: 90, name: "erasure_receipts", sql: erasureReceiptsSchema },
{ version: 91, name: "request_purpose", sql: requestPurposeSchema },
];
const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version;
+1
View File
@@ -207,6 +207,7 @@ async function runAudit(
fitContextBudget(instructions, messages, auditTools, auditor.inputBudgetTokens, iteration);
throwIfRunStopped();
const turn = await untilStopped(args.providers.runToolTurn({
purpose: "goal_audit",
orgId: args.ctx.orgId,
providerId: auditor.providerId,
model: auditor.model,
+1
View File
@@ -72,6 +72,7 @@ export async function runAgentLoop(deps: AgentRunLoopDeps, input: AgentRunLoopIn
fitContextBudget(input.instructions, messages, allTools, input.inputBudgetTokens, iteration);
throwIfRunStopped();
const turn = await untilStopped(deps.providers.runToolTurn({
purpose: "agent_turn",
orgId: input.orgId,
providerId: input.providerId,
workspaceId: input.workspaceId,
@@ -65,6 +65,7 @@ export class ProviderBackgroundService {
inputCharCount: promptChars(execution.prompt, execution.instructions),
executionMode: PROVIDER_EXECUTION_MODE.BACKGROUND,
createdAt: new Date().toISOString(),
purpose: "manual",
});
input.onRequestCreated?.(requestId);
const startedAt = Date.now();
+17 -2
View File
@@ -1,3 +1,4 @@
import type { RunRequestRecord } from "../../shared/provider-runtime-domain";
import type { DatabaseSync } from "node:sqlite";
import { PROVIDER_EXECUTION_MODE, PROVIDER_REQUEST_STATE, type ProviderExecutionMode } from "../../shared/domain";
import { createId } from "../../shared/ids";
@@ -29,6 +30,19 @@ export interface CreateProviderRequestInput {
taskRunId?: string | null;
/** The agent's department at call time, which its spend is charged to. */
department?: string | null;
/** What the call was for: a run's turn, a goal audit, a model probe, or a request made by hand. */
purpose: RequestPurpose;
}
export type RequestPurpose = "agent_turn" | "goal_audit" | "model_check" | "manual";
/** Every model call a run made, in the order it made them, whatever their outcome. */
export function requestsForRun(db: DatabaseSync, orgId: string, runId: string): RunRequestRecord[] {
return db.prepare(`
select id, purpose, model, state, error_message as errorMessage, input_tokens as inputTokens, output_tokens as outputTokens,
cost_cents as costCents, created_at as createdAt, completed_at as completedAt
from provider_requests where org_id = ? and task_run_id = ? order by rowid
`).all(orgId, runId) as unknown as RunRequestRecord[];
}
export interface ProviderRequestProgressInput {
@@ -116,9 +130,9 @@ export class ProviderRequestStore {
id, org_id, provider_id, provider_key, workspace_id, model, prompt_sha256,
response_text, raw_response_json, state, error_message, input_char_count,
output_char_count, execution_mode, policy_decision, external_response_id,
stream_event_count, cancelled_at, created_at, completed_at, agent_id, task_run_id, department
stream_event_count, cancelled_at, created_at, completed_at, agent_id, task_run_id, department, purpose
)
values (?, ?, ?, ?, ?, ?, ?, null, null, ?, null, ?, 0, ?, ?, null, 0, null, ?, null, ?, ?, ?)
values (?, ?, ?, ?, ?, ?, ?, null, null, ?, null, ?, 0, ?, ?, null, 0, null, ?, null, ?, ?, ?, ?)
`).run(
requestId,
input.orgId,
@@ -135,6 +149,7 @@ export class ProviderRequestStore {
input.agentId ?? null,
input.taskRunId ?? null,
input.department ?? null,
input.purpose,
);
return requestId;
}
@@ -9,7 +9,7 @@ import type { OperationsService } from "./operations-service";
import type { ProviderAdapterRegistry } from "./provider-adapters";
import type { ProviderBackgroundService } from "./provider-background-service";
import type { ProviderRateLimitRegistry } from "./provider-rate-limit-registry";
import type { ProviderRequestStore } from "./provider-request-store";
import type { ProviderRequestStore, RequestPurpose } from "./provider-request-store";
import type { ProviderResponseObserver } from "./provider-response-observer";
import { hashPrompt, readProviderResponseId, requireStreamText } from "./provider-response-utils";
import { incompleteTurn, type AgentTurnMessage, type AgentTurnResult, type ToolDefinition, type ToolTurnFn } from "./provider-tool-turn";
@@ -116,6 +116,8 @@ export class ProviderResponseRunner {
tools: ToolDefinition[];
/** A model chosen for this call (a goal auditor's), outside the session's pinned model and account. */
model?: string;
/** What the call is for, recorded with the request so a run's chain reads plainly. */
purpose: "agent_turn" | "goal_audit";
}): Promise<AgentTurnResult & { requestId: string }> {
const provider = loadConfiguredProvider(this.deps.db, input.orgId, input.providerId);
const workspaceId = this.normalizeWorkspaceId(input.orgId, input.workspaceId ?? null);
@@ -145,13 +147,13 @@ export class ProviderResponseRunner {
const known = readModelCheck(this.deps.db, provider.id, model, requireCredentialId(provider));
const problem = known ? (known.passed ? null : known.detail) : await this.probeModel(caller);
if (problem) throw new Error(modelCheckFailure(provider.provider_key, model, problem));
return this.callTurn(caller, { instructions: input.instructions, messages: input.messages, tools: input.tools });
return this.callTurn(caller, { instructions: input.instructions, messages: input.messages, tools: input.tools }, input.purpose);
}
/** Probes a model for agent work and remembers the verdict; a request that fails records nothing. */
private async probeModel(caller: TurnCaller): Promise<string | null> {
const probe = modelCheckTurn();
const problem = modelCheckProblem(await this.callTurn(caller, probe), probe.code);
const problem = modelCheckProblem(await this.callTurn(caller, probe, "model_check"), probe.code);
const { owner } = caller;
recordModelCheck(this.deps.db, { orgId: caller.orgId, providerId: owner.id, model: owner.model, credentialId: requireCredentialId(owner), problem, now: new Date() });
console.info(`[model-check] ${owner.provider_key} ${owner.model}: ${problem ?? "passed"}`);
@@ -171,7 +173,7 @@ export class ProviderResponseRunner {
return problem === null ? { passed: true } : { passed: false, detail: problem };
}
private async callTurn(caller: TurnCaller, request: { instructions: string | null; messages: AgentTurnMessage[]; tools: ToolDefinition[] }): Promise<AgentTurnResult & { requestId: string }> {
private async callTurn(caller: TurnCaller, request: { instructions: string | null; messages: AgentTurnMessage[]; tools: ToolDefinition[] }, purpose: RequestPurpose): Promise<AgentTurnResult & { requestId: string }> {
const { orgId, owner, workspaceId, agent, taskRunId, turn } = caller;
const model = owner.model;
// Every request, a probe or the work, passes the provider's policy and quota and the agent's budget.
@@ -192,6 +194,7 @@ export class ProviderResponseRunner {
inputCharCount: turnInputChars,
executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND,
createdAt: new Date().toISOString(),
purpose,
});
const startedAt = Date.now();
try {
@@ -395,6 +398,7 @@ export class ProviderResponseRunner {
inputCharCount: promptChars(execution.prompt, execution.instructions),
executionMode: execution.executionMode,
createdAt: new Date().toISOString(),
purpose: "manual",
});
}
+1
View File
@@ -95,6 +95,7 @@ const api: DesktopApi = {
disconnectComposioToolkit: (input) => invoke(IPC_CHANNELS.disconnectComposioToolkit, input),
grantConnector: (input) => invoke(IPC_CHANNELS.grantConnector, input),
approvalDrifts: (input) => invoke(IPC_CHANNELS.approvalDrifts, input),
runRequests: (input) => invoke(IPC_CHANNELS.runRequests, input),
revokeConnector: (input) => invoke(IPC_CHANNELS.revokeConnector, input),
refreshComposioAccounts: (input) => invoke(IPC_CHANNELS.refreshComposioAccounts, input),
search: (input) => invoke(IPC_CHANNELS.search, input),
+7 -1
View File
@@ -172,6 +172,13 @@ export const zh: Record<string, string> = {
"Check": "检查",
"Choose": "选择",
"No findings": "未发现问题",
"Model calls ({count})": "模型调用({count})",
"This run made no model call.": "本次运行没有调用模型。",
"Turn": "轮次",
"Goal audit": "目标审查",
"Model check": "模型检查",
"Manual": "手动",
"Earlier call": "早期调用",
"Partly scanned": "部分扫描",
"Scanned before whole-file checks existed: only its first MiB was checked.": "在全文件检查之前扫描:只检查了前 1 MiB。",
"Contents not inspected": "内容未检查",
@@ -395,7 +402,6 @@ export const zh: Record<string, string> = {
"Replaced by an edit": "已因编辑被替换",
"Changed since approval": "批准后已改动",
"Removed since approval": "批准后已删除",
"Goal audit": "目标审查",
"CEO: hires officers, grants connectors and skills, and closes goals. This rank cannot be changed.": "CEO:招聘高管、授予连接器和技能、关闭目标。此职级不可更改。",
"An independent model checks every goal before it closes. Pick one your agents do not use, ideally from another provider.": "每个目标关闭前都由一个独立模型检查。请选择 Agent 不使用的模型,最好来自另一家服务商。",
"No auditor is set, so goals cannot be closed yet.": "尚未设置审查模型,目标暂时无法关闭。",
+3
View File
@@ -1,3 +1,4 @@
import { RunRequestList } from "./runs/RunRequestList";
import { useMemo, useState } from "react";
import { Avatar, Button, Pill, Segmented } from "../shared/ui";
import type { AgentRecord, ProviderRecord, TaskRecord, TaskRunRecord } from "../../../shared/domain";
@@ -147,6 +148,8 @@ function RunDetail({ run, tasks, agents, providers, providerRequests, actions }:
<MetaLabel>{t("Context")}</MetaLabel><div className="mono" style={{ fontSize: 12, overflow: "hidden", textOverflow: "ellipsis" }}>{run.executionContextId}</div>
</div>
<RunRequestList orgId={run.orgId} runId={run.id} state={run.state} />
{run.state === "superseded" && run.errorMessage && (
<div className="card" style={{ padding: 16, marginBottom: 16 }}>
<div className="section-label" style={{ marginBottom: 8 }}>{t("Why it did not run")}</div>
@@ -0,0 +1,54 @@
import { useEffect, useState } from "react";
import type { RunRequestRecord } from "../../../../shared/provider-runtime-domain";
import { Pill } from "../../shared/ui";
import { t } from "../../i18n/i18n";
function purposeLabel(purpose: string): string {
if (purpose === "agent_turn") return t("Turn");
if (purpose === "goal_audit") return t("Goal audit");
if (purpose === "model_check") return t("Model check");
if (purpose === "manual") return t("Manual");
return t("Earlier call");
}
/** Every model call the run made, in order, so a turn or an audit can be found by its task. */
export function RunRequestList({ orgId, runId, state }: { orgId: string; runId: string; state: string }) {
// Fetched calls are kept with the run they belong to, so switching runs never shows another run's chain.
const [loaded, setLoaded] = useState<{ runId: string; requests: RunRequestRecord[] } | { runId: string; error: string } | null>(null);
useEffect(() => {
let cancelled = false;
window.agentCompany.runRequests({ orgId, runId })
.then((result) => {
if (cancelled) return;
setLoaded(result.ok ? { runId, requests: result.data } : { runId, error: result.error.message });
})
.catch((reason: unknown) => {
if (!cancelled) setLoaded({ runId, error: reason instanceof Error ? reason.message : String(reason) });
});
return () => {
cancelled = true;
};
}, [orgId, runId, state]);
if (!loaded || loaded.runId !== runId) return null;
if ("error" in loaded) return <div className="form-error" role="alert">{loaded.error}</div>;
const { requests } = loaded;
return (
<div style={{ marginBottom: 20 }}>
<div className="section-label" style={{ marginBottom: 8 }}>{t("Model calls ({count})", { count: String(requests.length) })}</div>
{requests.length === 0 && <div style={{ fontSize: 13, color: "var(--text-3)" }}>{t("This run made no model call.")}</div>}
{requests.map((request, index) => (
<div key={request.id} className="row" style={{ gap: 10, fontSize: 13, padding: "6px 0", borderBottom: "1px solid var(--border-soft)" }}>
<span className="mono" style={{ color: "var(--text-3)", width: 24 }}>{index + 1}</span>
<span style={{ width: 96 }}>{purposeLabel(request.purpose)}</span>
<span className="mono" style={{ flex: 1, minWidth: 0, overflow: "hidden", textOverflow: "ellipsis" }}>{request.model}</span>
<Pill tone={request.state === "failed" ? "red" : request.state === "succeeded" ? "green" : "amber"}>{request.state}</Pill>
<span className="mono" style={{ color: "var(--text-3)", width: 120, textAlign: "right" }}>
{request.inputTokens ?? "—"} / {request.outputTokens ?? "—"}
</span>
{request.errorMessage && <span style={{ color: "var(--red)", flexBasis: "100%", whiteSpace: "pre-wrap" }}>{request.errorMessage}</span>}
</div>
))}
</div>
);
}
+3
View File
@@ -1,3 +1,4 @@
import type { RunRequestRecord } from "./provider-runtime-domain";
import type { ApprovalDrift } from "./approval-pin";
import type { DesktopResult } from "./result";
import type { AppSnapshot, SearchResultRecord } from "./domain";
@@ -207,6 +208,8 @@ export interface DesktopApi {
grantConnector(input: GrantConnectorInput): Promise<DesktopResult<AppSnapshot>>;
/** Approved deliverables whose content changed or vanished since approval, checked on disk now. */
approvalDrifts(input: { orgId: string }): Promise<DesktopResult<Record<string, ApprovalDrift>>>;
/** Every model call a task run made, in order: turns, goal audits and model checks. */
runRequests(input: { orgId: string; runId: string }): Promise<DesktopResult<RunRequestRecord[]>>;
revokeConnector(input: RevokeConnectorInput): Promise<DesktopResult<AppSnapshot>>;
search(input: SearchInput): Promise<DesktopResult<SearchResultRecord[]>>;
subscribe(callback: (event: RendererEvent) => void): () => void;
+1
View File
@@ -88,6 +88,7 @@ export const IPC_CHANNELS = {
refreshComposioAccounts: "agentcompany:refresh-composio-accounts",
grantConnector: "agentcompany:grant-connector",
approvalDrifts: "agentcompany:approval-drifts",
runRequests: "agentcompany:run-requests",
revokeConnector: "agentcompany:revoke-connector",
search: "agentcompany:search",
rendererEvent: "agentcompany:renderer-event",
+14
View File
@@ -43,3 +43,17 @@ export function currentProviderUsageWindow(now = new Date()): { start: string; e
const end = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + 1, 1));
return { start: start.toISOString(), end: end.toISOString() };
}
/** One model call of a task run, as the run's detail lists it. */
export interface RunRequestRecord {
id: string;
purpose: string;
model: string;
state: string;
errorMessage: string | null;
inputTokens: number | null;
outputTokens: number | null;
costCents: number | null;
createdAt: string;
completedAt: string | null;
}
+50
View File
@@ -0,0 +1,50 @@
import { randomUUID } from "node:crypto";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, test } from "vitest";
import { DatabaseConnection } from "../../src/main/database/connection";
import { OnboardingService } from "../../src/main/services/onboarding-service";
import { ProviderService } from "../../src/main/services/provider-service";
import { providerAdapters } from "../../src/main/services/provider-adapters";
import { TaskExecutionService } from "../../src/main/services/task-execution-service";
import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
import { requestsForRun } from "../../src/main/services/provider-request-store";
import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
import { answeringModelCheck } from "../helpers/model-check";
const testVault = {
encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
decryptSecret: (ciphertext: Buffer) => ciphertext.toString("utf8"),
};
describe("a run's model calls", () => {
test("every call a run made is listed in order with its purpose, including the turn that failed", async () => {
let turns = 0;
const turn = async (_input: ProviderToolTurnInput): Promise<AgentTurnResult> => {
turns += 1;
if (turns === 1) return { text: "Checking in.", toolCalls: [{ id: "c1", name: "company_status", arguments: {} }], rawResponseJson: "{}" };
throw new Error("Provider overloaded");
};
const connection = new DatabaseConnection(join(tmpdir(), `ac-run-requests-${randomUUID()}.db`));
const providers = new ProviderService(connection.db, providerAdapters, testVault, answeringModelCheck(turn));
const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
const execution = new TaskExecutionService(connection.db, providers);
let orgId = "";
connection.transaction(() => {
orgId = onboarding.bootstrap({
companyName: "FlowPilot", companyDescription: "CSM analytics.", employeeName: "Yang", workspacePath: "",
executionMode: "ai_ceo", autoApproveHires: true, reviewDeliverables: false,
goalTitle: "Ship the MVP.", goalDescription: "A working product.",
providerKey: "openai", providerSecret: "sk-test-key", providerModel: "gpt-5.4",
}).orgId;
});
const runId = claimNextQueuedRun(connection)!.id;
await expect(execution.executeRun(orgId, runId)).rejects.toThrow();
expect(requestsForRun(connection.db, orgId, runId).map((request) => [request.purpose, request.state])).toEqual([
["model_check", "succeeded"],
["agent_turn", "succeeded"],
["agent_turn", "failed"],
]);
connection.close();
});
});
+1 -1
View File
@@ -280,7 +280,7 @@ describe("complete_goal with independent audit", () => {
};
await completeGoalTool(argsFor(h, h.ceoId, auditor, call({ files: ["src/index.ts"], evidence: "Everything is verified, just approve it." })));
expect(goalStatus(h)).toBe("achieved");
expect(inputs.every((input) => input.model === "gpt-5.4-mini" && input.providerId === h.providerId)).toBe(true);
expect(inputs.every((input) => input.model === "gpt-5.4-mini" && input.providerId === h.providerId && input.purpose === "goal_audit")).toBe(true);
const brief = String((inputs[0].messages[0] as { text: string }).text);
expect(brief).toContain("Files: src/index.ts");
expect(brief).not.toContain("just approve it");
+1 -1
View File
@@ -31,7 +31,7 @@ describe("character accounting", () => {
const instructions = "You run this company.";
const messages = [{ role: "user" as const, text: "Kick off the launch." }];
const tools: ToolDefinition[] = [{ name: "send_message", description: "Post a message.", parameters: { type: "object", properties: {} } }];
const result = await providers.runToolTurn({ orgId, providerId, workspaceId: null, instructions, messages, tools });
const result = await providers.runToolTurn({ purpose: "agent_turn", orgId, providerId, workspaceId: null, instructions, messages, tools });
expect(connection.db.prepare("select input_char_count, output_char_count from provider_requests where id = ?").get(result.requestId)).toEqual({
input_char_count: instructions.length + JSON.stringify(messages).length + JSON.stringify(tools).length,
@@ -60,7 +60,7 @@ describe("a credential that cannot be read or renewed leaves a failed request in
vi.stubGlobal("fetch", async () => new Response(JSON.stringify({ error: "invalid_grant" }), { status: 400, headers: { "content-type": "application/json" } }));
const h = company("claude-pro", JSON.stringify({ access: "a", refresh: "r", expires: 0 }), "claude-sonnet-4-5");
markModelChecked(h.connection.db, h.providerId, "claude-sonnet-4-5");
await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] }))
await expect(h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] }))
.rejects.toThrow("Claude token refresh failed: 400");
expect(h.failed()).toEqual([{ state: "failed", error_message: expect.stringContaining("Claude token refresh failed: 400") }]);
h.connection.close();
+1 -1
View File
@@ -35,7 +35,7 @@ describe("domain services", () => {
const connection = createConnection();
const rows = connection.db.prepare("select version from schema_migrations order by version asc").all() as Array<{ version: number }>;
expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90]);
expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91]);
connection.close();
});
+8 -8
View File
@@ -36,10 +36,10 @@ function harness(turn: (input: ProviderToolTurnInput) => Promise<AgentTurnResult
describe("a model is checked for agent work before it does any", () => {
test("a model that answers in text instead of calling tools is refused, and the verdict is kept", async () => {
const h = harness(TEXT_ONLY);
await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK }))
await expect(h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK }))
.rejects.toThrow("gpt-5.4 on openai failed the agent check: it answered in text instead of calling the tool it was given");
expect(h.requests()).toBe(1);
await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("failed the agent check");
await expect(h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("failed the agent check");
expect(h.requests()).toBe(1);
h.connection.close();
});
@@ -47,13 +47,13 @@ describe("a model is checked for agent work before it does any", () => {
test("a capable model is probed once, then works; the probe is a request like any other and counts against the quota", async () => {
const seen: ProviderToolTurnInput[] = [];
const h = harness(answeringModelCheck(async (input) => { seen.push(input); return { text: "Done.", toolCalls: [], rawResponseJson: "{}" }; }));
expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
expect((await h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
expect((await h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
expect([h.requests(), seen.length]).toEqual([3, 2]);
const tight = harness(answeringModelCheck(async () => ({ text: "Done.", toolCalls: [], rawResponseJson: "{}" })));
tight.providers.updatePolicy({ orgId: tight.orgId, providerId: tight.providerId, dataPolicy: PROVIDER_DATA_POLICY.STANDARD, backgroundEnabled: false, monthlyRequestLimit: 1, workspaceScope: "organization", contextWindowTokens: 128_000 });
await expect(tight.providers.runToolTurn({ orgId: tight.orgId, providerId: tight.providerId, ...WORK })).rejects.toThrow(PROVIDER_MONTHLY_LIMIT_MESSAGE);
await expect(tight.providers.runToolTurn({ purpose: "agent_turn", orgId: tight.orgId, providerId: tight.providerId, ...WORK })).rejects.toThrow(PROVIDER_MONTHLY_LIMIT_MESSAGE);
expect(tight.requests()).toBe(1);
h.connection.close();
tight.connection.close();
@@ -66,7 +66,7 @@ describe("a model is checked for agent work before it does any", () => {
if (mode === "down") throw new Error("connect ECONNREFUSED");
return mode === "text" ? TEXT_ONLY() : capable(input);
});
await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("ECONNREFUSED");
await expect(h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("ECONNREFUSED");
expect(h.connection.db.prepare("select count(*) as n from model_checks").get()).toEqual({ n: 0 });
mode = "text";
@@ -74,11 +74,11 @@ describe("a model is checked for agent work before it does any", () => {
.toEqual({ passed: false, detail: "it answered in text instead of calling the tool it was given" });
mode = "capable";
expect(await h.providers.checkModel({ orgId: h.orgId, providerId: h.providerId, model: "gpt-5.4" })).toEqual({ passed: true });
expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("ok");
expect((await h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("ok");
h.connection.db.prepare("update providers set model = 'gpt-5.5' where id = ?").run(h.providerId);
mode = "text";
await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("gpt-5.5 on openai failed the agent check");
await expect(h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("gpt-5.5 on openai failed the agent check");
h.connection.close();
});
});
+1 -1
View File
@@ -118,7 +118,7 @@ describe("provider errors keep their status and machine code", () => {
providers.saveCredential({ orgId, providerId, secret: "sk-test", model: "gpt-5.4" });
markModelChecked(connection.db, providerId, "gpt-5.4");
});
const call = () => providers.runToolTurn({ orgId, providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] });
const call = () => providers.runToolTurn({ purpose: "agent_turn", orgId, providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] });
await expect(call()).rejects.toThrow("Overloaded");
expect(connection.db.prepare("select error_status, error_code from provider_requests").get()).toEqual({ error_status: 529, error_code: "overloaded_error" });
+1 -1
View File
@@ -37,7 +37,7 @@ describe("monthly quota reservation", () => {
markModelChecked(connection.db, providerId, "gpt-5.4");
providers.updatePolicy({ orgId, providerId, dataPolicy: PROVIDER_DATA_POLICY.STANDARD, backgroundEnabled: false, monthlyRequestLimit: 1, workspaceScope: "organization", contextWindowTokens: 128_000 });
});
const call = () => providers.runToolTurn({ orgId, providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] });
const call = () => providers.runToolTurn({ purpose: "agent_turn", orgId, providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] });
const first = call();
const second = call();
await expect(second).rejects.toThrow(PROVIDER_MONTHLY_LIMIT_MESSAGE);
+1 -1
View File
@@ -22,7 +22,7 @@ describe("usage settlement", () => {
const store = new ProviderRequestStore(connection.db);
const call = () => store.create({
orgId, provider: provider as never, model: "claude", workspaceId: null, promptSha256: "sha", inputCharCount: 40,
executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND, createdAt: new Date().toISOString(),
executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND, createdAt: new Date().toISOString(), purpose: "manual",
});
const priced = call();
store.settleUsage({