Q128: every model call of a run is recorded with its purpose and listed in the run
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1f484df894
commit
6afd7bef2a
25 files changed
+189
-22
No files matched your search
@@ -79,6 +79,7 @@ export const platformContracts = {
|
||||
grantConnector: z.strictObject({ orgId: id, agentId: id, toolkitSlug: z.string(), accountLabel: z.string(), employeeId: id }) satisfies z.ZodType<C.GrantConnectorInput>,
|
||||
revokeConnector: z.strictObject({ orgId: id, agentId: id, toolkitSlug: z.string() }) satisfies z.ZodType<C.RevokeConnectorInput>,
|
||||
approvalDrifts: z.strictObject({ orgId: id }),
|
||||
runRequests: z.strictObject({ orgId: id, runId: id }),
|
||||
search: z.strictObject({
|
||||
orgId: id, employeeId: id, query: z.string(), entityTypes: z.array(z.enum(SEARCH_ENTITY_TYPES)).optional(), limit: z.number().optional(),
|
||||
}) satisfies z.ZodType<C.SearchInput>,
|
||||
|
||||
@@ -31,6 +31,7 @@ import { SprintService } from "../services/sprint-service";
|
||||
import { ComposioConnectionService } from "../services/composio/composio-connection-service";
|
||||
import { ConnectorGrantsService } from "../services/composio/connector-grants";
|
||||
import { approvalDrifts } from "../services/approval-drift";
|
||||
import { requestsForRun } from "../services/provider-request-store";
|
||||
import { setGoalAuditor } from "../services/goal-audit-inputs";
|
||||
import { IntegrationKeyStore } from "../services/integration-key-store";
|
||||
import { importStagedSkills, stageBuiltinSkillsLibrary } from "../services/builtin-skills";
|
||||
@@ -425,6 +426,7 @@ export function registerIpcHandlers(deps: IpcDeps): void {
|
||||
}
|
||||
});
|
||||
handle("grantConnector", (input) => mutate(() => new ConnectorGrantsService(deps.connection.db).grantFromBoard(input)));
|
||||
handle("runRequests", (input) => ok(requestsForRun(deps.connection.db, input.orgId, input.runId)));
|
||||
handle("approvalDrifts", (input) => {
|
||||
try {
|
||||
return ok(approvalDrifts(deps.connection.db, input.orgId));
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
// Requests from before purposes were recorded stay 'unrecorded' rather than being guessed.
|
||||
export const requestPurposeSchema = `
|
||||
alter table provider_requests add column purpose text not null default 'unrecorded';
|
||||
create index if not exists provider_requests_run on provider_requests(task_run_id);
|
||||
|
||||
insert into schema_migrations(version, name, applied_at)
|
||||
values (91, 'request_purpose', datetime('now'));
|
||||
`;
|
||||
@@ -89,6 +89,7 @@ import { goalAuditorSchema } from "./migrations/087-goal-auditor";
|
||||
import { hostAccessNoteSchema } from "./migrations/088-host-access-note";
|
||||
import { fileVersionsSchema } from "./migrations/089-file-versions";
|
||||
import { erasureReceiptsSchema } from "./migrations/090-erasure-receipts";
|
||||
import { requestPurposeSchema } from "./migrations/091-request-purpose";
|
||||
|
||||
interface Migration {
|
||||
version: number;
|
||||
@@ -187,6 +188,7 @@ const MIGRATIONS: Migration[] = [
|
||||
{ version: 88, name: "host_access_note", sql: hostAccessNoteSchema },
|
||||
{ version: 89, name: "file_versions", sql: fileVersionsSchema },
|
||||
{ version: 90, name: "erasure_receipts", sql: erasureReceiptsSchema },
|
||||
{ version: 91, name: "request_purpose", sql: requestPurposeSchema },
|
||||
];
|
||||
|
||||
const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version;
|
||||
|
||||
@@ -207,6 +207,7 @@ async function runAudit(
|
||||
fitContextBudget(instructions, messages, auditTools, auditor.inputBudgetTokens, iteration);
|
||||
throwIfRunStopped();
|
||||
const turn = await untilStopped(args.providers.runToolTurn({
|
||||
purpose: "goal_audit",
|
||||
orgId: args.ctx.orgId,
|
||||
providerId: auditor.providerId,
|
||||
model: auditor.model,
|
||||
|
||||
@@ -72,6 +72,7 @@ export async function runAgentLoop(deps: AgentRunLoopDeps, input: AgentRunLoopIn
|
||||
fitContextBudget(input.instructions, messages, allTools, input.inputBudgetTokens, iteration);
|
||||
throwIfRunStopped();
|
||||
const turn = await untilStopped(deps.providers.runToolTurn({
|
||||
purpose: "agent_turn",
|
||||
orgId: input.orgId,
|
||||
providerId: input.providerId,
|
||||
workspaceId: input.workspaceId,
|
||||
|
||||
@@ -65,6 +65,7 @@ export class ProviderBackgroundService {
|
||||
inputCharCount: promptChars(execution.prompt, execution.instructions),
|
||||
executionMode: PROVIDER_EXECUTION_MODE.BACKGROUND,
|
||||
createdAt: new Date().toISOString(),
|
||||
purpose: "manual",
|
||||
});
|
||||
input.onRequestCreated?.(requestId);
|
||||
const startedAt = Date.now();
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { RunRequestRecord } from "../../shared/provider-runtime-domain";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { PROVIDER_EXECUTION_MODE, PROVIDER_REQUEST_STATE, type ProviderExecutionMode } from "../../shared/domain";
|
||||
import { createId } from "../../shared/ids";
|
||||
@@ -29,6 +30,19 @@ export interface CreateProviderRequestInput {
|
||||
taskRunId?: string | null;
|
||||
/** The agent's department at call time, which its spend is charged to. */
|
||||
department?: string | null;
|
||||
/** What the call was for: a run's turn, a goal audit, a model probe, or a request made by hand. */
|
||||
purpose: RequestPurpose;
|
||||
}
|
||||
|
||||
export type RequestPurpose = "agent_turn" | "goal_audit" | "model_check" | "manual";
|
||||
|
||||
/** Every model call a run made, in the order it made them, whatever their outcome. */
|
||||
export function requestsForRun(db: DatabaseSync, orgId: string, runId: string): RunRequestRecord[] {
|
||||
return db.prepare(`
|
||||
select id, purpose, model, state, error_message as errorMessage, input_tokens as inputTokens, output_tokens as outputTokens,
|
||||
cost_cents as costCents, created_at as createdAt, completed_at as completedAt
|
||||
from provider_requests where org_id = ? and task_run_id = ? order by rowid
|
||||
`).all(orgId, runId) as unknown as RunRequestRecord[];
|
||||
}
|
||||
|
||||
export interface ProviderRequestProgressInput {
|
||||
@@ -116,9 +130,9 @@ export class ProviderRequestStore {
|
||||
id, org_id, provider_id, provider_key, workspace_id, model, prompt_sha256,
|
||||
response_text, raw_response_json, state, error_message, input_char_count,
|
||||
output_char_count, execution_mode, policy_decision, external_response_id,
|
||||
stream_event_count, cancelled_at, created_at, completed_at, agent_id, task_run_id, department
|
||||
stream_event_count, cancelled_at, created_at, completed_at, agent_id, task_run_id, department, purpose
|
||||
)
|
||||
values (?, ?, ?, ?, ?, ?, ?, null, null, ?, null, ?, 0, ?, ?, null, 0, null, ?, null, ?, ?, ?)
|
||||
values (?, ?, ?, ?, ?, ?, ?, null, null, ?, null, ?, 0, ?, ?, null, 0, null, ?, null, ?, ?, ?, ?)
|
||||
`).run(
|
||||
requestId,
|
||||
input.orgId,
|
||||
@@ -135,6 +149,7 @@ export class ProviderRequestStore {
|
||||
input.agentId ?? null,
|
||||
input.taskRunId ?? null,
|
||||
input.department ?? null,
|
||||
input.purpose,
|
||||
);
|
||||
return requestId;
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ import type { OperationsService } from "./operations-service";
|
||||
import type { ProviderAdapterRegistry } from "./provider-adapters";
|
||||
import type { ProviderBackgroundService } from "./provider-background-service";
|
||||
import type { ProviderRateLimitRegistry } from "./provider-rate-limit-registry";
|
||||
import type { ProviderRequestStore } from "./provider-request-store";
|
||||
import type { ProviderRequestStore, RequestPurpose } from "./provider-request-store";
|
||||
import type { ProviderResponseObserver } from "./provider-response-observer";
|
||||
import { hashPrompt, readProviderResponseId, requireStreamText } from "./provider-response-utils";
|
||||
import { incompleteTurn, type AgentTurnMessage, type AgentTurnResult, type ToolDefinition, type ToolTurnFn } from "./provider-tool-turn";
|
||||
@@ -116,6 +116,8 @@ export class ProviderResponseRunner {
|
||||
tools: ToolDefinition[];
|
||||
/** A model chosen for this call (a goal auditor's), outside the session's pinned model and account. */
|
||||
model?: string;
|
||||
/** What the call is for, recorded with the request so a run's chain reads plainly. */
|
||||
purpose: "agent_turn" | "goal_audit";
|
||||
}): Promise<AgentTurnResult & { requestId: string }> {
|
||||
const provider = loadConfiguredProvider(this.deps.db, input.orgId, input.providerId);
|
||||
const workspaceId = this.normalizeWorkspaceId(input.orgId, input.workspaceId ?? null);
|
||||
@@ -145,13 +147,13 @@ export class ProviderResponseRunner {
|
||||
const known = readModelCheck(this.deps.db, provider.id, model, requireCredentialId(provider));
|
||||
const problem = known ? (known.passed ? null : known.detail) : await this.probeModel(caller);
|
||||
if (problem) throw new Error(modelCheckFailure(provider.provider_key, model, problem));
|
||||
return this.callTurn(caller, { instructions: input.instructions, messages: input.messages, tools: input.tools });
|
||||
return this.callTurn(caller, { instructions: input.instructions, messages: input.messages, tools: input.tools }, input.purpose);
|
||||
}
|
||||
|
||||
/** Probes a model for agent work and remembers the verdict; a request that fails records nothing. */
|
||||
private async probeModel(caller: TurnCaller): Promise<string | null> {
|
||||
const probe = modelCheckTurn();
|
||||
const problem = modelCheckProblem(await this.callTurn(caller, probe), probe.code);
|
||||
const problem = modelCheckProblem(await this.callTurn(caller, probe, "model_check"), probe.code);
|
||||
const { owner } = caller;
|
||||
recordModelCheck(this.deps.db, { orgId: caller.orgId, providerId: owner.id, model: owner.model, credentialId: requireCredentialId(owner), problem, now: new Date() });
|
||||
console.info(`[model-check] ${owner.provider_key} ${owner.model}: ${problem ?? "passed"}`);
|
||||
@@ -171,7 +173,7 @@ export class ProviderResponseRunner {
|
||||
return problem === null ? { passed: true } : { passed: false, detail: problem };
|
||||
}
|
||||
|
||||
private async callTurn(caller: TurnCaller, request: { instructions: string | null; messages: AgentTurnMessage[]; tools: ToolDefinition[] }): Promise<AgentTurnResult & { requestId: string }> {
|
||||
private async callTurn(caller: TurnCaller, request: { instructions: string | null; messages: AgentTurnMessage[]; tools: ToolDefinition[] }, purpose: RequestPurpose): Promise<AgentTurnResult & { requestId: string }> {
|
||||
const { orgId, owner, workspaceId, agent, taskRunId, turn } = caller;
|
||||
const model = owner.model;
|
||||
// Every request, a probe or the work, passes the provider's policy and quota and the agent's budget.
|
||||
@@ -192,6 +194,7 @@ export class ProviderResponseRunner {
|
||||
inputCharCount: turnInputChars,
|
||||
executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND,
|
||||
createdAt: new Date().toISOString(),
|
||||
purpose,
|
||||
});
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
@@ -395,6 +398,7 @@ export class ProviderResponseRunner {
|
||||
inputCharCount: promptChars(execution.prompt, execution.instructions),
|
||||
executionMode: execution.executionMode,
|
||||
createdAt: new Date().toISOString(),
|
||||
purpose: "manual",
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -95,6 +95,7 @@ const api: DesktopApi = {
|
||||
disconnectComposioToolkit: (input) => invoke(IPC_CHANNELS.disconnectComposioToolkit, input),
|
||||
grantConnector: (input) => invoke(IPC_CHANNELS.grantConnector, input),
|
||||
approvalDrifts: (input) => invoke(IPC_CHANNELS.approvalDrifts, input),
|
||||
runRequests: (input) => invoke(IPC_CHANNELS.runRequests, input),
|
||||
revokeConnector: (input) => invoke(IPC_CHANNELS.revokeConnector, input),
|
||||
refreshComposioAccounts: (input) => invoke(IPC_CHANNELS.refreshComposioAccounts, input),
|
||||
search: (input) => invoke(IPC_CHANNELS.search, input),
|
||||
|
||||
@@ -172,6 +172,13 @@ export const zh: Record<string, string> = {
|
||||
"Check": "检查",
|
||||
"Choose": "选择",
|
||||
"No findings": "未发现问题",
|
||||
"Model calls ({count})": "模型调用({count})",
|
||||
"This run made no model call.": "本次运行没有调用模型。",
|
||||
"Turn": "轮次",
|
||||
"Goal audit": "目标审查",
|
||||
"Model check": "模型检查",
|
||||
"Manual": "手动",
|
||||
"Earlier call": "早期调用",
|
||||
"Partly scanned": "部分扫描",
|
||||
"Scanned before whole-file checks existed: only its first MiB was checked.": "在全文件检查之前扫描:只检查了前 1 MiB。",
|
||||
"Contents not inspected": "内容未检查",
|
||||
@@ -395,7 +402,6 @@ export const zh: Record<string, string> = {
|
||||
"Replaced by an edit": "已因编辑被替换",
|
||||
"Changed since approval": "批准后已改动",
|
||||
"Removed since approval": "批准后已删除",
|
||||
"Goal audit": "目标审查",
|
||||
"CEO: hires officers, grants connectors and skills, and closes goals. This rank cannot be changed.": "CEO:招聘高管、授予连接器和技能、关闭目标。此职级不可更改。",
|
||||
"An independent model checks every goal before it closes. Pick one your agents do not use, ideally from another provider.": "每个目标关闭前都由一个独立模型检查。请选择 Agent 不使用的模型,最好来自另一家服务商。",
|
||||
"No auditor is set, so goals cannot be closed yet.": "尚未设置审查模型,目标暂时无法关闭。",
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { RunRequestList } from "./runs/RunRequestList";
|
||||
import { useMemo, useState } from "react";
|
||||
import { Avatar, Button, Pill, Segmented } from "../shared/ui";
|
||||
import type { AgentRecord, ProviderRecord, TaskRecord, TaskRunRecord } from "../../../shared/domain";
|
||||
@@ -147,6 +148,8 @@ function RunDetail({ run, tasks, agents, providers, providerRequests, actions }:
|
||||
<MetaLabel>{t("Context")}</MetaLabel><div className="mono" style={{ fontSize: 12, overflow: "hidden", textOverflow: "ellipsis" }}>{run.executionContextId}</div>
|
||||
</div>
|
||||
|
||||
<RunRequestList orgId={run.orgId} runId={run.id} state={run.state} />
|
||||
|
||||
{run.state === "superseded" && run.errorMessage && (
|
||||
<div className="card" style={{ padding: 16, marginBottom: 16 }}>
|
||||
<div className="section-label" style={{ marginBottom: 8 }}>{t("Why it did not run")}</div>
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import type { RunRequestRecord } from "../../../../shared/provider-runtime-domain";
|
||||
import { Pill } from "../../shared/ui";
|
||||
import { t } from "../../i18n/i18n";
|
||||
|
||||
function purposeLabel(purpose: string): string {
|
||||
if (purpose === "agent_turn") return t("Turn");
|
||||
if (purpose === "goal_audit") return t("Goal audit");
|
||||
if (purpose === "model_check") return t("Model check");
|
||||
if (purpose === "manual") return t("Manual");
|
||||
return t("Earlier call");
|
||||
}
|
||||
|
||||
/** Every model call the run made, in order, so a turn or an audit can be found by its task. */
|
||||
export function RunRequestList({ orgId, runId, state }: { orgId: string; runId: string; state: string }) {
|
||||
// Fetched calls are kept with the run they belong to, so switching runs never shows another run's chain.
|
||||
const [loaded, setLoaded] = useState<{ runId: string; requests: RunRequestRecord[] } | { runId: string; error: string } | null>(null);
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
window.agentCompany.runRequests({ orgId, runId })
|
||||
.then((result) => {
|
||||
if (cancelled) return;
|
||||
setLoaded(result.ok ? { runId, requests: result.data } : { runId, error: result.error.message });
|
||||
})
|
||||
.catch((reason: unknown) => {
|
||||
if (!cancelled) setLoaded({ runId, error: reason instanceof Error ? reason.message : String(reason) });
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [orgId, runId, state]);
|
||||
|
||||
if (!loaded || loaded.runId !== runId) return null;
|
||||
if ("error" in loaded) return <div className="form-error" role="alert">{loaded.error}</div>;
|
||||
const { requests } = loaded;
|
||||
return (
|
||||
<div style={{ marginBottom: 20 }}>
|
||||
<div className="section-label" style={{ marginBottom: 8 }}>{t("Model calls ({count})", { count: String(requests.length) })}</div>
|
||||
{requests.length === 0 && <div style={{ fontSize: 13, color: "var(--text-3)" }}>{t("This run made no model call.")}</div>}
|
||||
{requests.map((request, index) => (
|
||||
<div key={request.id} className="row" style={{ gap: 10, fontSize: 13, padding: "6px 0", borderBottom: "1px solid var(--border-soft)" }}>
|
||||
<span className="mono" style={{ color: "var(--text-3)", width: 24 }}>{index + 1}</span>
|
||||
<span style={{ width: 96 }}>{purposeLabel(request.purpose)}</span>
|
||||
<span className="mono" style={{ flex: 1, minWidth: 0, overflow: "hidden", textOverflow: "ellipsis" }}>{request.model}</span>
|
||||
<Pill tone={request.state === "failed" ? "red" : request.state === "succeeded" ? "green" : "amber"}>{request.state}</Pill>
|
||||
<span className="mono" style={{ color: "var(--text-3)", width: 120, textAlign: "right" }}>
|
||||
{request.inputTokens ?? "—"} / {request.outputTokens ?? "—"}
|
||||
</span>
|
||||
{request.errorMessage && <span style={{ color: "var(--red)", flexBasis: "100%", whiteSpace: "pre-wrap" }}>{request.errorMessage}</span>}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { RunRequestRecord } from "./provider-runtime-domain";
|
||||
import type { ApprovalDrift } from "./approval-pin";
|
||||
import type { DesktopResult } from "./result";
|
||||
import type { AppSnapshot, SearchResultRecord } from "./domain";
|
||||
@@ -207,6 +208,8 @@ export interface DesktopApi {
|
||||
grantConnector(input: GrantConnectorInput): Promise<DesktopResult<AppSnapshot>>;
|
||||
/** Approved deliverables whose content changed or vanished since approval, checked on disk now. */
|
||||
approvalDrifts(input: { orgId: string }): Promise<DesktopResult<Record<string, ApprovalDrift>>>;
|
||||
/** Every model call a task run made, in order: turns, goal audits and model checks. */
|
||||
runRequests(input: { orgId: string; runId: string }): Promise<DesktopResult<RunRequestRecord[]>>;
|
||||
revokeConnector(input: RevokeConnectorInput): Promise<DesktopResult<AppSnapshot>>;
|
||||
search(input: SearchInput): Promise<DesktopResult<SearchResultRecord[]>>;
|
||||
subscribe(callback: (event: RendererEvent) => void): () => void;
|
||||
|
||||
@@ -88,6 +88,7 @@ export const IPC_CHANNELS = {
|
||||
refreshComposioAccounts: "agentcompany:refresh-composio-accounts",
|
||||
grantConnector: "agentcompany:grant-connector",
|
||||
approvalDrifts: "agentcompany:approval-drifts",
|
||||
runRequests: "agentcompany:run-requests",
|
||||
revokeConnector: "agentcompany:revoke-connector",
|
||||
search: "agentcompany:search",
|
||||
rendererEvent: "agentcompany:renderer-event",
|
||||
|
||||
@@ -43,3 +43,17 @@ export function currentProviderUsageWindow(now = new Date()): { start: string; e
|
||||
const end = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + 1, 1));
|
||||
return { start: start.toISOString(), end: end.toISOString() };
|
||||
}
|
||||
|
||||
/** One model call of a task run, as the run's detail lists it. */
|
||||
export interface RunRequestRecord {
|
||||
id: string;
|
||||
purpose: string;
|
||||
model: string;
|
||||
state: string;
|
||||
errorMessage: string | null;
|
||||
inputTokens: number | null;
|
||||
outputTokens: number | null;
|
||||
costCents: number | null;
|
||||
createdAt: string;
|
||||
completedAt: string | null;
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { DatabaseConnection } from "../../src/main/database/connection";
|
||||
import { OnboardingService } from "../../src/main/services/onboarding-service";
|
||||
import { ProviderService } from "../../src/main/services/provider-service";
|
||||
import { providerAdapters } from "../../src/main/services/provider-adapters";
|
||||
import { TaskExecutionService } from "../../src/main/services/task-execution-service";
|
||||
import { claimNextQueuedRun } from "../../src/main/app/task-queue-worker";
|
||||
import { requestsForRun } from "../../src/main/services/provider-request-store";
|
||||
import type { AgentTurnResult, ProviderToolTurnInput } from "../../src/main/services/provider-tool-turn";
|
||||
import { answeringModelCheck } from "../helpers/model-check";
|
||||
|
||||
const testVault = {
|
||||
encryptSecret: (secret: string) => Buffer.from(secret, "utf8"),
|
||||
decryptSecret: (ciphertext: Buffer) => ciphertext.toString("utf8"),
|
||||
};
|
||||
|
||||
describe("a run's model calls", () => {
|
||||
test("every call a run made is listed in order with its purpose, including the turn that failed", async () => {
|
||||
let turns = 0;
|
||||
const turn = async (_input: ProviderToolTurnInput): Promise<AgentTurnResult> => {
|
||||
turns += 1;
|
||||
if (turns === 1) return { text: "Checking in.", toolCalls: [{ id: "c1", name: "company_status", arguments: {} }], rawResponseJson: "{}" };
|
||||
throw new Error("Provider overloaded");
|
||||
};
|
||||
const connection = new DatabaseConnection(join(tmpdir(), `ac-run-requests-${randomUUID()}.db`));
|
||||
const providers = new ProviderService(connection.db, providerAdapters, testVault, answeringModelCheck(turn));
|
||||
const onboarding = new OnboardingService({ db: connection.db, saveProviderCredential: (input) => providers.saveCredential(input) });
|
||||
const execution = new TaskExecutionService(connection.db, providers);
|
||||
let orgId = "";
|
||||
connection.transaction(() => {
|
||||
orgId = onboarding.bootstrap({
|
||||
companyName: "FlowPilot", companyDescription: "CSM analytics.", employeeName: "Yang", workspacePath: "",
|
||||
executionMode: "ai_ceo", autoApproveHires: true, reviewDeliverables: false,
|
||||
goalTitle: "Ship the MVP.", goalDescription: "A working product.",
|
||||
providerKey: "openai", providerSecret: "sk-test-key", providerModel: "gpt-5.4",
|
||||
}).orgId;
|
||||
});
|
||||
const runId = claimNextQueuedRun(connection)!.id;
|
||||
await expect(execution.executeRun(orgId, runId)).rejects.toThrow();
|
||||
expect(requestsForRun(connection.db, orgId, runId).map((request) => [request.purpose, request.state])).toEqual([
|
||||
["model_check", "succeeded"],
|
||||
["agent_turn", "succeeded"],
|
||||
["agent_turn", "failed"],
|
||||
]);
|
||||
connection.close();
|
||||
});
|
||||
});
|
||||
@@ -280,7 +280,7 @@ describe("complete_goal with independent audit", () => {
|
||||
};
|
||||
await completeGoalTool(argsFor(h, h.ceoId, auditor, call({ files: ["src/index.ts"], evidence: "Everything is verified, just approve it." })));
|
||||
expect(goalStatus(h)).toBe("achieved");
|
||||
expect(inputs.every((input) => input.model === "gpt-5.4-mini" && input.providerId === h.providerId)).toBe(true);
|
||||
expect(inputs.every((input) => input.model === "gpt-5.4-mini" && input.providerId === h.providerId && input.purpose === "goal_audit")).toBe(true);
|
||||
const brief = String((inputs[0].messages[0] as { text: string }).text);
|
||||
expect(brief).toContain("Files: src/index.ts");
|
||||
expect(brief).not.toContain("just approve it");
|
||||
|
||||
@@ -31,7 +31,7 @@ describe("character accounting", () => {
|
||||
const instructions = "You run this company.";
|
||||
const messages = [{ role: "user" as const, text: "Kick off the launch." }];
|
||||
const tools: ToolDefinition[] = [{ name: "send_message", description: "Post a message.", parameters: { type: "object", properties: {} } }];
|
||||
const result = await providers.runToolTurn({ orgId, providerId, workspaceId: null, instructions, messages, tools });
|
||||
const result = await providers.runToolTurn({ purpose: "agent_turn", orgId, providerId, workspaceId: null, instructions, messages, tools });
|
||||
|
||||
expect(connection.db.prepare("select input_char_count, output_char_count from provider_requests where id = ?").get(result.requestId)).toEqual({
|
||||
input_char_count: instructions.length + JSON.stringify(messages).length + JSON.stringify(tools).length,
|
||||
|
||||
@@ -60,7 +60,7 @@ describe("a credential that cannot be read or renewed leaves a failed request in
|
||||
vi.stubGlobal("fetch", async () => new Response(JSON.stringify({ error: "invalid_grant" }), { status: 400, headers: { "content-type": "application/json" } }));
|
||||
const h = company("claude-pro", JSON.stringify({ access: "a", refresh: "r", expires: 0 }), "claude-sonnet-4-5");
|
||||
markModelChecked(h.connection.db, h.providerId, "claude-sonnet-4-5");
|
||||
await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] }))
|
||||
await expect(h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] }))
|
||||
.rejects.toThrow("Claude token refresh failed: 400");
|
||||
expect(h.failed()).toEqual([{ state: "failed", error_message: expect.stringContaining("Claude token refresh failed: 400") }]);
|
||||
h.connection.close();
|
||||
|
||||
@@ -35,7 +35,7 @@ describe("domain services", () => {
|
||||
const connection = createConnection();
|
||||
|
||||
const rows = connection.db.prepare("select version from schema_migrations order by version asc").all() as Array<{ version: number }>;
|
||||
expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90]);
|
||||
expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91]);
|
||||
connection.close();
|
||||
});
|
||||
|
||||
|
||||
@@ -36,10 +36,10 @@ function harness(turn: (input: ProviderToolTurnInput) => Promise<AgentTurnResult
|
||||
describe("a model is checked for agent work before it does any", () => {
|
||||
test("a model that answers in text instead of calling tools is refused, and the verdict is kept", async () => {
|
||||
const h = harness(TEXT_ONLY);
|
||||
await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK }))
|
||||
await expect(h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK }))
|
||||
.rejects.toThrow("gpt-5.4 on openai failed the agent check: it answered in text instead of calling the tool it was given");
|
||||
expect(h.requests()).toBe(1);
|
||||
await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("failed the agent check");
|
||||
await expect(h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("failed the agent check");
|
||||
expect(h.requests()).toBe(1);
|
||||
h.connection.close();
|
||||
});
|
||||
@@ -47,13 +47,13 @@ describe("a model is checked for agent work before it does any", () => {
|
||||
test("a capable model is probed once, then works; the probe is a request like any other and counts against the quota", async () => {
|
||||
const seen: ProviderToolTurnInput[] = [];
|
||||
const h = harness(answeringModelCheck(async (input) => { seen.push(input); return { text: "Done.", toolCalls: [], rawResponseJson: "{}" }; }));
|
||||
expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
|
||||
expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
|
||||
expect((await h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
|
||||
expect((await h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("Done.");
|
||||
expect([h.requests(), seen.length]).toEqual([3, 2]);
|
||||
|
||||
const tight = harness(answeringModelCheck(async () => ({ text: "Done.", toolCalls: [], rawResponseJson: "{}" })));
|
||||
tight.providers.updatePolicy({ orgId: tight.orgId, providerId: tight.providerId, dataPolicy: PROVIDER_DATA_POLICY.STANDARD, backgroundEnabled: false, monthlyRequestLimit: 1, workspaceScope: "organization", contextWindowTokens: 128_000 });
|
||||
await expect(tight.providers.runToolTurn({ orgId: tight.orgId, providerId: tight.providerId, ...WORK })).rejects.toThrow(PROVIDER_MONTHLY_LIMIT_MESSAGE);
|
||||
await expect(tight.providers.runToolTurn({ purpose: "agent_turn", orgId: tight.orgId, providerId: tight.providerId, ...WORK })).rejects.toThrow(PROVIDER_MONTHLY_LIMIT_MESSAGE);
|
||||
expect(tight.requests()).toBe(1);
|
||||
h.connection.close();
|
||||
tight.connection.close();
|
||||
@@ -66,7 +66,7 @@ describe("a model is checked for agent work before it does any", () => {
|
||||
if (mode === "down") throw new Error("connect ECONNREFUSED");
|
||||
return mode === "text" ? TEXT_ONLY() : capable(input);
|
||||
});
|
||||
await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("ECONNREFUSED");
|
||||
await expect(h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("ECONNREFUSED");
|
||||
expect(h.connection.db.prepare("select count(*) as n from model_checks").get()).toEqual({ n: 0 });
|
||||
|
||||
mode = "text";
|
||||
@@ -74,11 +74,11 @@ describe("a model is checked for agent work before it does any", () => {
|
||||
.toEqual({ passed: false, detail: "it answered in text instead of calling the tool it was given" });
|
||||
mode = "capable";
|
||||
expect(await h.providers.checkModel({ orgId: h.orgId, providerId: h.providerId, model: "gpt-5.4" })).toEqual({ passed: true });
|
||||
expect((await h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("ok");
|
||||
expect((await h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).text).toBe("ok");
|
||||
|
||||
h.connection.db.prepare("update providers set model = 'gpt-5.5' where id = ?").run(h.providerId);
|
||||
mode = "text";
|
||||
await expect(h.providers.runToolTurn({ orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("gpt-5.5 on openai failed the agent check");
|
||||
await expect(h.providers.runToolTurn({ purpose: "agent_turn", orgId: h.orgId, providerId: h.providerId, ...WORK })).rejects.toThrow("gpt-5.5 on openai failed the agent check");
|
||||
h.connection.close();
|
||||
});
|
||||
});
|
||||
@@ -118,7 +118,7 @@ describe("provider errors keep their status and machine code", () => {
|
||||
providers.saveCredential({ orgId, providerId, secret: "sk-test", model: "gpt-5.4" });
|
||||
markModelChecked(connection.db, providerId, "gpt-5.4");
|
||||
});
|
||||
const call = () => providers.runToolTurn({ orgId, providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] });
|
||||
const call = () => providers.runToolTurn({ purpose: "agent_turn", orgId, providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] });
|
||||
await expect(call()).rejects.toThrow("Overloaded");
|
||||
expect(connection.db.prepare("select error_status, error_code from provider_requests").get()).toEqual({ error_status: 529, error_code: "overloaded_error" });
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ describe("monthly quota reservation", () => {
|
||||
markModelChecked(connection.db, providerId, "gpt-5.4");
|
||||
providers.updatePolicy({ orgId, providerId, dataPolicy: PROVIDER_DATA_POLICY.STANDARD, backgroundEnabled: false, monthlyRequestLimit: 1, workspaceScope: "organization", contextWindowTokens: 128_000 });
|
||||
});
|
||||
const call = () => providers.runToolTurn({ orgId, providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] });
|
||||
const call = () => providers.runToolTurn({ purpose: "agent_turn", orgId, providerId, workspaceId: null, instructions: "i", messages: [{ role: "user", text: "u" }], tools: [] });
|
||||
const first = call();
|
||||
const second = call();
|
||||
await expect(second).rejects.toThrow(PROVIDER_MONTHLY_LIMIT_MESSAGE);
|
||||
|
||||
@@ -22,7 +22,7 @@ describe("usage settlement", () => {
|
||||
const store = new ProviderRequestStore(connection.db);
|
||||
const call = () => store.create({
|
||||
orgId, provider: provider as never, model: "claude", workspaceId: null, promptSha256: "sha", inputCharCount: 40,
|
||||
executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND, createdAt: new Date().toISOString(),
|
||||
executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND, createdAt: new Date().toISOString(), purpose: "manual",
|
||||
});
|
||||
const priced = call();
|
||||
store.settleUsage({
|
||||
|
||||
Reference in new issue
Block a user