Q128: every model call of a run is recorded with its purpose and listed in the run

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ZacharyZhang-NYandClaude Opus 5.5 committed 2026-10-08 20:55:18 -04:00
1 parent 1f484df894
commit 6afd7bef2a
25 files changed
+189 -22

No files matched your search

+1
View File
@@ -79,6 +79,7 @@ export const platformContracts = {
grantConnector: z.strictObject({ orgId: id, agentId: id, toolkitSlug: z.string(), accountLabel: z.string(), employeeId: id }) satisfies z.ZodType<C.GrantConnectorInput>,
revokeConnector: z.strictObject({ orgId: id, agentId: id, toolkitSlug: z.string() }) satisfies z.ZodType<C.RevokeConnectorInput>,
approvalDrifts: z.strictObject({ orgId: id }),
runRequests: z.strictObject({ orgId: id, runId: id }),
search: z.strictObject({
orgId: id, employeeId: id, query: z.string(), entityTypes: z.array(z.enum(SEARCH_ENTITY_TYPES)).optional(), limit: z.number().optional(),
}) satisfies z.ZodType<C.SearchInput>,
+2
View File
@@ -31,6 +31,7 @@ import { SprintService } from "../services/sprint-service";
import { ComposioConnectionService } from "../services/composio/composio-connection-service";
import { ConnectorGrantsService } from "../services/composio/connector-grants";
import { approvalDrifts } from "../services/approval-drift";
import { requestsForRun } from "../services/provider-request-store";
import { setGoalAuditor } from "../services/goal-audit-inputs";
import { IntegrationKeyStore } from "../services/integration-key-store";
import { importStagedSkills, stageBuiltinSkillsLibrary } from "../services/builtin-skills";
@@ -425,6 +426,7 @@ export function registerIpcHandlers(deps: IpcDeps): void {
}
});
handle("grantConnector", (input) => mutate(() => new ConnectorGrantsService(deps.connection.db).grantFromBoard(input)));
handle("runRequests", (input) => ok(requestsForRun(deps.connection.db, input.orgId, input.runId)));
handle("approvalDrifts", (input) => {
try {
return ok(approvalDrifts(deps.connection.db, input.orgId));
@@ -0,0 +1,8 @@
// Requests from before purposes were recorded stay 'unrecorded' rather than being guessed.
export const requestPurposeSchema = `
alter table provider_requests add column purpose text not null default 'unrecorded';
create index if not exists provider_requests_run on provider_requests(task_run_id);
insert into schema_migrations(version, name, applied_at)
values (91, 'request_purpose', datetime('now'));
`;
+2
View File
@@ -89,6 +89,7 @@ import { goalAuditorSchema } from "./migrations/087-goal-auditor";
import { hostAccessNoteSchema } from "./migrations/088-host-access-note";
import { fileVersionsSchema } from "./migrations/089-file-versions";
import { erasureReceiptsSchema } from "./migrations/090-erasure-receipts";
import { requestPurposeSchema } from "./migrations/091-request-purpose";
interface Migration {
version: number;
@@ -187,6 +188,7 @@ const MIGRATIONS: Migration[] = [
{ version: 88, name: "host_access_note", sql: hostAccessNoteSchema },
{ version: 89, name: "file_versions", sql: fileVersionsSchema },
{ version: 90, name: "erasure_receipts", sql: erasureReceiptsSchema },
{ version: 91, name: "request_purpose", sql: requestPurposeSchema },
];
const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version;
+1
View File
@@ -207,6 +207,7 @@ async function runAudit(
fitContextBudget(instructions, messages, auditTools, auditor.inputBudgetTokens, iteration);
throwIfRunStopped();
const turn = await untilStopped(args.providers.runToolTurn({
purpose: "goal_audit",
orgId: args.ctx.orgId,
providerId: auditor.providerId,
model: auditor.model,
+1
View File
@@ -72,6 +72,7 @@ export async function runAgentLoop(deps: AgentRunLoopDeps, input: AgentRunLoopIn
fitContextBudget(input.instructions, messages, allTools, input.inputBudgetTokens, iteration);
throwIfRunStopped();
const turn = await untilStopped(deps.providers.runToolTurn({
purpose: "agent_turn",
orgId: input.orgId,
providerId: input.providerId,
workspaceId: input.workspaceId,
@@ -65,6 +65,7 @@ export class ProviderBackgroundService {
inputCharCount: promptChars(execution.prompt, execution.instructions),
executionMode: PROVIDER_EXECUTION_MODE.BACKGROUND,
createdAt: new Date().toISOString(),
purpose: "manual",
});
input.onRequestCreated?.(requestId);
const startedAt = Date.now();
+17 -2
View File
@@ -1,3 +1,4 @@
import type { RunRequestRecord } from "../../shared/provider-runtime-domain";
import type { DatabaseSync } from "node:sqlite";
import { PROVIDER_EXECUTION_MODE, PROVIDER_REQUEST_STATE, type ProviderExecutionMode } from "../../shared/domain";
import { createId } from "../../shared/ids";
@@ -29,6 +30,19 @@ export interface CreateProviderRequestInput {
taskRunId?: string | null;
/** The agent's department at call time, which its spend is charged to. */
department?: string | null;
/** What the call was for: a run's turn, a goal audit, a model probe, or a request made by hand. */
purpose: RequestPurpose;
}
export type RequestPurpose = "agent_turn" | "goal_audit" | "model_check" | "manual";
/** Every model call a run made, in the order it made them, whatever their outcome. */
export function requestsForRun(db: DatabaseSync, orgId: string, runId: string): RunRequestRecord[] {
return db.prepare(`
select id, purpose, model, state, error_message as errorMessage, input_tokens as inputTokens, output_tokens as outputTokens,
cost_cents as costCents, created_at as createdAt, completed_at as completedAt
from provider_requests where org_id = ? and task_run_id = ? order by rowid
`).all(orgId, runId) as unknown as RunRequestRecord[];
}
export interface ProviderRequestProgressInput {
@@ -116,9 +130,9 @@ export class ProviderRequestStore {
id, org_id, provider_id, provider_key, workspace_id, model, prompt_sha256,
response_text, raw_response_json, state, error_message, input_char_count,
output_char_count, execution_mode, policy_decision, external_response_id,
stream_event_count, cancelled_at, created_at, completed_at, agent_id, task_run_id, department
stream_event_count, cancelled_at, created_at, completed_at, agent_id, task_run_id, department, purpose
)
values (?, ?, ?, ?, ?, ?, ?, null, null, ?, null, ?, 0, ?, ?, null, 0, null, ?, null, ?, ?, ?)
values (?, ?, ?, ?, ?, ?, ?, null, null, ?, null, ?, 0, ?, ?, null, 0, null, ?, null, ?, ?, ?, ?)
`).run(
requestId,
input.orgId,
@@ -135,6 +149,7 @@ export class ProviderRequestStore {
input.agentId ?? null,
input.taskRunId ?? null,
input.department ?? null,
input.purpose,
);
return requestId;
}
@@ -9,7 +9,7 @@ import type { OperationsService } from "./operations-service";
import type { ProviderAdapterRegistry } from "./provider-adapters";
import type { ProviderBackgroundService } from "./provider-background-service";
import type { ProviderRateLimitRegistry } from "./provider-rate-limit-registry";
import type { ProviderRequestStore } from "./provider-request-store";
import type { ProviderRequestStore, RequestPurpose } from "./provider-request-store";
import type { ProviderResponseObserver } from "./provider-response-observer";
import { hashPrompt, readProviderResponseId, requireStreamText } from "./provider-response-utils";
import { incompleteTurn, type AgentTurnMessage, type AgentTurnResult, type ToolDefinition, type ToolTurnFn } from "./provider-tool-turn";
@@ -116,6 +116,8 @@ export class ProviderResponseRunner {
tools: ToolDefinition[];
/** A model chosen for this call (a goal auditor's), outside the session's pinned model and account. */
model?: string;
/** What the call is for, recorded with the request so a run's chain reads plainly. */
purpose: "agent_turn" | "goal_audit";
}): Promise<AgentTurnResult & { requestId: string }> {
const provider = loadConfiguredProvider(this.deps.db, input.orgId, input.providerId);
const workspaceId = this.normalizeWorkspaceId(input.orgId, input.workspaceId ?? null);
@@ -145,13 +147,13 @@ export class ProviderResponseRunner {
const known = readModelCheck(this.deps.db, provider.id, model, requireCredentialId(provider));
const problem = known ? (known.passed ? null : known.detail) : await this.probeModel(caller);
if (problem) throw new Error(modelCheckFailure(provider.provider_key, model, problem));
return this.callTurn(caller, { instructions: input.instructions, messages: input.messages, tools: input.tools });
return this.callTurn(caller, { instructions: input.instructions, messages: input.messages, tools: input.tools }, input.purpose);
}
/** Probes a model for agent work and remembers the verdict; a request that fails records nothing. */
private async probeModel(caller: TurnCaller): Promise<string | null> {
const probe = modelCheckTurn();
const problem = modelCheckProblem(await this.callTurn(caller, probe), probe.code);
const problem = modelCheckProblem(await this.callTurn(caller, probe, "model_check"), probe.code);
const { owner } = caller;
recordModelCheck(this.deps.db, { orgId: caller.orgId, providerId: owner.id, model: owner.model, credentialId: requireCredentialId(owner), problem, now: new Date() });
console.info(`[model-check] ${owner.provider_key} ${owner.model}: ${problem ?? "passed"}`);
@@ -171,7 +173,7 @@ export class ProviderResponseRunner {
return problem === null ? { passed: true } : { passed: false, detail: problem };
}
private async callTurn(caller: TurnCaller, request: { instructions: string | null; messages: AgentTurnMessage[]; tools: ToolDefinition[] }): Promise<AgentTurnResult & { requestId: string }> {
private async callTurn(caller: TurnCaller, request: { instructions: string | null; messages: AgentTurnMessage[]; tools: ToolDefinition[] }, purpose: RequestPurpose): Promise<AgentTurnResult & { requestId: string }> {
const { orgId, owner, workspaceId, agent, taskRunId, turn } = caller;
const model = owner.model;
// Every request, a probe or the work, passes the provider's policy and quota and the agent's budget.
@@ -192,6 +194,7 @@ export class ProviderResponseRunner {
inputCharCount: turnInputChars,
executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND,
createdAt: new Date().toISOString(),
purpose,
});
const startedAt = Date.now();
try {
@@ -395,6 +398,7 @@ export class ProviderResponseRunner {
inputCharCount: promptChars(execution.prompt, execution.instructions),
executionMode: execution.executionMode,
createdAt: new Date().toISOString(),
purpose: "manual",
});
}
+1
View File
@@ -95,6 +95,7 @@ const api: DesktopApi = {
disconnectComposioToolkit: (input) => invoke(IPC_CHANNELS.disconnectComposioToolkit, input),
grantConnector: (input) => invoke(IPC_CHANNELS.grantConnector, input),
approvalDrifts: (input) => invoke(IPC_CHANNELS.approvalDrifts, input),
runRequests: (input) => invoke(IPC_CHANNELS.runRequests, input),
revokeConnector: (input) => invoke(IPC_CHANNELS.revokeConnector, input),
refreshComposioAccounts: (input) => invoke(IPC_CHANNELS.refreshComposioAccounts, input),
search: (input) => invoke(IPC_CHANNELS.search, input),
+7 -1
View File
@@ -172,6 +172,13 @@ export const zh: Record<string, string> = {
"Check": "检查",
"Choose": "选择",
"No findings": "未发现问题",
"Model calls ({count})": "模型调用({count})",
"This run made no model call.": "本次运行没有调用模型。",
"Turn": "轮次",
"Goal audit": "目标审查",
"Model check": "模型检查",
"Manual": "手动",
"Earlier call": "早期调用",
"Partly scanned": "部分扫描",
"Scanned before whole-file checks existed: only its first MiB was checked.": "在全文件检查之前扫描:只检查了前 1 MiB。",
"Contents not inspected": "内容未检查",
@@ -395,7 +402,6 @@ export const zh: Record<string, string> = {
"Replaced by an edit": "已因编辑被替换",
"Changed since approval": "批准后已改动",
"Removed since approval": "批准后已删除",
"Goal audit": "目标审查",
"CEO: hires officers, grants connectors and skills, and closes goals. This rank cannot be changed.": "CEO:招聘高管、授予连接器和技能、关闭目标。此职级不可更改。",
"An independent model checks every goal before it closes. Pick one your agents do not use, ideally from another provider.": "每个目标关闭前都由一个独立模型检查。请选择 Agent 不使用的模型,最好来自另一家服务商。",
"No auditor is set, so goals cannot be closed yet.": "尚未设置审查模型,目标暂时无法关闭。",
+3
View File
@@ -1,3 +1,4 @@
import { RunRequestList } from "./runs/RunRequestList";
import { useMemo, useState } from "react";
import { Avatar, Button, Pill, Segmented } from "../shared/ui";
import type { AgentRecord, ProviderRecord, TaskRecord, TaskRunRecord } from "../../../shared/domain";
@@ -147,6 +148,8 @@ function RunDetail({ run, tasks, agents, providers, providerRequests, actions }:
<MetaLabel>{t("Context")}</MetaLabel><div className="mono" style={{ fontSize: 12, overflow: "hidden", textOverflow: "ellipsis" }}>{run.executionContextId}</div>
</div>
<RunRequestList orgId={run.orgId} runId={run.id} state={run.state} />
{run.state === "superseded" && run.errorMessage && (
<div className="card" style={{ padding: 16, marginBottom: 16 }}>
<div className="section-label" style={{ marginBottom: 8 }}>{t("Why it did not run")}</div>
@@ -0,0 +1,54 @@
import { useEffect, useState } from "react";
import type { RunRequestRecord } from "../../../../shared/provider-runtime-domain";
import { Pill } from "../../shared/ui";
import { t } from "../../i18n/i18n";
function purposeLabel(purpose: string): string {
if (purpose === "agent_turn") return t("Turn");
if (purpose === "goal_audit") return t("Goal audit");
if (purpose === "model_check") return t("Model check");
if (purpose === "manual") return t("Manual");
return t("Earlier call");
}
/** Every model call the run made, in order, so a turn or an audit can be found by its task. */
export function RunRequestList({ orgId, runId, state }: { orgId: string; runId: string; state: string }) {
// Fetched calls are kept with the run they belong to, so switching runs never shows another run's chain.
const [loaded, setLoaded] = useState<{ runId: string; requests: RunRequestRecord[] } | { runId: string; error: string } | null>(null);
useEffect(() => {
let cancelled = false;
window.agentCompany.runRequests({ orgId, runId })
.then((result) => {
if (cancelled) return;
setLoaded(result.ok ? { runId, requests: result.data } : { runId, error: result.error.message });
})
.catch((reason: unknown) => {
if (!cancelled) setLoaded({ runId, error: reason instanceof Error ? reason.message : String(reason) });
});
return () => {
cancelled = true;
};
}, [orgId, runId, state]);
if (!loaded || loaded.runId !== runId) return null;
if ("error" in loaded) return <div className="form-error" role="alert">{loaded.error}</div>;
const { requests } = loaded;
return (
<div style={{ marginBottom: 20 }}>
<div className="section-label" style={{ marginBottom: 8 }}>{t("Model calls ({count})", { count: String(requests.length) })}</div>
{requests.length === 0 && <div style={{ fontSize: 13, color: "var(--text-3)" }}>{t("This run made no model call.")}</div>}
{requests.map((request, index) => (
<div key={request.id} className="row" style={{ gap: 10, fontSize: 13, padding: "6px 0", borderBottom: "1px solid var(--border-soft)" }}>
<span className="mono" style={{ color: "var(--text-3)", width: 24 }}>{index + 1}</span>
<span style={{ width: 96 }}>{purposeLabel(request.purpose)}</span>
<span className="mono" style={{ flex: 1, minWidth: 0, overflow: "hidden", textOverflow: "ellipsis" }}>{request.model}</span>
<Pill tone={request.state === "failed" ? "red" : request.state === "succeeded" ? "green" : "amber"}>{request.state}</Pill>
<span className="mono" style={{ color: "var(--text-3)", width: 120, textAlign: "right" }}>
{request.inputTokens ?? "—"} / {request.outputTokens ?? "—"}
</span>
{request.errorMessage && <span style={{ color: "var(--red)", flexBasis: "100%", whiteSpace: "pre-wrap" }}>{request.errorMessage}</span>}
</div>
))}
</div>
);
}
+3
View File
@@ -1,3 +1,4 @@
import type { RunRequestRecord } from "./provider-runtime-domain";
import type { ApprovalDrift } from "./approval-pin";
import type { DesktopResult } from "./result";
import type { AppSnapshot, SearchResultRecord } from "./domain";
@@ -207,6 +208,8 @@ export interface DesktopApi {
grantConnector(input: GrantConnectorInput): Promise<DesktopResult<AppSnapshot>>;
/** Approved deliverables whose content changed or vanished since approval, checked on disk now. */
approvalDrifts(input: { orgId: string }): Promise<DesktopResult<Record<string, ApprovalDrift>>>;
/** Every model call a task run made, in order: turns, goal audits and model checks. */
runRequests(input: { orgId: string; runId: string }): Promise<DesktopResult<RunRequestRecord[]>>;
revokeConnector(input: RevokeConnectorInput): Promise<DesktopResult<AppSnapshot>>;
search(input: SearchInput): Promise<DesktopResult<SearchResultRecord[]>>;
subscribe(callback: (event: RendererEvent) => void): () => void;
+1
View File
@@ -88,6 +88,7 @@ export const IPC_CHANNELS = {
refreshComposioAccounts: "agentcompany:refresh-composio-accounts",
grantConnector: "agentcompany:grant-connector",
approvalDrifts: "agentcompany:approval-drifts",
runRequests: "agentcompany:run-requests",
revokeConnector: "agentcompany:revoke-connector",
search: "agentcompany:search",
rendererEvent: "agentcompany:renderer-event",
+14
View File
@@ -43,3 +43,17 @@ export function currentProviderUsageWindow(now = new Date()): { start: string; e
const end = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + 1, 1));
return { start: start.toISOString(), end: end.toISOString() };
}
/** One model call of a task run, as the run's detail lists it. */
export interface RunRequestRecord {
id: string;
purpose: string;
model: string;
state: string;
errorMessage: string | null;
inputTokens: number | null;
outputTokens: number | null;
costCents: number | null;
createdAt: string;
completedAt: string | null;
}