Q128: every model call of a run is recorded with its purpose and listed in the run

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ZacharyZhang-NYandClaude Opus 5.5 committed 2026-10-08 20:55:18 -04:00
1 parent 1f484df894
commit 6afd7bef2a
25 files changed
+189 -22

No files matched your search

+1
View File
@@ -79,6 +79,7 @@ export const platformContracts = {
grantConnector: z.strictObject({ orgId: id, agentId: id, toolkitSlug: z.string(), accountLabel: z.string(), employeeId: id }) satisfies z.ZodType<C.GrantConnectorInput>,
revokeConnector: z.strictObject({ orgId: id, agentId: id, toolkitSlug: z.string() }) satisfies z.ZodType<C.RevokeConnectorInput>,
approvalDrifts: z.strictObject({ orgId: id }),
runRequests: z.strictObject({ orgId: id, runId: id }),
search: z.strictObject({
orgId: id, employeeId: id, query: z.string(), entityTypes: z.array(z.enum(SEARCH_ENTITY_TYPES)).optional(), limit: z.number().optional(),
}) satisfies z.ZodType<C.SearchInput>,
+2
View File
@@ -31,6 +31,7 @@ import { SprintService } from "../services/sprint-service";
import { ComposioConnectionService } from "../services/composio/composio-connection-service";
import { ConnectorGrantsService } from "../services/composio/connector-grants";
import { approvalDrifts } from "../services/approval-drift";
import { requestsForRun } from "../services/provider-request-store";
import { setGoalAuditor } from "../services/goal-audit-inputs";
import { IntegrationKeyStore } from "../services/integration-key-store";
import { importStagedSkills, stageBuiltinSkillsLibrary } from "../services/builtin-skills";
@@ -425,6 +426,7 @@ export function registerIpcHandlers(deps: IpcDeps): void {
}
});
handle("grantConnector", (input) => mutate(() => new ConnectorGrantsService(deps.connection.db).grantFromBoard(input)));
handle("runRequests", (input) => ok(requestsForRun(deps.connection.db, input.orgId, input.runId)));
handle("approvalDrifts", (input) => {
try {
return ok(approvalDrifts(deps.connection.db, input.orgId));
@@ -0,0 +1,8 @@
// Requests from before purposes were recorded stay 'unrecorded' rather than being guessed.
export const requestPurposeSchema = `
alter table provider_requests add column purpose text not null default 'unrecorded';
create index if not exists provider_requests_run on provider_requests(task_run_id);
insert into schema_migrations(version, name, applied_at)
values (91, 'request_purpose', datetime('now'));
`;
+2
View File
@@ -89,6 +89,7 @@ import { goalAuditorSchema } from "./migrations/087-goal-auditor";
import { hostAccessNoteSchema } from "./migrations/088-host-access-note";
import { fileVersionsSchema } from "./migrations/089-file-versions";
import { erasureReceiptsSchema } from "./migrations/090-erasure-receipts";
import { requestPurposeSchema } from "./migrations/091-request-purpose";
interface Migration {
version: number;
@@ -187,6 +188,7 @@ const MIGRATIONS: Migration[] = [
{ version: 88, name: "host_access_note", sql: hostAccessNoteSchema },
{ version: 89, name: "file_versions", sql: fileVersionsSchema },
{ version: 90, name: "erasure_receipts", sql: erasureReceiptsSchema },
{ version: 91, name: "request_purpose", sql: requestPurposeSchema },
];
const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version;
+1
View File
@@ -207,6 +207,7 @@ async function runAudit(
fitContextBudget(instructions, messages, auditTools, auditor.inputBudgetTokens, iteration);
throwIfRunStopped();
const turn = await untilStopped(args.providers.runToolTurn({
purpose: "goal_audit",
orgId: args.ctx.orgId,
providerId: auditor.providerId,
model: auditor.model,
+1
View File
@@ -72,6 +72,7 @@ export async function runAgentLoop(deps: AgentRunLoopDeps, input: AgentRunLoopIn
fitContextBudget(input.instructions, messages, allTools, input.inputBudgetTokens, iteration);
throwIfRunStopped();
const turn = await untilStopped(deps.providers.runToolTurn({
purpose: "agent_turn",
orgId: input.orgId,
providerId: input.providerId,
workspaceId: input.workspaceId,
@@ -65,6 +65,7 @@ export class ProviderBackgroundService {
inputCharCount: promptChars(execution.prompt, execution.instructions),
executionMode: PROVIDER_EXECUTION_MODE.BACKGROUND,
createdAt: new Date().toISOString(),
purpose: "manual",
});
input.onRequestCreated?.(requestId);
const startedAt = Date.now();
+17 -2
View File
@@ -1,3 +1,4 @@
import type { RunRequestRecord } from "../../shared/provider-runtime-domain";
import type { DatabaseSync } from "node:sqlite";
import { PROVIDER_EXECUTION_MODE, PROVIDER_REQUEST_STATE, type ProviderExecutionMode } from "../../shared/domain";
import { createId } from "../../shared/ids";
@@ -29,6 +30,19 @@ export interface CreateProviderRequestInput {
taskRunId?: string | null;
/** The agent's department at call time, which its spend is charged to. */
department?: string | null;
/** What the call was for: a run's turn, a goal audit, a model probe, or a request made by hand. */
purpose: RequestPurpose;
}
export type RequestPurpose = "agent_turn" | "goal_audit" | "model_check" | "manual";
/** Every model call a run made, in the order it made them, whatever their outcome. */
export function requestsForRun(db: DatabaseSync, orgId: string, runId: string): RunRequestRecord[] {
return db.prepare(`
select id, purpose, model, state, error_message as errorMessage, input_tokens as inputTokens, output_tokens as outputTokens,
cost_cents as costCents, created_at as createdAt, completed_at as completedAt
from provider_requests where org_id = ? and task_run_id = ? order by rowid
`).all(orgId, runId) as unknown as RunRequestRecord[];
}
export interface ProviderRequestProgressInput {
@@ -116,9 +130,9 @@ export class ProviderRequestStore {
id, org_id, provider_id, provider_key, workspace_id, model, prompt_sha256,
response_text, raw_response_json, state, error_message, input_char_count,
output_char_count, execution_mode, policy_decision, external_response_id,
stream_event_count, cancelled_at, created_at, completed_at, agent_id, task_run_id, department
stream_event_count, cancelled_at, created_at, completed_at, agent_id, task_run_id, department, purpose
)
values (?, ?, ?, ?, ?, ?, ?, null, null, ?, null, ?, 0, ?, ?, null, 0, null, ?, null, ?, ?, ?)
values (?, ?, ?, ?, ?, ?, ?, null, null, ?, null, ?, 0, ?, ?, null, 0, null, ?, null, ?, ?, ?, ?)
`).run(
requestId,
input.orgId,
@@ -135,6 +149,7 @@ export class ProviderRequestStore {
input.agentId ?? null,
input.taskRunId ?? null,
input.department ?? null,
input.purpose,
);
return requestId;
}
@@ -9,7 +9,7 @@ import type { OperationsService } from "./operations-service";
import type { ProviderAdapterRegistry } from "./provider-adapters";
import type { ProviderBackgroundService } from "./provider-background-service";
import type { ProviderRateLimitRegistry } from "./provider-rate-limit-registry";
import type { ProviderRequestStore } from "./provider-request-store";
import type { ProviderRequestStore, RequestPurpose } from "./provider-request-store";
import type { ProviderResponseObserver } from "./provider-response-observer";
import { hashPrompt, readProviderResponseId, requireStreamText } from "./provider-response-utils";
import { incompleteTurn, type AgentTurnMessage, type AgentTurnResult, type ToolDefinition, type ToolTurnFn } from "./provider-tool-turn";
@@ -116,6 +116,8 @@ export class ProviderResponseRunner {
tools: ToolDefinition[];
/** A model chosen for this call (a goal auditor's), outside the session's pinned model and account. */
model?: string;
/** What the call is for, recorded with the request so a run's chain reads plainly. */
purpose: "agent_turn" | "goal_audit";
}): Promise<AgentTurnResult & { requestId: string }> {
const provider = loadConfiguredProvider(this.deps.db, input.orgId, input.providerId);
const workspaceId = this.normalizeWorkspaceId(input.orgId, input.workspaceId ?? null);
@@ -145,13 +147,13 @@ export class ProviderResponseRunner {
const known = readModelCheck(this.deps.db, provider.id, model, requireCredentialId(provider));
const problem = known ? (known.passed ? null : known.detail) : await this.probeModel(caller);
if (problem) throw new Error(modelCheckFailure(provider.provider_key, model, problem));
return this.callTurn(caller, { instructions: input.instructions, messages: input.messages, tools: input.tools });
return this.callTurn(caller, { instructions: input.instructions, messages: input.messages, tools: input.tools }, input.purpose);
}
/** Probes a model for agent work and remembers the verdict; a request that fails records nothing. */
private async probeModel(caller: TurnCaller): Promise<string | null> {
const probe = modelCheckTurn();
const problem = modelCheckProblem(await this.callTurn(caller, probe), probe.code);
const problem = modelCheckProblem(await this.callTurn(caller, probe, "model_check"), probe.code);
const { owner } = caller;
recordModelCheck(this.deps.db, { orgId: caller.orgId, providerId: owner.id, model: owner.model, credentialId: requireCredentialId(owner), problem, now: new Date() });
console.info(`[model-check] ${owner.provider_key} ${owner.model}: ${problem ?? "passed"}`);
@@ -171,7 +173,7 @@ export class ProviderResponseRunner {
return problem === null ? { passed: true } : { passed: false, detail: problem };
}
private async callTurn(caller: TurnCaller, request: { instructions: string | null; messages: AgentTurnMessage[]; tools: ToolDefinition[] }): Promise<AgentTurnResult & { requestId: string }> {
private async callTurn(caller: TurnCaller, request: { instructions: string | null; messages: AgentTurnMessage[]; tools: ToolDefinition[] }, purpose: RequestPurpose): Promise<AgentTurnResult & { requestId: string }> {
const { orgId, owner, workspaceId, agent, taskRunId, turn } = caller;
const model = owner.model;
// Every request, a probe or the work, passes the provider's policy and quota and the agent's budget.
@@ -192,6 +194,7 @@ export class ProviderResponseRunner {
inputCharCount: turnInputChars,
executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND,
createdAt: new Date().toISOString(),
purpose,
});
const startedAt = Date.now();
try {
@@ -395,6 +398,7 @@ export class ProviderResponseRunner {
inputCharCount: promptChars(execution.prompt, execution.instructions),
executionMode: execution.executionMode,
createdAt: new Date().toISOString(),
purpose: "manual",
});
}