Q128: every model call of a run is recorded with its purpose and listed in the run
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1f484df894
commit
6afd7bef2a
25 files changed
+189
-22
No files matched your search
@@ -79,6 +79,7 @@ export const platformContracts = {
|
||||
grantConnector: z.strictObject({ orgId: id, agentId: id, toolkitSlug: z.string(), accountLabel: z.string(), employeeId: id }) satisfies z.ZodType<C.GrantConnectorInput>,
|
||||
revokeConnector: z.strictObject({ orgId: id, agentId: id, toolkitSlug: z.string() }) satisfies z.ZodType<C.RevokeConnectorInput>,
|
||||
approvalDrifts: z.strictObject({ orgId: id }),
|
||||
runRequests: z.strictObject({ orgId: id, runId: id }),
|
||||
search: z.strictObject({
|
||||
orgId: id, employeeId: id, query: z.string(), entityTypes: z.array(z.enum(SEARCH_ENTITY_TYPES)).optional(), limit: z.number().optional(),
|
||||
}) satisfies z.ZodType<C.SearchInput>,
|
||||
|
||||
@@ -31,6 +31,7 @@ import { SprintService } from "../services/sprint-service";
|
||||
import { ComposioConnectionService } from "../services/composio/composio-connection-service";
|
||||
import { ConnectorGrantsService } from "../services/composio/connector-grants";
|
||||
import { approvalDrifts } from "../services/approval-drift";
|
||||
import { requestsForRun } from "../services/provider-request-store";
|
||||
import { setGoalAuditor } from "../services/goal-audit-inputs";
|
||||
import { IntegrationKeyStore } from "../services/integration-key-store";
|
||||
import { importStagedSkills, stageBuiltinSkillsLibrary } from "../services/builtin-skills";
|
||||
@@ -425,6 +426,7 @@ export function registerIpcHandlers(deps: IpcDeps): void {
|
||||
}
|
||||
});
|
||||
handle("grantConnector", (input) => mutate(() => new ConnectorGrantsService(deps.connection.db).grantFromBoard(input)));
|
||||
handle("runRequests", (input) => ok(requestsForRun(deps.connection.db, input.orgId, input.runId)));
|
||||
handle("approvalDrifts", (input) => {
|
||||
try {
|
||||
return ok(approvalDrifts(deps.connection.db, input.orgId));
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
// Requests from before purposes were recorded stay 'unrecorded' rather than being guessed.
|
||||
export const requestPurposeSchema = `
|
||||
alter table provider_requests add column purpose text not null default 'unrecorded';
|
||||
create index if not exists provider_requests_run on provider_requests(task_run_id);
|
||||
|
||||
insert into schema_migrations(version, name, applied_at)
|
||||
values (91, 'request_purpose', datetime('now'));
|
||||
`;
|
||||
@@ -89,6 +89,7 @@ import { goalAuditorSchema } from "./migrations/087-goal-auditor";
|
||||
import { hostAccessNoteSchema } from "./migrations/088-host-access-note";
|
||||
import { fileVersionsSchema } from "./migrations/089-file-versions";
|
||||
import { erasureReceiptsSchema } from "./migrations/090-erasure-receipts";
|
||||
import { requestPurposeSchema } from "./migrations/091-request-purpose";
|
||||
|
||||
interface Migration {
|
||||
version: number;
|
||||
@@ -187,6 +188,7 @@ const MIGRATIONS: Migration[] = [
|
||||
{ version: 88, name: "host_access_note", sql: hostAccessNoteSchema },
|
||||
{ version: 89, name: "file_versions", sql: fileVersionsSchema },
|
||||
{ version: 90, name: "erasure_receipts", sql: erasureReceiptsSchema },
|
||||
{ version: 91, name: "request_purpose", sql: requestPurposeSchema },
|
||||
];
|
||||
|
||||
const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version;
|
||||
|
||||
@@ -207,6 +207,7 @@ async function runAudit(
|
||||
fitContextBudget(instructions, messages, auditTools, auditor.inputBudgetTokens, iteration);
|
||||
throwIfRunStopped();
|
||||
const turn = await untilStopped(args.providers.runToolTurn({
|
||||
purpose: "goal_audit",
|
||||
orgId: args.ctx.orgId,
|
||||
providerId: auditor.providerId,
|
||||
model: auditor.model,
|
||||
|
||||
@@ -72,6 +72,7 @@ export async function runAgentLoop(deps: AgentRunLoopDeps, input: AgentRunLoopIn
|
||||
fitContextBudget(input.instructions, messages, allTools, input.inputBudgetTokens, iteration);
|
||||
throwIfRunStopped();
|
||||
const turn = await untilStopped(deps.providers.runToolTurn({
|
||||
purpose: "agent_turn",
|
||||
orgId: input.orgId,
|
||||
providerId: input.providerId,
|
||||
workspaceId: input.workspaceId,
|
||||
|
||||
@@ -65,6 +65,7 @@ export class ProviderBackgroundService {
|
||||
inputCharCount: promptChars(execution.prompt, execution.instructions),
|
||||
executionMode: PROVIDER_EXECUTION_MODE.BACKGROUND,
|
||||
createdAt: new Date().toISOString(),
|
||||
purpose: "manual",
|
||||
});
|
||||
input.onRequestCreated?.(requestId);
|
||||
const startedAt = Date.now();
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { RunRequestRecord } from "../../shared/provider-runtime-domain";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { PROVIDER_EXECUTION_MODE, PROVIDER_REQUEST_STATE, type ProviderExecutionMode } from "../../shared/domain";
|
||||
import { createId } from "../../shared/ids";
|
||||
@@ -29,6 +30,19 @@ export interface CreateProviderRequestInput {
|
||||
taskRunId?: string | null;
|
||||
/** The agent's department at call time, which its spend is charged to. */
|
||||
department?: string | null;
|
||||
/** What the call was for: a run's turn, a goal audit, a model probe, or a request made by hand. */
|
||||
purpose: RequestPurpose;
|
||||
}
|
||||
|
||||
export type RequestPurpose = "agent_turn" | "goal_audit" | "model_check" | "manual";
|
||||
|
||||
/** Every model call a run made, in the order it made them, whatever their outcome. */
|
||||
export function requestsForRun(db: DatabaseSync, orgId: string, runId: string): RunRequestRecord[] {
|
||||
return db.prepare(`
|
||||
select id, purpose, model, state, error_message as errorMessage, input_tokens as inputTokens, output_tokens as outputTokens,
|
||||
cost_cents as costCents, created_at as createdAt, completed_at as completedAt
|
||||
from provider_requests where org_id = ? and task_run_id = ? order by rowid
|
||||
`).all(orgId, runId) as unknown as RunRequestRecord[];
|
||||
}
|
||||
|
||||
export interface ProviderRequestProgressInput {
|
||||
@@ -116,9 +130,9 @@ export class ProviderRequestStore {
|
||||
id, org_id, provider_id, provider_key, workspace_id, model, prompt_sha256,
|
||||
response_text, raw_response_json, state, error_message, input_char_count,
|
||||
output_char_count, execution_mode, policy_decision, external_response_id,
|
||||
stream_event_count, cancelled_at, created_at, completed_at, agent_id, task_run_id, department
|
||||
stream_event_count, cancelled_at, created_at, completed_at, agent_id, task_run_id, department, purpose
|
||||
)
|
||||
values (?, ?, ?, ?, ?, ?, ?, null, null, ?, null, ?, 0, ?, ?, null, 0, null, ?, null, ?, ?, ?)
|
||||
values (?, ?, ?, ?, ?, ?, ?, null, null, ?, null, ?, 0, ?, ?, null, 0, null, ?, null, ?, ?, ?, ?)
|
||||
`).run(
|
||||
requestId,
|
||||
input.orgId,
|
||||
@@ -135,6 +149,7 @@ export class ProviderRequestStore {
|
||||
input.agentId ?? null,
|
||||
input.taskRunId ?? null,
|
||||
input.department ?? null,
|
||||
input.purpose,
|
||||
);
|
||||
return requestId;
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ import type { OperationsService } from "./operations-service";
|
||||
import type { ProviderAdapterRegistry } from "./provider-adapters";
|
||||
import type { ProviderBackgroundService } from "./provider-background-service";
|
||||
import type { ProviderRateLimitRegistry } from "./provider-rate-limit-registry";
|
||||
import type { ProviderRequestStore } from "./provider-request-store";
|
||||
import type { ProviderRequestStore, RequestPurpose } from "./provider-request-store";
|
||||
import type { ProviderResponseObserver } from "./provider-response-observer";
|
||||
import { hashPrompt, readProviderResponseId, requireStreamText } from "./provider-response-utils";
|
||||
import { incompleteTurn, type AgentTurnMessage, type AgentTurnResult, type ToolDefinition, type ToolTurnFn } from "./provider-tool-turn";
|
||||
@@ -116,6 +116,8 @@ export class ProviderResponseRunner {
|
||||
tools: ToolDefinition[];
|
||||
/** A model chosen for this call (a goal auditor's), outside the session's pinned model and account. */
|
||||
model?: string;
|
||||
/** What the call is for, recorded with the request so a run's chain reads plainly. */
|
||||
purpose: "agent_turn" | "goal_audit";
|
||||
}): Promise<AgentTurnResult & { requestId: string }> {
|
||||
const provider = loadConfiguredProvider(this.deps.db, input.orgId, input.providerId);
|
||||
const workspaceId = this.normalizeWorkspaceId(input.orgId, input.workspaceId ?? null);
|
||||
@@ -145,13 +147,13 @@ export class ProviderResponseRunner {
|
||||
const known = readModelCheck(this.deps.db, provider.id, model, requireCredentialId(provider));
|
||||
const problem = known ? (known.passed ? null : known.detail) : await this.probeModel(caller);
|
||||
if (problem) throw new Error(modelCheckFailure(provider.provider_key, model, problem));
|
||||
return this.callTurn(caller, { instructions: input.instructions, messages: input.messages, tools: input.tools });
|
||||
return this.callTurn(caller, { instructions: input.instructions, messages: input.messages, tools: input.tools }, input.purpose);
|
||||
}
|
||||
|
||||
/** Probes a model for agent work and remembers the verdict; a request that fails records nothing. */
|
||||
private async probeModel(caller: TurnCaller): Promise<string | null> {
|
||||
const probe = modelCheckTurn();
|
||||
const problem = modelCheckProblem(await this.callTurn(caller, probe), probe.code);
|
||||
const problem = modelCheckProblem(await this.callTurn(caller, probe, "model_check"), probe.code);
|
||||
const { owner } = caller;
|
||||
recordModelCheck(this.deps.db, { orgId: caller.orgId, providerId: owner.id, model: owner.model, credentialId: requireCredentialId(owner), problem, now: new Date() });
|
||||
console.info(`[model-check] ${owner.provider_key} ${owner.model}: ${problem ?? "passed"}`);
|
||||
@@ -171,7 +173,7 @@ export class ProviderResponseRunner {
|
||||
return problem === null ? { passed: true } : { passed: false, detail: problem };
|
||||
}
|
||||
|
||||
private async callTurn(caller: TurnCaller, request: { instructions: string | null; messages: AgentTurnMessage[]; tools: ToolDefinition[] }): Promise<AgentTurnResult & { requestId: string }> {
|
||||
private async callTurn(caller: TurnCaller, request: { instructions: string | null; messages: AgentTurnMessage[]; tools: ToolDefinition[] }, purpose: RequestPurpose): Promise<AgentTurnResult & { requestId: string }> {
|
||||
const { orgId, owner, workspaceId, agent, taskRunId, turn } = caller;
|
||||
const model = owner.model;
|
||||
// Every request, a probe or the work, passes the provider's policy and quota and the agent's budget.
|
||||
@@ -192,6 +194,7 @@ export class ProviderResponseRunner {
|
||||
inputCharCount: turnInputChars,
|
||||
executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND,
|
||||
createdAt: new Date().toISOString(),
|
||||
purpose,
|
||||
});
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
@@ -395,6 +398,7 @@ export class ProviderResponseRunner {
|
||||
inputCharCount: promptChars(execution.prompt, execution.instructions),
|
||||
executionMode: execution.executionMode,
|
||||
createdAt: new Date().toISOString(),
|
||||
purpose: "manual",
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user