diff --git a/src/main/services/provider-background-service.ts b/src/main/services/provider-background-service.ts index 0933734f..0bc544b3 100644 --- a/src/main/services/provider-background-service.ts +++ b/src/main/services/provider-background-service.ts @@ -19,6 +19,7 @@ import { ProviderRateLimitRegistry } from "./provider-rate-limit-registry"; import { ProviderRequestStore, type ProviderBackgroundRequestRow } from "./provider-request-store"; import { ProviderResponseObserver } from "./provider-response-observer"; import { hashPrompt, readProviderFailureMessage, requireBackgroundText } from "./provider-response-utils"; +import { promptChars } from "./spend"; const AUTH_STATE_CONFIGURED = "configured"; const HEALTH_STATE_HEALTHY = "healthy"; @@ -60,7 +61,7 @@ export class ProviderBackgroundService { model: execution.provider.model, workspaceId: execution.workspaceId, promptSha256: hashPrompt(execution.prompt), - inputCharCount: execution.prompt.length, + inputCharCount: promptChars(execution.prompt, execution.instructions), executionMode: PROVIDER_EXECUTION_MODE.BACKGROUND, createdAt: new Date().toISOString(), }); @@ -99,7 +100,7 @@ export class ProviderBackgroundService { requestId, executionMode: PROVIDER_EXECUTION_MODE.BACKGROUND, startedAt, - inputCharCount: execution.prompt.length, + inputCharCount: promptChars(execution.prompt, execution.instructions), error, }); throw error; diff --git a/src/main/services/provider-response-runner.ts b/src/main/services/provider-response-runner.ts index 5ec1de0a..abea2dce 100644 --- a/src/main/services/provider-response-runner.ts +++ b/src/main/services/provider-response-runner.ts @@ -17,7 +17,7 @@ import { type AgentTurnMessage, type AgentTurnResult, type ToolDefinition, type import { selectToolTurn, supportsToolTurn } from "./provider-tool-turns/router"; import { resolveTurnCredential } from "./provider-tool-turns/credential-resolver"; import { currentRunId } from "./run-control"; -import { budgetBlock, isPriced, providerPrices } from "./spend"; +import { budgetBlock, isPriced, promptChars, providerPrices } from "./spend"; import { HEALTH_STATE_HEALTHY, WORKSPACE_SCOPES, @@ -86,7 +86,7 @@ export class ProviderResponseRunner { }); this.deps.requestStore.settleUsage({ orgId: input.orgId, requestId, provider: execution.provider, rawResponseJson: result.rawResponseJson, - inputCharCount: execution.prompt.length, outputCharCount: result.text.length, + inputCharCount: promptChars(execution.prompt, execution.instructions), outputCharCount: result.text.length, }); return requestId; } catch (error) { @@ -96,7 +96,7 @@ export class ProviderResponseRunner { requestId, executionMode: execution.executionMode, startedAt, - inputCharCount: execution.prompt.length, + inputCharCount: promptChars(execution.prompt, execution.instructions), error, }); throw error; @@ -129,7 +129,8 @@ export class ProviderResponseRunner { const blocked = agent ? budgetBlock(this.deps.db, agent.id, isPriced(providerPrices(this.deps.db, provider.id)), new Date()) : null; if (blocked) throw new Error(blocked); const credential = await resolveTurnCredential({ authBroker: this.deps.authBroker, integrationKeys: this.deps.integrationKeys, orgId: input.orgId, provider }); - const promptChars = JSON.stringify(input.messages).length; + // Everything the model reads: instructions, the conversation, and the tool definitions. + const turnInputChars = (input.instructions?.length ?? 0) + JSON.stringify(input.messages).length + JSON.stringify(input.tools).length; const requestId = this.deps.requestStore.create({ agentId: agent?.id ?? null, taskRunId, @@ -139,19 +140,21 @@ export class ProviderResponseRunner { model, workspaceId, promptSha256: hashPrompt(JSON.stringify(input.messages)), - inputCharCount: promptChars, + inputCharCount: turnInputChars, executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND, createdAt: new Date().toISOString(), }); const startedAt = Date.now(); try { const result = await turn({ credential, model, instructions: input.instructions, messages: input.messages, tools: input.tools }); + // Tool calls are output the model wrote, as much as its text. + const outputChars = result.text.length + (result.toolCalls.length > 0 ? JSON.stringify(result.toolCalls).length : 0); this.deps.requestStore.markSucceeded({ orgId: input.orgId, requestId, responseText: result.text, rawResponseJson: result.rawResponseJson, - outputCharCount: result.text.length, + outputCharCount: outputChars, externalResponseId: readProviderResponseId(result.rawResponseJson), completedAt: new Date().toISOString(), }); @@ -165,7 +168,7 @@ export class ProviderResponseRunner { }); this.deps.requestStore.settleUsage({ orgId: input.orgId, requestId, provider: owner, rawResponseJson: result.rawResponseJson, - inputCharCount: promptChars, outputCharCount: result.text.length, + inputCharCount: turnInputChars, outputCharCount: outputChars, }); return { ...result, requestId }; } catch (error) { @@ -175,7 +178,7 @@ export class ProviderResponseRunner { requestId, executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND, startedAt, - inputCharCount: promptChars, + inputCharCount: turnInputChars, error, }); throw error; @@ -253,7 +256,7 @@ export class ProviderResponseRunner { }); this.deps.requestStore.settleUsage({ orgId: input.orgId, requestId, provider: execution.provider, rawResponseJson: JSON.stringify(rawEvents), - inputCharCount: execution.prompt.length, outputCharCount: responseText.length, + inputCharCount: promptChars(execution.prompt, execution.instructions), outputCharCount: responseText.length, }); return requestId; } catch (error) { @@ -263,7 +266,7 @@ export class ProviderResponseRunner { requestId, executionMode: execution.executionMode, startedAt, - inputCharCount: execution.prompt.length, + inputCharCount: promptChars(execution.prompt, execution.instructions), error, }); throw error; @@ -332,7 +335,7 @@ export class ProviderResponseRunner { model: execution.provider.model, workspaceId: execution.workspaceId, promptSha256: hashPrompt(execution.prompt), - inputCharCount: execution.prompt.length, + inputCharCount: promptChars(execution.prompt, execution.instructions), executionMode: execution.executionMode, createdAt: new Date().toISOString(), }); diff --git a/src/main/services/spend.ts b/src/main/services/spend.ts index 3df37610..7ce2ca97 100644 --- a/src/main/services/spend.ts +++ b/src/main/services/spend.ts @@ -50,6 +50,11 @@ export function providerPrices(db: DatabaseSync, providerId: string): TokenPrice return { inputCentsPerMtok: row.input_cents_per_mtok, cachedInputCentsPerMtok: row.cached_input_cents_per_mtok, outputCentsPerMtok: row.output_cents_per_mtok }; } +/** Characters a call sends: the instructions count as much as the prompt. */ +export function promptChars(prompt: string, instructions: string | null): number { + return prompt.length + (instructions?.length ?? 0); +} + export function monthStartIso(now: Date): string { return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1)).toISOString(); } diff --git a/tests/unit/char-accounting.test.ts b/tests/unit/char-accounting.test.ts new file mode 100644 index 00000000..daaa84e8 --- /dev/null +++ b/tests/unit/char-accounting.test.ts @@ -0,0 +1,40 @@ +import { randomUUID } from "node:crypto"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, test } from "vitest"; +import { DatabaseConnection } from "../../src/main/database/connection"; +import { OrganizationService } from "../../src/main/services/organization-service"; +import { ProviderService } from "../../src/main/services/provider-service"; +import { providerAdapters } from "../../src/main/services/provider-adapters"; +import type { AgentTurnResult, ToolDefinition } from "../../src/main/services/provider-tool-turn"; + +const testVault = { + encryptSecret: (secret: string) => Buffer.from(secret, "utf8"), + decryptSecret: (ciphertext: Buffer) => ciphertext.toString("utf8"), +}; + +describe("character accounting", () => { + test("a tool turn counts instructions, conversation and tool definitions in, and text plus tool calls out", async () => { + const toolCalls = [{ id: "c1", name: "send_message", arguments: { channel: "general", body: "Shipping today." } }]; + const turn = async (): Promise => ({ text: "Posting.", toolCalls, rawResponseJson: "{}" }); + const connection = new DatabaseConnection(join(tmpdir(), `ac-chars-${randomUUID()}.db`)); + const providers = new ProviderService(connection.db, providerAdapters, testVault, undefined, turn); + let orgId = ""; + let providerId = ""; + connection.transaction(() => { + orgId = new OrganizationService(connection.db).createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" }); + providerId = (connection.db.prepare("select id from providers where org_id = ? and provider_key = 'openai'").get(orgId) as { id: string }).id; + providers.saveCredential({ orgId, providerId, secret: "sk-test", model: "gpt-5.4" }); + }); + const instructions = "You run this company."; + const messages = [{ role: "user" as const, text: "Kick off the launch." }]; + const tools: ToolDefinition[] = [{ name: "send_message", description: "Post a message.", parameters: { type: "object", properties: {} } }]; + const result = await providers.runToolTurn({ orgId, providerId, workspaceId: null, instructions, messages, tools }); + + expect(connection.db.prepare("select input_char_count, output_char_count from provider_requests where id = ?").get(result.requestId)).toEqual({ + input_char_count: instructions.length + JSON.stringify(messages).length + JSON.stringify(tools).length, + output_char_count: "Posting.".length + JSON.stringify(toolCalls).length, + }); + connection.close(); + }); +}); diff --git a/tests/unit/provider-background-service.test.ts b/tests/unit/provider-background-service.test.ts index 4ac978b0..97cd3c63 100644 --- a/tests/unit/provider-background-service.test.ts +++ b/tests/unit/provider-background-service.test.ts @@ -88,11 +88,11 @@ describe("provider background service", () => { state: PROVIDER_REQUEST_STATE.SUCCEEDED, response_text: "Background analysis complete.", external_response_id: "resp_bg_1", - input_char_count: 17, + input_char_count: "Run long research".length + "Use verified citations.".length, output_char_count: 29, }); expect(JSON.parse(request.raw_response_json)).toMatchObject({ id: "resp_bg_1", status: "completed" }); - expect(usage).toEqual({ request_count: 1, input_char_count: 17, output_char_count: 29 }); + expect(usage).toEqual({ request_count: 1, input_char_count: 40, output_char_count: 29 }); expect(signal).toEqual({ success: 1 }); connection.close(); });