Q65: every call path records real tokens, cache, reasoning and cost

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/unit/domain-services.test.ts

@@ -24,7 +24,7 @@ describe("domain services", () => {
     const connection = createConnection();
 
     const rows = connection.db.prepare("select version from schema_migrations order by version asc").all() as Array<{ version: number }>;
-    expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72]);
+    expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73]);
     connection.close();
   });
 

tests/unit/provider-streaming.test.ts

@@ -48,6 +48,7 @@ describe("provider streaming", () => {
       message(null, { choices: [{ delta: { content: "Hel" } }] }),
       message(null, { choices: [{ delta: { content: "lo" } }] }),
       message(null, { choices: [{ delta: {}, finish_reason: "stop" }] }),
+      message(null, { choices: [], usage: { prompt_tokens: 12, completion_tokens: 3 } }),
     ])));
 
     expect(events.map((event) => [event.type, event.textDelta])).toEqual([
@@ -55,6 +56,7 @@ describe("provider streaming", () => {
       ["text_delta", "Hel"],
       ["text_delta", "lo"],
       ["message_stop", null],
+      ["usage", null],
     ]);
   });
 
@@ -63,6 +65,7 @@ describe("provider streaming", () => {
       message("message_start", { type: "message_start" }),
       message("content_block_delta", { type: "content_block_delta", delta: { type: "text_delta", text: "Hel" } }),
       message("content_block_delta", { type: "content_block_delta", delta: { type: "text_delta", text: "lo" } }),
+      message("message_delta", { type: "message_delta", usage: { output_tokens: 7 } }),
       message("message_stop", { type: "message_stop" }),
     ])));
 
@@ -70,6 +73,7 @@ describe("provider streaming", () => {
       ["message_start", null],
       ["text_delta", "Hel"],
       ["text_delta", "lo"],
+      ["usage", null],
       ["message_stop", null],
     ]);
   });

tests/unit/spend.test.ts

@@ -17,6 +17,13 @@ describe("token usage and cost", () => {
     expect(readTokenUsage(JSON.stringify(responsesStream))).toEqual({ inputTokens: 10, outputTokens: 5, cachedInputTokens: 0, reasoningTokens: 0 });
     const moonshotStream = ["{\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\",\"usage\":{\"prompt_tokens\":40,\"completion_tokens\":9}}]}"];
     expect(readTokenUsage(JSON.stringify(moonshotStream))).toEqual({ inputTokens: 40, outputTokens: 9, cachedInputTokens: 0, reasoningTokens: 0 });
+    const anthropicStream = [
+      "{\"type\":\"message_start\",\"message\":{\"usage\":{\"input_tokens\":700,\"cache_read_input_tokens\":300,\"output_tokens\":1}}}",
+      "{\"type\":\"content_block_delta\",\"delta\":{\"type\":\"text_delta\",\"text\":\"hi\"}}",
+      "{\"type\":\"message_delta\",\"usage\":{\"output_tokens\":42}}",
+      "{\"type\":\"message_stop\"}",
+    ];
+    expect(readTokenUsage(JSON.stringify(anthropicStream))).toEqual({ inputTokens: 1000, outputTokens: 42, cachedInputTokens: 300, reasoningTokens: 0 });
     expect(readTokenUsage(JSON.stringify({ output: [] }))).toBeNull();
   });
 

tests/unit/stream-usage-request.test.ts

@@ -0,0 +1,32 @@
+import { afterEach, describe, expect, test, vi } from "vitest";
+import { providerAdapters } from "../../src/main/services/provider-adapters";
+import { makeChatCompletionsTurn } from "../../src/main/services/provider-tool-turns/turns";
+
+const bodies: Array<Record<string, unknown>> = [];
+
+function sseResponse(): Response {
+  const text = "data: {\"choices\":[{\"delta\":{\"content\":\"ok\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n";
+  return new Response(text, { status: 200, headers: { "content-type": "text/event-stream" } });
+}
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+  bodies.length = 0;
+});
+
+describe("stream usage requests", () => {
+  test("xAI and Groq streams ask for token usage; Mistral and Cerebras streams do not", async () => {
+    vi.stubGlobal("fetch", async (_url: string, init: RequestInit) => {
+      bodies.push(JSON.parse(String(init.body)) as Record<string, unknown>);
+      return sseResponse();
+    });
+    for (const key of ["xai", "groq", "mistral", "cerebras"] as const) {
+      const result = await providerAdapters[key].streamResponse({ authProfile: { type: "api_key", apiKey: "k" }, model: "m", prompt: "p", instructions: null });
+      for await (const event of result.events) void event;
+    }
+    expect(bodies.map((body) => body.stream_options ?? null)).toEqual([{ include_usage: true }, { include_usage: true }, null, null]);
+
+    await makeChatCompletionsTurn("https://api.x.ai/v1/chat/completions", "xAI", true)({ credential: { apiKey: "k" }, model: "m", instructions: null, messages: [{ role: "user", text: "hi" }], tools: [] });
+    expect(bodies.at(-1)?.stream_options).toEqual({ include_usage: true });
+  });
+});

tests/unit/usage-settlement.test.ts

@@ -0,0 +1,44 @@
+import { randomUUID } from "node:crypto";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, test } from "vitest";
+import { DatabaseConnection } from "../../src/main/database/connection";
+import { OrganizationService } from "../../src/main/services/organization-service";
+import { ProviderRequestStore } from "../../src/main/services/provider-request-store";
+import { PROVIDER_EXECUTION_MODE } from "../../src/shared/domain";
+
+describe("usage settlement", () => {
+  test("every settled call records real tokens and cost on the request and in the month's totals", () => {
+    const connection = new DatabaseConnection(join(tmpdir(), `ac-settle-${randomUUID()}.db`));
+    const organization = new OrganizationService(connection.db);
+    let orgId = "";
+    connection.transaction(() => {
+      orgId = organization.createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" });
+    });
+    const provider = connection.db.prepare("select id, provider_key, data_policy, workspace_scope from providers where org_id = ? and provider_key = 'anthropic'").get(orgId) as
+      { id: string; provider_key: string; data_policy: string; workspace_scope: string };
+    connection.db.prepare("update providers set input_cents_per_mtok = 300, cached_input_cents_per_mtok = 30, output_cents_per_mtok = 1500 where id = ?").run(provider.id);
+    const store = new ProviderRequestStore(connection.db);
+    const call = () => store.create({
+      orgId, provider: provider as never, model: "claude", workspaceId: null, promptSha256: "sha", inputCharCount: 40,
+      executionMode: PROVIDER_EXECUTION_MODE.FOREGROUND, createdAt: new Date().toISOString(),
+    });
+    const priced = call();
+    store.settleUsage({
+      orgId, requestId: priced, provider, inputCharCount: 40, outputCharCount: 12,
+      rawResponseJson: JSON.stringify({ usage: { input_tokens: 600_000, output_tokens: 100_000, cache_read_input_tokens: 400_000 } }),
+    });
+    const silent = call();
+    store.settleUsage({ orgId, requestId: silent, provider, inputCharCount: 10, outputCharCount: 3, rawResponseJson: JSON.stringify({ content: [] }) });
+
+    expect(connection.db.prepare("select input_tokens, output_tokens, cached_input_tokens, cost_cents from provider_requests where id = ?").get(priced))
+      .toEqual({ input_tokens: 1_000_000, output_tokens: 100_000, cached_input_tokens: 400_000, cost_cents: 180 + 12 + 150 });
+    expect(connection.db.prepare(`
+      select request_count, input_tokens, output_tokens, cached_input_tokens, cost_cents, requests_without_usage, unpriced_requests
+      from provider_usage_windows where provider_id = ?
+    `).get(provider.id)).toEqual({
+      request_count: 2, input_tokens: 1_000_000, output_tokens: 100_000, cached_input_tokens: 400_000, cost_cents: 342, requests_without_usage: 1, unpriced_requests: 0,
+    });
+    connection.close();
+  });
+});