From 403c57785ca5d05b804bd1692af39644baa43d0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Thu, 8 Oct 2026 21:43:21 -0400 Subject: [PATCH] Q134: a signal's category comes from its runtime source, so self-checks cannot pass as real traffic Co-Authored-By: Claude Opus 5.5 --- .../migrations/097-drop-drill-only-slo.ts | 7 ++++++ src/main/database/schema.ts | 2 ++ src/main/services/migration-service.ts | 2 -- src/main/services/operations-service.ts | 24 ++++++++++--------- src/main/services/org-governance-service.ts | 3 +-- .../services/provider-response-observer.ts | 4 +--- src/main/services/skill-service.ts | 2 -- src/shared/operations-domain.ts | 1 - tests/eval/replay-eval.test.ts | 11 --------- tests/unit/domain-services.test.ts | 2 +- tests/unit/operations-service.test.ts | 20 +++++++++++++++- 11 files changed, 44 insertions(+), 34 deletions(-) create mode 100644 src/main/database/migrations/097-drop-drill-only-slo.ts diff --git a/src/main/database/migrations/097-drop-drill-only-slo.ts b/src/main/database/migrations/097-drop-drill-only-slo.ts new file mode 100644 index 00000000..7a832166 --- /dev/null +++ b/src/main/database/migrations/097-drop-drill-only-slo.ts @@ -0,0 +1,7 @@ +// The browser-session SLO was fed only by the removed chaos drill, so no runtime signal could ever meet or miss it. +export const dropDrillOnlySloSchema = ` + delete from slo_targets where signal_key = 'browser_session_boundary'; + + insert into schema_migrations(version, name, applied_at) + values (97, 'drop_drill_only_slo', datetime('now')); +`; diff --git a/src/main/database/schema.ts b/src/main/database/schema.ts index 6490f94b..201fa3b9 100644 --- a/src/main/database/schema.ts +++ b/src/main/database/schema.ts @@ -95,6 +95,7 @@ import { taskEnqueueErrorSchema } from "./migrations/093-task-enqueue-error"; import { dropConfigCountEvalsSchema } from "./migrations/094-drop-config-count-evals"; import { dropPrBotGatesSchema } from "./migrations/095-drop-pr-bot-gates"; import { dropChaosDrillsSchema } from "./migrations/096-drop-chaos-drills"; +import { dropDrillOnlySloSchema } from "./migrations/097-drop-drill-only-slo"; interface Migration { version: number; @@ -199,6 +200,7 @@ const MIGRATIONS: Migration[] = [ { version: 94, name: "drop_config_count_evals", sql: dropConfigCountEvalsSchema }, { version: 95, name: "drop_pr_bot_gates", sql: dropPrBotGatesSchema }, { version: 96, name: "drop_chaos_drills", sql: dropChaosDrillsSchema }, + { version: 97, name: "drop_drill_only_slo", sql: dropDrillOnlySloSchema }, ]; const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version; diff --git a/src/main/services/migration-service.ts b/src/main/services/migration-service.ts index 0a2bf3b4..1689c05a 100644 --- a/src/main/services/migration-service.ts +++ b/src/main/services/migration-service.ts @@ -3,7 +3,6 @@ import { createHash } from "node:crypto"; import { resolve } from "node:path"; import { DatabaseSync, type StatementSync } from "node:sqlite"; import type { CreateMigrationReportInput } from "../../shared/commands"; -import { SLO_SIGNAL_KEY } from "../../shared/domain"; import { createId } from "../../shared/ids"; import { AuditService } from "./audit-service"; import { buildMigrationValidationSummary, type MigrationValidationSummary } from "./migration-validation-summary"; @@ -343,7 +342,6 @@ export class MigrationService { }); this.operations.recordSignal({ orgId: input.orgId, - signalKey: SLO_SIGNAL_KEY.MIGRATION_VALIDATION_SUCCESS, source: "migration_validation", success: validationSummary.passed, durationMs: null, diff --git a/src/main/services/operations-service.ts b/src/main/services/operations-service.ts index a311a7c7..86c96d98 100644 --- a/src/main/services/operations-service.ts +++ b/src/main/services/operations-service.ts @@ -1,6 +1,6 @@ import type { DatabaseSync } from "node:sqlite"; import type { SetFeatureFlagInput } from "../../shared/commands"; -import { FEATURE_FLAG_KEY, SLO_SIGNAL_KEY, type FeatureFlagKey, type SloSignalKey } from "../../shared/domain"; +import { FEATURE_FLAG_KEY, SLO_SIGNAL_KEY, type FeatureFlagKey } from "../../shared/domain"; import { createId } from "../../shared/ids"; import { AuditService } from "./audit-service"; @@ -10,10 +10,18 @@ const FEATURE_FLAGS = [ { key: FEATURE_FLAG_KEY.BOARD_INTERVENTION, label: "Board intervention", enabled: true }, ] as const; +/** Each runtime source and the one signal it reports; drills and self-checks have no source here, so they cannot pass as real traffic. */ +const SIGNAL_OF_SOURCE = { + provider_gateway: SLO_SIGNAL_KEY.PROVIDER_REQUEST_SUCCESS, + approval_routing: SLO_SIGNAL_KEY.APPROVAL_ROUTE_COVERAGE, + skill_resolver: SLO_SIGNAL_KEY.SKILL_RESOLVE_SUCCESS, + migration_validation: SLO_SIGNAL_KEY.MIGRATION_VALIDATION_SUCCESS, +} as const; +export type SignalSource = keyof typeof SIGNAL_OF_SOURCE; + const SLO_TARGETS = [ { signalKey: SLO_SIGNAL_KEY.PROVIDER_REQUEST_SUCCESS, label: "Provider requests", targetBasisPoints: 9950, windowMinutes: 43200 }, { signalKey: SLO_SIGNAL_KEY.APPROVAL_ROUTE_COVERAGE, label: "Approval routing", targetBasisPoints: 10000, windowMinutes: 43200 }, - { signalKey: SLO_SIGNAL_KEY.BROWSER_SESSION_BOUNDARY, label: "Browser session boundary", targetBasisPoints: 10000, windowMinutes: 43200 }, { signalKey: SLO_SIGNAL_KEY.SKILL_RESOLVE_SUCCESS, label: "Skill resolve", targetBasisPoints: 9950, windowMinutes: 43200 }, { signalKey: SLO_SIGNAL_KEY.MIGRATION_VALIDATION_SUCCESS, label: "Migration validation", targetBasisPoints: 10000, windowMinutes: 43200 }, ] as const; @@ -68,21 +76,15 @@ export class OperationsService { return row.enabled === 1; } - recordSignal(input: { - orgId: string; - signalKey: SloSignalKey; - source: string; - success: boolean; - durationMs: number | null; - detail: unknown; - }): void { + recordSignal(input: { orgId: string; source: SignalSource; success: boolean; durationMs: number | null; detail: unknown }): void { + if (!Object.hasOwn(SIGNAL_OF_SOURCE, input.source)) throw new Error(`"${input.source}" is not a runtime signal source.`); this.db.prepare(` insert into operational_events(id, org_id, signal_key, source, success, duration_ms, detail_json, created_at) values (?, ?, ?, ?, ?, ?, ?, ?) `).run( createId("ope"), input.orgId, - input.signalKey, + SIGNAL_OF_SOURCE[input.source], input.source, input.success ? 1 : 0, input.durationMs, diff --git a/src/main/services/org-governance-service.ts b/src/main/services/org-governance-service.ts index a572ec43..6a85f9d9 100644 --- a/src/main/services/org-governance-service.ts +++ b/src/main/services/org-governance-service.ts @@ -1,6 +1,6 @@ import type { DatabaseSync } from "node:sqlite"; import type { CreateAgentInput } from "../../shared/commands"; -import { RISK_LEVEL, ROUTE_TO_TYPE, SLO_SIGNAL_KEY, type RiskLevel } from "../../shared/domain"; +import { RISK_LEVEL, ROUTE_TO_TYPE, type RiskLevel } from "../../shared/domain"; import { createId } from "../../shared/ids"; import { AuditService } from "./audit-service"; import { OperationsService } from "./operations-service"; @@ -117,7 +117,6 @@ export class OrgGovernanceService { }); this.operations.recordSignal({ orgId: input.orgId, - signalKey: SLO_SIGNAL_KEY.APPROVAL_ROUTE_COVERAGE, source: "approval_routing", success: true, durationMs: null, diff --git a/src/main/services/provider-response-observer.ts b/src/main/services/provider-response-observer.ts index dfb4df6d..f7e2011c 100644 --- a/src/main/services/provider-response-observer.ts +++ b/src/main/services/provider-response-observer.ts @@ -1,5 +1,5 @@ import type { DatabaseSync } from "node:sqlite"; -import { SLO_SIGNAL_KEY, type ProviderExecutionMode } from "../../shared/domain"; +import type { ProviderExecutionMode } from "../../shared/domain"; import type { ProviderObservedRateLimitState } from "../../shared/provider-runtime-domain"; import { AuditService } from "./audit-service"; import { OperationsService } from "./operations-service"; @@ -59,7 +59,6 @@ export class ProviderResponseObserver { }); this.operations.recordSignal({ orgId: input.orgId, - signalKey: SLO_SIGNAL_KEY.PROVIDER_REQUEST_SUCCESS, source: "provider_gateway", success: true, durationMs: Date.now() - input.startedAt, @@ -85,7 +84,6 @@ export class ProviderResponseObserver { this.recordProviderHealth(input.provider.id, input.orgId, HEALTH_STATE_FAILED, message); this.operations.recordSignal({ orgId: input.orgId, - signalKey: SLO_SIGNAL_KEY.PROVIDER_REQUEST_SUCCESS, source: "provider_gateway", success: false, durationMs: Date.now() - input.startedAt, diff --git a/src/main/services/skill-service.ts b/src/main/services/skill-service.ts index aeba39cc..e0aaf6e4 100644 --- a/src/main/services/skill-service.ts +++ b/src/main/services/skill-service.ts @@ -1,7 +1,6 @@ import { existsSync, readFileSync } from "node:fs"; import type { DatabaseSync } from "node:sqlite"; import type { BindSkillInput, ImportSkillDirectoryInput, ResolveAgentSkillsInput, RunSkillTestsInput } from "../../shared/commands"; -import { SLO_SIGNAL_KEY } from "../../shared/domain"; import { createId } from "../../shared/ids"; import { AuditService } from "./audit-service"; import { OperationsService } from "./operations-service"; @@ -390,7 +389,6 @@ export class SkillService { }); this.operations.recordSignal({ orgId: input.orgId, - signalKey: SLO_SIGNAL_KEY.SKILL_RESOLVE_SUCCESS, source: "skill_resolver", success: true, durationMs: null, diff --git a/src/shared/operations-domain.ts b/src/shared/operations-domain.ts index 2a10829c..6c9ad279 100644 --- a/src/shared/operations-domain.ts +++ b/src/shared/operations-domain.ts @@ -9,7 +9,6 @@ export const FEATURE_FLAG_KEY = { export const SLO_SIGNAL_KEY = { PROVIDER_REQUEST_SUCCESS: "provider_request_success", APPROVAL_ROUTE_COVERAGE: "approval_route_coverage", - BROWSER_SESSION_BOUNDARY: "browser_session_boundary", SKILL_RESOLVE_SUCCESS: "skill_resolve_success", MIGRATION_VALIDATION_SUCCESS: "migration_validation_success", } as const; diff --git a/tests/eval/replay-eval.test.ts b/tests/eval/replay-eval.test.ts index 1aaadfb7..6fce3f04 100644 --- a/tests/eval/replay-eval.test.ts +++ b/tests/eval/replay-eval.test.ts @@ -5,10 +5,8 @@ import { describe, expect, test } from "vitest"; import { DatabaseConnection } from "../../src/main/database/connection"; import { AuditService } from "../../src/main/services/audit-service"; import { ChatService } from "../../src/main/services/chat-service"; -import { OperationsService } from "../../src/main/services/operations-service"; import { OrganizationService } from "../../src/main/services/organization-service"; import { SnapshotService } from "../../src/main/services/snapshot-service"; -import { SLO_SIGNAL_KEY } from "../../src/shared/domain"; function createConnection(): DatabaseConnection { return new DatabaseConnection(join(tmpdir(), `agentcompany-new-replay-${randomUUID()}.db`)); @@ -29,7 +27,6 @@ describe("replay eval", () => { const organization = new OrganizationService(connection.db); const chat = new ChatService(connection.db); const audit = new AuditService(connection.db); - const operations = new OperationsService(connection.db); const snapshot = new SnapshotService(connection.db); let orgId = ""; let employeeId = ""; @@ -57,14 +54,6 @@ describe("replay eval", () => { }); threadId = (connection.db.prepare("select thread_id from chat_messages where id = ?").get(messageId) as { thread_id: string }).thread_id; replayRunId = audit.createThreadReplay({ orgId, threadId, createdByEmployeeId: employeeId }); - operations.recordSignal({ - orgId, - signalKey: SLO_SIGNAL_KEY.APPROVAL_ROUTE_COVERAGE, - source: "replay_eval", - success: true, - durationMs: 12, - detail: { threadId, replayRunId }, - }); }); const state = snapshot.load(); diff --git a/tests/unit/domain-services.test.ts b/tests/unit/domain-services.test.ts index 4ecc1495..ba37e8e6 100644 --- a/tests/unit/domain-services.test.ts +++ b/tests/unit/domain-services.test.ts @@ -35,7 +35,7 @@ describe("domain services", () => { const connection = createConnection(); const rows = connection.db.prepare("select version from schema_migrations order by version asc").all() as Array<{ version: number }>; - expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96]); + expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97]); connection.close(); }); diff --git a/tests/unit/operations-service.test.ts b/tests/unit/operations-service.test.ts index 358a739f..150756ed 100644 --- a/tests/unit/operations-service.test.ts +++ b/tests/unit/operations-service.test.ts @@ -49,7 +49,10 @@ describe("operations service", () => { "provider_background_execution", "social_publish_approval", ]); - expect(state.sloTargets.map((target) => target.signalKey)).toContain("browser_session_boundary"); + // Every SLO target is fed by a runtime source; none waits on a signal only a drill would write. + expect(state.sloTargets.map((target) => target.signalKey).sort()).toEqual([ + "approval_route_coverage", "migration_validation_success", "provider_request_success", "skill_resolve_success", + ]); expect(connection.db.prepare("select name from sqlite_master where type = 'table' and name like 'chaos_drill%'").all()).toEqual([]); connection.close(); }); @@ -97,4 +100,19 @@ describe("operations service", () => { })).rejects.toThrow("Provider background execution is disabled by feature flag."); connection.close(); }); + + test("a signal's category comes from its runtime source, so a self-check cannot be filed as real traffic", () => { + const connection = createConnection(); + const operations = new OperationsService(connection.db); + let orgId = ""; + connection.transaction(() => { + orgId = createOrganization(new OrganizationService(connection.db)); + }); + operations.recordSignal({ orgId, source: "skill_resolver", success: true, durationMs: null, detail: {} }); + expect(connection.db.prepare("select signal_key, source from operational_events where org_id = ?").all(orgId)) + .toEqual([{ signal_key: "skill_resolve_success", source: "skill_resolver" }]); + expect(() => operations.recordSignal({ orgId, source: "replay_eval" as "skill_resolver", success: true, durationMs: null, detail: {} })) + .toThrow('"replay_eval" is not a runtime signal source.'); + connection.close(); + }); });