Q133: remove chaos drills that never injected a fault, and the signals they wrote
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e562444487
commit
3849b96192
17 files changed
+20
-220
No files matched your search
@@ -56,7 +56,6 @@ export const platformContracts = {
|
||||
setRollbackSnapshotActive: z.strictObject({ orgId: id, snapshotId: id, active: z.boolean(), employeeId: id }) satisfies z.ZodType<C.SetRollbackSnapshotActiveInput>,
|
||||
resolveMigrationReconciliation: z.strictObject({ orgId: id, reconciliationItemId: id, employeeId: id }) satisfies z.ZodType<C.ResolveMigrationReconciliationInput>,
|
||||
setFeatureFlag: z.strictObject({ orgId: id, flagId: id, enabled: z.boolean() }) satisfies z.ZodType<C.SetFeatureFlagInput>,
|
||||
runChaosDrill: z.strictObject({ orgId: id, drillId: id }) satisfies z.ZodType<C.RunChaosDrillInput>,
|
||||
createComplianceExport: z.strictObject({
|
||||
orgId: id, subjectType: z.enum(["employee", "agent"]), subjectId: id, exportType: z.enum(COMPLIANCE_EXPORT_TYPE), requestedByEmployeeId: id,
|
||||
}) satisfies z.ZodType<C.CreateComplianceExportInput>,
|
||||
|
||||
@@ -376,7 +376,6 @@ export function registerIpcHandlers(deps: IpcDeps): void {
|
||||
handle("setRollbackSnapshotActive", (input) => mutate(() => rollback.setActive(input)));
|
||||
handle("resolveMigrationReconciliation", (input) => mutate(() => rollback.resolveReconciliationItem(input)));
|
||||
handle("setFeatureFlag", (input) => mutate(() => operations.setFeatureFlag(input)));
|
||||
handle("runChaosDrill", (input) => mutate(() => operations.runChaosDrill(input)));
|
||||
handle("createComplianceExport", (input) => mutate(() => compliance.createExport(input)));
|
||||
handle("createAuditReplay", (input) => mutate(() => audit.createThreadReplay(input)));
|
||||
handle("createLegalHold", (input) => mutate(() => compliance.createLegalHold(input)));
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
// Chaos drills scored configuration rows as fault handling and never injected a fault; their flag and the signals they wrote go with them.
|
||||
export const dropChaosDrillsSchema = `
|
||||
drop table if exists chaos_drill_runs;
|
||||
drop table if exists chaos_drills;
|
||||
delete from feature_flags where key = 'chaos_drills';
|
||||
delete from operational_events where source = 'chaos_drill';
|
||||
|
||||
insert into schema_migrations(version, name, applied_at)
|
||||
values (96, 'drop_chaos_drills', datetime('now'));
|
||||
`;
|
||||
@@ -94,6 +94,7 @@ import { runActionTargetsSchema } from "./migrations/092-run-action-targets";
|
||||
import { taskEnqueueErrorSchema } from "./migrations/093-task-enqueue-error";
|
||||
import { dropConfigCountEvalsSchema } from "./migrations/094-drop-config-count-evals";
|
||||
import { dropPrBotGatesSchema } from "./migrations/095-drop-pr-bot-gates";
|
||||
import { dropChaosDrillsSchema } from "./migrations/096-drop-chaos-drills";
|
||||
|
||||
interface Migration {
|
||||
version: number;
|
||||
@@ -197,6 +198,7 @@ const MIGRATIONS: Migration[] = [
|
||||
{ version: 93, name: "task_enqueue_error", sql: taskEnqueueErrorSchema },
|
||||
{ version: 94, name: "drop_config_count_evals", sql: dropConfigCountEvalsSchema },
|
||||
{ version: 95, name: "drop_pr_bot_gates", sql: dropPrBotGatesSchema },
|
||||
{ version: 96, name: "drop_chaos_drills", sql: dropChaosDrillsSchema },
|
||||
];
|
||||
|
||||
const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version;
|
||||
|
||||
@@ -30,8 +30,6 @@ const LEGACY_TABLES = [
|
||||
"feature_flags",
|
||||
"slo_targets",
|
||||
"operational_events",
|
||||
"chaos_drills",
|
||||
"chaos_drill_runs",
|
||||
"chat_spaces",
|
||||
"chat_messages",
|
||||
"chat_threads",
|
||||
@@ -155,9 +153,6 @@ const RELATION_CHECKS: RelationCheck[] = [
|
||||
{ table: "feature_flags", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
||||
{ table: "slo_targets", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
||||
{ table: "operational_events", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
||||
{ table: "chaos_drills", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
||||
{ table: "chaos_drill_runs", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
||||
{ table: "chaos_drill_runs", column: "drill_id", referencedTable: "chaos_drills", referencedColumn: "id", reason: "dangling_drill_id" },
|
||||
{ table: "chat_spaces", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
||||
{ table: "chat_messages", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
||||
{ table: "chat_messages", column: "space_id", referencedTable: "chat_spaces", referencedColumn: "id", reason: "dangling_space_id" },
|
||||
|
||||
@@ -1,14 +1,6 @@
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import type { RunChaosDrillInput, SetFeatureFlagInput } from "../../shared/commands";
|
||||
import {
|
||||
CHAOS_DRILL_SCENARIO,
|
||||
CHAOS_DRILL_STATE,
|
||||
FEATURE_FLAG_KEY,
|
||||
SLO_SIGNAL_KEY,
|
||||
type ChaosDrillScenario,
|
||||
type FeatureFlagKey,
|
||||
type SloSignalKey,
|
||||
} from "../../shared/domain";
|
||||
import type { SetFeatureFlagInput } from "../../shared/commands";
|
||||
import { FEATURE_FLAG_KEY, SLO_SIGNAL_KEY, type FeatureFlagKey, type SloSignalKey } from "../../shared/domain";
|
||||
import { createId } from "../../shared/ids";
|
||||
import { AuditService } from "./audit-service";
|
||||
|
||||
@@ -16,7 +8,6 @@ const FEATURE_FLAGS = [
|
||||
{ key: FEATURE_FLAG_KEY.PROVIDER_BACKGROUND_EXECUTION, label: "Provider background execution", enabled: true },
|
||||
{ key: FEATURE_FLAG_KEY.SOCIAL_PUBLISH_APPROVAL, label: "Social publish approval", enabled: true },
|
||||
{ key: FEATURE_FLAG_KEY.BOARD_INTERVENTION, label: "Board intervention", enabled: true },
|
||||
{ key: FEATURE_FLAG_KEY.CHAOS_DRILLS, label: "Chaos drills", enabled: true },
|
||||
] as const;
|
||||
|
||||
const SLO_TARGETS = [
|
||||
@@ -27,24 +18,6 @@ const SLO_TARGETS = [
|
||||
{ signalKey: SLO_SIGNAL_KEY.MIGRATION_VALIDATION_SUCCESS, label: "Migration validation", targetBasisPoints: 10000, windowMinutes: 43200 },
|
||||
] as const;
|
||||
|
||||
const CHAOS_DRILLS = [
|
||||
{ key: "provider-outage-policy", label: "Provider outage policy", scenarioKey: CHAOS_DRILL_SCENARIO.PROVIDER_OUTAGE_POLICY_GATE },
|
||||
{ key: "approval-backlog-routing", label: "Approval backlog routing", scenarioKey: CHAOS_DRILL_SCENARIO.APPROVAL_BACKLOG_GATE },
|
||||
{ key: "browser-session-boundary", label: "Browser session boundary", scenarioKey: CHAOS_DRILL_SCENARIO.BROWSER_SESSION_BOUNDARY },
|
||||
{ key: "migration-rollback-pack", label: "Migration rollback pack", scenarioKey: CHAOS_DRILL_SCENARIO.MIGRATION_ROLLBACK_PACK },
|
||||
] as const;
|
||||
|
||||
interface DrillOutcome {
|
||||
success: boolean;
|
||||
checks: Record<string, number | boolean | string>;
|
||||
}
|
||||
|
||||
interface ChaosDrillRow {
|
||||
id: string;
|
||||
scenario_key: ChaosDrillScenario;
|
||||
enabled: number;
|
||||
}
|
||||
|
||||
export class OperationsService {
|
||||
private readonly audit: AuditService;
|
||||
|
||||
@@ -67,13 +40,6 @@ export class OperationsService {
|
||||
on conflict(org_id, signal_key) do nothing
|
||||
`).run(createId("slo"), orgId, target.signalKey, target.label, target.targetBasisPoints, target.windowMinutes, now, now);
|
||||
}
|
||||
for (const drill of CHAOS_DRILLS) {
|
||||
this.db.prepare(`
|
||||
insert into chaos_drills(id, org_id, key, label, scenario_key, enabled, created_at, updated_at)
|
||||
values (?, ?, ?, ?, ?, 1, ?, ?)
|
||||
on conflict(org_id, key) do nothing
|
||||
`).run(createId("cdr"), orgId, drill.key, drill.label, drill.scenarioKey, now, now);
|
||||
}
|
||||
}
|
||||
|
||||
setFeatureFlag(input: SetFeatureFlagInput): void {
|
||||
@@ -95,48 +61,6 @@ export class OperationsService {
|
||||
});
|
||||
}
|
||||
|
||||
runChaosDrill(input: RunChaosDrillInput): string {
|
||||
if (!this.isFeatureEnabled(input.orgId, FEATURE_FLAG_KEY.CHAOS_DRILLS)) throw new Error("Chaos drills are disabled.");
|
||||
const drill = this.db.prepare("select id, scenario_key, enabled from chaos_drills where id = ? and org_id = ?")
|
||||
.get(input.drillId, input.orgId) as ChaosDrillRow | undefined;
|
||||
if (!drill) throw new Error("Chaos drill was not found.");
|
||||
if (drill.enabled !== 1) throw new Error("Chaos drill is disabled.");
|
||||
const startedAt = new Date().toISOString();
|
||||
const outcome = this.evaluateScenario(input.orgId, drill.scenario_key);
|
||||
const runId = createId("cdrun");
|
||||
const completedAt = new Date().toISOString();
|
||||
this.db.prepare(`
|
||||
insert into chaos_drill_runs(id, org_id, drill_id, state, result_json, started_at, completed_at)
|
||||
values (?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
runId,
|
||||
input.orgId,
|
||||
input.drillId,
|
||||
outcome.success ? CHAOS_DRILL_STATE.PASSED : CHAOS_DRILL_STATE.FAILED,
|
||||
JSON.stringify(outcome.checks),
|
||||
startedAt,
|
||||
completedAt,
|
||||
);
|
||||
this.recordSignal({
|
||||
orgId: input.orgId,
|
||||
signalKey: this.signalForScenario(drill.scenario_key),
|
||||
source: "chaos_drill",
|
||||
success: outcome.success,
|
||||
durationMs: new Date(completedAt).getTime() - new Date(startedAt).getTime(),
|
||||
detail: { drillId: input.drillId, scenarioKey: drill.scenario_key },
|
||||
});
|
||||
this.audit.record({
|
||||
orgId: input.orgId,
|
||||
entityType: "chaos_drill",
|
||||
entityId: input.drillId,
|
||||
actorType: "system",
|
||||
actorId: null,
|
||||
action: "run",
|
||||
payload: { runId, state: outcome.success ? CHAOS_DRILL_STATE.PASSED : CHAOS_DRILL_STATE.FAILED },
|
||||
});
|
||||
return runId;
|
||||
}
|
||||
|
||||
isFeatureEnabled(orgId: string, key: FeatureFlagKey): boolean {
|
||||
const row = this.db.prepare("select enabled from feature_flags where org_id = ? and key = ?")
|
||||
.get(orgId, key) as { enabled: number } | undefined;
|
||||
@@ -166,68 +90,4 @@ export class OperationsService {
|
||||
new Date().toISOString(),
|
||||
);
|
||||
}
|
||||
|
||||
private evaluateScenario(orgId: string, scenario: ChaosDrillScenario): DrillOutcome {
|
||||
if (scenario === CHAOS_DRILL_SCENARIO.PROVIDER_OUTAGE_POLICY_GATE) return this.providerOutagePolicyGate(orgId);
|
||||
if (scenario === CHAOS_DRILL_SCENARIO.APPROVAL_BACKLOG_GATE) return this.approvalBacklogGate(orgId);
|
||||
if (scenario === CHAOS_DRILL_SCENARIO.BROWSER_SESSION_BOUNDARY) return this.browserSessionBoundary(orgId);
|
||||
if (scenario === CHAOS_DRILL_SCENARIO.MIGRATION_ROLLBACK_PACK) return this.migrationRollbackPack(orgId);
|
||||
throw new Error("Chaos drill scenario is invalid.");
|
||||
}
|
||||
|
||||
private providerOutagePolicyGate(orgId: string): DrillOutcome {
|
||||
const total = count(this.db, "select count(*) as count from providers where org_id = ?", orgId);
|
||||
const policyConflicts = count(this.db, `
|
||||
select count(*) as count
|
||||
from providers
|
||||
where org_id = ? and data_policy = 'zero_data_retention' and background_enabled = 1
|
||||
`, orgId);
|
||||
const invalidLimits = count(this.db, "select count(*) as count from providers where org_id = ? and monthly_request_limit < 1", orgId);
|
||||
return {
|
||||
success: total > 0 && policyConflicts === 0 && invalidLimits === 0,
|
||||
checks: { providers: total, policyConflicts, invalidLimits },
|
||||
};
|
||||
}
|
||||
|
||||
private approvalBacklogGate(orgId: string): DrillOutcome {
|
||||
const routes = count(this.db, "select count(*) as count from approval_routing_rules where org_id = ?", orgId);
|
||||
const unassigned = count(this.db, `
|
||||
select count(*) as count
|
||||
from approvals
|
||||
where org_id = ? and id not in (
|
||||
select approval_id from approval_assignments where org_id = ?
|
||||
)
|
||||
`, orgId, orgId);
|
||||
return { success: routes > 0 && unassigned === 0, checks: { routes, unassignedApprovals: unassigned } };
|
||||
}
|
||||
|
||||
private browserSessionBoundary(orgId: string): DrillOutcome {
|
||||
const accounts = count(this.db, "select count(*) as count from social_accounts where org_id = ?", orgId);
|
||||
const grants = count(this.db, "select count(*) as count from social_account_grants where org_id = ?", orgId);
|
||||
const actions = count(this.db, "select count(*) as count from browser_actions where org_id = ?", orgId);
|
||||
return { success: grants >= accounts, checks: { accounts, grants, actions } };
|
||||
}
|
||||
|
||||
private migrationRollbackPack(orgId: string): DrillOutcome {
|
||||
const reports = this.db.prepare("select report_json from migration_reports where org_id = ?").all(orgId) as Array<{ report_json: string }>;
|
||||
const complete = reports.filter((report) => hasRollbackPack(report.report_json)).length;
|
||||
return { success: reports.length === complete, checks: { reports: reports.length, rollbackPacks: complete } };
|
||||
}
|
||||
|
||||
private signalForScenario(scenario: ChaosDrillScenario): SloSignalKey {
|
||||
if (scenario === CHAOS_DRILL_SCENARIO.PROVIDER_OUTAGE_POLICY_GATE) return SLO_SIGNAL_KEY.PROVIDER_REQUEST_SUCCESS;
|
||||
if (scenario === CHAOS_DRILL_SCENARIO.APPROVAL_BACKLOG_GATE) return SLO_SIGNAL_KEY.APPROVAL_ROUTE_COVERAGE;
|
||||
if (scenario === CHAOS_DRILL_SCENARIO.BROWSER_SESSION_BOUNDARY) return SLO_SIGNAL_KEY.BROWSER_SESSION_BOUNDARY;
|
||||
if (scenario === CHAOS_DRILL_SCENARIO.MIGRATION_ROLLBACK_PACK) return SLO_SIGNAL_KEY.MIGRATION_VALIDATION_SUCCESS;
|
||||
throw new Error("Chaos drill scenario is invalid.");
|
||||
}
|
||||
}
|
||||
|
||||
function count(db: DatabaseSync, sql: string, ...params: string[]): number {
|
||||
return (db.prepare(sql).get(...params) as { count: number }).count;
|
||||
}
|
||||
|
||||
function hasRollbackPack(reportJson: string): boolean {
|
||||
const parsed = JSON.parse(reportJson) as { rollbackPack?: unknown };
|
||||
return typeof parsed.rollbackPack === "object" && parsed.rollbackPack !== null;
|
||||
}
|
||||
@@ -79,8 +79,6 @@ export class SnapshotService {
|
||||
.map((flag) => ({ ...flag, enabled: toBool(flag.enabled) }));
|
||||
const operationalEvents = list<AppSnapshot["operationalEvents"][number]>(this.db, "operational_events", "created_at desc")
|
||||
.map((event) => ({ ...event, success: toBool(event.success) }));
|
||||
const chaosDrills = list<AppSnapshot["chaosDrills"][number]>(this.db, "chaos_drills", "key asc")
|
||||
.map((drill) => ({ ...drill, enabled: toBool(drill.enabled) }));
|
||||
const rollbackSnapshots = list<AppSnapshot["rollbackSnapshots"][number]>(this.db, "rollback_snapshots", "created_at desc")
|
||||
.map((snapshot) => ({ ...snapshot, active: toBool(snapshot.active) }));
|
||||
|
||||
@@ -158,8 +156,6 @@ export class SnapshotService {
|
||||
featureFlags,
|
||||
sloTargets: list(this.db, "slo_targets", "signal_key asc"),
|
||||
operationalEvents,
|
||||
chaosDrills,
|
||||
chaosDrillRuns: list(this.db, "chaos_drill_runs", "started_at desc"),
|
||||
auditReplayRuns: list(this.db, "audit_replay_runs", "created_at desc"),
|
||||
auditEvents: recentList<AppSnapshot["auditEvents"][number]>(this.db, "audit_events", "created_at desc", 200),
|
||||
documents: list(this.db, "documents", "updated_at desc"),
|
||||
|
||||
@@ -80,7 +80,6 @@ const api: DesktopApi = {
|
||||
setRollbackSnapshotActive: (input) => invoke(IPC_CHANNELS.setRollbackSnapshotActive, input),
|
||||
resolveMigrationReconciliation: (input) => invoke(IPC_CHANNELS.resolveMigrationReconciliation, input),
|
||||
setFeatureFlag: (input) => invoke(IPC_CHANNELS.setFeatureFlag, input),
|
||||
runChaosDrill: (input) => invoke(IPC_CHANNELS.runChaosDrill, input),
|
||||
createComplianceExport: (input) => invoke(IPC_CHANNELS.createComplianceExport, input),
|
||||
createAuditReplay: (input) => invoke(IPC_CHANNELS.createAuditReplay, input),
|
||||
createLegalHold: (input) => invoke(IPC_CHANNELS.createLegalHold, input),
|
||||
|
||||
@@ -78,8 +78,3 @@ export interface SetFeatureFlagInput {
|
||||
flagId: string;
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
export interface RunChaosDrillInput {
|
||||
orgId: string;
|
||||
drillId: string;
|
||||
}
|
||||
@@ -65,7 +65,6 @@ import type {
|
||||
GrantRoleInput,
|
||||
LaunchSocialLoginInput,
|
||||
ResolveMigrationReconciliationInput,
|
||||
RunChaosDrillInput,
|
||||
RunTaskInput,
|
||||
RevokeProviderCredentialInput,
|
||||
SaveProviderCredentialInput,
|
||||
@@ -188,7 +187,6 @@ export interface DesktopApi {
|
||||
setRollbackSnapshotActive(input: SetRollbackSnapshotActiveInput): Promise<DesktopResult<AppSnapshot>>;
|
||||
resolveMigrationReconciliation(input: ResolveMigrationReconciliationInput): Promise<DesktopResult<AppSnapshot>>;
|
||||
setFeatureFlag(input: SetFeatureFlagInput): Promise<DesktopResult<AppSnapshot>>;
|
||||
runChaosDrill(input: RunChaosDrillInput): Promise<DesktopResult<AppSnapshot>>;
|
||||
createComplianceExport(input: CreateComplianceExportInput): Promise<DesktopResult<AppSnapshot>>;
|
||||
createAuditReplay(input: CreateAuditReplayInput): Promise<DesktopResult<AppSnapshot>>;
|
||||
createLegalHold(input: CreateLegalHoldInput): Promise<DesktopResult<AppSnapshot>>;
|
||||
|
||||
@@ -5,7 +5,7 @@ export type { AuditReplayRunRecord } from "./audit-domain";
|
||||
export { COMPLIANCE_EXPORT_STATE, COMPLIANCE_EXPORT_TYPE, type ComplianceExportRecord, type ComplianceExportState, type ComplianceExportType, type ComplianceSubjectType, type LegalHoldEntityType, type LegalHoldRecord, type RetentionFreezeRecord, type RetentionFreezeScopeType } from "./compliance-domain";
|
||||
export { ACCESS_DECISION, IDENTITY_SOURCE, ROLE_KEY, type AccessDecision, type AccessDecisionRecord, type EmployeeIdentityRecord, type IdentitySource, type PrincipalType, type RoleGrantRecord, type RoleKey, type ServiceIdentityRecord } from "./identity-domain";
|
||||
export { ROUTE_TO_TYPE, type ApprovalAssignmentRecord, type ApprovalRoutingRuleRecord, type HiringPolicyRecord, type OrgDepartmentRecord, type RouteToType } from "./org-governance-domain";
|
||||
export { CHAOS_DRILL_SCENARIO, CHAOS_DRILL_STATE, FEATURE_FLAG_KEY, SLO_SIGNAL_KEY, type ChaosDrillRecord, type ChaosDrillRunRecord, type ChaosDrillScenario, type ChaosDrillState, type FeatureFlagKey, type FeatureFlagRecord, type OperationalEventRecord, type SloSignalKey, type SloTargetRecord } from "./operations-domain";
|
||||
export { FEATURE_FLAG_KEY, SLO_SIGNAL_KEY, type FeatureFlagKey, type FeatureFlagRecord, type OperationalEventRecord, type SloSignalKey, type SloTargetRecord } from "./operations-domain";
|
||||
export type { ProviderAuthProfileRecord, ProviderAuthSessionRecord } from "./provider-auth-domain";
|
||||
export { PROVIDER_DATA_POLICY, PROVIDER_EXECUTION_MODE, type ProviderDataPolicy, type ProviderExecutionMode } from "./provider-policy-domain";
|
||||
export { PROVIDER_REQUEST_STATE, type ProviderRequestRecord, type ProviderRequestState, type ProviderStreamRendererEvent } from "./provider-request-domain";
|
||||
|
||||
@@ -53,7 +53,6 @@ export const IPC_CHANNELS = {
|
||||
setRollbackSnapshotActive: "agentcompany:set-rollback-snapshot-active",
|
||||
resolveMigrationReconciliation: "agentcompany:resolve-migration-reconciliation",
|
||||
setFeatureFlag: "agentcompany:set-feature-flag",
|
||||
runChaosDrill: "agentcompany:run-chaos-drill",
|
||||
createComplianceExport: "agentcompany:create-compliance-export",
|
||||
createAuditReplay: "agentcompany:create-audit-replay",
|
||||
createLegalHold: "agentcompany:create-legal-hold",
|
||||
|
||||
@@ -4,7 +4,6 @@ export const FEATURE_FLAG_KEY = {
|
||||
PROVIDER_BACKGROUND_EXECUTION: "provider_background_execution",
|
||||
SOCIAL_PUBLISH_APPROVAL: "social_publish_approval",
|
||||
BOARD_INTERVENTION: "board_intervention",
|
||||
CHAOS_DRILLS: "chaos_drills",
|
||||
} as const;
|
||||
|
||||
export const SLO_SIGNAL_KEY = {
|
||||
@@ -15,19 +14,8 @@ export const SLO_SIGNAL_KEY = {
|
||||
MIGRATION_VALIDATION_SUCCESS: "migration_validation_success",
|
||||
} as const;
|
||||
|
||||
export const CHAOS_DRILL_SCENARIO = {
|
||||
PROVIDER_OUTAGE_POLICY_GATE: "provider_outage_policy_gate",
|
||||
APPROVAL_BACKLOG_GATE: "approval_backlog_gate",
|
||||
BROWSER_SESSION_BOUNDARY: "browser_session_boundary",
|
||||
MIGRATION_ROLLBACK_PACK: "migration_rollback_pack",
|
||||
} as const;
|
||||
|
||||
export const CHAOS_DRILL_STATE = { PASSED: "passed", FAILED: "failed" } as const;
|
||||
|
||||
export type FeatureFlagKey = ValueOf<typeof FEATURE_FLAG_KEY>;
|
||||
export type SloSignalKey = ValueOf<typeof SLO_SIGNAL_KEY>;
|
||||
export type ChaosDrillScenario = ValueOf<typeof CHAOS_DRILL_SCENARIO>;
|
||||
export type ChaosDrillState = ValueOf<typeof CHAOS_DRILL_STATE>;
|
||||
|
||||
export interface FeatureFlagRecord {
|
||||
id: string; orgId: string; key: FeatureFlagKey; label: string; enabled: boolean; createdAt: string; updatedAt: string;
|
||||
@@ -40,11 +28,3 @@ export interface SloTargetRecord {
|
||||
export interface OperationalEventRecord {
|
||||
id: string; orgId: string; signalKey: SloSignalKey; source: string; success: boolean; durationMs: number | null; detailJson: string; createdAt: string;
|
||||
}
|
||||
|
||||
export interface ChaosDrillRecord {
|
||||
id: string; orgId: string; key: string; label: string; scenarioKey: ChaosDrillScenario; enabled: boolean; createdAt: string; updatedAt: string;
|
||||
}
|
||||
|
||||
export interface ChaosDrillRunRecord {
|
||||
id: string; orgId: string; drillId: string; state: ChaosDrillState; resultJson: string; startedAt: string; completedAt: string;
|
||||
}
|
||||
@@ -44,7 +44,7 @@ import type {
|
||||
} from "./domain";
|
||||
import type { AccessDecisionRecord, EmployeeIdentityRecord, RoleGrantRecord, ServiceIdentityRecord } from "./identity-domain";
|
||||
import type { ApprovalAssignmentRecord, ApprovalRoutingRuleRecord, HiringPolicyRecord, OrgDepartmentRecord } from "./org-governance-domain";
|
||||
import type { ChaosDrillRecord, ChaosDrillRunRecord, FeatureFlagRecord, OperationalEventRecord, SloTargetRecord } from "./operations-domain";
|
||||
import type { FeatureFlagRecord, OperationalEventRecord, SloTargetRecord } from "./operations-domain";
|
||||
import type { ProviderAuthProfileRecord, ProviderAuthSessionRecord } from "./provider-auth-domain";
|
||||
import type { ProviderRequestRecord } from "./provider-request-domain";
|
||||
import type { ProviderRateLimitObservationRecord, ProviderRateLimitStateRecord } from "./provider-runtime-domain";
|
||||
@@ -136,8 +136,6 @@ export interface AppSnapshot {
|
||||
featureFlags: FeatureFlagRecord[];
|
||||
sloTargets: SloTargetRecord[];
|
||||
operationalEvents: OperationalEventRecord[];
|
||||
chaosDrills: ChaosDrillRecord[];
|
||||
chaosDrillRuns: ChaosDrillRunRecord[];
|
||||
auditReplayRuns: AuditReplayRunRecord[];
|
||||
auditEvents: AuditEventRecord[];
|
||||
documents: DocumentRecord[];
|
||||
|
||||
@@ -35,7 +35,7 @@ describe("domain services", () => {
|
||||
const connection = createConnection();
|
||||
|
||||
const rows = connection.db.prepare("select version from schema_migrations order by version asc").all() as Array<{ version: number }>;
|
||||
expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95]);
|
||||
expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96]);
|
||||
connection.close();
|
||||
});
|
||||
|
||||
@@ -60,9 +60,8 @@ describe("domain services", () => {
|
||||
expect(state.orgDepartments.map((department) => department.name)).toContain("Engineering");
|
||||
expect(state.hiringPolicies.some((policy) => policy.departmentName === "Engineering")).toBe(true);
|
||||
expect(state.approvalRoutingRules[0]).toMatchObject({ kind: "*", minimumRisk: "low", routeToType: "employee" });
|
||||
expect(state.featureFlags.map((flag) => flag.key)).toContain("chaos_drills");
|
||||
expect(state.featureFlags.map((flag) => flag.key)).toContain("board_intervention");
|
||||
expect(state.sloTargets.map((target) => target.signalKey)).toContain("provider_request_success");
|
||||
expect(state.chaosDrills.map((drill) => drill.scenarioKey)).toContain("approval_backlog_gate");
|
||||
expect(state.chatSpaces.map((space) => space.name)).toContain("board");
|
||||
expect(state.policyRules[0]?.action).toBe("task_create");
|
||||
expect(state.auditEvents[0]?.action).toBe("created");
|
||||
|
||||
@@ -130,8 +130,6 @@ describe("migration service", () => {
|
||||
create table feature_flags(id text primary key, org_id text);
|
||||
create table slo_targets(id text primary key, org_id text);
|
||||
create table operational_events(id text primary key, org_id text);
|
||||
create table chaos_drills(id text primary key, org_id text);
|
||||
create table chaos_drill_runs(id text primary key, org_id text, drill_id text);
|
||||
create table chat_spaces(id text primary key, org_id text);
|
||||
create table chat_messages(id text primary key, org_id text, space_id text, thread_id text);
|
||||
create table chat_threads(id text primary key, org_id text, space_id text, root_message_id text);
|
||||
@@ -202,8 +200,6 @@ describe("migration service", () => {
|
||||
insert into feature_flags(id, org_id) values ('flag-1', 'org-1');
|
||||
insert into slo_targets(id, org_id) values ('slo-1', 'org-1');
|
||||
insert into operational_events(id, org_id) values ('op-1', 'org-1');
|
||||
insert into chaos_drills(id, org_id) values ('drill-1', 'org-1');
|
||||
insert into chaos_drill_runs(id, org_id, drill_id) values ('drill-run-1', 'org-1', 'drill-1');
|
||||
insert into chat_threads(id, org_id, space_id, root_message_id) values ('thread-1', 'org-1', 'space-1', null);
|
||||
insert into chat_messages(id, org_id, space_id, thread_id) values ('msg-1', 'org-1', 'space-1', 'thread-1');
|
||||
insert into chat_memberships(space_id, principal_type, principal_id) values ('space-1', 'employee', 'emp-1');
|
||||
@@ -269,7 +265,6 @@ describe("migration service", () => {
|
||||
expect(body.tableCounts.org_departments).toBe(1);
|
||||
expect(body.tableCounts.hiring_policies).toBe(1);
|
||||
expect(body.tableCounts.feature_flags).toBe(1);
|
||||
expect(body.tableCounts.chaos_drill_runs).toBe(1);
|
||||
expect(body.tableCounts.role_grants).toBe(1);
|
||||
expect(body.tableCounts.skill_artifacts).toBe(1);
|
||||
expect(body.tableCounts.chat_read_state).toBe(1);
|
||||
@@ -297,7 +292,6 @@ describe("migration service", () => {
|
||||
expect(body.idMaps.access_decisions).toEqual(["access-1"]);
|
||||
expect(body.idMaps.approval_assignments).toEqual(["assign-1"]);
|
||||
expect(body.idMaps.operational_events).toEqual(["op-1"]);
|
||||
expect(body.idMaps.chaos_drill_runs).toEqual(["drill-run-1"]);
|
||||
expect(body.idMaps.board_interventions).toEqual(["bdi-1"]);
|
||||
expect(body.idMaps.file_handoff_records).toEqual(["handoff-1"]);
|
||||
expect(body.idMaps.retention_freezes).toEqual(["freeze-1"]);
|
||||
|
||||
@@ -34,7 +34,7 @@ const testVault = {
|
||||
};
|
||||
|
||||
describe("operations service", () => {
|
||||
test("seeds feature flags, SLO targets, and chaos drills for new organizations", () => {
|
||||
test("seeds feature flags and SLO targets, and no drill that scores configuration as fault handling", () => {
|
||||
const connection = createConnection();
|
||||
const organization = new OrganizationService(connection.db);
|
||||
const snapshot = new SnapshotService(connection.db);
|
||||
@@ -46,12 +46,11 @@ describe("operations service", () => {
|
||||
const state = snapshot.load();
|
||||
expect(state.featureFlags.map((flag) => flag.key).sort()).toEqual([
|
||||
"board_intervention",
|
||||
"chaos_drills",
|
||||
"provider_background_execution",
|
||||
"social_publish_approval",
|
||||
]);
|
||||
expect(state.sloTargets.map((target) => target.signalKey)).toContain("browser_session_boundary");
|
||||
expect(state.chaosDrills).toHaveLength(4);
|
||||
expect(connection.db.prepare("select name from sqlite_master where type = 'table' and name like 'chaos_drill%'").all()).toEqual([]);
|
||||
connection.close();
|
||||
});
|
||||
|
||||
@@ -98,26 +97,4 @@ describe("operations service", () => {
|
||||
})).rejects.toThrow("Provider background execution is disabled by feature flag.");
|
||||
connection.close();
|
||||
});
|
||||
|
||||
test("runs chaos drills and records operational signals", () => {
|
||||
const connection = createConnection();
|
||||
const organization = new OrganizationService(connection.db);
|
||||
const operations = new OperationsService(connection.db);
|
||||
const snapshot = new SnapshotService(connection.db);
|
||||
let orgId = "";
|
||||
let drillId = "";
|
||||
|
||||
connection.transaction(() => {
|
||||
orgId = createOrganization(organization);
|
||||
drillId = (connection.db.prepare("select id from chaos_drills where org_id = ? and scenario_key = ?")
|
||||
.get(orgId, "approval_backlog_gate") as { id: string }).id;
|
||||
operations.runChaosDrill({ orgId, drillId });
|
||||
});
|
||||
|
||||
const state = snapshot.load();
|
||||
expect(state.chaosDrillRuns[0]).toMatchObject({ orgId, drillId, state: "passed" });
|
||||
expect(state.operationalEvents[0]).toMatchObject({ orgId, signalKey: "approval_route_coverage", source: "chaos_drill", success: true });
|
||||
expect(state.auditEvents.some((event) => event.entityType === "chaos_drill" && event.action === "run")).toBe(true);
|
||||
connection.close();
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user