Add migration cutover reconciliation journal

This commit is contained in:
ZacharyZhang-NY committed 2026-05-23 21:30:44 -04:00
1 parent a21d35a165
commit 62ac397459
12 files changed
+319 -6

No files matched your search

@@ -0,0 +1,62 @@
export const migrationCutoverJournalSchema = `
create table if not exists migration_cutover_events (
id text primary key,
org_id text not null references organizations(id) on delete cascade,
migration_report_id text not null references migration_reports(id) on delete cascade,
rollback_snapshot_id text references rollback_snapshots(id) on delete set null,
event_type text not null,
actor_type text not null,
actor_id text,
payload_json text not null,
created_at text not null
);
create table if not exists migration_reconciliation_items (
id text primary key,
org_id text not null references organizations(id) on delete cascade,
migration_report_id text not null references migration_reports(id) on delete cascade,
rollback_snapshot_id text references rollback_snapshots(id) on delete set null,
source_event_id text not null,
target_type text not null,
target_id text not null,
reason text not null,
state text not null,
payload_json text not null,
created_at text not null,
resolved_at text,
resolved_by_employee_id text references employees(id) on delete set null,
unique(rollback_snapshot_id, source_event_id)
);
insert into migration_cutover_events(
id, org_id, migration_report_id, rollback_snapshot_id, event_type,
actor_type, actor_id, payload_json, created_at
)
select
'mce_' || lower(hex(randomblob(16))),
org_id,
migration_report_id,
id,
case active when 1 then 'rollback_snapshot_activated' else 'rollback_snapshot_created' end,
'system',
null,
json_object('sourceFingerprint', source_fingerprint, 'sourceDatabaseSha256', source_database_sha256),
created_at
from rollback_snapshots
where not exists (
select 1 from migration_cutover_events
where migration_cutover_events.rollback_snapshot_id = rollback_snapshots.id
);
create index if not exists idx_migration_cutover_events_org_created
on migration_cutover_events(org_id, created_at);
create index if not exists idx_migration_cutover_events_report
on migration_cutover_events(migration_report_id, created_at);
create index if not exists idx_migration_reconciliation_items_org_state
on migration_reconciliation_items(org_id, state, created_at);
create index if not exists idx_migration_reconciliation_items_report
on migration_reconciliation_items(migration_report_id, created_at);
insert into schema_migrations(version, name, applied_at)
values (28, 'migration_cutover_journal', datetime('now'));
`;
@@ -26,6 +26,7 @@ import { taskRunProviderRequestSchema } from "./migrations/024-task-run-provider
import { chatMessageDeliverySchema } from "./migrations/025-chat-message-delivery";
import { browserPlatformGovernanceSchema } from "./migrations/026-browser-platform-governance";
import { socialAuditRecordsSchema } from "./migrations/027-social-audit-records";
import { migrationCutoverJournalSchema } from "./migrations/028-migration-cutover-journal";
interface Migration {
version: number;
@@ -61,6 +62,7 @@ const MIGRATIONS: Migration[] = [
{ version: 25, name: "chat_message_delivery", sql: chatMessageDeliverySchema },
{ version: 26, name: "browser_platform_governance", sql: browserPlatformGovernanceSchema },
{ version: 27, name: "social_audit_records", sql: socialAuditRecordsSchema },
{ version: 28, name: "migration_cutover_journal", sql: migrationCutoverJournalSchema },
];
const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version;
@@ -0,0 +1,103 @@
import type { DatabaseSync } from "node:sqlite";
import { createId } from "../../shared/ids";
const REPLAYABLE_ENTITY_TYPES = new Set([
"agent",
"approval",
"browser_action",
"chat_message",
"chat_space",
"file",
"scheduled_post",
"social_account",
"task",
"workspace",
]);
export interface CutoverDeltaEvent {
id: string;
entityType: string;
entityId: string;
actorType: string;
actorId: string | null;
action: string;
payload: Record<string, unknown>;
createdAt: string;
}
export interface RecordCutoverEventInput {
orgId: string;
migrationReportId: string;
rollbackSnapshotId: string | null;
eventType: string;
actorType: string;
actorId: string | null;
payload: Record<string, unknown>;
createdAt?: string;
}
export class MigrationCutoverJournalService {
constructor(private readonly db: DatabaseSync) {}
recordEvent(input: RecordCutoverEventInput): string {
const id = createId("mce");
this.db.prepare(`
insert into migration_cutover_events(
id, org_id, migration_report_id, rollback_snapshot_id, event_type,
actor_type, actor_id, payload_json, created_at
) values (?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(
id,
input.orgId,
input.migrationReportId,
input.rollbackSnapshotId,
input.eventType,
input.actorType,
input.actorId,
JSON.stringify(input.payload),
input.createdAt ?? new Date().toISOString(),
);
return id;
}
queueReconciliation(input: {
orgId: string;
migrationReportId: string;
rollbackSnapshotId: string;
events: CutoverDeltaEvent[];
createdAt?: string;
}): number {
let queued = 0;
const createdAt = input.createdAt ?? new Date().toISOString();
const statement = this.db.prepare(`
insert or ignore into migration_reconciliation_items(
id, org_id, migration_report_id, rollback_snapshot_id, source_event_id,
target_type, target_id, reason, state, payload_json, created_at, resolved_at, resolved_by_employee_id
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, null, null)
`);
for (const event of input.events) {
if (!REPLAYABLE_ENTITY_TYPES.has(event.entityType)) continue;
const result = statement.run(
createId("mri"),
input.orgId,
input.migrationReportId,
input.rollbackSnapshotId,
event.id,
event.entityType,
event.entityId,
"manual_delta_replay",
"open",
JSON.stringify({
action: event.action,
actorType: event.actorType,
actorId: event.actorId,
payload: event.payload,
createdAt: event.createdAt,
}),
createdAt,
);
queued += Number(result.changes);
}
return queued;
}
}
@@ -77,6 +77,8 @@ const LEGACY_TABLES = [
"search_documents",
"audit_replay_runs",
"rollback_snapshots",
"migration_cutover_events",
"migration_reconciliation_items",
"audit_events",
];
@@ -248,6 +250,13 @@ const RELATION_CHECKS: RelationCheck[] = [
{ table: "rollback_snapshots", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
{ table: "rollback_snapshots", column: "migration_report_id", referencedTable: "migration_reports", referencedColumn: "id", reason: "dangling_migration_report_id" },
{ table: "rollback_snapshots", column: "created_by_employee_id", referencedTable: "employees", referencedColumn: "id", reason: "dangling_creator_employee_id" },
{ table: "migration_cutover_events", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
{ table: "migration_cutover_events", column: "migration_report_id", referencedTable: "migration_reports", referencedColumn: "id", reason: "dangling_migration_report_id" },
{ table: "migration_cutover_events", column: "rollback_snapshot_id", referencedTable: "rollback_snapshots", referencedColumn: "id", reason: "dangling_rollback_snapshot_id" },
{ table: "migration_reconciliation_items", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
{ table: "migration_reconciliation_items", column: "migration_report_id", referencedTable: "migration_reports", referencedColumn: "id", reason: "dangling_migration_report_id" },
{ table: "migration_reconciliation_items", column: "rollback_snapshot_id", referencedTable: "rollback_snapshots", referencedColumn: "id", reason: "dangling_rollback_snapshot_id" },
{ table: "migration_reconciliation_items", column: "resolved_by_employee_id", referencedTable: "employees", referencedColumn: "id", reason: "dangling_resolver_employee_id" },
{ table: "audit_events", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
];
@@ -4,6 +4,7 @@ import type { DatabaseSync } from "node:sqlite";
import type { CreateRollbackSnapshotInput, SetRollbackSnapshotActiveInput } from "../../shared/commands";
import { createId } from "../../shared/ids";
import { AuditService } from "./audit-service";
import { MigrationCutoverJournalService, type CutoverDeltaEvent } from "./migration-cutover-journal-service";
interface MigrationReportRow {
id: string;
@@ -31,6 +32,13 @@ interface WorkspaceRow {
updated_at: string;
}
interface RollbackSnapshotRow {
id: string;
migration_report_id: string;
source_fingerprint: string;
delta_journal_json: string;
}
interface FileRow {
id: string;
name: string;
@@ -54,9 +62,11 @@ interface AuditRow {
export class RollbackService {
private readonly audit: AuditService;
private readonly cutoverJournal: MigrationCutoverJournalService;
constructor(private readonly db: DatabaseSync) {
this.audit = new AuditService(db);
this.cutoverJournal = new MigrationCutoverJournalService(db);
}
createSnapshot(input: CreateRollbackSnapshotInput): string {
@@ -109,13 +119,26 @@ export class RollbackService {
action: "created",
payload: { migrationReportId: report.id, sourceFingerprint: body.sourceFingerprint },
});
this.cutoverJournal.recordEvent({
orgId: input.orgId,
migrationReportId: report.id,
rollbackSnapshotId: id,
eventType: "rollback_snapshot_created",
actorType: "employee",
actorId: input.createdByEmployeeId,
payload: { sourceFingerprint: body.sourceFingerprint, sourceDatabaseSha256, workspaceManifestSha256 },
createdAt: now,
});
return id;
}
setActive(input: SetRollbackSnapshotActiveInput): void {
this.requireEmployee(input.orgId, input.employeeId);
const snapshot = this.db.prepare("select id from rollback_snapshots where id = ? and org_id = ?")
.get(input.snapshotId, input.orgId) as { id: string } | undefined;
const snapshot = this.db.prepare(`
select id, migration_report_id, source_fingerprint, delta_journal_json
from rollback_snapshots
where id = ? and org_id = ?
`).get(input.snapshotId, input.orgId) as RollbackSnapshotRow | undefined;
if (!snapshot) throw new Error("Rollback snapshot was not found.");
const now = new Date().toISOString();
if (input.active) this.db.prepare("update rollback_snapshots set active = 0, activated_at = null where org_id = ?").run(input.orgId);
@@ -131,6 +154,25 @@ export class RollbackService {
action: input.active ? "activated" : "deactivated",
payload: { active: input.active },
});
const queuedReconciliationItems = input.active
? this.cutoverJournal.queueReconciliation({
orgId: input.orgId,
migrationReportId: snapshot.migration_report_id,
rollbackSnapshotId: snapshot.id,
events: parseDeltaJournal(snapshot.delta_journal_json),
createdAt: now,
})
: 0;
this.cutoverJournal.recordEvent({
orgId: input.orgId,
migrationReportId: snapshot.migration_report_id,
rollbackSnapshotId: snapshot.id,
eventType: input.active ? "rollback_snapshot_activated" : "rollback_snapshot_deactivated",
actorType: "employee",
actorId: input.employeeId,
payload: { active: input.active, sourceFingerprint: snapshot.source_fingerprint, queuedReconciliationItems },
createdAt: now,
});
}
private requireReport(orgId: string, reportId: string): MigrationReportRow {
@@ -248,3 +290,34 @@ function parseRecord(json: string): Record<string, unknown> {
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error("JSON record is invalid.");
return parsed as Record<string, unknown>;
}
function parseDeltaJournal(json: string): CutoverDeltaEvent[] {
const parsed = JSON.parse(json) as unknown;
if (!Array.isArray(parsed)) throw new Error("Rollback delta journal is invalid.");
return parsed.map((item) => {
if (typeof item !== "object" || item === null || Array.isArray(item)) throw new Error("Rollback delta journal entry is invalid.");
const record = item as Record<string, unknown>;
if (
typeof record.id !== "string"
|| typeof record.entityType !== "string"
|| typeof record.entityId !== "string"
|| typeof record.actorType !== "string"
|| typeof record.action !== "string"
|| typeof record.createdAt !== "string"
) {
throw new Error("Rollback delta journal entry is invalid.");
}
const payload = record.payload;
if (typeof payload !== "object" || payload === null || Array.isArray(payload)) throw new Error("Rollback delta journal entry is invalid.");
return {
id: record.id,
entityType: record.entityType,
entityId: record.entityId,
actorType: record.actorType,
actorId: typeof record.actorId === "string" ? record.actorId : null,
action: record.action,
payload: payload as Record<string, unknown>,
createdAt: record.createdAt,
};
});
}
@@ -262,7 +262,7 @@ export class SdlcService {
{ key: "provider_adapter_coverage", passed: missing.length === 0, value: `${adapterKeys.length}/${catalogKeys.length}` },
{ key: "provider_rate_limit_registry", passed: rateLimitRegistry === 1, value: rateLimitRegistry },
{ key: "provider_runtime_contract", passed: runtimeContractReady, value: runtimeContractReady },
{ key: "schema_version", passed: schemaVersion >= 27, value: schemaVersion },
{ key: "schema_version", passed: schemaVersion >= 28, value: schemaVersion },
];
}
@@ -126,6 +126,8 @@ export class SnapshotService {
policyRules: list(this.db, "policy_rules", "created_at asc"),
migrationReports: list(this.db, "migration_reports", "created_at desc"),
rollbackSnapshots,
migrationCutoverEvents: list(this.db, "migration_cutover_events", "created_at desc"),
migrationReconciliationItems: list(this.db, "migration_reconciliation_items", "created_at desc"),
complianceExports: list(this.db, "compliance_exports", "created_at desc"),
legalHolds,
retentionFreezes,
+4 -2
View File
@@ -8,7 +8,7 @@ import type { ProviderAuthProfileRecord } from "./provider-auth-domain";
import type { ProviderDataPolicy } from "./provider-policy-domain";
import type { ProviderRequestRecord } from "./provider-request-domain";
import type { ProviderRateLimitObservationRecord, ProviderRateLimitStateRecord } from "./provider-runtime-domain";
import type { RollbackSnapshotRecord } from "./rollback-domain";
import type { MigrationCutoverEventRecord, MigrationReconciliationItemRecord, RollbackSnapshotRecord } from "./rollback-domain";
import type { EvalCaseRecord, EvalRunRecord, EvalSuiteRecord, PrBotGateRecord, SdlcReviewAgentRecord, SdlcReviewRunRecord } from "./sdlc-domain";
import type { BrowserActionPolicyRecord, BrowserActionRecord, BrowserSessionContainerRecord, SocialAccountGrantRecord, SocialAccountRecord, SocialApprovalRouteRecord, SocialAuditRecord } from "./social-domain";
export { BOARD_INTERVENTION_ACTION, BOARD_INTERVENTION_MODE, CHAT_THREAD_BOARD_STATE, type BoardInterventionAction, type BoardInterventionMode, type BoardInterventionRecord, type ChatThreadBoardState } from "./board-domain";
@@ -21,7 +21,7 @@ export type { ProviderAuthProfileRecord } from "./provider-auth-domain";
export { PROVIDER_DATA_POLICY, PROVIDER_EXECUTION_MODE, type ProviderDataPolicy, type ProviderExecutionMode } from "./provider-policy-domain";
export { PROVIDER_REQUEST_STATE, type ProviderRequestRecord, type ProviderRequestState, type ProviderStreamRendererEvent } from "./provider-request-domain";
export type { ProviderRateLimitObservationRecord, ProviderRateLimitStateRecord } from "./provider-runtime-domain";
export type { RollbackSnapshotRecord } from "./rollback-domain";
export type { MigrationCutoverEventRecord, MigrationReconciliationItemRecord, RollbackSnapshotRecord } from "./rollback-domain";
export { EVAL_SUITE_KEY, PR_BOT_GATE_KEY, SDLC_REVIEW_AGENT_KEY, SDLC_REVIEW_TARGET_TYPE, SDLC_RUN_STATE, type EvalCaseRecord, type EvalRunRecord, type EvalSuiteKey, type EvalSuiteRecord, type PrBotGateKey, type PrBotGateRecord, type SdlcReviewAgentKey, type SdlcReviewAgentRecord, type SdlcReviewRunRecord, type SdlcReviewTargetType, type SdlcRunState } from "./sdlc-domain";
export { SEARCH_ENTITY_TYPES, type SearchEntityType, type SearchResultRecord } from "./search-domain";
export type { BrowserActionPolicyRecord, BrowserActionRecord, BrowserSessionContainerRecord, SocialAccountGrantRecord, SocialAccountRecord, SocialApprovalRouteRecord, SocialAuditRecord } from "./social-domain";
@@ -453,6 +453,8 @@ export interface AppSnapshot {
policyRules: PolicyRuleRecord[];
migrationReports: MigrationReportRecord[];
rollbackSnapshots: RollbackSnapshotRecord[];
migrationCutoverEvents: MigrationCutoverEventRecord[];
migrationReconciliationItems: MigrationReconciliationItemRecord[];
complianceExports: ComplianceExportRecord[]; legalHolds: LegalHoldRecord[]; retentionFreezes: RetentionFreezeRecord[];
employeeIdentities: EmployeeIdentityRecord[]; serviceIdentities: ServiceIdentityRecord[]; roleGrants: RoleGrantRecord[]; accessDecisions: AccessDecisionRecord[];
orgDepartments: OrgDepartmentRecord[]; hiringPolicies: HiringPolicyRecord[]; approvalRoutingRules: ApprovalRoutingRuleRecord[]; approvalAssignments: ApprovalAssignmentRecord[];
@@ -14,3 +14,31 @@ export interface RollbackSnapshotRecord {
createdAt: string;
activatedAt: string | null;
}
export interface MigrationCutoverEventRecord {
id: string;
orgId: string;
migrationReportId: string;
rollbackSnapshotId: string | null;
eventType: string;
actorType: string;
actorId: string | null;
payloadJson: string;
createdAt: string;
}
export interface MigrationReconciliationItemRecord {
id: string;
orgId: string;
migrationReportId: string;
rollbackSnapshotId: string | null;
sourceEventId: string;
targetType: string;
targetId: string;
reason: string;
state: string;
payloadJson: string;
createdAt: string;
resolvedAt: string | null;
resolvedByEmployeeId: string | null;
}
@@ -23,7 +23,7 @@ describe("domain services", () => {
const connection = createConnection();
const rows = connection.db.prepare("select version from schema_migrations order by version asc").all() as Array<{ version: number }>;
expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]);
expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]);
connection.close();
});
@@ -159,6 +159,8 @@ describe("migration service", () => {
create table search_documents(entity_id text);
create table audit_replay_runs(id text primary key, org_id text, thread_id text, created_by_employee_id text);
create table rollback_snapshots(id text primary key, org_id text, migration_report_id text, created_by_employee_id text);
create table migration_cutover_events(id text primary key, org_id text, migration_report_id text, rollback_snapshot_id text);
create table migration_reconciliation_items(id text primary key, org_id text, migration_report_id text, rollback_snapshot_id text, resolved_by_employee_id text);
create table audit_events(id text primary key, org_id text);
insert into organizations(id) values ('org-1');
insert into employees(id, org_id) values ('emp-1', 'org-1');
@@ -228,6 +230,8 @@ describe("migration service", () => {
insert into search_documents(entity_id) values ('task-1');
insert into audit_replay_runs(id, org_id, thread_id, created_by_employee_id) values ('replay-1', 'org-1', 'thread-1', 'emp-1');
insert into rollback_snapshots(id, org_id, migration_report_id, created_by_employee_id) values ('rollback-1', 'org-1', 'migration-1', 'emp-1');
insert into migration_cutover_events(id, org_id, migration_report_id, rollback_snapshot_id) values ('cutover-1', 'org-1', 'migration-1', 'rollback-1');
insert into migration_reconciliation_items(id, org_id, migration_report_id, rollback_snapshot_id, resolved_by_employee_id) values ('reconcile-1', 'org-1', 'migration-1', 'rollback-1', 'emp-1');
insert into audit_events(id, org_id) values ('audit-1', 'org-1');
`);
let orgId = "";
@@ -262,6 +266,8 @@ describe("migration service", () => {
expect(body.tableCounts.compliance_exports).toBe(1);
expect(body.tableCounts.audit_replay_runs).toBe(1);
expect(body.tableCounts.rollback_snapshots).toBe(1);
expect(body.tableCounts.migration_cutover_events).toBe(1);
expect(body.tableCounts.migration_reconciliation_items).toBe(1);
expect(body.tableCounts.search_documents).toBe(1);
expect(body.tableChecksums.audit_events).toHaveLength(64);
expect(body.idMaps.skill_runtime_snapshots).toEqual(["snapshot-1"]);
@@ -276,6 +282,8 @@ describe("migration service", () => {
expect(body.idMaps.retention_freezes).toEqual(["freeze-1"]);
expect(body.idMaps.audit_replay_runs).toEqual(["replay-1"]);
expect(body.idMaps.rollback_snapshots).toEqual(["rollback-1"]);
expect(body.idMaps.migration_cutover_events).toEqual(["cutover-1"]);
expect(body.idMaps.migration_reconciliation_items).toEqual(["reconcile-1"]);
expect(body.idMaps.provider_usage_windows).toEqual(["usage-1"]);
expect(body.idMaps.provider_rate_limit_observations).toEqual(["quota-1"]);
expect(body.idMaps.browser_session_containers).toEqual(["container-1"]);
@@ -4,6 +4,7 @@ import { join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { describe, expect, test } from "vitest";
import { DatabaseConnection } from "../../src/main/database/connection";
import { AuditService } from "../../src/main/services/audit-service";
import { MigrationService } from "../../src/main/services/migration-service";
import { OrganizationService } from "../../src/main/services/organization-service";
import { RollbackService } from "../../src/main/services/rollback-service";
@@ -31,6 +32,7 @@ describe("rollback service", () => {
const migrations = new MigrationService(connection.db);
const rollback = new RollbackService(connection.db);
const snapshot = new SnapshotService(connection.db);
const audit = new AuditService(connection.db);
const legacyPath = createLegacyDatabase();
let orgId = "";
let employeeId = "";
@@ -47,6 +49,15 @@ describe("rollback service", () => {
employeeId = (connection.db.prepare("select id from employees where org_id = ?").get(orgId) as { id: string }).id;
seedWorkspace(connection.db, orgId);
reportId = migrations.createReport({ orgId, sourceDatabasePath: legacyPath });
audit.record({
orgId,
entityType: "task",
entityId: "task-cutover",
actorType: "employee",
actorId: employeeId,
action: "created",
payload: { title: "Reconcile release delta" },
});
rollbackId = rollback.createSnapshot({ orgId, migrationReportId: reportId, createdByEmployeeId: employeeId });
rollback.setActive({ orgId, snapshotId: rollbackId, active: true, employeeId });
});
@@ -64,6 +75,19 @@ describe("rollback service", () => {
expect(workspaceManifest.files).toHaveLength(1);
expect(deltaJournal.length).toBeGreaterThan(0);
expect(restorePlan).toMatchObject({ migrationReportId: reportId, restoreStrategy: "legacy_snapshot_restore" });
expect(state.migrationCutoverEvents.map((event) => event.eventType).sort()).toEqual([
"rollback_snapshot_activated",
"rollback_snapshot_created",
]);
expect(state.migrationReconciliationItems).toContainEqual(expect.objectContaining({
orgId,
migrationReportId: reportId,
rollbackSnapshotId: rollbackId,
targetType: "task",
targetId: "task-cutover",
reason: "manual_delta_replay",
state: "open",
}));
expect(state.auditEvents.some((event) => event.entityType === "rollback_snapshot" && event.action === "activated")).toBe(true);
connection.close();
});