Q131: remove review agents and eval suites whose pass only counted configuration rows
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
074a1fc8e3
commit
fe45cd4d46
20 files changed
+32
-476
No files matched your search
@@ -4,7 +4,6 @@ import { INTEGRATION_KEY_NAMES } from "../../../shared/commands";
|
|||||||
import { COMPLIANCE_EXPORT_TYPE } from "../../../shared/compliance-domain";
|
import { COMPLIANCE_EXPORT_TYPE } from "../../../shared/compliance-domain";
|
||||||
import { SOCIAL_PLATFORM } from "../../../shared/domain";
|
import { SOCIAL_PLATFORM } from "../../../shared/domain";
|
||||||
import { PROVIDER_DATA_POLICY, PROVIDER_EXECUTION_MODE } from "../../../shared/provider-policy-domain";
|
import { PROVIDER_DATA_POLICY, PROVIDER_EXECUTION_MODE } from "../../../shared/provider-policy-domain";
|
||||||
import { SDLC_REVIEW_TARGET_TYPE } from "../../../shared/sdlc-domain";
|
|
||||||
import { SEARCH_ENTITY_TYPES } from "../../../shared/search-domain";
|
import { SEARCH_ENTITY_TYPES } from "../../../shared/search-domain";
|
||||||
|
|
||||||
const id = z.string();
|
const id = z.string();
|
||||||
@@ -58,8 +57,6 @@ export const platformContracts = {
|
|||||||
resolveMigrationReconciliation: z.strictObject({ orgId: id, reconciliationItemId: id, employeeId: id }) satisfies z.ZodType<C.ResolveMigrationReconciliationInput>,
|
resolveMigrationReconciliation: z.strictObject({ orgId: id, reconciliationItemId: id, employeeId: id }) satisfies z.ZodType<C.ResolveMigrationReconciliationInput>,
|
||||||
setFeatureFlag: z.strictObject({ orgId: id, flagId: id, enabled: z.boolean() }) satisfies z.ZodType<C.SetFeatureFlagInput>,
|
setFeatureFlag: z.strictObject({ orgId: id, flagId: id, enabled: z.boolean() }) satisfies z.ZodType<C.SetFeatureFlagInput>,
|
||||||
runChaosDrill: z.strictObject({ orgId: id, drillId: id }) satisfies z.ZodType<C.RunChaosDrillInput>,
|
runChaosDrill: z.strictObject({ orgId: id, drillId: id }) satisfies z.ZodType<C.RunChaosDrillInput>,
|
||||||
runSdlcReview: z.strictObject({ orgId: id, reviewAgentId: id, targetType: z.enum(SDLC_REVIEW_TARGET_TYPE), targetRef: z.string() }) satisfies z.ZodType<C.RunSdlcReviewInput>,
|
|
||||||
runEvalSuite: z.strictObject({ orgId: id, suiteId: id }) satisfies z.ZodType<C.RunEvalSuiteInput>,
|
|
||||||
setPrBotGate: z.strictObject({ orgId: id, gateId: id, enabled: z.boolean() }) satisfies z.ZodType<C.SetPrBotGateInput>,
|
setPrBotGate: z.strictObject({ orgId: id, gateId: id, enabled: z.boolean() }) satisfies z.ZodType<C.SetPrBotGateInput>,
|
||||||
createComplianceExport: z.strictObject({
|
createComplianceExport: z.strictObject({
|
||||||
orgId: id, subjectType: z.enum(["employee", "agent"]), subjectId: id, exportType: z.enum(COMPLIANCE_EXPORT_TYPE), requestedByEmployeeId: id,
|
orgId: id, subjectType: z.enum(["employee", "agent"]), subjectId: id, exportType: z.enum(COMPLIANCE_EXPORT_TYPE), requestedByEmployeeId: id,
|
||||||
|
|||||||
@@ -379,8 +379,6 @@ export function registerIpcHandlers(deps: IpcDeps): void {
|
|||||||
handle("resolveMigrationReconciliation", (input) => mutate(() => rollback.resolveReconciliationItem(input)));
|
handle("resolveMigrationReconciliation", (input) => mutate(() => rollback.resolveReconciliationItem(input)));
|
||||||
handle("setFeatureFlag", (input) => mutate(() => operations.setFeatureFlag(input)));
|
handle("setFeatureFlag", (input) => mutate(() => operations.setFeatureFlag(input)));
|
||||||
handle("runChaosDrill", (input) => mutate(() => operations.runChaosDrill(input)));
|
handle("runChaosDrill", (input) => mutate(() => operations.runChaosDrill(input)));
|
||||||
handle("runSdlcReview", (input) => mutate(() => sdlc.runReview(input)));
|
|
||||||
handle("runEvalSuite", (input) => mutate(() => sdlc.runEvalSuite(input)));
|
|
||||||
handle("setPrBotGate", (input) => mutate(() => sdlc.setPrBotGate(input)));
|
handle("setPrBotGate", (input) => mutate(() => sdlc.setPrBotGate(input)));
|
||||||
handle("createComplianceExport", (input) => mutate(() => compliance.createExport(input)));
|
handle("createComplianceExport", (input) => mutate(() => compliance.createExport(input)));
|
||||||
handle("createAuditReplay", (input) => mutate(() => audit.createThreadReplay(input)));
|
handle("createAuditReplay", (input) => mutate(() => audit.createThreadReplay(input)));
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
// The SDLC review agents and eval suites only counted configuration rows, so their pass proved nothing about code or behavior.
|
||||||
|
export const dropConfigCountEvalsSchema = `
|
||||||
|
drop table if exists sdlc_review_runs;
|
||||||
|
drop table if exists sdlc_review_agents;
|
||||||
|
drop table if exists eval_runs;
|
||||||
|
drop table if exists eval_cases;
|
||||||
|
drop table if exists eval_suites;
|
||||||
|
|
||||||
|
insert into schema_migrations(version, name, applied_at)
|
||||||
|
values (94, 'drop_config_count_evals', datetime('now'));
|
||||||
|
`;
|
||||||
@@ -92,6 +92,7 @@ import { erasureReceiptsSchema } from "./migrations/090-erasure-receipts";
|
|||||||
import { requestPurposeSchema } from "./migrations/091-request-purpose";
|
import { requestPurposeSchema } from "./migrations/091-request-purpose";
|
||||||
import { runActionTargetsSchema } from "./migrations/092-run-action-targets";
|
import { runActionTargetsSchema } from "./migrations/092-run-action-targets";
|
||||||
import { taskEnqueueErrorSchema } from "./migrations/093-task-enqueue-error";
|
import { taskEnqueueErrorSchema } from "./migrations/093-task-enqueue-error";
|
||||||
|
import { dropConfigCountEvalsSchema } from "./migrations/094-drop-config-count-evals";
|
||||||
|
|
||||||
interface Migration {
|
interface Migration {
|
||||||
version: number;
|
version: number;
|
||||||
@@ -193,6 +194,7 @@ const MIGRATIONS: Migration[] = [
|
|||||||
{ version: 91, name: "request_purpose", sql: requestPurposeSchema },
|
{ version: 91, name: "request_purpose", sql: requestPurposeSchema },
|
||||||
{ version: 92, name: "run_action_targets", sql: runActionTargetsSchema },
|
{ version: 92, name: "run_action_targets", sql: runActionTargetsSchema },
|
||||||
{ version: 93, name: "task_enqueue_error", sql: taskEnqueueErrorSchema },
|
{ version: 93, name: "task_enqueue_error", sql: taskEnqueueErrorSchema },
|
||||||
|
{ version: 94, name: "drop_config_count_evals", sql: dropConfigCountEvalsSchema },
|
||||||
];
|
];
|
||||||
|
|
||||||
const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version;
|
const SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1].version;
|
||||||
|
|||||||
@@ -32,11 +32,6 @@ const LEGACY_TABLES = [
|
|||||||
"operational_events",
|
"operational_events",
|
||||||
"chaos_drills",
|
"chaos_drills",
|
||||||
"chaos_drill_runs",
|
"chaos_drill_runs",
|
||||||
"sdlc_review_agents",
|
|
||||||
"sdlc_review_runs",
|
|
||||||
"eval_suites",
|
|
||||||
"eval_cases",
|
|
||||||
"eval_runs",
|
|
||||||
"pr_bot_gates",
|
"pr_bot_gates",
|
||||||
"chat_spaces",
|
"chat_spaces",
|
||||||
"chat_messages",
|
"chat_messages",
|
||||||
@@ -164,14 +159,6 @@ const RELATION_CHECKS: RelationCheck[] = [
|
|||||||
{ table: "chaos_drills", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
{ table: "chaos_drills", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
||||||
{ table: "chaos_drill_runs", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
{ table: "chaos_drill_runs", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
||||||
{ table: "chaos_drill_runs", column: "drill_id", referencedTable: "chaos_drills", referencedColumn: "id", reason: "dangling_drill_id" },
|
{ table: "chaos_drill_runs", column: "drill_id", referencedTable: "chaos_drills", referencedColumn: "id", reason: "dangling_drill_id" },
|
||||||
{ table: "sdlc_review_agents", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
|
||||||
{ table: "sdlc_review_runs", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
|
||||||
{ table: "sdlc_review_runs", column: "review_agent_id", referencedTable: "sdlc_review_agents", referencedColumn: "id", reason: "dangling_review_agent_id" },
|
|
||||||
{ table: "eval_suites", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
|
||||||
{ table: "eval_cases", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
|
||||||
{ table: "eval_cases", column: "suite_id", referencedTable: "eval_suites", referencedColumn: "id", reason: "dangling_suite_id" },
|
|
||||||
{ table: "eval_runs", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
|
||||||
{ table: "eval_runs", column: "suite_id", referencedTable: "eval_suites", referencedColumn: "id", reason: "dangling_suite_id" },
|
|
||||||
{ table: "pr_bot_gates", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
{ table: "pr_bot_gates", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
||||||
{ table: "chat_spaces", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
{ table: "chat_spaces", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
||||||
{ table: "chat_messages", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
{ table: "chat_messages", column: "org_id", referencedTable: "organizations", referencedColumn: "id", reason: "dangling_org_id" },
|
||||||
|
|||||||
@@ -1,40 +1,8 @@
|
|||||||
import type { DatabaseSync } from "node:sqlite";
|
import type { DatabaseSync } from "node:sqlite";
|
||||||
import type { RunEvalSuiteInput, RunSdlcReviewInput, SetPrBotGateInput } from "../../shared/commands";
|
import type { SetPrBotGateInput } from "../../shared/commands";
|
||||||
import {
|
import { PR_BOT_GATE_KEY } from "../../shared/domain";
|
||||||
EVAL_SUITE_KEY,
|
|
||||||
PR_BOT_GATE_KEY,
|
|
||||||
SDLC_REVIEW_AGENT_KEY,
|
|
||||||
SDLC_REVIEW_TARGET_TYPE,
|
|
||||||
SDLC_RUN_STATE,
|
|
||||||
type EvalSuiteKey,
|
|
||||||
type SdlcReviewAgentKey,
|
|
||||||
type SdlcReviewTargetType,
|
|
||||||
} from "../../shared/domain";
|
|
||||||
import { PROVIDER_CATALOG } from "../../shared/provider-catalog";
|
|
||||||
import { createId } from "../../shared/ids";
|
import { createId } from "../../shared/ids";
|
||||||
import { AuditService } from "./audit-service";
|
import { AuditService } from "./audit-service";
|
||||||
import { providerAdapters } from "./provider-adapters";
|
|
||||||
|
|
||||||
const REVIEW_AGENTS = [
|
|
||||||
{ key: SDLC_REVIEW_AGENT_KEY.CODE_REVIEWER, label: "Code Review Agent", specialty: "static quality and architecture" },
|
|
||||||
{ key: SDLC_REVIEW_AGENT_KEY.MIGRATION_REVIEWER, label: "Migration Review Agent", specialty: "schema diff and rollback evidence" },
|
|
||||||
{ key: SDLC_REVIEW_AGENT_KEY.POLICY_REVIEWER, label: "Policy Review Agent", specialty: "provider, approval, and access policy" },
|
|
||||||
] as const;
|
|
||||||
|
|
||||||
const EVAL_SUITES = [
|
|
||||||
{ key: EVAL_SUITE_KEY.CONTRACT, label: "Contract checks", scope: "provider adapters, event schemas, slash commands" },
|
|
||||||
{ key: EVAL_SUITE_KEY.INTEGRATION, label: "Integration checks", scope: "chat, approval, file, skill resolution flows" },
|
|
||||||
{ key: EVAL_SUITE_KEY.REPLAY, label: "Replay checks", scope: "audit timeline and historical decision regression" },
|
|
||||||
] as const;
|
|
||||||
|
|
||||||
const EVAL_CASES = [
|
|
||||||
{ suiteKey: EVAL_SUITE_KEY.CONTRACT, title: "Provider catalog has adapters", sourceType: "provider_gateway", sourceRef: "providers", expectedSignal: "adapter_coverage" },
|
|
||||||
{ suiteKey: EVAL_SUITE_KEY.CONTRACT, title: "Schema ledger is current", sourceType: "database", sourceRef: "schema_migrations", expectedSignal: "migration_ledger" },
|
|
||||||
{ suiteKey: EVAL_SUITE_KEY.INTEGRATION, title: "Approval routes are assigned", sourceType: "org_governance", sourceRef: "approval_assignments", expectedSignal: "route_coverage" },
|
|
||||||
{ suiteKey: EVAL_SUITE_KEY.INTEGRATION, title: "Skills can resolve runtime snapshots", sourceType: "skills_platform", sourceRef: "skill_runtime_snapshots", expectedSignal: "skill_resolution" },
|
|
||||||
{ suiteKey: EVAL_SUITE_KEY.REPLAY, title: "Audit timeline has domain events", sourceType: "audit", sourceRef: "audit_events", expectedSignal: "replayable_timeline" },
|
|
||||||
{ suiteKey: EVAL_SUITE_KEY.REPLAY, title: "Operational signals are captured", sourceType: "operations", sourceRef: "operational_events", expectedSignal: "slo_signal" },
|
|
||||||
] as const;
|
|
||||||
|
|
||||||
const PR_BOT_GATES = [
|
const PR_BOT_GATES = [
|
||||||
{ key: PR_BOT_GATE_KEY.STATIC_CHECKS, label: "Static checks", gateType: "typecheck_lint_test" },
|
{ key: PR_BOT_GATE_KEY.STATIC_CHECKS, label: "Static checks", gateType: "typecheck_lint_test" },
|
||||||
@@ -43,23 +11,6 @@ const PR_BOT_GATES = [
|
|||||||
{ key: PR_BOT_GATE_KEY.HUMAN_APPROVAL, label: "Human approval", gateType: "approval_required" },
|
{ key: PR_BOT_GATE_KEY.HUMAN_APPROVAL, label: "Human approval", gateType: "approval_required" },
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
interface ReviewAgentRow {
|
|
||||||
id: string;
|
|
||||||
key: SdlcReviewAgentKey;
|
|
||||||
enabled: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface EvalSuiteRow {
|
|
||||||
id: string;
|
|
||||||
key: EvalSuiteKey;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface GateCheck {
|
|
||||||
key: string;
|
|
||||||
passed: boolean;
|
|
||||||
value: number | string | boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class SdlcService {
|
export class SdlcService {
|
||||||
private readonly audit: AuditService;
|
private readonly audit: AuditService;
|
||||||
|
|
||||||
@@ -68,28 +19,6 @@ export class SdlcService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
seedOrganizationSdlc(orgId: string, now: string): void {
|
seedOrganizationSdlc(orgId: string, now: string): void {
|
||||||
for (const agent of REVIEW_AGENTS) {
|
|
||||||
this.db.prepare(`
|
|
||||||
insert into sdlc_review_agents(id, org_id, key, label, specialty, enabled, created_at, updated_at)
|
|
||||||
values (?, ?, ?, ?, ?, 1, ?, ?)
|
|
||||||
on conflict(org_id, key) do nothing
|
|
||||||
`).run(createId("sra"), orgId, agent.key, agent.label, agent.specialty, now, now);
|
|
||||||
}
|
|
||||||
for (const suite of EVAL_SUITES) {
|
|
||||||
this.db.prepare(`
|
|
||||||
insert into eval_suites(id, org_id, key, label, scope, created_at, updated_at)
|
|
||||||
values (?, ?, ?, ?, ?, ?, ?)
|
|
||||||
on conflict(org_id, key) do nothing
|
|
||||||
`).run(createId("evs"), orgId, suite.key, suite.label, suite.scope, now, now);
|
|
||||||
}
|
|
||||||
for (const item of EVAL_CASES) {
|
|
||||||
const suite = this.requireSuiteByKey(orgId, item.suiteKey);
|
|
||||||
this.db.prepare(`
|
|
||||||
insert into eval_cases(id, org_id, suite_id, title, source_type, source_ref, expected_signal, created_at)
|
|
||||||
values (?, ?, ?, ?, ?, ?, ?, ?)
|
|
||||||
on conflict(suite_id, title) do nothing
|
|
||||||
`).run(createId("evc"), orgId, suite.id, item.title, item.sourceType, item.sourceRef, item.expectedSignal, now);
|
|
||||||
}
|
|
||||||
for (const gate of PR_BOT_GATES) {
|
for (const gate of PR_BOT_GATES) {
|
||||||
this.db.prepare(`
|
this.db.prepare(`
|
||||||
insert into pr_bot_gates(id, org_id, key, label, gate_type, enabled, created_at, updated_at)
|
insert into pr_bot_gates(id, org_id, key, label, gate_type, enabled, created_at, updated_at)
|
||||||
@@ -99,58 +28,6 @@ export class SdlcService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
runReview(input: RunSdlcReviewInput): string {
|
|
||||||
const agent = this.db.prepare("select id, key, enabled from sdlc_review_agents where id = ? and org_id = ?")
|
|
||||||
.get(input.reviewAgentId, input.orgId) as ReviewAgentRow | undefined;
|
|
||||||
if (!agent) throw new Error("SDLC review agent was not found.");
|
|
||||||
if (agent.enabled !== 1) throw new Error("SDLC review agent is disabled.");
|
|
||||||
this.assertTarget(input.orgId, input.targetType, input.targetRef);
|
|
||||||
const checks = this.reviewChecks(input.orgId, agent.key, input.targetType);
|
|
||||||
const state = checks.every((check) => check.passed) ? SDLC_RUN_STATE.PASSED : SDLC_RUN_STATE.FAILED;
|
|
||||||
const id = createId("srr");
|
|
||||||
const now = new Date().toISOString();
|
|
||||||
this.db.prepare(`
|
|
||||||
insert into sdlc_review_runs(id, org_id, review_agent_id, target_type, target_ref, state, result_json, created_at, completed_at)
|
|
||||||
values (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
||||||
`).run(id, input.orgId, input.reviewAgentId, input.targetType, input.targetRef, state, JSON.stringify({ checks }), now, now);
|
|
||||||
this.audit.record({
|
|
||||||
orgId: input.orgId,
|
|
||||||
entityType: "sdlc_review_run",
|
|
||||||
entityId: id,
|
|
||||||
actorType: "system",
|
|
||||||
actorId: null,
|
|
||||||
action: "created",
|
|
||||||
payload: { reviewAgentKey: agent.key, state, targetType: input.targetType },
|
|
||||||
});
|
|
||||||
return id;
|
|
||||||
}
|
|
||||||
|
|
||||||
runEvalSuite(input: RunEvalSuiteInput): string {
|
|
||||||
const suite = this.db.prepare("select id, key from eval_suites where id = ? and org_id = ?")
|
|
||||||
.get(input.suiteId, input.orgId) as EvalSuiteRow | undefined;
|
|
||||||
if (!suite) throw new Error("Eval suite was not found.");
|
|
||||||
const checks = this.evalChecks(input.orgId, suite.key);
|
|
||||||
const passedCount = checks.filter((check) => check.passed).length;
|
|
||||||
const failedCount = checks.length - passedCount;
|
|
||||||
const state = failedCount === 0 ? SDLC_RUN_STATE.PASSED : SDLC_RUN_STATE.FAILED;
|
|
||||||
const id = createId("evr");
|
|
||||||
const now = new Date().toISOString();
|
|
||||||
this.db.prepare(`
|
|
||||||
insert into eval_runs(id, org_id, suite_id, state, passed_count, failed_count, result_json, created_at, completed_at)
|
|
||||||
values (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
||||||
`).run(id, input.orgId, input.suiteId, state, passedCount, failedCount, JSON.stringify({ checks }), now, now);
|
|
||||||
this.audit.record({
|
|
||||||
orgId: input.orgId,
|
|
||||||
entityType: "eval_run",
|
|
||||||
entityId: id,
|
|
||||||
actorType: "system",
|
|
||||||
actorId: null,
|
|
||||||
action: "created",
|
|
||||||
payload: { suiteKey: suite.key, state, passedCount, failedCount },
|
|
||||||
});
|
|
||||||
return id;
|
|
||||||
}
|
|
||||||
|
|
||||||
setPrBotGate(input: SetPrBotGateInput): void {
|
setPrBotGate(input: SetPrBotGateInput): void {
|
||||||
const gate = this.db.prepare("select key from pr_bot_gates where id = ? and org_id = ?")
|
const gate = this.db.prepare("select key from pr_bot_gates where id = ? and org_id = ?")
|
||||||
.get(input.gateId, input.orgId) as { key: string } | undefined;
|
.get(input.gateId, input.orgId) as { key: string } | undefined;
|
||||||
@@ -169,141 +46,4 @@ export class SdlcService {
|
|||||||
payload: { key: gate.key, enabled: input.enabled },
|
payload: { key: gate.key, enabled: input.enabled },
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private requireSuiteByKey(orgId: string, key: EvalSuiteKey): { id: string } {
|
|
||||||
const suite = this.db.prepare("select id from eval_suites where org_id = ? and key = ?")
|
|
||||||
.get(orgId, key) as { id: string } | undefined;
|
|
||||||
if (!suite) throw new Error("Eval suite is required.");
|
|
||||||
return suite;
|
|
||||||
}
|
|
||||||
|
|
||||||
private assertTarget(orgId: string, type: SdlcReviewTargetType, ref: string): void {
|
|
||||||
if (!Object.values(SDLC_REVIEW_TARGET_TYPE).includes(type)) throw new Error("SDLC review target type is invalid.");
|
|
||||||
const targetRef = ref.trim();
|
|
||||||
if (!targetRef) throw new Error("SDLC review target is required.");
|
|
||||||
if (type === SDLC_REVIEW_TARGET_TYPE.WORKSPACE) {
|
|
||||||
const org = this.db.prepare("select id from organizations where id = ?").get(targetRef);
|
|
||||||
if (!org || targetRef !== orgId) throw new Error("SDLC workspace target is invalid.");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (type === SDLC_REVIEW_TARGET_TYPE.MIGRATION) {
|
|
||||||
const report = this.db.prepare("select id from migration_reports where id = ? and org_id = ?").get(targetRef, orgId);
|
|
||||||
if (!report) throw new Error("SDLC migration target was not found.");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const policy = this.db.prepare("select id from policy_rules where id = ? and org_id = ?").get(targetRef, orgId);
|
|
||||||
if (!policy) throw new Error("SDLC policy target was not found.");
|
|
||||||
}
|
|
||||||
|
|
||||||
private reviewChecks(orgId: string, agentKey: SdlcReviewAgentKey, targetType: SdlcReviewTargetType): GateCheck[] {
|
|
||||||
if (agentKey === SDLC_REVIEW_AGENT_KEY.MIGRATION_REVIEWER) return this.migrationReviewChecks(orgId);
|
|
||||||
if (agentKey === SDLC_REVIEW_AGENT_KEY.POLICY_REVIEWER) return this.policyReviewChecks(orgId);
|
|
||||||
return this.codeReviewChecks(orgId, targetType);
|
|
||||||
}
|
|
||||||
|
|
||||||
private codeReviewChecks(orgId: string, targetType: SdlcReviewTargetType): GateCheck[] {
|
|
||||||
const enabledGates = count(this.db, "select count(*) as count from pr_bot_gates where org_id = ? and enabled = 1", orgId);
|
|
||||||
const auditEvents = count(this.db, "select count(*) as count from audit_events where org_id = ?", orgId);
|
|
||||||
return [
|
|
||||||
{ key: "target_type", passed: targetType === SDLC_REVIEW_TARGET_TYPE.WORKSPACE, value: targetType },
|
|
||||||
{ key: "enabled_pr_gates", passed: enabledGates >= 3, value: enabledGates },
|
|
||||||
{ key: "recent_audit_events", passed: auditEvents > 0, value: auditEvents },
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
private migrationReviewChecks(orgId: string): GateCheck[] {
|
|
||||||
const reports = this.db.prepare("select report_json from migration_reports where org_id = ?").all(orgId) as Array<{ report_json: string }>;
|
|
||||||
const rollbackPacks = reports.filter((report) => hasObjectField(report.report_json, "rollbackPack")).length;
|
|
||||||
const migrationGateEnabled = this.isGateEnabled(orgId, PR_BOT_GATE_KEY.MIGRATION_DIFF);
|
|
||||||
return [
|
|
||||||
{ key: "migration_gate_enabled", passed: migrationGateEnabled, value: migrationGateEnabled },
|
|
||||||
{ key: "rollback_pack_coverage", passed: reports.length === rollbackPacks, value: `${rollbackPacks}/${reports.length}` },
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
private policyReviewChecks(orgId: string): GateCheck[] {
|
|
||||||
const providerConflicts = count(this.db, "select count(*) as count from providers where org_id = ? and data_policy = 'zero_data_retention' and background_enabled = 1", orgId);
|
|
||||||
const routeRules = count(this.db, "select count(*) as count from approval_routing_rules where org_id = ?", orgId);
|
|
||||||
const policyGateEnabled = this.isGateEnabled(orgId, PR_BOT_GATE_KEY.POLICY_DIFF);
|
|
||||||
return [
|
|
||||||
{ key: "policy_gate_enabled", passed: policyGateEnabled, value: policyGateEnabled },
|
|
||||||
{ key: "provider_policy_conflicts", passed: providerConflicts === 0, value: providerConflicts },
|
|
||||||
{ key: "approval_routes", passed: routeRules > 0, value: routeRules },
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
private evalChecks(orgId: string, suiteKey: EvalSuiteKey): GateCheck[] {
|
|
||||||
if (suiteKey === EVAL_SUITE_KEY.CONTRACT) return this.contractEvalChecks();
|
|
||||||
if (suiteKey === EVAL_SUITE_KEY.INTEGRATION) return this.integrationEvalChecks(orgId);
|
|
||||||
return this.replayEvalChecks(orgId);
|
|
||||||
}
|
|
||||||
|
|
||||||
private contractEvalChecks(): GateCheck[] {
|
|
||||||
const catalogKeys = PROVIDER_CATALOG.map((provider) => provider.key);
|
|
||||||
const adapterKeys = Object.keys(providerAdapters);
|
|
||||||
const missing = catalogKeys.filter((key) => !adapterKeys.includes(key));
|
|
||||||
const runtimeContractReady = PROVIDER_CATALOG.every((provider) => {
|
|
||||||
const adapter = providerAdapters[provider.key];
|
|
||||||
return typeof adapter.authorize === "function"
|
|
||||||
&& typeof adapter.refresh === "function"
|
|
||||||
&& typeof adapter.revoke === "function"
|
|
||||||
&& typeof adapter.listModels === "function"
|
|
||||||
&& typeof adapter.getRateLimitState === "function"
|
|
||||||
&& typeof adapter.streamResponse === "function"
|
|
||||||
&& typeof adapter.cancelResponse === "function";
|
|
||||||
});
|
|
||||||
const schemaVersion = count(this.db, "select coalesce(max(version), 0) as count from schema_migrations");
|
|
||||||
const rateLimitRegistry = count(this.db, `
|
|
||||||
select count(*) as count
|
|
||||||
from sqlite_master
|
|
||||||
where type = 'table' and name = 'provider_rate_limit_observations'
|
|
||||||
`);
|
|
||||||
return [
|
|
||||||
{ key: "provider_adapter_coverage", passed: missing.length === 0, value: `${adapterKeys.length}/${catalogKeys.length}` },
|
|
||||||
{ key: "provider_rate_limit_registry", passed: rateLimitRegistry === 1, value: rateLimitRegistry },
|
|
||||||
{ key: "provider_runtime_contract", passed: runtimeContractReady, value: runtimeContractReady },
|
|
||||||
{ key: "schema_version", passed: schemaVersion >= 31, value: schemaVersion },
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
private integrationEvalChecks(orgId: string): GateCheck[] {
|
|
||||||
const chatSpaces = count(this.db, "select count(*) as count from chat_spaces where org_id = ?", orgId);
|
|
||||||
const approvalRoutes = count(this.db, "select count(*) as count from approval_routing_rules where org_id = ?", orgId);
|
|
||||||
const skillSnapshots = count(this.db, `
|
|
||||||
select count(*) as count
|
|
||||||
from skill_runtime_snapshots
|
|
||||||
join agents on agents.id = skill_runtime_snapshots.agent_id
|
|
||||||
where agents.org_id = ?
|
|
||||||
`, orgId);
|
|
||||||
return [
|
|
||||||
{ key: "chat_spaces", passed: chatSpaces > 0, value: chatSpaces },
|
|
||||||
{ key: "approval_routes", passed: approvalRoutes > 0, value: approvalRoutes },
|
|
||||||
{ key: "skill_runtime_snapshots", passed: skillSnapshots > 0, value: skillSnapshots },
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
private replayEvalChecks(orgId: string): GateCheck[] {
|
|
||||||
const auditEvents = count(this.db, "select count(*) as count from audit_events where org_id = ?", orgId);
|
|
||||||
const operationalEvents = count(this.db, "select count(*) as count from operational_events where org_id = ?", orgId);
|
|
||||||
return [
|
|
||||||
{ key: "audit_events", passed: auditEvents > 0, value: auditEvents },
|
|
||||||
{ key: "operational_events", passed: operationalEvents > 0, value: operationalEvents },
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
private isGateEnabled(orgId: string, key: string): boolean {
|
|
||||||
const row = this.db.prepare("select enabled from pr_bot_gates where org_id = ? and key = ?")
|
|
||||||
.get(orgId, key) as { enabled: number } | undefined;
|
|
||||||
if (!row) throw new Error("PR bot gate is required.");
|
|
||||||
return row.enabled === 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function count(db: DatabaseSync, sql: string, ...params: string[]): number {
|
|
||||||
return (db.prepare(sql).get(...params) as { count: number }).count;
|
|
||||||
}
|
|
||||||
|
|
||||||
function hasObjectField(json: string, field: string): boolean {
|
|
||||||
const parsed = JSON.parse(json) as Record<string, unknown>;
|
|
||||||
return typeof parsed[field] === "object" && parsed[field] !== null;
|
|
||||||
}
|
}
|
||||||
@@ -81,8 +81,6 @@ export class SnapshotService {
|
|||||||
.map((event) => ({ ...event, success: toBool(event.success) }));
|
.map((event) => ({ ...event, success: toBool(event.success) }));
|
||||||
const chaosDrills = list<AppSnapshot["chaosDrills"][number]>(this.db, "chaos_drills", "key asc")
|
const chaosDrills = list<AppSnapshot["chaosDrills"][number]>(this.db, "chaos_drills", "key asc")
|
||||||
.map((drill) => ({ ...drill, enabled: toBool(drill.enabled) }));
|
.map((drill) => ({ ...drill, enabled: toBool(drill.enabled) }));
|
||||||
const sdlcReviewAgents = list<AppSnapshot["sdlcReviewAgents"][number]>(this.db, "sdlc_review_agents", "key asc")
|
|
||||||
.map((agent) => ({ ...agent, enabled: toBool(agent.enabled) }));
|
|
||||||
const prBotGates = list<AppSnapshot["prBotGates"][number]>(this.db, "pr_bot_gates", "key asc")
|
const prBotGates = list<AppSnapshot["prBotGates"][number]>(this.db, "pr_bot_gates", "key asc")
|
||||||
.map((gate) => ({ ...gate, enabled: toBool(gate.enabled) }));
|
.map((gate) => ({ ...gate, enabled: toBool(gate.enabled) }));
|
||||||
const rollbackSnapshots = list<AppSnapshot["rollbackSnapshots"][number]>(this.db, "rollback_snapshots", "created_at desc")
|
const rollbackSnapshots = list<AppSnapshot["rollbackSnapshots"][number]>(this.db, "rollback_snapshots", "created_at desc")
|
||||||
@@ -164,11 +162,6 @@ export class SnapshotService {
|
|||||||
operationalEvents,
|
operationalEvents,
|
||||||
chaosDrills,
|
chaosDrills,
|
||||||
chaosDrillRuns: list(this.db, "chaos_drill_runs", "started_at desc"),
|
chaosDrillRuns: list(this.db, "chaos_drill_runs", "started_at desc"),
|
||||||
sdlcReviewAgents,
|
|
||||||
sdlcReviewRuns: list(this.db, "sdlc_review_runs", "created_at desc"),
|
|
||||||
evalSuites: list(this.db, "eval_suites", "key asc"),
|
|
||||||
evalCases: list(this.db, "eval_cases", "title asc"),
|
|
||||||
evalRuns: list(this.db, "eval_runs", "created_at desc"),
|
|
||||||
prBotGates,
|
prBotGates,
|
||||||
auditReplayRuns: list(this.db, "audit_replay_runs", "created_at desc"),
|
auditReplayRuns: list(this.db, "audit_replay_runs", "created_at desc"),
|
||||||
auditEvents: recentList<AppSnapshot["auditEvents"][number]>(this.db, "audit_events", "created_at desc", 200),
|
auditEvents: recentList<AppSnapshot["auditEvents"][number]>(this.db, "audit_events", "created_at desc", 200),
|
||||||
|
|||||||
@@ -81,8 +81,6 @@ const api: DesktopApi = {
|
|||||||
resolveMigrationReconciliation: (input) => invoke(IPC_CHANNELS.resolveMigrationReconciliation, input),
|
resolveMigrationReconciliation: (input) => invoke(IPC_CHANNELS.resolveMigrationReconciliation, input),
|
||||||
setFeatureFlag: (input) => invoke(IPC_CHANNELS.setFeatureFlag, input),
|
setFeatureFlag: (input) => invoke(IPC_CHANNELS.setFeatureFlag, input),
|
||||||
runChaosDrill: (input) => invoke(IPC_CHANNELS.runChaosDrill, input),
|
runChaosDrill: (input) => invoke(IPC_CHANNELS.runChaosDrill, input),
|
||||||
runSdlcReview: (input) => invoke(IPC_CHANNELS.runSdlcReview, input),
|
|
||||||
runEvalSuite: (input) => invoke(IPC_CHANNELS.runEvalSuite, input),
|
|
||||||
setPrBotGate: (input) => invoke(IPC_CHANNELS.setPrBotGate, input),
|
setPrBotGate: (input) => invoke(IPC_CHANNELS.setPrBotGate, input),
|
||||||
createComplianceExport: (input) => invoke(IPC_CHANNELS.createComplianceExport, input),
|
createComplianceExport: (input) => invoke(IPC_CHANNELS.createComplianceExport, input),
|
||||||
createAuditReplay: (input) => invoke(IPC_CHANNELS.createAuditReplay, input),
|
createAuditReplay: (input) => invoke(IPC_CHANNELS.createAuditReplay, input),
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ import type {
|
|||||||
ComplianceSubjectType,
|
ComplianceSubjectType,
|
||||||
LegalHoldEntityType,
|
LegalHoldEntityType,
|
||||||
RetentionFreezeScopeType,
|
RetentionFreezeScopeType,
|
||||||
SdlcReviewTargetType,
|
|
||||||
} from "../domain";
|
} from "../domain";
|
||||||
|
|
||||||
export interface CreateMigrationReportInput {
|
export interface CreateMigrationReportInput {
|
||||||
@@ -85,18 +84,6 @@ export interface RunChaosDrillInput {
|
|||||||
drillId: string;
|
drillId: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RunSdlcReviewInput {
|
|
||||||
orgId: string;
|
|
||||||
reviewAgentId: string;
|
|
||||||
targetType: SdlcReviewTargetType;
|
|
||||||
targetRef: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface RunEvalSuiteInput {
|
|
||||||
orgId: string;
|
|
||||||
suiteId: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SetPrBotGateInput {
|
export interface SetPrBotGateInput {
|
||||||
orgId: string;
|
orgId: string;
|
||||||
gateId: string;
|
gateId: string;
|
||||||
|
|||||||
@@ -66,8 +66,6 @@ import type {
|
|||||||
LaunchSocialLoginInput,
|
LaunchSocialLoginInput,
|
||||||
ResolveMigrationReconciliationInput,
|
ResolveMigrationReconciliationInput,
|
||||||
RunChaosDrillInput,
|
RunChaosDrillInput,
|
||||||
RunEvalSuiteInput,
|
|
||||||
RunSdlcReviewInput,
|
|
||||||
RunTaskInput,
|
RunTaskInput,
|
||||||
RevokeProviderCredentialInput,
|
RevokeProviderCredentialInput,
|
||||||
SaveProviderCredentialInput,
|
SaveProviderCredentialInput,
|
||||||
@@ -192,8 +190,6 @@ export interface DesktopApi {
|
|||||||
resolveMigrationReconciliation(input: ResolveMigrationReconciliationInput): Promise<DesktopResult<AppSnapshot>>;
|
resolveMigrationReconciliation(input: ResolveMigrationReconciliationInput): Promise<DesktopResult<AppSnapshot>>;
|
||||||
setFeatureFlag(input: SetFeatureFlagInput): Promise<DesktopResult<AppSnapshot>>;
|
setFeatureFlag(input: SetFeatureFlagInput): Promise<DesktopResult<AppSnapshot>>;
|
||||||
runChaosDrill(input: RunChaosDrillInput): Promise<DesktopResult<AppSnapshot>>;
|
runChaosDrill(input: RunChaosDrillInput): Promise<DesktopResult<AppSnapshot>>;
|
||||||
runSdlcReview(input: RunSdlcReviewInput): Promise<DesktopResult<AppSnapshot>>;
|
|
||||||
runEvalSuite(input: RunEvalSuiteInput): Promise<DesktopResult<AppSnapshot>>;
|
|
||||||
setPrBotGate(input: SetPrBotGateInput): Promise<DesktopResult<AppSnapshot>>;
|
setPrBotGate(input: SetPrBotGateInput): Promise<DesktopResult<AppSnapshot>>;
|
||||||
createComplianceExport(input: CreateComplianceExportInput): Promise<DesktopResult<AppSnapshot>>;
|
createComplianceExport(input: CreateComplianceExportInput): Promise<DesktopResult<AppSnapshot>>;
|
||||||
createAuditReplay(input: CreateAuditReplayInput): Promise<DesktopResult<AppSnapshot>>;
|
createAuditReplay(input: CreateAuditReplayInput): Promise<DesktopResult<AppSnapshot>>;
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ export { PROVIDER_DATA_POLICY, PROVIDER_EXECUTION_MODE, type ProviderDataPolicy,
|
|||||||
export { PROVIDER_REQUEST_STATE, type ProviderRequestRecord, type ProviderRequestState, type ProviderStreamRendererEvent } from "./provider-request-domain";
|
export { PROVIDER_REQUEST_STATE, type ProviderRequestRecord, type ProviderRequestState, type ProviderStreamRendererEvent } from "./provider-request-domain";
|
||||||
export type { ProviderRateLimitObservationRecord, ProviderRateLimitStateRecord } from "./provider-runtime-domain";
|
export type { ProviderRateLimitObservationRecord, ProviderRateLimitStateRecord } from "./provider-runtime-domain";
|
||||||
export type { MigrationCutoverEventRecord, MigrationReconciliationItemRecord, RollbackSnapshotRecord } from "./rollback-domain";
|
export type { MigrationCutoverEventRecord, MigrationReconciliationItemRecord, RollbackSnapshotRecord } from "./rollback-domain";
|
||||||
export { EVAL_SUITE_KEY, PR_BOT_GATE_KEY, SDLC_REVIEW_AGENT_KEY, SDLC_REVIEW_TARGET_TYPE, SDLC_RUN_STATE, type EvalCaseRecord, type EvalRunRecord, type EvalSuiteKey, type EvalSuiteRecord, type PrBotGateKey, type PrBotGateRecord, type SdlcReviewAgentKey, type SdlcReviewAgentRecord, type SdlcReviewRunRecord, type SdlcReviewTargetType, type SdlcRunState } from "./sdlc-domain";
|
export { PR_BOT_GATE_KEY, type PrBotGateKey, type PrBotGateRecord } from "./sdlc-domain";
|
||||||
export { SEARCH_ENTITY_TYPES, type SearchEntityType, type SearchResultRecord } from "./search-domain";
|
export { SEARCH_ENTITY_TYPES, type SearchEntityType, type SearchResultRecord } from "./search-domain";
|
||||||
export type { BrowserActionPolicyRecord, BrowserActionRecord, BrowserSessionContainerRecord, SocialAccountGrantRecord, SocialAccountRecord, SocialApprovalRouteRecord, SocialAuditRecord } from "./social-domain";
|
export type { BrowserActionPolicyRecord, BrowserActionRecord, BrowserSessionContainerRecord, SocialAccountGrantRecord, SocialAccountRecord, SocialApprovalRouteRecord, SocialAuditRecord } from "./social-domain";
|
||||||
export type { AppSnapshot, UiLanguage, UiPreferences, UiTheme } from "./snapshot-domain";
|
export type { AppSnapshot, UiLanguage, UiPreferences, UiTheme } from "./snapshot-domain";
|
||||||
|
|||||||
@@ -54,8 +54,6 @@ export const IPC_CHANNELS = {
|
|||||||
resolveMigrationReconciliation: "agentcompany:resolve-migration-reconciliation",
|
resolveMigrationReconciliation: "agentcompany:resolve-migration-reconciliation",
|
||||||
setFeatureFlag: "agentcompany:set-feature-flag",
|
setFeatureFlag: "agentcompany:set-feature-flag",
|
||||||
runChaosDrill: "agentcompany:run-chaos-drill",
|
runChaosDrill: "agentcompany:run-chaos-drill",
|
||||||
runSdlcReview: "agentcompany:run-sdlc-review",
|
|
||||||
runEvalSuite: "agentcompany:run-eval-suite",
|
|
||||||
setPrBotGate: "agentcompany:set-pr-bot-gate",
|
setPrBotGate: "agentcompany:set-pr-bot-gate",
|
||||||
createComplianceExport: "agentcompany:create-compliance-export",
|
createComplianceExport: "agentcompany:create-compliance-export",
|
||||||
createAuditReplay: "agentcompany:create-audit-replay",
|
createAuditReplay: "agentcompany:create-audit-replay",
|
||||||
|
|||||||
@@ -1,14 +1,5 @@
|
|||||||
type ValueOf<T> = T[keyof T];
|
type ValueOf<T> = T[keyof T];
|
||||||
|
|
||||||
export const SDLC_REVIEW_AGENT_KEY = {
|
|
||||||
CODE_REVIEWER: "code_reviewer",
|
|
||||||
MIGRATION_REVIEWER: "migration_reviewer",
|
|
||||||
POLICY_REVIEWER: "policy_reviewer",
|
|
||||||
} as const;
|
|
||||||
|
|
||||||
export const SDLC_REVIEW_TARGET_TYPE = { WORKSPACE: "workspace", MIGRATION: "migration", POLICY: "policy" } as const;
|
|
||||||
export const EVAL_SUITE_KEY = { CONTRACT: "contract", INTEGRATION: "integration", REPLAY: "replay" } as const;
|
|
||||||
export const SDLC_RUN_STATE = { PASSED: "passed", FAILED: "failed" } as const;
|
|
||||||
export const PR_BOT_GATE_KEY = {
|
export const PR_BOT_GATE_KEY = {
|
||||||
STATIC_CHECKS: "static_checks",
|
STATIC_CHECKS: "static_checks",
|
||||||
MIGRATION_DIFF: "migration_diff",
|
MIGRATION_DIFF: "migration_diff",
|
||||||
@@ -16,33 +7,8 @@ export const PR_BOT_GATE_KEY = {
|
|||||||
HUMAN_APPROVAL: "human_approval",
|
HUMAN_APPROVAL: "human_approval",
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
export type SdlcReviewAgentKey = ValueOf<typeof SDLC_REVIEW_AGENT_KEY>;
|
|
||||||
export type SdlcReviewTargetType = ValueOf<typeof SDLC_REVIEW_TARGET_TYPE>;
|
|
||||||
export type EvalSuiteKey = ValueOf<typeof EVAL_SUITE_KEY>;
|
|
||||||
export type SdlcRunState = ValueOf<typeof SDLC_RUN_STATE>;
|
|
||||||
export type PrBotGateKey = ValueOf<typeof PR_BOT_GATE_KEY>;
|
export type PrBotGateKey = ValueOf<typeof PR_BOT_GATE_KEY>;
|
||||||
|
|
||||||
export interface SdlcReviewAgentRecord {
|
|
||||||
id: string; orgId: string; key: SdlcReviewAgentKey; label: string; specialty: string; enabled: boolean; createdAt: string; updatedAt: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SdlcReviewRunRecord {
|
|
||||||
id: string; orgId: string; reviewAgentId: string; targetType: SdlcReviewTargetType; targetRef: string;
|
|
||||||
state: SdlcRunState; resultJson: string; createdAt: string; completedAt: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface EvalSuiteRecord {
|
|
||||||
id: string; orgId: string; key: EvalSuiteKey; label: string; scope: string; createdAt: string; updatedAt: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface EvalCaseRecord {
|
|
||||||
id: string; orgId: string; suiteId: string; title: string; sourceType: string; sourceRef: string; expectedSignal: string; createdAt: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface EvalRunRecord {
|
|
||||||
id: string; orgId: string; suiteId: string; state: SdlcRunState; passedCount: number; failedCount: number; resultJson: string; createdAt: string; completedAt: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface PrBotGateRecord {
|
export interface PrBotGateRecord {
|
||||||
id: string; orgId: string; key: PrBotGateKey; label: string; gateType: string; enabled: boolean; createdAt: string; updatedAt: string;
|
id: string; orgId: string; key: PrBotGateKey; label: string; gateType: string; enabled: boolean; createdAt: string; updatedAt: string;
|
||||||
}
|
}
|
||||||
@@ -49,7 +49,7 @@ import type { ProviderAuthProfileRecord, ProviderAuthSessionRecord } from "./pro
|
|||||||
import type { ProviderRequestRecord } from "./provider-request-domain";
|
import type { ProviderRequestRecord } from "./provider-request-domain";
|
||||||
import type { ProviderRateLimitObservationRecord, ProviderRateLimitStateRecord } from "./provider-runtime-domain";
|
import type { ProviderRateLimitObservationRecord, ProviderRateLimitStateRecord } from "./provider-runtime-domain";
|
||||||
import type { MigrationCutoverEventRecord, MigrationReconciliationItemRecord, RollbackSnapshotRecord } from "./rollback-domain";
|
import type { MigrationCutoverEventRecord, MigrationReconciliationItemRecord, RollbackSnapshotRecord } from "./rollback-domain";
|
||||||
import type { EvalCaseRecord, EvalRunRecord, EvalSuiteRecord, PrBotGateRecord, SdlcReviewAgentRecord, SdlcReviewRunRecord } from "./sdlc-domain";
|
import type { PrBotGateRecord } from "./sdlc-domain";
|
||||||
import type {
|
import type {
|
||||||
BrowserActionPolicyRecord,
|
BrowserActionPolicyRecord,
|
||||||
BrowserActionRecord,
|
BrowserActionRecord,
|
||||||
@@ -139,11 +139,6 @@ export interface AppSnapshot {
|
|||||||
operationalEvents: OperationalEventRecord[];
|
operationalEvents: OperationalEventRecord[];
|
||||||
chaosDrills: ChaosDrillRecord[];
|
chaosDrills: ChaosDrillRecord[];
|
||||||
chaosDrillRuns: ChaosDrillRunRecord[];
|
chaosDrillRuns: ChaosDrillRunRecord[];
|
||||||
sdlcReviewAgents: SdlcReviewAgentRecord[];
|
|
||||||
sdlcReviewRuns: SdlcReviewRunRecord[];
|
|
||||||
evalSuites: EvalSuiteRecord[];
|
|
||||||
evalCases: EvalCaseRecord[];
|
|
||||||
evalRuns: EvalRunRecord[];
|
|
||||||
prBotGates: PrBotGateRecord[];
|
prBotGates: PrBotGateRecord[];
|
||||||
auditReplayRuns: AuditReplayRunRecord[];
|
auditReplayRuns: AuditReplayRunRecord[];
|
||||||
auditEvents: AuditEventRecord[];
|
auditEvents: AuditEventRecord[];
|
||||||
|
|||||||
@@ -1,12 +1,6 @@
|
|||||||
import { randomUUID } from "node:crypto";
|
|
||||||
import { tmpdir } from "node:os";
|
|
||||||
import { join } from "node:path";
|
|
||||||
import { describe, expect, test } from "vitest";
|
import { describe, expect, test } from "vitest";
|
||||||
import { DatabaseConnection } from "../../src/main/database/connection";
|
|
||||||
import { OrganizationService } from "../../src/main/services/organization-service";
|
|
||||||
import { providerAdapters } from "../../src/main/services/provider-adapters";
|
import { providerAdapters } from "../../src/main/services/provider-adapters";
|
||||||
import { SdlcService } from "../../src/main/services/sdlc-service";
|
import { PROVIDER_AUTH_TYPE } from "../../src/shared/domain";
|
||||||
import { EVAL_SUITE_KEY, PROVIDER_AUTH_TYPE, SDLC_RUN_STATE } from "../../src/shared/domain";
|
|
||||||
import { PROVIDER_CATALOG } from "../../src/shared/provider-catalog";
|
import { PROVIDER_CATALOG } from "../../src/shared/provider-catalog";
|
||||||
|
|
||||||
const PROVIDER_SECRET_SAMPLES: Record<string, string> = {
|
const PROVIDER_SECRET_SAMPLES: Record<string, string> = {
|
||||||
@@ -42,19 +36,6 @@ const PROVIDER_SECRET_SAMPLES: Record<string, string> = {
|
|||||||
"custom-anthropic": JSON.stringify({ baseUrl: "https://llm.example.com", secret: "sk-custom" }),
|
"custom-anthropic": JSON.stringify({ baseUrl: "https://llm.example.com", secret: "sk-custom" }),
|
||||||
};
|
};
|
||||||
|
|
||||||
function createConnection(): DatabaseConnection {
|
|
||||||
return new DatabaseConnection(join(tmpdir(), `agentcompany-new-contract-${randomUUID()}.db`));
|
|
||||||
}
|
|
||||||
|
|
||||||
function createOrganization(organization: OrganizationService): string {
|
|
||||||
return organization.createOrganization({
|
|
||||||
organizationName: "Acme Operations",
|
|
||||||
employeeName: "Dana Lee",
|
|
||||||
mission: "Coordinate agents with auditable approvals.",
|
|
||||||
goal: "Ship a verified control plane.",
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("provider contracts", () => {
|
describe("provider contracts", () => {
|
||||||
test("keeps provider catalog and adapters aligned", () => {
|
test("keeps provider catalog and adapters aligned", () => {
|
||||||
expect(Object.keys(providerAdapters).sort()).toEqual(PROVIDER_CATALOG.map((provider) => provider.key).sort());
|
expect(Object.keys(providerAdapters).sort()).toEqual(PROVIDER_CATALOG.map((provider) => provider.key).sort());
|
||||||
@@ -139,33 +120,4 @@ describe("provider contracts", () => {
|
|||||||
await expect(adapter.refresh({ authProfile: profile })).rejects.toThrow("does not support token refresh");
|
await expect(adapter.refresh({ authProfile: profile })).rejects.toThrow("does not support token refresh");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test("runs the seeded contract eval suite", () => {
|
|
||||||
const connection = createConnection();
|
|
||||||
const organization = new OrganizationService(connection.db);
|
|
||||||
const sdlc = new SdlcService(connection.db);
|
|
||||||
let orgId = "";
|
|
||||||
let suiteId = "";
|
|
||||||
let runId = "";
|
|
||||||
|
|
||||||
connection.transaction(() => {
|
|
||||||
orgId = createOrganization(organization);
|
|
||||||
suiteId = (connection.db.prepare("select id from eval_suites where org_id = ? and key = ?")
|
|
||||||
.get(orgId, EVAL_SUITE_KEY.CONTRACT) as { id: string }).id;
|
|
||||||
runId = sdlc.runEvalSuite({ orgId, suiteId });
|
|
||||||
});
|
|
||||||
|
|
||||||
const run = connection.db.prepare("select state, failed_count, result_json from eval_runs where id = ?")
|
|
||||||
.get(runId) as { state: string; failed_count: number; result_json: string };
|
|
||||||
const result = JSON.parse(run.result_json) as { checks: Array<{ key: string; passed: boolean }> };
|
|
||||||
expect(run).toMatchObject({ state: SDLC_RUN_STATE.PASSED, failed_count: 0 });
|
|
||||||
expect(result.checks.map((check) => check.key).sort()).toEqual([
|
|
||||||
"provider_adapter_coverage",
|
|
||||||
"provider_rate_limit_registry",
|
|
||||||
"provider_runtime_contract",
|
|
||||||
"schema_version",
|
|
||||||
]);
|
|
||||||
expect(result.checks.every((check) => check.passed)).toBe(true);
|
|
||||||
connection.close();
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
@@ -7,9 +7,8 @@ import { AuditService } from "../../src/main/services/audit-service";
|
|||||||
import { ChatService } from "../../src/main/services/chat-service";
|
import { ChatService } from "../../src/main/services/chat-service";
|
||||||
import { OperationsService } from "../../src/main/services/operations-service";
|
import { OperationsService } from "../../src/main/services/operations-service";
|
||||||
import { OrganizationService } from "../../src/main/services/organization-service";
|
import { OrganizationService } from "../../src/main/services/organization-service";
|
||||||
import { SdlcService } from "../../src/main/services/sdlc-service";
|
|
||||||
import { SnapshotService } from "../../src/main/services/snapshot-service";
|
import { SnapshotService } from "../../src/main/services/snapshot-service";
|
||||||
import { EVAL_SUITE_KEY, SDLC_RUN_STATE, SLO_SIGNAL_KEY } from "../../src/shared/domain";
|
import { SLO_SIGNAL_KEY } from "../../src/shared/domain";
|
||||||
|
|
||||||
function createConnection(): DatabaseConnection {
|
function createConnection(): DatabaseConnection {
|
||||||
return new DatabaseConnection(join(tmpdir(), `agentcompany-new-replay-${randomUUID()}.db`));
|
return new DatabaseConnection(join(tmpdir(), `agentcompany-new-replay-${randomUUID()}.db`));
|
||||||
@@ -25,19 +24,17 @@ function createOrganization(organization: OrganizationService): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("replay eval", () => {
|
describe("replay eval", () => {
|
||||||
test("creates deterministic thread replay evidence and runs replay eval suite", () => {
|
test("creates deterministic thread replay evidence and records the replay digest", () => {
|
||||||
const connection = createConnection();
|
const connection = createConnection();
|
||||||
const organization = new OrganizationService(connection.db);
|
const organization = new OrganizationService(connection.db);
|
||||||
const chat = new ChatService(connection.db);
|
const chat = new ChatService(connection.db);
|
||||||
const audit = new AuditService(connection.db);
|
const audit = new AuditService(connection.db);
|
||||||
const operations = new OperationsService(connection.db);
|
const operations = new OperationsService(connection.db);
|
||||||
const sdlc = new SdlcService(connection.db);
|
|
||||||
const snapshot = new SnapshotService(connection.db);
|
const snapshot = new SnapshotService(connection.db);
|
||||||
let orgId = "";
|
let orgId = "";
|
||||||
let employeeId = "";
|
let employeeId = "";
|
||||||
let threadId = "";
|
let threadId = "";
|
||||||
let replayRunId = "";
|
let replayRunId = "";
|
||||||
let evalRunId = "";
|
|
||||||
|
|
||||||
connection.transaction(() => {
|
connection.transaction(() => {
|
||||||
orgId = createOrganization(organization);
|
orgId = createOrganization(organization);
|
||||||
@@ -68,23 +65,17 @@ describe("replay eval", () => {
|
|||||||
durationMs: 12,
|
durationMs: 12,
|
||||||
detail: { threadId, replayRunId },
|
detail: { threadId, replayRunId },
|
||||||
});
|
});
|
||||||
const suiteId = (connection.db.prepare("select id from eval_suites where org_id = ? and key = ?")
|
|
||||||
.get(orgId, EVAL_SUITE_KEY.REPLAY) as { id: string }).id;
|
|
||||||
evalRunId = sdlc.runEvalSuite({ orgId, suiteId });
|
|
||||||
});
|
});
|
||||||
|
|
||||||
const state = snapshot.load();
|
const state = snapshot.load();
|
||||||
const replayRun = state.auditReplayRuns.find((run) => run.id === replayRunId);
|
const replayRun = state.auditReplayRuns.find((run) => run.id === replayRunId);
|
||||||
const evalRun = state.evalRuns.find((run) => run.id === evalRunId);
|
|
||||||
if (!replayRun) throw new Error("Audit replay run was not created.");
|
if (!replayRun) throw new Error("Audit replay run was not created.");
|
||||||
if (!evalRun) throw new Error("Replay eval run was not created.");
|
|
||||||
const replay = JSON.parse(replayRun.replayJson) as { events: Array<{ entityType: string; action: string }> };
|
const replay = JSON.parse(replayRun.replayJson) as { events: Array<{ entityType: string; action: string }> };
|
||||||
expect(replayRun.threadId).toBe(threadId);
|
expect(replayRun.threadId).toBe(threadId);
|
||||||
expect(replayRun.eventDigest).toBe(createHash("sha256").update(replayRun.replayJson).digest("hex"));
|
expect(replayRun.eventDigest).toBe(createHash("sha256").update(replayRun.replayJson).digest("hex"));
|
||||||
expect(replayRun.eventCount).toBe(replay.events.length);
|
expect(replayRun.eventCount).toBe(replay.events.length);
|
||||||
expect(replay.events).toContainEqual(expect.objectContaining({ entityType: "chat_message", action: "sent" }));
|
expect(replay.events).toContainEqual(expect.objectContaining({ entityType: "chat_message", action: "sent" }));
|
||||||
expect(replay.events).toContainEqual(expect.objectContaining({ entityType: "approval", action: "requested" }));
|
expect(replay.events).toContainEqual(expect.objectContaining({ entityType: "approval", action: "requested" }));
|
||||||
expect(evalRun).toMatchObject({ state: SDLC_RUN_STATE.PASSED, failedCount: 0 });
|
|
||||||
connection.close();
|
connection.close();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -7,10 +7,9 @@ import { DatabaseConnection } from "../../src/main/database/connection";
|
|||||||
import { ApprovalService } from "../../src/main/services/approval-service";
|
import { ApprovalService } from "../../src/main/services/approval-service";
|
||||||
import { ChatService } from "../../src/main/services/chat-service";
|
import { ChatService } from "../../src/main/services/chat-service";
|
||||||
import { OrganizationService } from "../../src/main/services/organization-service";
|
import { OrganizationService } from "../../src/main/services/organization-service";
|
||||||
import { SdlcService } from "../../src/main/services/sdlc-service";
|
|
||||||
import { SkillService } from "../../src/main/services/skill-service";
|
import { SkillService } from "../../src/main/services/skill-service";
|
||||||
import { SnapshotService } from "../../src/main/services/snapshot-service";
|
import { SnapshotService } from "../../src/main/services/snapshot-service";
|
||||||
import { APPROVAL_STATE, EVAL_SUITE_KEY, SDLC_RUN_STATE } from "../../src/shared/domain";
|
import { APPROVAL_STATE } from "../../src/shared/domain";
|
||||||
|
|
||||||
function createConnection(): DatabaseConnection {
|
function createConnection(): DatabaseConnection {
|
||||||
return new DatabaseConnection(join(tmpdir(), `agentcompany-new-integration-${randomUUID()}.db`));
|
return new DatabaseConnection(join(tmpdir(), `agentcompany-new-integration-${randomUUID()}.db`));
|
||||||
@@ -52,7 +51,6 @@ describe("collaboration integration flow", () => {
|
|||||||
const approvals = new ApprovalService(connection.db);
|
const approvals = new ApprovalService(connection.db);
|
||||||
const chat = new ChatService(connection.db);
|
const chat = new ChatService(connection.db);
|
||||||
const skills = new SkillService(connection.db, join(tmpdir(), `agentcompany-new-artifacts-${randomUUID()}`));
|
const skills = new SkillService(connection.db, join(tmpdir(), `agentcompany-new-artifacts-${randomUUID()}`));
|
||||||
const sdlc = new SdlcService(connection.db);
|
|
||||||
const snapshot = new SnapshotService(connection.db);
|
const snapshot = new SnapshotService(connection.db);
|
||||||
let orgId = "";
|
let orgId = "";
|
||||||
let employeeId = "";
|
let employeeId = "";
|
||||||
@@ -60,8 +58,6 @@ describe("collaboration integration flow", () => {
|
|||||||
let approvalId = "";
|
let approvalId = "";
|
||||||
let skillId = "";
|
let skillId = "";
|
||||||
let revisionId = "";
|
let revisionId = "";
|
||||||
let suiteId = "";
|
|
||||||
let evalRunId = "";
|
|
||||||
|
|
||||||
connection.transaction(() => {
|
connection.transaction(() => {
|
||||||
orgId = createOrganization(organization);
|
orgId = createOrganization(organization);
|
||||||
@@ -99,23 +95,17 @@ describe("collaboration integration flow", () => {
|
|||||||
skillId = (connection.db.prepare("select skill_id from skill_revisions where id = ?").get(revisionId) as { skill_id: string }).skill_id;
|
skillId = (connection.db.prepare("select skill_id from skill_revisions where id = ?").get(revisionId) as { skill_id: string }).skill_id;
|
||||||
skills.bind({ orgId, agentId, skillId, revisionId });
|
skills.bind({ orgId, agentId, skillId, revisionId });
|
||||||
skills.resolveRuntimeSnapshot({ orgId, agentId, runId: `run_${randomUUID().replaceAll("-", "")}`, maxTokens: 44_800, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" }, houseStyle: true });
|
skills.resolveRuntimeSnapshot({ orgId, agentId, runId: `run_${randomUUID().replaceAll("-", "")}`, maxTokens: 44_800, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" }, houseStyle: true });
|
||||||
suiteId = (connection.db.prepare("select id from eval_suites where org_id = ? and key = ?")
|
|
||||||
.get(orgId, EVAL_SUITE_KEY.INTEGRATION) as { id: string }).id;
|
|
||||||
evalRunId = sdlc.runEvalSuite({ orgId, suiteId });
|
|
||||||
});
|
});
|
||||||
|
|
||||||
const state = snapshot.load();
|
const state = snapshot.load();
|
||||||
const evalRun = state.evalRuns.find((run) => run.id === evalRunId);
|
|
||||||
// Compiled runtime snapshots live in the database only - the renderer
|
// Compiled runtime snapshots live in the database only - the renderer
|
||||||
// snapshot must not carry per-run compiled prompts.
|
// snapshot must not carry per-run compiled prompts.
|
||||||
const snapshotRecord = connection.db.prepare("select compiled_prompt from skill_runtime_snapshots limit 1")
|
const snapshotRecord = connection.db.prepare("select compiled_prompt from skill_runtime_snapshots limit 1")
|
||||||
.get() as { compiled_prompt: string } | undefined;
|
.get() as { compiled_prompt: string } | undefined;
|
||||||
if (!evalRun) throw new Error("Integration eval run was not created.");
|
|
||||||
if (!snapshotRecord) throw new Error("Skill runtime snapshot was not created.");
|
if (!snapshotRecord) throw new Error("Skill runtime snapshot was not created.");
|
||||||
expect(state.approvals.find((approval) => approval.id === approvalId)?.state).toBe(APPROVAL_STATE.APPROVED);
|
expect(state.approvals.find((approval) => approval.id === approvalId)?.state).toBe(APPROVAL_STATE.APPROVED);
|
||||||
expect(state.approvalAssignments.some((assignment) => assignment.approvalId === approvalId)).toBe(true);
|
expect(state.approvalAssignments.some((assignment) => assignment.approvalId === approvalId)).toBe(true);
|
||||||
expect(snapshotRecord.compiled_prompt).toContain("Release Gate");
|
expect(snapshotRecord.compiled_prompt).toContain("Release Gate");
|
||||||
expect(evalRun).toMatchObject({ state: SDLC_RUN_STATE.PASSED, failedCount: 0 });
|
|
||||||
expect(state.operationalEvents.some((event) => event.signalKey === "skill_resolve_success")).toBe(true);
|
expect(state.operationalEvents.some((event) => event.signalKey === "skill_resolve_success")).toBe(true);
|
||||||
connection.close();
|
connection.close();
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ describe("domain services", () => {
|
|||||||
const connection = createConnection();
|
const connection = createConnection();
|
||||||
|
|
||||||
const rows = connection.db.prepare("select version from schema_migrations order by version asc").all() as Array<{ version: number }>;
|
const rows = connection.db.prepare("select version from schema_migrations order by version asc").all() as Array<{ version: number }>;
|
||||||
expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93]);
|
expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94]);
|
||||||
connection.close();
|
connection.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -63,8 +63,6 @@ describe("domain services", () => {
|
|||||||
expect(state.featureFlags.map((flag) => flag.key)).toContain("chaos_drills");
|
expect(state.featureFlags.map((flag) => flag.key)).toContain("chaos_drills");
|
||||||
expect(state.sloTargets.map((target) => target.signalKey)).toContain("provider_request_success");
|
expect(state.sloTargets.map((target) => target.signalKey)).toContain("provider_request_success");
|
||||||
expect(state.chaosDrills.map((drill) => drill.scenarioKey)).toContain("approval_backlog_gate");
|
expect(state.chaosDrills.map((drill) => drill.scenarioKey)).toContain("approval_backlog_gate");
|
||||||
expect(state.sdlcReviewAgents.map((agent) => agent.key)).toContain("code_reviewer");
|
|
||||||
expect(state.evalSuites.map((suite) => suite.key)).toContain("contract");
|
|
||||||
expect(state.prBotGates.map((gate) => gate.key)).toContain("static_checks");
|
expect(state.prBotGates.map((gate) => gate.key)).toContain("static_checks");
|
||||||
expect(state.chatSpaces.map((space) => space.name)).toContain("board");
|
expect(state.chatSpaces.map((space) => space.name)).toContain("board");
|
||||||
expect(state.policyRules[0]?.action).toBe("task_create");
|
expect(state.policyRules[0]?.action).toBe("task_create");
|
||||||
|
|||||||
@@ -132,11 +132,6 @@ describe("migration service", () => {
|
|||||||
create table operational_events(id text primary key, org_id text);
|
create table operational_events(id text primary key, org_id text);
|
||||||
create table chaos_drills(id text primary key, org_id text);
|
create table chaos_drills(id text primary key, org_id text);
|
||||||
create table chaos_drill_runs(id text primary key, org_id text, drill_id text);
|
create table chaos_drill_runs(id text primary key, org_id text, drill_id text);
|
||||||
create table sdlc_review_agents(id text primary key, org_id text);
|
|
||||||
create table sdlc_review_runs(id text primary key, org_id text, review_agent_id text);
|
|
||||||
create table eval_suites(id text primary key, org_id text);
|
|
||||||
create table eval_cases(id text primary key, org_id text, suite_id text);
|
|
||||||
create table eval_runs(id text primary key, org_id text, suite_id text);
|
|
||||||
create table pr_bot_gates(id text primary key, org_id text);
|
create table pr_bot_gates(id text primary key, org_id text);
|
||||||
create table chat_spaces(id text primary key, org_id text);
|
create table chat_spaces(id text primary key, org_id text);
|
||||||
create table chat_messages(id text primary key, org_id text, space_id text, thread_id text);
|
create table chat_messages(id text primary key, org_id text, space_id text, thread_id text);
|
||||||
@@ -210,11 +205,6 @@ describe("migration service", () => {
|
|||||||
insert into operational_events(id, org_id) values ('op-1', 'org-1');
|
insert into operational_events(id, org_id) values ('op-1', 'org-1');
|
||||||
insert into chaos_drills(id, org_id) values ('drill-1', 'org-1');
|
insert into chaos_drills(id, org_id) values ('drill-1', 'org-1');
|
||||||
insert into chaos_drill_runs(id, org_id, drill_id) values ('drill-run-1', 'org-1', 'drill-1');
|
insert into chaos_drill_runs(id, org_id, drill_id) values ('drill-run-1', 'org-1', 'drill-1');
|
||||||
insert into sdlc_review_agents(id, org_id) values ('reviewer-1', 'org-1');
|
|
||||||
insert into sdlc_review_runs(id, org_id, review_agent_id) values ('review-run-1', 'org-1', 'reviewer-1');
|
|
||||||
insert into eval_suites(id, org_id) values ('suite-1', 'org-1');
|
|
||||||
insert into eval_cases(id, org_id, suite_id) values ('case-1', 'org-1', 'suite-1');
|
|
||||||
insert into eval_runs(id, org_id, suite_id) values ('eval-run-1', 'org-1', 'suite-1');
|
|
||||||
insert into pr_bot_gates(id, org_id) values ('gate-1', 'org-1');
|
insert into pr_bot_gates(id, org_id) values ('gate-1', 'org-1');
|
||||||
insert into chat_threads(id, org_id, space_id, root_message_id) values ('thread-1', 'org-1', 'space-1', null);
|
insert into chat_threads(id, org_id, space_id, root_message_id) values ('thread-1', 'org-1', 'space-1', null);
|
||||||
insert into chat_messages(id, org_id, space_id, thread_id) values ('msg-1', 'org-1', 'space-1', 'thread-1');
|
insert into chat_messages(id, org_id, space_id, thread_id) values ('msg-1', 'org-1', 'space-1', 'thread-1');
|
||||||
@@ -282,8 +272,6 @@ describe("migration service", () => {
|
|||||||
expect(body.tableCounts.hiring_policies).toBe(1);
|
expect(body.tableCounts.hiring_policies).toBe(1);
|
||||||
expect(body.tableCounts.feature_flags).toBe(1);
|
expect(body.tableCounts.feature_flags).toBe(1);
|
||||||
expect(body.tableCounts.chaos_drill_runs).toBe(1);
|
expect(body.tableCounts.chaos_drill_runs).toBe(1);
|
||||||
expect(body.tableCounts.sdlc_review_runs).toBe(1);
|
|
||||||
expect(body.tableCounts.eval_cases).toBe(1);
|
|
||||||
expect(body.tableCounts.role_grants).toBe(1);
|
expect(body.tableCounts.role_grants).toBe(1);
|
||||||
expect(body.tableCounts.skill_artifacts).toBe(1);
|
expect(body.tableCounts.skill_artifacts).toBe(1);
|
||||||
expect(body.tableCounts.chat_read_state).toBe(1);
|
expect(body.tableCounts.chat_read_state).toBe(1);
|
||||||
@@ -312,8 +300,6 @@ describe("migration service", () => {
|
|||||||
expect(body.idMaps.approval_assignments).toEqual(["assign-1"]);
|
expect(body.idMaps.approval_assignments).toEqual(["assign-1"]);
|
||||||
expect(body.idMaps.operational_events).toEqual(["op-1"]);
|
expect(body.idMaps.operational_events).toEqual(["op-1"]);
|
||||||
expect(body.idMaps.chaos_drill_runs).toEqual(["drill-run-1"]);
|
expect(body.idMaps.chaos_drill_runs).toEqual(["drill-run-1"]);
|
||||||
expect(body.idMaps.sdlc_review_runs).toEqual(["review-run-1"]);
|
|
||||||
expect(body.idMaps.eval_runs).toEqual(["eval-run-1"]);
|
|
||||||
expect(body.idMaps.board_interventions).toEqual(["bdi-1"]);
|
expect(body.idMaps.board_interventions).toEqual(["bdi-1"]);
|
||||||
expect(body.idMaps.file_handoff_records).toEqual(["handoff-1"]);
|
expect(body.idMaps.file_handoff_records).toEqual(["handoff-1"]);
|
||||||
expect(body.idMaps.retention_freezes).toEqual(["freeze-1"]);
|
expect(body.idMaps.retention_freezes).toEqual(["freeze-1"]);
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import { DatabaseConnection } from "../../src/main/database/connection";
|
|||||||
import { OrganizationService } from "../../src/main/services/organization-service";
|
import { OrganizationService } from "../../src/main/services/organization-service";
|
||||||
import { SdlcService } from "../../src/main/services/sdlc-service";
|
import { SdlcService } from "../../src/main/services/sdlc-service";
|
||||||
import { SnapshotService } from "../../src/main/services/snapshot-service";
|
import { SnapshotService } from "../../src/main/services/snapshot-service";
|
||||||
import { SDLC_REVIEW_TARGET_TYPE } from "../../src/shared/domain";
|
import { IPC_CHANNELS } from "../../src/shared/ipc";
|
||||||
|
|
||||||
function createConnection(): DatabaseConnection {
|
function createConnection(): DatabaseConnection {
|
||||||
return new DatabaseConnection(join(tmpdir(), `agentcompany-new-sdlc-${randomUUID()}.db`));
|
return new DatabaseConnection(join(tmpdir(), `agentcompany-new-sdlc-${randomUUID()}.db`));
|
||||||
@@ -22,65 +22,36 @@ function createOrganization(organization: OrganizationService): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("SDLC service", () => {
|
describe("SDLC service", () => {
|
||||||
test("seeds review agents, eval suites, eval cases, and PR bot gates", () => {
|
test("offers no review or eval whose pass only counts configuration rows", () => {
|
||||||
const connection = createConnection();
|
const connection = createConnection();
|
||||||
const organization = new OrganizationService(connection.db);
|
|
||||||
const snapshot = new SnapshotService(connection.db);
|
|
||||||
|
|
||||||
connection.transaction(() => {
|
connection.transaction(() => {
|
||||||
createOrganization(organization);
|
createOrganization(new OrganizationService(connection.db));
|
||||||
});
|
});
|
||||||
|
const tables = connection.db.prepare(`
|
||||||
const state = snapshot.load();
|
select name from sqlite_master where type = 'table' and name in ('sdlc_review_agents', 'sdlc_review_runs', 'eval_suites', 'eval_cases', 'eval_runs')
|
||||||
expect(state.sdlcReviewAgents.map((agent) => agent.key).sort()).toEqual(["code_reviewer", "migration_reviewer", "policy_reviewer"]);
|
`).all();
|
||||||
expect(state.evalSuites.map((suite) => suite.key).sort()).toEqual(["contract", "integration", "replay"]);
|
expect(tables).toEqual([]);
|
||||||
expect(state.evalCases).toHaveLength(6);
|
expect(Object.keys(IPC_CHANNELS).filter((name) => name === "runSdlcReview" || name === "runEvalSuite")).toEqual([]);
|
||||||
expect(state.prBotGates.map((gate) => gate.key).sort()).toEqual(["human_approval", "migration_diff", "policy_diff", "static_checks"]);
|
|
||||||
connection.close();
|
connection.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
test("runs review agents against validated workspace targets", () => {
|
test("seeds PR bot gates and toggles them", () => {
|
||||||
const connection = createConnection();
|
const connection = createConnection();
|
||||||
const organization = new OrganizationService(connection.db);
|
const organization = new OrganizationService(connection.db);
|
||||||
const sdlc = new SdlcService(connection.db);
|
const sdlc = new SdlcService(connection.db);
|
||||||
const snapshot = new SnapshotService(connection.db);
|
const snapshot = new SnapshotService(connection.db);
|
||||||
let orgId = "";
|
let orgId = "";
|
||||||
let reviewAgentId = "";
|
|
||||||
|
|
||||||
connection.transaction(() => {
|
|
||||||
orgId = createOrganization(organization);
|
|
||||||
reviewAgentId = (connection.db.prepare("select id from sdlc_review_agents where org_id = ? and key = ?")
|
|
||||||
.get(orgId, "code_reviewer") as { id: string }).id;
|
|
||||||
sdlc.runReview({ orgId, reviewAgentId, targetType: SDLC_REVIEW_TARGET_TYPE.WORKSPACE, targetRef: orgId });
|
|
||||||
});
|
|
||||||
|
|
||||||
const state = snapshot.load();
|
|
||||||
expect(state.sdlcReviewRuns[0]).toMatchObject({ orgId, reviewAgentId, targetType: SDLC_REVIEW_TARGET_TYPE.WORKSPACE, state: "passed" });
|
|
||||||
expect(state.auditEvents.some((event) => event.entityType === "sdlc_review_run" && event.action === "created")).toBe(true);
|
|
||||||
connection.close();
|
|
||||||
});
|
|
||||||
|
|
||||||
test("runs contract eval suites and toggles PR bot gates", () => {
|
|
||||||
const connection = createConnection();
|
|
||||||
const organization = new OrganizationService(connection.db);
|
|
||||||
const sdlc = new SdlcService(connection.db);
|
|
||||||
const snapshot = new SnapshotService(connection.db);
|
|
||||||
let orgId = "";
|
|
||||||
let suiteId = "";
|
|
||||||
let gateId = "";
|
let gateId = "";
|
||||||
|
|
||||||
connection.transaction(() => {
|
connection.transaction(() => {
|
||||||
orgId = createOrganization(organization);
|
orgId = createOrganization(organization);
|
||||||
suiteId = (connection.db.prepare("select id from eval_suites where org_id = ? and key = ?")
|
|
||||||
.get(orgId, "contract") as { id: string }).id;
|
|
||||||
gateId = (connection.db.prepare("select id from pr_bot_gates where org_id = ? and key = ?")
|
gateId = (connection.db.prepare("select id from pr_bot_gates where org_id = ? and key = ?")
|
||||||
.get(orgId, "static_checks") as { id: string }).id;
|
.get(orgId, "static_checks") as { id: string }).id;
|
||||||
sdlc.runEvalSuite({ orgId, suiteId });
|
|
||||||
sdlc.setPrBotGate({ orgId, gateId, enabled: false });
|
sdlc.setPrBotGate({ orgId, gateId, enabled: false });
|
||||||
});
|
});
|
||||||
|
|
||||||
const state = snapshot.load();
|
const state = snapshot.load();
|
||||||
expect(state.evalRuns[0]).toMatchObject({ orgId, suiteId, state: "passed", failedCount: 0 });
|
expect(state.prBotGates.map((gate) => gate.key).sort()).toEqual(["human_approval", "migration_diff", "policy_diff", "static_checks"]);
|
||||||
expect(state.prBotGates.find((gate) => gate.id === gateId)?.enabled).toBe(false);
|
expect(state.prBotGates.find((gate) => gate.id === gateId)?.enabled).toBe(false);
|
||||||
expect(state.auditEvents.some((event) => event.entityType === "pr_bot_gate" && event.action === "updated")).toBe(true);
|
expect(state.auditEvents.some((event) => event.entityType === "pr_bot_gate" && event.action === "updated")).toBe(true);
|
||||||
connection.close();
|
connection.close();
|
||||||
|
|||||||
Reference in new issue
Block a user