Add SDLC review and eval controls

This commit is contained in:
ZacharyZhang-NY committed 2026-05-23 18:39:05 -04:00
1 parent bc14a2fbcb
commit bbf90a1a62
19 files changed
+927 -2

No files matched your search

@@ -23,7 +23,7 @@ describe("domain services", () => {
const connection = createConnection();
const rows = connection.db.prepare("select version from schema_migrations order by version asc").all() as Array<{ version: number }>;
expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]);
expect(rows.map((row) => row.version)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]);
connection.close();
});
@@ -51,6 +51,9 @@ describe("domain services", () => {
expect(state.featureFlags.map((flag) => flag.key)).toContain("chaos_drills");
expect(state.sloTargets.map((target) => target.signalKey)).toContain("provider_request_success");
expect(state.chaosDrills.map((drill) => drill.scenarioKey)).toContain("approval_backlog_gate");
expect(state.sdlcReviewAgents.map((agent) => agent.key)).toContain("code_reviewer");
expect(state.evalSuites.map((suite) => suite.key)).toContain("contract");
expect(state.prBotGates.map((gate) => gate.key)).toContain("static_checks");
expect(state.chatSpaces.map((space) => space.name)).toContain("board");
expect(state.policyRules[0]?.action).toBe("task_create");
expect(state.auditEvents[0]?.action).toBe("created");
@@ -113,6 +113,12 @@ describe("migration service", () => {
create table operational_events(id text primary key, org_id text);
create table chaos_drills(id text primary key, org_id text);
create table chaos_drill_runs(id text primary key, org_id text, drill_id text);
create table sdlc_review_agents(id text primary key, org_id text);
create table sdlc_review_runs(id text primary key, org_id text, review_agent_id text);
create table eval_suites(id text primary key, org_id text);
create table eval_cases(id text primary key, org_id text, suite_id text);
create table eval_runs(id text primary key, org_id text, suite_id text);
create table pr_bot_gates(id text primary key, org_id text);
create table chat_spaces(id text primary key, org_id text);
create table chat_messages(id text primary key, org_id text, space_id text, thread_id text);
create table chat_threads(id text primary key, org_id text, space_id text, root_message_id text);
@@ -173,6 +179,12 @@ describe("migration service", () => {
insert into operational_events(id, org_id) values ('op-1', 'org-1');
insert into chaos_drills(id, org_id) values ('drill-1', 'org-1');
insert into chaos_drill_runs(id, org_id, drill_id) values ('drill-run-1', 'org-1', 'drill-1');
insert into sdlc_review_agents(id, org_id) values ('reviewer-1', 'org-1');
insert into sdlc_review_runs(id, org_id, review_agent_id) values ('review-run-1', 'org-1', 'reviewer-1');
insert into eval_suites(id, org_id) values ('suite-1', 'org-1');
insert into eval_cases(id, org_id, suite_id) values ('case-1', 'org-1', 'suite-1');
insert into eval_runs(id, org_id, suite_id) values ('eval-run-1', 'org-1', 'suite-1');
insert into pr_bot_gates(id, org_id) values ('gate-1', 'org-1');
insert into chat_threads(id, org_id, space_id, root_message_id) values ('thread-1', 'org-1', 'space-1', null);
insert into chat_messages(id, org_id, space_id, thread_id) values ('msg-1', 'org-1', 'space-1', 'thread-1');
insert into chat_memberships(space_id, principal_type, principal_id) values ('space-1', 'employee', 'emp-1');
@@ -219,6 +231,8 @@ describe("migration service", () => {
expect(body.tableCounts.hiring_policies).toBe(1);
expect(body.tableCounts.feature_flags).toBe(1);
expect(body.tableCounts.chaos_drill_runs).toBe(1);
expect(body.tableCounts.sdlc_review_runs).toBe(1);
expect(body.tableCounts.eval_cases).toBe(1);
expect(body.tableCounts.role_grants).toBe(1);
expect(body.tableCounts.skill_artifacts).toBe(1);
expect(body.tableCounts.chat_read_state).toBe(1);
@@ -235,6 +249,8 @@ describe("migration service", () => {
expect(body.idMaps.approval_assignments).toEqual(["assign-1"]);
expect(body.idMaps.operational_events).toEqual(["op-1"]);
expect(body.idMaps.chaos_drill_runs).toEqual(["drill-run-1"]);
expect(body.idMaps.sdlc_review_runs).toEqual(["review-run-1"]);
expect(body.idMaps.eval_runs).toEqual(["eval-run-1"]);
expect(body.idMaps.board_interventions).toEqual(["bdi-1"]);
expect(body.idMaps.retention_freezes).toEqual(["freeze-1"]);
expect(body.idMaps.provider_usage_windows).toEqual(["usage-1"]);
@@ -0,0 +1,88 @@
import { randomUUID } from "node:crypto";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, test } from "vitest";
import { DatabaseConnection } from "../../src/main/database/connection";
import { OrganizationService } from "../../src/main/services/organization-service";
import { SdlcService } from "../../src/main/services/sdlc-service";
import { SnapshotService } from "../../src/main/services/snapshot-service";
import { SDLC_REVIEW_TARGET_TYPE } from "../../src/shared/domain";
function createConnection(): DatabaseConnection {
return new DatabaseConnection(join(tmpdir(), `agentcompany-new-sdlc-${randomUUID()}.db`));
}
function createOrganization(organization: OrganizationService): string {
return organization.createOrganization({
organizationName: "Acme Operations",
employeeName: "Dana Lee",
mission: "Coordinate agents with auditable approvals.",
goal: "Ship a verified control plane.",
});
}
describe("SDLC service", () => {
test("seeds review agents, eval suites, eval cases, and PR bot gates", () => {
const connection = createConnection();
const organization = new OrganizationService(connection.db);
const snapshot = new SnapshotService(connection.db);
connection.transaction(() => {
createOrganization(organization);
});
const state = snapshot.load();
expect(state.sdlcReviewAgents.map((agent) => agent.key).sort()).toEqual(["code_reviewer", "migration_reviewer", "policy_reviewer"]);
expect(state.evalSuites.map((suite) => suite.key).sort()).toEqual(["contract", "integration", "replay"]);
expect(state.evalCases).toHaveLength(6);
expect(state.prBotGates.map((gate) => gate.key).sort()).toEqual(["human_approval", "migration_diff", "policy_diff", "static_checks"]);
connection.close();
});
test("runs review agents against validated workspace targets", () => {
const connection = createConnection();
const organization = new OrganizationService(connection.db);
const sdlc = new SdlcService(connection.db);
const snapshot = new SnapshotService(connection.db);
let orgId = "";
let reviewAgentId = "";
connection.transaction(() => {
orgId = createOrganization(organization);
reviewAgentId = (connection.db.prepare("select id from sdlc_review_agents where org_id = ? and key = ?")
.get(orgId, "code_reviewer") as { id: string }).id;
sdlc.runReview({ orgId, reviewAgentId, targetType: SDLC_REVIEW_TARGET_TYPE.WORKSPACE, targetRef: orgId });
});
const state = snapshot.load();
expect(state.sdlcReviewRuns[0]).toMatchObject({ orgId, reviewAgentId, targetType: SDLC_REVIEW_TARGET_TYPE.WORKSPACE, state: "passed" });
expect(state.auditEvents.some((event) => event.entityType === "sdlc_review_run" && event.action === "created")).toBe(true);
connection.close();
});
test("runs contract eval suites and toggles PR bot gates", () => {
const connection = createConnection();
const organization = new OrganizationService(connection.db);
const sdlc = new SdlcService(connection.db);
const snapshot = new SnapshotService(connection.db);
let orgId = "";
let suiteId = "";
let gateId = "";
connection.transaction(() => {
orgId = createOrganization(organization);
suiteId = (connection.db.prepare("select id from eval_suites where org_id = ? and key = ?")
.get(orgId, "contract") as { id: string }).id;
gateId = (connection.db.prepare("select id from pr_bot_gates where org_id = ? and key = ?")
.get(orgId, "static_checks") as { id: string }).id;
sdlc.runEvalSuite({ orgId, suiteId });
sdlc.setPrBotGate({ orgId, gateId, enabled: false });
});
const state = snapshot.load();
expect(state.evalRuns[0]).toMatchObject({ orgId, suiteId, state: "passed", failedCount: 0 });
expect(state.prBotGates.find((gate) => gate.id === gateId)?.enabled).toBe(false);
expect(state.auditEvents.some((event) => event.entityType === "pr_bot_gate" && event.action === "updated")).toBe(true);
connection.close();
});
});