Add layered CI quality gates

AgentCompany-New/package.json

@@ -14,9 +14,14 @@
     "build": "electron-vite build",
     "typecheck": "tsc --noEmit",
     "lint": "eslint .",
-    "test": "vitest run",
+    "test:unit": "vitest run tests/unit",
+    "test:contract": "vitest run tests/contract",
+    "test:integration": "vitest run tests/integration",
+    "test:replay-eval": "vitest run tests/eval",
+    "test": "pnpm test:unit && pnpm test:contract && pnpm test:integration && pnpm test:replay-eval",
     "test:e2e": "pnpm build && playwright test -c tests/e2e/playwright.config.ts",
-    "verify": "pnpm typecheck && pnpm lint && pnpm test && pnpm build",
+    "verify": "pnpm verify:ci",
+    "verify:ci": "pnpm typecheck && pnpm lint && pnpm test:unit && pnpm test:contract && pnpm test:integration && pnpm test:replay-eval && pnpm test:e2e",
     "package:dir": "pnpm build && electron-builder --dir --publish never",
     "package:win": "pnpm build && electron-builder --win nsis --x64 --publish never",
     "package:mac:arm": "pnpm build && electron-builder --mac dmg --arm64 --publish never",

AgentCompany-New/tests/contract/provider-contracts.test.ts

@@ -0,0 +1,107 @@
+import { randomUUID } from "node:crypto";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, test } from "vitest";
+import { DatabaseConnection } from "../../src/main/database/connection";
+import { OrganizationService } from "../../src/main/services/organization-service";
+import { providerAdapters } from "../../src/main/services/provider-adapters";
+import { SdlcService } from "../../src/main/services/sdlc-service";
+import { EVAL_SUITE_KEY, PROVIDER_AUTH_TYPE, SDLC_RUN_STATE } from "../../src/shared/domain";
+import { PROVIDER_CATALOG } from "../../src/shared/provider-catalog";
+
+function createConnection(): DatabaseConnection {
+  return new DatabaseConnection(join(tmpdir(), `agentcompany-new-contract-${randomUUID()}.db`));
+}
+
+function createOrganization(organization: OrganizationService): string {
+  return organization.createOrganization({
+    organizationName: "Acme Operations",
+    employeeName: "Dana Lee",
+    mission: "Coordinate agents with auditable approvals.",
+    goal: "Ship a verified control plane.",
+  });
+}
+
+describe("provider contracts", () => {
+  test("keeps provider catalog and adapters aligned", () => {
+    expect(Object.keys(providerAdapters).sort()).toEqual(PROVIDER_CATALOG.map((provider) => provider.key).sort());
+    for (const provider of PROVIDER_CATALOG) {
+      const adapter = providerAdapters[provider.key];
+      expect(adapter.providerKey).toBe(provider.key);
+      expect(adapter.authType).toBe(provider.authType);
+    }
+  });
+
+  test("parses credential profiles with required fields", () => {
+    const samples: Record<string, string> = {
+      openai: "sk-live-provider",
+      anthropic: "sk-live-provider",
+      xai: "sk-live-provider",
+      groq: "sk-live-provider",
+      "github-copilot": "ghu_live_provider",
+      "google-antigravity": "antigravity-session",
+      "gpt-account": "chatgpt-session",
+      "azure-openai": JSON.stringify({
+        baseUrl: "https://acme.openai.azure.com",
+        deployment: "gpt-5.4",
+        secret: "azure-key",
+      }),
+      "aws-bedrock": JSON.stringify({
+        region: "us-east-1",
+        secret: JSON.stringify({ accessKeyId: "AKIA123", secretAccessKey: "aws-secret" }),
+      }),
+      "vertex-ai": JSON.stringify({
+        projectId: "acme-ai",
+        region: "us-central1",
+        secret: JSON.stringify({ client_email: "agent@acme.iam.gserviceaccount.com", private_key: "key" }),
+      }),
+    };
+
+    for (const provider of PROVIDER_CATALOG) {
+      const profile = providerAdapters[provider.key].parseAuthProfile(samples[provider.key]);
+      if (provider.authType === PROVIDER_AUTH_TYPE.API_KEY) expect(profile.type).toBe("api_key");
+      if (provider.authType === PROVIDER_AUTH_TYPE.OAUTH_PKCE) expect(profile.type).toBe("access_token");
+      if (provider.authType === PROVIDER_AUTH_TYPE.ACCOUNT_SUBSCRIPTION) expect(profile.type).toBe("account_token");
+      if (provider.key === "azure-openai") expect(profile.type).toBe("azure_openai");
+      if (provider.key === "aws-bedrock") expect(profile.type).toBe("aws_bedrock");
+      if (provider.key === "vertex-ai") expect(profile.type).toBe("vertex_ai");
+    }
+  });
+
+  test("rejects incomplete credential profiles", () => {
+    for (const provider of PROVIDER_CATALOG) {
+      const adapter = providerAdapters[provider.key];
+      const secret = provider.authType === PROVIDER_AUTH_TYPE.CLOUD ? "{}" : " ";
+      expect(() => adapter.parseAuthProfile(secret)).toThrow();
+    }
+    expect(() => providerAdapters["azure-openai"].parseAuthProfile(JSON.stringify({
+      baseUrl: "http://acme.openai.azure.com",
+      deployment: "gpt-5.4",
+      secret: "azure-key",
+    }))).toThrow("Azure base URL must use HTTPS.");
+  });
+
+  test("runs the seeded contract eval suite", () => {
+    const connection = createConnection();
+    const organization = new OrganizationService(connection.db);
+    const sdlc = new SdlcService(connection.db);
+    let orgId = "";
+    let suiteId = "";
+    let runId = "";
+
+    connection.transaction(() => {
+      orgId = createOrganization(organization);
+      suiteId = (connection.db.prepare("select id from eval_suites where org_id = ? and key = ?")
+        .get(orgId, EVAL_SUITE_KEY.CONTRACT) as { id: string }).id;
+      runId = sdlc.runEvalSuite({ orgId, suiteId });
+    });
+
+    const run = connection.db.prepare("select state, failed_count, result_json from eval_runs where id = ?")
+      .get(runId) as { state: string; failed_count: number; result_json: string };
+    const result = JSON.parse(run.result_json) as { checks: Array<{ key: string; passed: boolean }> };
+    expect(run).toMatchObject({ state: SDLC_RUN_STATE.PASSED, failed_count: 0 });
+    expect(result.checks.map((check) => check.key).sort()).toEqual(["provider_adapter_coverage", "schema_version"]);
+    expect(result.checks.every((check) => check.passed)).toBe(true);
+    connection.close();
+  });
+});

AgentCompany-New/tests/eval/replay-eval.test.ts

@@ -0,0 +1,90 @@
+import { createHash, randomUUID } from "node:crypto";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, test } from "vitest";
+import { DatabaseConnection } from "../../src/main/database/connection";
+import { AuditService } from "../../src/main/services/audit-service";
+import { ChatService } from "../../src/main/services/chat-service";
+import { OperationsService } from "../../src/main/services/operations-service";
+import { OrganizationService } from "../../src/main/services/organization-service";
+import { SdlcService } from "../../src/main/services/sdlc-service";
+import { SnapshotService } from "../../src/main/services/snapshot-service";
+import { EVAL_SUITE_KEY, SDLC_RUN_STATE, SLO_SIGNAL_KEY } from "../../src/shared/domain";
+
+function createConnection(): DatabaseConnection {
+  return new DatabaseConnection(join(tmpdir(), `agentcompany-new-replay-${randomUUID()}.db`));
+}
+
+function createOrganization(organization: OrganizationService): string {
+  return organization.createOrganization({
+    organizationName: "Acme Operations",
+    employeeName: "Dana Lee",
+    mission: "Coordinate agents with auditable approvals.",
+    goal: "Ship a verified control plane.",
+  });
+}
+
+describe("replay eval", () => {
+  test("creates deterministic thread replay evidence and runs replay eval suite", () => {
+    const connection = createConnection();
+    const organization = new OrganizationService(connection.db);
+    const chat = new ChatService(connection.db);
+    const audit = new AuditService(connection.db);
+    const operations = new OperationsService(connection.db);
+    const sdlc = new SdlcService(connection.db);
+    const snapshot = new SnapshotService(connection.db);
+    let orgId = "";
+    let employeeId = "";
+    let threadId = "";
+    let replayRunId = "";
+    let evalRunId = "";
+
+    connection.transaction(() => {
+      orgId = createOrganization(organization);
+      employeeId = (connection.db.prepare("select id from employees where org_id = ?").get(orgId) as { id: string }).id;
+      const spaceId = chat.createSpace({
+        orgId,
+        spaceType: "project",
+        name: "release",
+        topic: "Release coordination",
+        visibility: "workspace",
+      });
+      const messageId = chat.sendMessage({
+        orgId,
+        spaceId,
+        threadId: null,
+        senderType: "employee",
+        senderId: employeeId,
+        bodyMd: "/escalate replay the release approval trail",
+        attachmentFileIds: [],
+      });
+      threadId = (connection.db.prepare("select thread_id from chat_messages where id = ?").get(messageId) as { thread_id: string }).thread_id;
+      replayRunId = audit.createThreadReplay({ orgId, threadId, createdByEmployeeId: employeeId });
+      operations.recordSignal({
+        orgId,
+        signalKey: SLO_SIGNAL_KEY.APPROVAL_ROUTE_COVERAGE,
+        source: "replay_eval",
+        success: true,
+        durationMs: 12,
+        detail: { threadId, replayRunId },
+      });
+      const suiteId = (connection.db.prepare("select id from eval_suites where org_id = ? and key = ?")
+        .get(orgId, EVAL_SUITE_KEY.REPLAY) as { id: string }).id;
+      evalRunId = sdlc.runEvalSuite({ orgId, suiteId });
+    });
+
+    const state = snapshot.load();
+    const replayRun = state.auditReplayRuns.find((run) => run.id === replayRunId);
+    const evalRun = state.evalRuns.find((run) => run.id === evalRunId);
+    if (!replayRun) throw new Error("Audit replay run was not created.");
+    if (!evalRun) throw new Error("Replay eval run was not created.");
+    const replay = JSON.parse(replayRun.replayJson) as { events: Array<{ entityType: string; action: string }> };
+    expect(replayRun.threadId).toBe(threadId);
+    expect(replayRun.eventDigest).toBe(createHash("sha256").update(replayRun.replayJson).digest("hex"));
+    expect(replayRun.eventCount).toBe(replay.events.length);
+    expect(replay.events).toContainEqual(expect.objectContaining({ entityType: "chat_message", action: "sent" }));
+    expect(replay.events).toContainEqual(expect.objectContaining({ entityType: "approval", action: "requested" }));
+    expect(evalRun).toMatchObject({ state: SDLC_RUN_STATE.PASSED, failedCount: 0 });
+    connection.close();
+  });
+});

AgentCompany-New/tests/integration/collaboration-flow.test.ts

@@ -0,0 +1,119 @@
+import { mkdirSync, writeFileSync } from "node:fs";
+import { randomUUID } from "node:crypto";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, test } from "vitest";
+import { DatabaseConnection } from "../../src/main/database/connection";
+import { ApprovalService } from "../../src/main/services/approval-service";
+import { ChatService } from "../../src/main/services/chat-service";
+import { OrganizationService } from "../../src/main/services/organization-service";
+import { SdlcService } from "../../src/main/services/sdlc-service";
+import { SkillService } from "../../src/main/services/skill-service";
+import { SnapshotService } from "../../src/main/services/snapshot-service";
+import { APPROVAL_STATE, EVAL_SUITE_KEY, SDLC_RUN_STATE } from "../../src/shared/domain";
+
+function createConnection(): DatabaseConnection {
+  return new DatabaseConnection(join(tmpdir(), `agentcompany-new-integration-${randomUUID()}.db`));
+}
+
+function createOrganization(organization: OrganizationService): string {
+  return organization.createOrganization({
+    organizationName: "Acme Operations",
+    employeeName: "Dana Lee",
+    mission: "Coordinate agents with auditable approvals.",
+    goal: "Ship a verified control plane.",
+  });
+}
+
+function createSkillDirectory(): string {
+  const root = join(tmpdir(), `agentcompany-new-skill-${randomUUID()}`);
+  mkdirSync(join(root, "checks"), { recursive: true });
+  writeFileSync(join(root, "SKILL.md"), "Review release evidence and produce a concise gate decision.\n", "utf8");
+  writeFileSync(join(root, "checks", "acceptance.txt"), "release evidence accepted\n", "utf8");
+  writeFileSync(join(root, "skill.manifest.json"), JSON.stringify({
+    name: "Release Gate",
+    version: "1.0.0",
+    entrypoint: "SKILL.md",
+    capabilities: ["release-review"],
+    required_tools: ["git"],
+    compatible_models: ["gpt-5.4"],
+    risk_level: "low",
+    tests: ["checks/acceptance.txt"],
+    inputs: {},
+    outputs: {},
+  }, null, 2), "utf8");
+  return root;
+}
+
+describe("collaboration integration flow", () => {
+  test("connects chat escalation, approval routing, skill resolution, and integration eval", () => {
+    const connection = createConnection();
+    const organization = new OrganizationService(connection.db);
+    const approvals = new ApprovalService(connection.db);
+    const chat = new ChatService(connection.db);
+    const skills = new SkillService(connection.db, join(tmpdir(), `agentcompany-new-artifacts-${randomUUID()}`));
+    const sdlc = new SdlcService(connection.db);
+    const snapshot = new SnapshotService(connection.db);
+    let orgId = "";
+    let employeeId = "";
+    let agentId = "";
+    let approvalId = "";
+    let skillId = "";
+    let revisionId = "";
+    let suiteId = "";
+    let evalRunId = "";
+
+    connection.transaction(() => {
+      orgId = createOrganization(organization);
+      employeeId = (connection.db.prepare("select id from employees where org_id = ?").get(orgId) as { id: string }).id;
+      agentId = organization.createAgent({
+        orgId,
+        name: "Ada",
+        role: "Release reviewer",
+        department: "Engineering",
+        modelProviderId: null,
+        modelId: null,
+        budgetCents: 10000,
+        managerAgentId: null,
+      });
+      const spaceId = chat.createSpace({
+        orgId,
+        spaceType: "project",
+        name: "release",
+        topic: "Release coordination",
+        visibility: "workspace",
+      });
+      chat.sendMessage({
+        orgId,
+        spaceId,
+        threadId: null,
+        senderType: "employee",
+        senderId: employeeId,
+        bodyMd: "/escalate review release acceptance package",
+        attachmentFileIds: [],
+      });
+      approvalId = (connection.db.prepare("select id from approvals where org_id = ? and kind = ?")
+        .get(orgId, "chat_escalation") as { id: string }).id;
+      approvals.decide({ orgId, approvalId, state: APPROVAL_STATE.APPROVED, decidedByEmployeeId: employeeId });
+      revisionId = skills.importDirectory({ orgId, directoryPath: createSkillDirectory(), versionSemver: "1.0.0" });
+      skillId = (connection.db.prepare("select skill_id from skill_revisions where id = ?").get(revisionId) as { skill_id: string }).skill_id;
+      skills.bind({ orgId, agentId, skillId, revisionId });
+      skills.resolveRuntimeSnapshot({ orgId, agentId, runId: `run_${randomUUID().replaceAll("-", "")}` });
+      suiteId = (connection.db.prepare("select id from eval_suites where org_id = ? and key = ?")
+        .get(orgId, EVAL_SUITE_KEY.INTEGRATION) as { id: string }).id;
+      evalRunId = sdlc.runEvalSuite({ orgId, suiteId });
+    });
+
+    const state = snapshot.load();
+    const evalRun = state.evalRuns.find((run) => run.id === evalRunId);
+    const snapshotRecord = state.skillRuntimeSnapshots[0];
+    if (!evalRun) throw new Error("Integration eval run was not created.");
+    if (!snapshotRecord) throw new Error("Skill runtime snapshot was not created.");
+    expect(state.approvals.find((approval) => approval.id === approvalId)?.state).toBe(APPROVAL_STATE.APPROVED);
+    expect(state.approvalAssignments.some((assignment) => assignment.approvalId === approvalId)).toBe(true);
+    expect(snapshotRecord.compiledPrompt).toContain("Release Gate");
+    expect(evalRun).toMatchObject({ state: SDLC_RUN_STATE.PASSED, failedCount: 0 });
+    expect(state.operationalEvents.some((event) => event.signalKey === "skill_resolve_success")).toBe(true);
+    connection.close();
+  });
+});

AgentCompany-New/vitest.config.ts

@@ -3,6 +3,6 @@ import { defineConfig } from "vitest/config";
 export default defineConfig({
   test: {
     environment: "node",
-    include: ["tests/unit/**/*.test.ts"],
+    include: ["tests/{unit,contract,integration,eval}/**/*.test.ts"],
   },
 });