From 3132308b72e97629e85cd09622c9491790417634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Sat, 23 May 2026 19:05:32 -0400 Subject: [PATCH] Add layered CI quality gates --- AgentCompany-New/package.json | 9 +- .../tests/contract/provider-contracts.test.ts | 107 ++++++++++++++++ .../tests/eval/replay-eval.test.ts | 90 +++++++++++++ .../integration/collaboration-flow.test.ts | 119 ++++++++++++++++++ AgentCompany-New/vitest.config.ts | 2 +- 5 files changed, 324 insertions(+), 3 deletions(-) create mode 100644 AgentCompany-New/tests/contract/provider-contracts.test.ts create mode 100644 AgentCompany-New/tests/eval/replay-eval.test.ts create mode 100644 AgentCompany-New/tests/integration/collaboration-flow.test.ts diff --git a/AgentCompany-New/package.json b/AgentCompany-New/package.json index 8391e784..2bed2b1c 100644 --- a/AgentCompany-New/package.json +++ b/AgentCompany-New/package.json @@ -14,9 +14,14 @@ "build": "electron-vite build", "typecheck": "tsc --noEmit", "lint": "eslint .", - "test": "vitest run", + "test:unit": "vitest run tests/unit", + "test:contract": "vitest run tests/contract", + "test:integration": "vitest run tests/integration", + "test:replay-eval": "vitest run tests/eval", + "test": "pnpm test:unit && pnpm test:contract && pnpm test:integration && pnpm test:replay-eval", "test:e2e": "pnpm build && playwright test -c tests/e2e/playwright.config.ts", - "verify": "pnpm typecheck && pnpm lint && pnpm test && pnpm build", + "verify": "pnpm verify:ci", + "verify:ci": "pnpm typecheck && pnpm lint && pnpm test:unit && pnpm test:contract && pnpm test:integration && pnpm test:replay-eval && pnpm test:e2e", "package:dir": "pnpm build && electron-builder --dir --publish never", "package:win": "pnpm build && electron-builder --win nsis --x64 --publish never", "package:mac:arm": "pnpm build && electron-builder --mac dmg --arm64 --publish never", diff --git a/AgentCompany-New/tests/contract/provider-contracts.test.ts b/AgentCompany-New/tests/contract/provider-contracts.test.ts new file mode 100644 index 00000000..40464564 --- /dev/null +++ b/AgentCompany-New/tests/contract/provider-contracts.test.ts @@ -0,0 +1,107 @@ +import { randomUUID } from "node:crypto"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, test } from "vitest"; +import { DatabaseConnection } from "../../src/main/database/connection"; +import { OrganizationService } from "../../src/main/services/organization-service"; +import { providerAdapters } from "../../src/main/services/provider-adapters"; +import { SdlcService } from "../../src/main/services/sdlc-service"; +import { EVAL_SUITE_KEY, PROVIDER_AUTH_TYPE, SDLC_RUN_STATE } from "../../src/shared/domain"; +import { PROVIDER_CATALOG } from "../../src/shared/provider-catalog"; + +function createConnection(): DatabaseConnection { + return new DatabaseConnection(join(tmpdir(), `agentcompany-new-contract-${randomUUID()}.db`)); +} + +function createOrganization(organization: OrganizationService): string { + return organization.createOrganization({ + organizationName: "Acme Operations", + employeeName: "Dana Lee", + mission: "Coordinate agents with auditable approvals.", + goal: "Ship a verified control plane.", + }); +} + +describe("provider contracts", () => { + test("keeps provider catalog and adapters aligned", () => { + expect(Object.keys(providerAdapters).sort()).toEqual(PROVIDER_CATALOG.map((provider) => provider.key).sort()); + for (const provider of PROVIDER_CATALOG) { + const adapter = providerAdapters[provider.key]; + expect(adapter.providerKey).toBe(provider.key); + expect(adapter.authType).toBe(provider.authType); + } + }); + + test("parses credential profiles with required fields", () => { + const samples: Record = { + openai: "sk-live-provider", + anthropic: "sk-live-provider", + xai: "sk-live-provider", + groq: "sk-live-provider", + "github-copilot": "ghu_live_provider", + "google-antigravity": "antigravity-session", + "gpt-account": "chatgpt-session", + "azure-openai": JSON.stringify({ + baseUrl: "https://acme.openai.azure.com", + deployment: "gpt-5.4", + secret: "azure-key", + }), + "aws-bedrock": JSON.stringify({ + region: "us-east-1", + secret: JSON.stringify({ accessKeyId: "AKIA123", secretAccessKey: "aws-secret" }), + }), + "vertex-ai": JSON.stringify({ + projectId: "acme-ai", + region: "us-central1", + secret: JSON.stringify({ client_email: "agent@acme.iam.gserviceaccount.com", private_key: "key" }), + }), + }; + + for (const provider of PROVIDER_CATALOG) { + const profile = providerAdapters[provider.key].parseAuthProfile(samples[provider.key]); + if (provider.authType === PROVIDER_AUTH_TYPE.API_KEY) expect(profile.type).toBe("api_key"); + if (provider.authType === PROVIDER_AUTH_TYPE.OAUTH_PKCE) expect(profile.type).toBe("access_token"); + if (provider.authType === PROVIDER_AUTH_TYPE.ACCOUNT_SUBSCRIPTION) expect(profile.type).toBe("account_token"); + if (provider.key === "azure-openai") expect(profile.type).toBe("azure_openai"); + if (provider.key === "aws-bedrock") expect(profile.type).toBe("aws_bedrock"); + if (provider.key === "vertex-ai") expect(profile.type).toBe("vertex_ai"); + } + }); + + test("rejects incomplete credential profiles", () => { + for (const provider of PROVIDER_CATALOG) { + const adapter = providerAdapters[provider.key]; + const secret = provider.authType === PROVIDER_AUTH_TYPE.CLOUD ? "{}" : " "; + expect(() => adapter.parseAuthProfile(secret)).toThrow(); + } + expect(() => providerAdapters["azure-openai"].parseAuthProfile(JSON.stringify({ + baseUrl: "http://acme.openai.azure.com", + deployment: "gpt-5.4", + secret: "azure-key", + }))).toThrow("Azure base URL must use HTTPS."); + }); + + test("runs the seeded contract eval suite", () => { + const connection = createConnection(); + const organization = new OrganizationService(connection.db); + const sdlc = new SdlcService(connection.db); + let orgId = ""; + let suiteId = ""; + let runId = ""; + + connection.transaction(() => { + orgId = createOrganization(organization); + suiteId = (connection.db.prepare("select id from eval_suites where org_id = ? and key = ?") + .get(orgId, EVAL_SUITE_KEY.CONTRACT) as { id: string }).id; + runId = sdlc.runEvalSuite({ orgId, suiteId }); + }); + + const run = connection.db.prepare("select state, failed_count, result_json from eval_runs where id = ?") + .get(runId) as { state: string; failed_count: number; result_json: string }; + const result = JSON.parse(run.result_json) as { checks: Array<{ key: string; passed: boolean }> }; + expect(run).toMatchObject({ state: SDLC_RUN_STATE.PASSED, failed_count: 0 }); + expect(result.checks.map((check) => check.key).sort()).toEqual(["provider_adapter_coverage", "schema_version"]); + expect(result.checks.every((check) => check.passed)).toBe(true); + connection.close(); + }); +}); diff --git a/AgentCompany-New/tests/eval/replay-eval.test.ts b/AgentCompany-New/tests/eval/replay-eval.test.ts new file mode 100644 index 00000000..f39881b9 --- /dev/null +++ b/AgentCompany-New/tests/eval/replay-eval.test.ts @@ -0,0 +1,90 @@ +import { createHash, randomUUID } from "node:crypto"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, test } from "vitest"; +import { DatabaseConnection } from "../../src/main/database/connection"; +import { AuditService } from "../../src/main/services/audit-service"; +import { ChatService } from "../../src/main/services/chat-service"; +import { OperationsService } from "../../src/main/services/operations-service"; +import { OrganizationService } from "../../src/main/services/organization-service"; +import { SdlcService } from "../../src/main/services/sdlc-service"; +import { SnapshotService } from "../../src/main/services/snapshot-service"; +import { EVAL_SUITE_KEY, SDLC_RUN_STATE, SLO_SIGNAL_KEY } from "../../src/shared/domain"; + +function createConnection(): DatabaseConnection { + return new DatabaseConnection(join(tmpdir(), `agentcompany-new-replay-${randomUUID()}.db`)); +} + +function createOrganization(organization: OrganizationService): string { + return organization.createOrganization({ + organizationName: "Acme Operations", + employeeName: "Dana Lee", + mission: "Coordinate agents with auditable approvals.", + goal: "Ship a verified control plane.", + }); +} + +describe("replay eval", () => { + test("creates deterministic thread replay evidence and runs replay eval suite", () => { + const connection = createConnection(); + const organization = new OrganizationService(connection.db); + const chat = new ChatService(connection.db); + const audit = new AuditService(connection.db); + const operations = new OperationsService(connection.db); + const sdlc = new SdlcService(connection.db); + const snapshot = new SnapshotService(connection.db); + let orgId = ""; + let employeeId = ""; + let threadId = ""; + let replayRunId = ""; + let evalRunId = ""; + + connection.transaction(() => { + orgId = createOrganization(organization); + employeeId = (connection.db.prepare("select id from employees where org_id = ?").get(orgId) as { id: string }).id; + const spaceId = chat.createSpace({ + orgId, + spaceType: "project", + name: "release", + topic: "Release coordination", + visibility: "workspace", + }); + const messageId = chat.sendMessage({ + orgId, + spaceId, + threadId: null, + senderType: "employee", + senderId: employeeId, + bodyMd: "/escalate replay the release approval trail", + attachmentFileIds: [], + }); + threadId = (connection.db.prepare("select thread_id from chat_messages where id = ?").get(messageId) as { thread_id: string }).thread_id; + replayRunId = audit.createThreadReplay({ orgId, threadId, createdByEmployeeId: employeeId }); + operations.recordSignal({ + orgId, + signalKey: SLO_SIGNAL_KEY.APPROVAL_ROUTE_COVERAGE, + source: "replay_eval", + success: true, + durationMs: 12, + detail: { threadId, replayRunId }, + }); + const suiteId = (connection.db.prepare("select id from eval_suites where org_id = ? and key = ?") + .get(orgId, EVAL_SUITE_KEY.REPLAY) as { id: string }).id; + evalRunId = sdlc.runEvalSuite({ orgId, suiteId }); + }); + + const state = snapshot.load(); + const replayRun = state.auditReplayRuns.find((run) => run.id === replayRunId); + const evalRun = state.evalRuns.find((run) => run.id === evalRunId); + if (!replayRun) throw new Error("Audit replay run was not created."); + if (!evalRun) throw new Error("Replay eval run was not created."); + const replay = JSON.parse(replayRun.replayJson) as { events: Array<{ entityType: string; action: string }> }; + expect(replayRun.threadId).toBe(threadId); + expect(replayRun.eventDigest).toBe(createHash("sha256").update(replayRun.replayJson).digest("hex")); + expect(replayRun.eventCount).toBe(replay.events.length); + expect(replay.events).toContainEqual(expect.objectContaining({ entityType: "chat_message", action: "sent" })); + expect(replay.events).toContainEqual(expect.objectContaining({ entityType: "approval", action: "requested" })); + expect(evalRun).toMatchObject({ state: SDLC_RUN_STATE.PASSED, failedCount: 0 }); + connection.close(); + }); +}); diff --git a/AgentCompany-New/tests/integration/collaboration-flow.test.ts b/AgentCompany-New/tests/integration/collaboration-flow.test.ts new file mode 100644 index 00000000..4f1157c4 --- /dev/null +++ b/AgentCompany-New/tests/integration/collaboration-flow.test.ts @@ -0,0 +1,119 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { randomUUID } from "node:crypto"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, test } from "vitest"; +import { DatabaseConnection } from "../../src/main/database/connection"; +import { ApprovalService } from "../../src/main/services/approval-service"; +import { ChatService } from "../../src/main/services/chat-service"; +import { OrganizationService } from "../../src/main/services/organization-service"; +import { SdlcService } from "../../src/main/services/sdlc-service"; +import { SkillService } from "../../src/main/services/skill-service"; +import { SnapshotService } from "../../src/main/services/snapshot-service"; +import { APPROVAL_STATE, EVAL_SUITE_KEY, SDLC_RUN_STATE } from "../../src/shared/domain"; + +function createConnection(): DatabaseConnection { + return new DatabaseConnection(join(tmpdir(), `agentcompany-new-integration-${randomUUID()}.db`)); +} + +function createOrganization(organization: OrganizationService): string { + return organization.createOrganization({ + organizationName: "Acme Operations", + employeeName: "Dana Lee", + mission: "Coordinate agents with auditable approvals.", + goal: "Ship a verified control plane.", + }); +} + +function createSkillDirectory(): string { + const root = join(tmpdir(), `agentcompany-new-skill-${randomUUID()}`); + mkdirSync(join(root, "checks"), { recursive: true }); + writeFileSync(join(root, "SKILL.md"), "Review release evidence and produce a concise gate decision.\n", "utf8"); + writeFileSync(join(root, "checks", "acceptance.txt"), "release evidence accepted\n", "utf8"); + writeFileSync(join(root, "skill.manifest.json"), JSON.stringify({ + name: "Release Gate", + version: "1.0.0", + entrypoint: "SKILL.md", + capabilities: ["release-review"], + required_tools: ["git"], + compatible_models: ["gpt-5.4"], + risk_level: "low", + tests: ["checks/acceptance.txt"], + inputs: {}, + outputs: {}, + }, null, 2), "utf8"); + return root; +} + +describe("collaboration integration flow", () => { + test("connects chat escalation, approval routing, skill resolution, and integration eval", () => { + const connection = createConnection(); + const organization = new OrganizationService(connection.db); + const approvals = new ApprovalService(connection.db); + const chat = new ChatService(connection.db); + const skills = new SkillService(connection.db, join(tmpdir(), `agentcompany-new-artifacts-${randomUUID()}`)); + const sdlc = new SdlcService(connection.db); + const snapshot = new SnapshotService(connection.db); + let orgId = ""; + let employeeId = ""; + let agentId = ""; + let approvalId = ""; + let skillId = ""; + let revisionId = ""; + let suiteId = ""; + let evalRunId = ""; + + connection.transaction(() => { + orgId = createOrganization(organization); + employeeId = (connection.db.prepare("select id from employees where org_id = ?").get(orgId) as { id: string }).id; + agentId = organization.createAgent({ + orgId, + name: "Ada", + role: "Release reviewer", + department: "Engineering", + modelProviderId: null, + modelId: null, + budgetCents: 10000, + managerAgentId: null, + }); + const spaceId = chat.createSpace({ + orgId, + spaceType: "project", + name: "release", + topic: "Release coordination", + visibility: "workspace", + }); + chat.sendMessage({ + orgId, + spaceId, + threadId: null, + senderType: "employee", + senderId: employeeId, + bodyMd: "/escalate review release acceptance package", + attachmentFileIds: [], + }); + approvalId = (connection.db.prepare("select id from approvals where org_id = ? and kind = ?") + .get(orgId, "chat_escalation") as { id: string }).id; + approvals.decide({ orgId, approvalId, state: APPROVAL_STATE.APPROVED, decidedByEmployeeId: employeeId }); + revisionId = skills.importDirectory({ orgId, directoryPath: createSkillDirectory(), versionSemver: "1.0.0" }); + skillId = (connection.db.prepare("select skill_id from skill_revisions where id = ?").get(revisionId) as { skill_id: string }).skill_id; + skills.bind({ orgId, agentId, skillId, revisionId }); + skills.resolveRuntimeSnapshot({ orgId, agentId, runId: `run_${randomUUID().replaceAll("-", "")}` }); + suiteId = (connection.db.prepare("select id from eval_suites where org_id = ? and key = ?") + .get(orgId, EVAL_SUITE_KEY.INTEGRATION) as { id: string }).id; + evalRunId = sdlc.runEvalSuite({ orgId, suiteId }); + }); + + const state = snapshot.load(); + const evalRun = state.evalRuns.find((run) => run.id === evalRunId); + const snapshotRecord = state.skillRuntimeSnapshots[0]; + if (!evalRun) throw new Error("Integration eval run was not created."); + if (!snapshotRecord) throw new Error("Skill runtime snapshot was not created."); + expect(state.approvals.find((approval) => approval.id === approvalId)?.state).toBe(APPROVAL_STATE.APPROVED); + expect(state.approvalAssignments.some((assignment) => assignment.approvalId === approvalId)).toBe(true); + expect(snapshotRecord.compiledPrompt).toContain("Release Gate"); + expect(evalRun).toMatchObject({ state: SDLC_RUN_STATE.PASSED, failedCount: 0 }); + expect(state.operationalEvents.some((event) => event.signalKey === "skill_resolve_success")).toBe(true); + connection.close(); + }); +}); diff --git a/AgentCompany-New/vitest.config.ts b/AgentCompany-New/vitest.config.ts index 93cf2aa8..4e9e2444 100644 --- a/AgentCompany-New/vitest.config.ts +++ b/AgentCompany-New/vitest.config.ts @@ -3,6 +3,6 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { environment: "node", - include: ["tests/unit/**/*.test.ts"], + include: ["tests/{unit,contract,integration,eval}/**/*.test.ts"], }, });