diff --git a/AgentCompany-New/src/main/app/ipc-handlers.ts b/AgentCompany-New/src/main/app/ipc-handlers.ts index 920ad6f7..2d56d588 100644 --- a/AgentCompany-New/src/main/app/ipc-handlers.ts +++ b/AgentCompany-New/src/main/app/ipc-handlers.ts @@ -171,6 +171,7 @@ export function registerIpcHandlers(deps: IpcDeps): void { ipcMain.handle(IPC_CHANNELS.importSkillDirectory, (_event, input) => mutate(() => skills.importDirectory(input))); ipcMain.handle(IPC_CHANNELS.bindSkill, (_event, input) => mutate(() => skills.bind(input))); ipcMain.handle(IPC_CHANNELS.resolveAgentSkills, (_event, input) => mutate(() => skills.resolveRuntimeSnapshot(input))); + ipcMain.handle(IPC_CHANNELS.runSkillTests, (_event, input) => mutate(() => skills.runTests(input))); ipcMain.handle(IPC_CHANNELS.createSocialAccount, (_event, input) => mutate(() => social.createAccount(input))); ipcMain.handle(IPC_CHANNELS.createScheduledPost, (_event, input) => mutate(() => social.createScheduledPost(input))); ipcMain.handle(IPC_CHANNELS.setScheduledPostState, (_event, input) => mutate(() => social.setScheduledPostState(input))); diff --git a/AgentCompany-New/src/main/services/skill-service.ts b/AgentCompany-New/src/main/services/skill-service.ts index 2d24019e..db92dbd0 100644 --- a/AgentCompany-New/src/main/services/skill-service.ts +++ b/AgentCompany-New/src/main/services/skill-service.ts @@ -1,6 +1,8 @@ import { existsSync, readFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { normalize } from "node:path"; import type { DatabaseSync } from "node:sqlite"; -import type { BindSkillInput, ImportSkillDirectoryInput, ResolveAgentSkillsInput } from "../../shared/commands"; +import type { BindSkillInput, ImportSkillDirectoryInput, ResolveAgentSkillsInput, RunSkillTestsInput } from "../../shared/commands"; import { SLO_SIGNAL_KEY } from "../../shared/domain"; import { createId } from "../../shared/ids"; import { AuditService } from "./audit-service"; @@ -181,6 +183,53 @@ export class SkillService { return snapshotId; } + runTests(input: RunSkillTestsInput): string { + const revision = this.db.prepare(` + select sr.id, sr.version_semver, sd.slug + from skill_revisions sr + join skill_definitions sd on sd.id = sr.skill_id + where sr.id = ? and sd.org_id = ? + `).get(input.revisionId, input.orgId) as SkillRevisionTestRow | undefined; + if (!revision) throw new Error("Skill revision was not found."); + + const tests = this.db.prepare(` + select id, test_type, expected_output_ref + from skill_tests + where revision_id = ? + order by expected_output_ref asc + `).all(input.revisionId) as unknown as SkillTestRow[]; + const artifacts = this.db.prepare(` + select artifact_type, object_key, sha256, size_bytes + from skill_artifacts + where revision_id = ? + order by artifact_type asc, object_key asc + `).all(input.revisionId) as unknown as SkillArtifactRow[]; + const structureFailure = verifyRevisionStructure(artifacts); + if (structureFailure && tests.length === 0) throw new Error(structureFailure.failure ?? "Skill revision structure check failed."); + const now = new Date().toISOString(); + let passed = 0; + let failed = 0; + + for (const test of tests) { + const result = structureFailure ?? runManifestArtifactTest(test, artifacts); + if (result.status === "passed") passed += 1; + if (result.status === "failed") failed += 1; + this.db.prepare("update skill_tests set last_result = ?, last_run_at = ? where id = ?") + .run(JSON.stringify(result), now, test.id); + } + + this.audit.record({ + orgId: input.orgId, + entityType: "skill_revision", + entityId: revision.id, + actorType: "system", + actorId: null, + action: "tests_run", + payload: { slug: revision.slug, version: revision.version_semver, total: tests.length, passed, failed }, + }); + return revision.id; + } + private verifyBindingTargets(input: BindSkillInput): void { const agent = this.db.prepare("select id from agents where id = ? and org_id = ?") .get(input.agentId, input.orgId) as { id: string } | undefined; @@ -207,3 +256,103 @@ interface RuntimeSkillRow { manifest_json: string; object_key: string; } + +interface SkillRevisionTestRow { + id: string; + version_semver: string; + slug: string; +} + +interface SkillTestRow { + id: string; + test_type: string; + expected_output_ref: string; +} + +interface SkillArtifactRow { + artifact_type: string; + object_key: string; + sha256: string; + size_bytes: number; +} + +interface SkillTestResult { + status: "passed" | "failed"; + testType: string; + expectedOutputRef: string; + checks: Array<{ name: string; status: "passed" | "failed"; detail: string }>; + artifactSha256?: string; + failure?: string; +} + +function verifyRevisionStructure(artifacts: SkillArtifactRow[]): SkillTestResult | null { + const requiredTypes = ["manifest", "entrypoint"]; + for (const type of requiredTypes) { + if (!artifacts.some((artifact) => artifact.artifact_type === type)) { + return failedSkillTest("revision_structure", type, `Skill revision artifact is missing: ${type}`); + } + } + for (const artifact of artifacts) { + try { + verifyArtifactIntegrity(artifact); + } catch (error) { + return failedSkillTest("artifact_integrity", artifact.object_key, error instanceof Error ? error.message : String(error)); + } + } + return null; +} + +function runManifestArtifactTest(test: SkillTestRow, artifacts: SkillArtifactRow[]): SkillTestResult { + if (test.test_type !== "manifest") { + return failedSkillTest(test.test_type, test.expected_output_ref, `Unsupported skill test type: ${test.test_type}`); + } + const testArtifact = artifacts.find((artifact) => ( + artifact.artifact_type === "test" && artifactMatchesExpectedRef(artifact.object_key, test.expected_output_ref) + )); + if (!testArtifact) return failedSkillTest(test.test_type, test.expected_output_ref, "Skill test artifact was not found."); + + try { + const content = verifyArtifactIntegrity(testArtifact).toString("utf8"); + if (!content.trim()) throw new Error("Skill test artifact is empty."); + return { + status: "passed", + testType: test.test_type, + expectedOutputRef: test.expected_output_ref, + artifactSha256: testArtifact.sha256, + checks: [ + { name: "manifest_test_artifact", status: "passed", detail: "Artifact exists, checksum matches, and content is non-empty." }, + ], + }; + } catch (error) { + return failedSkillTest(test.test_type, test.expected_output_ref, error instanceof Error ? error.message : String(error)); + } +} + +function verifyArtifactIntegrity(artifact: SkillArtifactRow): Buffer { + if (!existsSync(artifact.object_key)) throw new Error(`Skill artifact is missing: ${artifact.object_key}`); + const body = readFileSync(artifact.object_key); + const sha256 = createHash("sha256").update(body).digest("hex"); + if (sha256 !== artifact.sha256) throw new Error(`Skill artifact checksum mismatch: ${artifact.object_key}`); + if (body.byteLength !== artifact.size_bytes) throw new Error(`Skill artifact size mismatch: ${artifact.object_key}`); + return body; +} + +function artifactMatchesExpectedRef(objectKey: string, expectedRef: string): boolean { + const expected = normalizeSkillPath(expectedRef); + const actual = normalizeSkillPath(objectKey); + return actual === expected || actual.endsWith(`/${expected}`); +} + +function normalizeSkillPath(value: string): string { + return normalize(value).replaceAll("\\", "/"); +} + +function failedSkillTest(testType: string, expectedOutputRef: string, failure: string): SkillTestResult { + return { + status: "failed", + testType, + expectedOutputRef, + checks: [{ name: "skill_test", status: "failed", detail: failure }], + failure, + }; +} diff --git a/AgentCompany-New/src/main/services/snapshot-service.ts b/AgentCompany-New/src/main/services/snapshot-service.ts index 5f58ec60..e2679966 100644 --- a/AgentCompany-New/src/main/services/snapshot-service.ts +++ b/AgentCompany-New/src/main/services/snapshot-service.ts @@ -115,6 +115,7 @@ export class SnapshotService { skillRevisions: list(this.db, "skill_revisions", "published_at desc"), skillBindings, skillRuntimeSnapshots: list(this.db, "skill_runtime_snapshots", "created_at desc"), + skillTests: list(this.db, "skill_tests", "last_run_at desc, id asc"), socialAccounts: list(this.db, "social_accounts", "created_at asc"), socialAccountGrants: list(this.db, "social_account_grants", "created_at asc"), browserSessionContainers: list(this.db, "browser_session_containers", "created_at asc") diff --git a/AgentCompany-New/src/preload/index.ts b/AgentCompany-New/src/preload/index.ts index cb0e6c68..f27580d1 100644 --- a/AgentCompany-New/src/preload/index.ts +++ b/AgentCompany-New/src/preload/index.ts @@ -37,6 +37,7 @@ const api: DesktopApi = { importSkillDirectory: (input) => invoke(IPC_CHANNELS.importSkillDirectory, input), bindSkill: (input) => invoke(IPC_CHANNELS.bindSkill, input), resolveAgentSkills: (input) => invoke(IPC_CHANNELS.resolveAgentSkills, input), + runSkillTests: (input) => invoke(IPC_CHANNELS.runSkillTests, input), createSocialAccount: (input) => invoke(IPC_CHANNELS.createSocialAccount, input), launchSocialLogin: (input) => invoke(IPC_CHANNELS.launchSocialLogin, input), createScheduledPost: (input) => invoke(IPC_CHANNELS.createScheduledPost, input), diff --git a/AgentCompany-New/src/renderer/src/pages/SettingsPage.tsx b/AgentCompany-New/src/renderer/src/pages/SettingsPage.tsx index 540d8153..86a1d898 100644 --- a/AgentCompany-New/src/renderer/src/pages/SettingsPage.tsx +++ b/AgentCompany-New/src/renderer/src/pages/SettingsPage.tsx @@ -71,6 +71,10 @@ export function SettingsPage({ snapshot, actions }: PageProps) { const skillIds = new Set(skills.map((skill) => skill.id)); return snapshot.skillRevisions.filter((revision) => skillIds.has(revision.skillId)); }, [snapshot.skillRevisions, skills]); + const skillTests = useMemo(() => { + const revisionIds = new Set(skillRevisions.map((revision) => revision.id)); + return snapshot.skillTests.filter((test) => revisionIds.has(test.revisionId)); + }, [snapshot.skillTests, skillRevisions]); const skillBindings = useMemo(() => snapshot.skillBindings.filter((item) => item.orgId === orgId), [snapshot.skillBindings, orgId]); const policies = useMemo(() => snapshot.policyRules.filter((item) => item.orgId === orgId), [snapshot.policyRules, orgId]); const workspaces = useMemo(() => snapshot.workspaces.filter((item) => item.orgId === orgId), [snapshot.workspaces, orgId]); @@ -147,6 +151,7 @@ export function SettingsPage({ snapshot, actions }: PageProps) { skills={skills} revisions={skillRevisions} bindings={skillBindings} + tests={skillTests} agents={agents} actions={actions} /> diff --git a/AgentCompany-New/src/renderer/src/pages/settings/SkillsPanel.tsx b/AgentCompany-New/src/renderer/src/pages/settings/SkillsPanel.tsx index 19de087b..7e128a00 100644 --- a/AgentCompany-New/src/renderer/src/pages/settings/SkillsPanel.tsx +++ b/AgentCompany-New/src/renderer/src/pages/settings/SkillsPanel.tsx @@ -6,6 +6,7 @@ import type { SkillBindingRecord, SkillDefinitionRecord, SkillRevisionRecord, + SkillTestRecord, } from "../../../../shared/domain"; import type { SnapshotActions } from "../../app/useSnapshot"; @@ -14,11 +15,12 @@ interface SkillsPanelProps { skills: SkillDefinitionRecord[]; revisions: SkillRevisionRecord[]; bindings: SkillBindingRecord[]; + tests: SkillTestRecord[]; agents: AgentRecord[]; actions: SnapshotActions; } -export function SkillsPanel({ orgId, skills, revisions, bindings, agents, actions }: SkillsPanelProps) { +export function SkillsPanel({ orgId, skills, revisions, bindings, tests, agents, actions }: SkillsPanelProps) { const [importOpen, setImportOpen] = useState(false); const importSkill = (event: FormEvent) => { @@ -37,6 +39,10 @@ export function SkillsPanel({ orgId, skills, revisions, bindings, agents, action void actions.run(() => window.agentCompany.bindSkill({ orgId, agentId, skillId: skill.id, revisionId: latest.id })); }; + const runTests = (revisionId: string) => { + void actions.run(() => window.agentCompany.runSkillTests({ orgId, revisionId })); + }; + return (
@@ -54,31 +60,36 @@ export function SkillsPanel({ orgId, skills, revisions, bindings, agents, action
-
Skill
Version
Bindings
Bind to agent
+
Skill
Version
Tests
Bindings
Actions
{skills.map((skill, index) => { const last = index === skills.length - 1; const latest = revisions.find((revision) => revision.skillId === skill.id); const skillBindings = bindings.filter((binding) => binding.skillId === skill.id && binding.enabled); + const revisionTests = latest ? tests.filter((test) => test.revisionId === latest.id) : []; return (
{skill.name}
{skill.slug}
{latest ? latest.versionSemver : "—"} + {testSummary(revisionTests)}
{skillBindings.map((binding) => ( {agentName(binding.agentId, agents)} ))}
- bind(skill, agentId)} disabled={!latest} /> +
+ + bind(skill, agentId)} disabled={!latest} /> +
); })} @@ -124,3 +135,23 @@ function BindAgent({ function agentName(agentId: string, agents: AgentRecord[]): string { return agents.find((agent) => agent.id === agentId)?.name ?? agentId; } + +function testSummary(tests: SkillTestRecord[]): string { + if (tests.length === 0) return "No tests"; + const completed = tests.filter((test) => test.lastResult); + if (completed.length === 0) return "Never run"; + const passed = completed.filter((test) => testStatus(test) === "passed").length; + const failed = completed.length - passed; + if (failed > 0) return `${failed} failed`; + return `${passed}/${tests.length} passed`; +} + +function testStatus(test: SkillTestRecord): "passed" | "failed" { + if (!test.lastResult) return "failed"; + try { + const result = JSON.parse(test.lastResult) as { status?: unknown }; + return result.status === "passed" ? "passed" : "failed"; + } catch { + return "failed"; + } +} diff --git a/AgentCompany-New/src/shared/commands.ts b/AgentCompany-New/src/shared/commands.ts index feda51fb..0e36b0ef 100644 --- a/AgentCompany-New/src/shared/commands.ts +++ b/AgentCompany-New/src/shared/commands.ts @@ -223,6 +223,11 @@ export interface ResolveAgentSkillsInput { runId: string; } +export interface RunSkillTestsInput { + orgId: string; + revisionId: string; +} + export interface CreateSocialAccountInput { orgId: string; platform: SocialPlatform; diff --git a/AgentCompany-New/src/shared/desktop-api.ts b/AgentCompany-New/src/shared/desktop-api.ts index ad35a77a..12946413 100644 --- a/AgentCompany-New/src/shared/desktop-api.ts +++ b/AgentCompany-New/src/shared/desktop-api.ts @@ -35,6 +35,7 @@ import type { RunChaosDrillInput, RunEvalSuiteInput, RunSdlcReviewInput, + RunSkillTestsInput, RunTaskInput, RevokeProviderCredentialInput, SaveProviderCredentialInput, @@ -90,6 +91,7 @@ export interface DesktopApi { importSkillDirectory(input: ImportSkillDirectoryInput): Promise>; bindSkill(input: BindSkillInput): Promise>; resolveAgentSkills(input: ResolveAgentSkillsInput): Promise>; + runSkillTests(input: RunSkillTestsInput): Promise>; createSocialAccount(input: CreateSocialAccountInput): Promise>; launchSocialLogin(input: LaunchSocialLoginInput): Promise>; createScheduledPost(input: CreateScheduledPostInput): Promise>; diff --git a/AgentCompany-New/src/shared/domain.ts b/AgentCompany-New/src/shared/domain.ts index c3b7a096..3da1652c 100644 --- a/AgentCompany-New/src/shared/domain.ts +++ b/AgentCompany-New/src/shared/domain.ts @@ -398,6 +398,15 @@ export interface SkillRuntimeSnapshotRecord { createdAt: string; } +export interface SkillTestRecord { + id: string; + revisionId: string; + testType: string; + expectedOutputRef: string; + lastResult: string | null; + lastRunAt: string | null; +} + export interface AuditEventRecord { id: string; orgId: string; @@ -464,6 +473,7 @@ export interface AppSnapshot { skillRevisions: SkillRevisionRecord[]; skillBindings: SkillBindingRecord[]; skillRuntimeSnapshots: SkillRuntimeSnapshotRecord[]; + skillTests: SkillTestRecord[]; socialAccounts: SocialAccountRecord[]; socialAccountGrants: SocialAccountGrantRecord[]; browserSessionContainers: BrowserSessionContainerRecord[]; diff --git a/AgentCompany-New/src/shared/ipc.ts b/AgentCompany-New/src/shared/ipc.ts index 0df8e9ad..6faabe5b 100644 --- a/AgentCompany-New/src/shared/ipc.ts +++ b/AgentCompany-New/src/shared/ipc.ts @@ -29,6 +29,7 @@ export const IPC_CHANNELS = { importSkillDirectory: "agentcompany:import-skill-directory", bindSkill: "agentcompany:bind-skill", resolveAgentSkills: "agentcompany:resolve-agent-skills", + runSkillTests: "agentcompany:run-skill-tests", createSocialAccount: "agentcompany:create-social-account", launchSocialLogin: "agentcompany:launch-social-login", createScheduledPost: "agentcompany:create-scheduled-post", diff --git a/AgentCompany-New/tests/e2e/desktop.spec.ts b/AgentCompany-New/tests/e2e/desktop.spec.ts index e1597980..e63c812e 100644 --- a/AgentCompany-New/tests/e2e/desktop.spec.ts +++ b/AgentCompany-New/tests/e2e/desktop.spec.ts @@ -1,5 +1,5 @@ import { _electron as electron, expect, test, type Page } from "@playwright/test"; -import { mkdirSync } from "node:fs"; +import { mkdirSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { randomUUID } from "node:crypto"; @@ -133,6 +133,14 @@ test("onboards, hires, and creates work in the Electron shell", async () => { await page.getByLabel("OpenAI monthly request limit").fill("25"); await page.getByRole("button", { name: "Save OpenAI policy" }).click(); await expect(page.getByText("0 / 25").first()).toBeVisible(); + await page.getByRole("button", { name: "Skills" }).click(); + await page.getByRole("button", { name: "Import" }).click(); + await page.getByLabel("Directory path").fill(createSkillDirectory()); + await page.locator(".modal").getByRole("button", { name: "Import" }).click(); + await expect(page.getByText("Release Gate").first()).toBeVisible(); + await expect(page.getByText("Never run").first()).toBeVisible(); + await page.getByRole("button", { name: "Run tests" }).click(); + await expect(page.getByText("1/1 passed").first()).toBeVisible(); await page.getByRole("button", { name: "Compliance" }).click(); await page.getByRole("button", { name: "Create" }).click(); await expect(page.getByText("gdpr export · employee").first()).toBeVisible(); @@ -167,6 +175,26 @@ function createLegacyDatabase(): string { return legacyPath; } +function createSkillDirectory(): string { + const root = join(tmpdir(), `agentcompany-skill-${randomUUID()}`); + mkdirSync(join(root, "checks"), { recursive: true }); + writeFileSync(join(root, "SKILL.md"), "Review release evidence and produce a concise gate decision.\n", "utf8"); + writeFileSync(join(root, "checks", "acceptance.txt"), "release evidence accepted\n", "utf8"); + writeFileSync(join(root, "skill.manifest.json"), JSON.stringify({ + name: "Release Gate", + version: "1.0.0", + entrypoint: "SKILL.md", + capabilities: ["release-review"], + required_tools: ["git"], + compatible_models: ["gpt-5.4"], + risk_level: "low", + tests: ["checks/acceptance.txt"], + inputs: {}, + outputs: {}, + }, null, 2), "utf8"); + return root; +} + async function verifyPagesDoNotOverflow(page: Page): Promise { for (const size of [{ width: 1280, height: 860 }, { width: 980, height: 700 }]) { await page.setViewportSize(size); diff --git a/AgentCompany-New/tests/unit/domain-services.test.ts b/AgentCompany-New/tests/unit/domain-services.test.ts index 12c92893..450a6a64 100644 --- a/AgentCompany-New/tests/unit/domain-services.test.ts +++ b/AgentCompany-New/tests/unit/domain-services.test.ts @@ -145,6 +145,7 @@ describe("domain services", () => { .get(orgId, "review-routes") as { id: string }; skills.bind({ orgId, agentId, skillId: skill.id, revisionId }); rmSync(skillDir, { recursive: true, force: true }); + skills.runTests({ orgId, revisionId }); skills.resolveRuntimeSnapshot({ orgId, agentId, runId: "run-review-1" }); }); @@ -153,8 +154,11 @@ describe("domain services", () => { expect(state.skillRevisions[0]?.versionSemver).toBe("1.0.0"); expect(state.skillRevisions[0]?.immutableChecksum).toHaveLength(64); expect(state.skillRevisions[0]?.manifestJson).toContain("architecture-review"); + expect(state.skillTests[0]?.lastResult).toContain("\"status\":\"passed\""); + expect(state.skillTests[0]?.lastRunAt).toBeTruthy(); expect(state.skillRuntimeSnapshots[0]?.sourceRevisionIds).toContain(revisionId); expect(state.skillRuntimeSnapshots[0]?.compiledPrompt).toContain("# Review Routes"); + expect(state.auditEvents.some((event) => event.entityType === "skill_revision" && event.action === "tests_run")).toBe(true); connection.close(); });