Run skill revision tests

AgentCompany-New/src/main/app/ipc-handlers.ts

@@ -171,6 +171,7 @@ export function registerIpcHandlers(deps: IpcDeps): void {
   ipcMain.handle(IPC_CHANNELS.importSkillDirectory, (_event, input) => mutate(() => skills.importDirectory(input)));
   ipcMain.handle(IPC_CHANNELS.bindSkill, (_event, input) => mutate(() => skills.bind(input)));
   ipcMain.handle(IPC_CHANNELS.resolveAgentSkills, (_event, input) => mutate(() => skills.resolveRuntimeSnapshot(input)));
+  ipcMain.handle(IPC_CHANNELS.runSkillTests, (_event, input) => mutate(() => skills.runTests(input)));
   ipcMain.handle(IPC_CHANNELS.createSocialAccount, (_event, input) => mutate(() => social.createAccount(input)));
   ipcMain.handle(IPC_CHANNELS.createScheduledPost, (_event, input) => mutate(() => social.createScheduledPost(input)));
   ipcMain.handle(IPC_CHANNELS.setScheduledPostState, (_event, input) => mutate(() => social.setScheduledPostState(input)));

AgentCompany-New/src/main/services/skill-service.ts

@@ -1,6 +1,8 @@
 import { existsSync, readFileSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { normalize } from "node:path";
 import type { DatabaseSync } from "node:sqlite";
-import type { BindSkillInput, ImportSkillDirectoryInput, ResolveAgentSkillsInput } from "../../shared/commands";
+import type { BindSkillInput, ImportSkillDirectoryInput, ResolveAgentSkillsInput, RunSkillTestsInput } from "../../shared/commands";
 import { SLO_SIGNAL_KEY } from "../../shared/domain";
 import { createId } from "../../shared/ids";
 import { AuditService } from "./audit-service";
@@ -181,6 +183,53 @@ export class SkillService {
     return snapshotId;
   }
 
+  runTests(input: RunSkillTestsInput): string {
+    const revision = this.db.prepare(`
+      select sr.id, sr.version_semver, sd.slug
+      from skill_revisions sr
+      join skill_definitions sd on sd.id = sr.skill_id
+      where sr.id = ? and sd.org_id = ?
+    `).get(input.revisionId, input.orgId) as SkillRevisionTestRow | undefined;
+    if (!revision) throw new Error("Skill revision was not found.");
+
+    const tests = this.db.prepare(`
+      select id, test_type, expected_output_ref
+      from skill_tests
+      where revision_id = ?
+      order by expected_output_ref asc
+    `).all(input.revisionId) as unknown as SkillTestRow[];
+    const artifacts = this.db.prepare(`
+      select artifact_type, object_key, sha256, size_bytes
+      from skill_artifacts
+      where revision_id = ?
+      order by artifact_type asc, object_key asc
+    `).all(input.revisionId) as unknown as SkillArtifactRow[];
+    const structureFailure = verifyRevisionStructure(artifacts);
+    if (structureFailure && tests.length === 0) throw new Error(structureFailure.failure ?? "Skill revision structure check failed.");
+    const now = new Date().toISOString();
+    let passed = 0;
+    let failed = 0;
+
+    for (const test of tests) {
+      const result = structureFailure ?? runManifestArtifactTest(test, artifacts);
+      if (result.status === "passed") passed += 1;
+      if (result.status === "failed") failed += 1;
+      this.db.prepare("update skill_tests set last_result = ?, last_run_at = ? where id = ?")
+        .run(JSON.stringify(result), now, test.id);
+    }
+
+    this.audit.record({
+      orgId: input.orgId,
+      entityType: "skill_revision",
+      entityId: revision.id,
+      actorType: "system",
+      actorId: null,
+      action: "tests_run",
+      payload: { slug: revision.slug, version: revision.version_semver, total: tests.length, passed, failed },
+    });
+    return revision.id;
+  }
+
   private verifyBindingTargets(input: BindSkillInput): void {
     const agent = this.db.prepare("select id from agents where id = ? and org_id = ?")
       .get(input.agentId, input.orgId) as { id: string } | undefined;
@@ -207,3 +256,103 @@ interface RuntimeSkillRow {
   manifest_json: string;
   object_key: string;
 }
+
+interface SkillRevisionTestRow {
+  id: string;
+  version_semver: string;
+  slug: string;
+}
+
+interface SkillTestRow {
+  id: string;
+  test_type: string;
+  expected_output_ref: string;
+}
+
+interface SkillArtifactRow {
+  artifact_type: string;
+  object_key: string;
+  sha256: string;
+  size_bytes: number;
+}
+
+interface SkillTestResult {
+  status: "passed" | "failed";
+  testType: string;
+  expectedOutputRef: string;
+  checks: Array<{ name: string; status: "passed" | "failed"; detail: string }>;
+  artifactSha256?: string;
+  failure?: string;
+}
+
+function verifyRevisionStructure(artifacts: SkillArtifactRow[]): SkillTestResult | null {
+  const requiredTypes = ["manifest", "entrypoint"];
+  for (const type of requiredTypes) {
+    if (!artifacts.some((artifact) => artifact.artifact_type === type)) {
+      return failedSkillTest("revision_structure", type, `Skill revision artifact is missing: ${type}`);
+    }
+  }
+  for (const artifact of artifacts) {
+    try {
+      verifyArtifactIntegrity(artifact);
+    } catch (error) {
+      return failedSkillTest("artifact_integrity", artifact.object_key, error instanceof Error ? error.message : String(error));
+    }
+  }
+  return null;
+}
+
+function runManifestArtifactTest(test: SkillTestRow, artifacts: SkillArtifactRow[]): SkillTestResult {
+  if (test.test_type !== "manifest") {
+    return failedSkillTest(test.test_type, test.expected_output_ref, `Unsupported skill test type: ${test.test_type}`);
+  }
+  const testArtifact = artifacts.find((artifact) => (
+    artifact.artifact_type === "test" && artifactMatchesExpectedRef(artifact.object_key, test.expected_output_ref)
+  ));
+  if (!testArtifact) return failedSkillTest(test.test_type, test.expected_output_ref, "Skill test artifact was not found.");
+
+  try {
+    const content = verifyArtifactIntegrity(testArtifact).toString("utf8");
+    if (!content.trim()) throw new Error("Skill test artifact is empty.");
+    return {
+      status: "passed",
+      testType: test.test_type,
+      expectedOutputRef: test.expected_output_ref,
+      artifactSha256: testArtifact.sha256,
+      checks: [
+        { name: "manifest_test_artifact", status: "passed", detail: "Artifact exists, checksum matches, and content is non-empty." },
+      ],
+    };
+  } catch (error) {
+    return failedSkillTest(test.test_type, test.expected_output_ref, error instanceof Error ? error.message : String(error));
+  }
+}
+
+function verifyArtifactIntegrity(artifact: SkillArtifactRow): Buffer {
+  if (!existsSync(artifact.object_key)) throw new Error(`Skill artifact is missing: ${artifact.object_key}`);
+  const body = readFileSync(artifact.object_key);
+  const sha256 = createHash("sha256").update(body).digest("hex");
+  if (sha256 !== artifact.sha256) throw new Error(`Skill artifact checksum mismatch: ${artifact.object_key}`);
+  if (body.byteLength !== artifact.size_bytes) throw new Error(`Skill artifact size mismatch: ${artifact.object_key}`);
+  return body;
+}
+
+function artifactMatchesExpectedRef(objectKey: string, expectedRef: string): boolean {
+  const expected = normalizeSkillPath(expectedRef);
+  const actual = normalizeSkillPath(objectKey);
+  return actual === expected || actual.endsWith(`/${expected}`);
+}
+
+function normalizeSkillPath(value: string): string {
+  return normalize(value).replaceAll("\\", "/");
+}
+
+function failedSkillTest(testType: string, expectedOutputRef: string, failure: string): SkillTestResult {
+  return {
+    status: "failed",
+    testType,
+    expectedOutputRef,
+    checks: [{ name: "skill_test", status: "failed", detail: failure }],
+    failure,
+  };
+}

AgentCompany-New/src/main/services/snapshot-service.ts

@@ -115,6 +115,7 @@ export class SnapshotService {
       skillRevisions: list(this.db, "skill_revisions", "published_at desc"),
       skillBindings,
       skillRuntimeSnapshots: list(this.db, "skill_runtime_snapshots", "created_at desc"),
+      skillTests: list(this.db, "skill_tests", "last_run_at desc, id asc"),
       socialAccounts: list(this.db, "social_accounts", "created_at asc"),
       socialAccountGrants: list(this.db, "social_account_grants", "created_at asc"),
       browserSessionContainers: list<AppSnapshot["browserSessionContainers"][number]>(this.db, "browser_session_containers", "created_at asc")

AgentCompany-New/src/preload/index.ts

@@ -37,6 +37,7 @@ const api: DesktopApi = {
   importSkillDirectory: (input) => invoke(IPC_CHANNELS.importSkillDirectory, input),
   bindSkill: (input) => invoke(IPC_CHANNELS.bindSkill, input),
   resolveAgentSkills: (input) => invoke(IPC_CHANNELS.resolveAgentSkills, input),
+  runSkillTests: (input) => invoke(IPC_CHANNELS.runSkillTests, input),
   createSocialAccount: (input) => invoke(IPC_CHANNELS.createSocialAccount, input),
   launchSocialLogin: (input) => invoke(IPC_CHANNELS.launchSocialLogin, input),
   createScheduledPost: (input) => invoke(IPC_CHANNELS.createScheduledPost, input),

AgentCompany-New/src/renderer/src/pages/SettingsPage.tsx

@@ -71,6 +71,10 @@ export function SettingsPage({ snapshot, actions }: PageProps) {
     const skillIds = new Set(skills.map((skill) => skill.id));
     return snapshot.skillRevisions.filter((revision) => skillIds.has(revision.skillId));
   }, [snapshot.skillRevisions, skills]);
+  const skillTests = useMemo(() => {
+    const revisionIds = new Set(skillRevisions.map((revision) => revision.id));
+    return snapshot.skillTests.filter((test) => revisionIds.has(test.revisionId));
+  }, [snapshot.skillTests, skillRevisions]);
   const skillBindings = useMemo(() => snapshot.skillBindings.filter((item) => item.orgId === orgId), [snapshot.skillBindings, orgId]);
   const policies = useMemo(() => snapshot.policyRules.filter((item) => item.orgId === orgId), [snapshot.policyRules, orgId]);
   const workspaces = useMemo(() => snapshot.workspaces.filter((item) => item.orgId === orgId), [snapshot.workspaces, orgId]);
@@ -147,6 +151,7 @@ export function SettingsPage({ snapshot, actions }: PageProps) {
               skills={skills}
               revisions={skillRevisions}
               bindings={skillBindings}
+              tests={skillTests}
               agents={agents}
               actions={actions}
             />

AgentCompany-New/src/renderer/src/pages/settings/SkillsPanel.tsx

@@ -6,6 +6,7 @@ import type {
   SkillBindingRecord,
   SkillDefinitionRecord,
   SkillRevisionRecord,
+  SkillTestRecord,
 } from "../../../../shared/domain";
 import type { SnapshotActions } from "../../app/useSnapshot";
 
@@ -14,11 +15,12 @@ interface SkillsPanelProps {
   skills: SkillDefinitionRecord[];
   revisions: SkillRevisionRecord[];
   bindings: SkillBindingRecord[];
+  tests: SkillTestRecord[];
   agents: AgentRecord[];
   actions: SnapshotActions;
 }
 
-export function SkillsPanel({ orgId, skills, revisions, bindings, agents, actions }: SkillsPanelProps) {
+export function SkillsPanel({ orgId, skills, revisions, bindings, tests, agents, actions }: SkillsPanelProps) {
   const [importOpen, setImportOpen] = useState(false);
 
   const importSkill = (event: FormEvent<HTMLFormElement>) => {
@@ -37,6 +39,10 @@ export function SkillsPanel({ orgId, skills, revisions, bindings, agents, action
     void actions.run(() => window.agentCompany.bindSkill({ orgId, agentId, skillId: skill.id, revisionId: latest.id }));
   };
 
+  const runTests = (revisionId: string) => {
+    void actions.run(() => window.agentCompany.runSkillTests({ orgId, revisionId }));
+  };
+
   return (
     <div className="page">
       <div className="row" style={{ marginBottom: 24 }}>
@@ -54,31 +60,36 @@ export function SkillsPanel({ orgId, skills, revisions, bindings, agents, action
       <div className="card" style={{ padding: 0, overflow: "hidden" }}>
         <div
           className="table-row head"
-          style={{ gridTemplateColumns: "minmax(0, 1.6fr) minmax(0, 1fr) minmax(0, 1.4fr) minmax(0, 1.6fr)" }}
+          style={{ gridTemplateColumns: "minmax(0, 1.4fr) 92px 132px minmax(0, 1.2fr) minmax(220px, 1.4fr)" }}
         >
-          <div>Skill</div><div>Version</div><div>Bindings</div><div>Bind to agent</div>
+          <div>Skill</div><div>Version</div><div>Tests</div><div>Bindings</div><div>Actions</div>
         </div>
         {skills.map((skill, index) => {
           const last = index === skills.length - 1;
           const latest = revisions.find((revision) => revision.skillId === skill.id);
           const skillBindings = bindings.filter((binding) => binding.skillId === skill.id && binding.enabled);
+          const revisionTests = latest ? tests.filter((test) => test.revisionId === latest.id) : [];
           return (
             <div
               key={skill.id}
               className="table-row"
-              style={{ gridTemplateColumns: "minmax(0, 1.6fr) minmax(0, 1fr) minmax(0, 1.4fr) minmax(0, 1.6fr)", borderBottom: last ? "none" : "1px solid var(--border-soft)" }}
+              style={{ gridTemplateColumns: "minmax(0, 1.4fr) 92px 132px minmax(0, 1.2fr) minmax(220px, 1.4fr)", borderBottom: last ? "none" : "1px solid var(--border-soft)" }}
             >
               <div style={{ minWidth: 0 }}>
                 <div style={{ fontWeight: 540, overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap" }}>{skill.name}</div>
                 <div style={{ color: "var(--text-3)", fontSize: 11 }}>{skill.slug}</div>
               </div>
               <Pill>{latest ? latest.versionSemver : "—"}</Pill>
+              <Pill>{testSummary(revisionTests)}</Pill>
               <div className="row" style={{ gap: 6, flexWrap: "wrap" }}>
                 {skillBindings.map((binding) => (
                   <Pill key={binding.id}>{agentName(binding.agentId, agents)}</Pill>
                 ))}
               </div>
-              <BindAgent skill={skill} agents={agents} onBind={(agentId) => bind(skill, agentId)} disabled={!latest} />
+              <div className="row" style={{ gap: 8, minWidth: 0 }}>
+                <Button size="small" onClick={() => { if (latest && revisionTests.length > 0) runTests(latest.id); }} disabled={!latest || revisionTests.length === 0}>Run tests</Button>
+                <BindAgent skill={skill} agents={agents} onBind={(agentId) => bind(skill, agentId)} disabled={!latest} />
+              </div>
             </div>
           );
         })}
@@ -124,3 +135,23 @@ function BindAgent({
 function agentName(agentId: string, agents: AgentRecord[]): string {
   return agents.find((agent) => agent.id === agentId)?.name ?? agentId;
 }
+
+function testSummary(tests: SkillTestRecord[]): string {
+  if (tests.length === 0) return "No tests";
+  const completed = tests.filter((test) => test.lastResult);
+  if (completed.length === 0) return "Never run";
+  const passed = completed.filter((test) => testStatus(test) === "passed").length;
+  const failed = completed.length - passed;
+  if (failed > 0) return `${failed} failed`;
+  return `${passed}/${tests.length} passed`;
+}
+
+function testStatus(test: SkillTestRecord): "passed" | "failed" {
+  if (!test.lastResult) return "failed";
+  try {
+    const result = JSON.parse(test.lastResult) as { status?: unknown };
+    return result.status === "passed" ? "passed" : "failed";
+  } catch {
+    return "failed";
+  }
+}

AgentCompany-New/src/shared/commands.ts

@@ -223,6 +223,11 @@ export interface ResolveAgentSkillsInput {
   runId: string;
 }
 
+export interface RunSkillTestsInput {
+  orgId: string;
+  revisionId: string;
+}
+
 export interface CreateSocialAccountInput {
   orgId: string;
   platform: SocialPlatform;

AgentCompany-New/src/shared/desktop-api.ts

@@ -35,6 +35,7 @@ import type {
   RunChaosDrillInput,
   RunEvalSuiteInput,
   RunSdlcReviewInput,
+  RunSkillTestsInput,
   RunTaskInput,
   RevokeProviderCredentialInput,
   SaveProviderCredentialInput,
@@ -90,6 +91,7 @@ export interface DesktopApi {
   importSkillDirectory(input: ImportSkillDirectoryInput): Promise<DesktopResult<AppSnapshot>>;
   bindSkill(input: BindSkillInput): Promise<DesktopResult<AppSnapshot>>;
   resolveAgentSkills(input: ResolveAgentSkillsInput): Promise<DesktopResult<AppSnapshot>>;
+  runSkillTests(input: RunSkillTestsInput): Promise<DesktopResult<AppSnapshot>>;
   createSocialAccount(input: CreateSocialAccountInput): Promise<DesktopResult<AppSnapshot>>;
   launchSocialLogin(input: LaunchSocialLoginInput): Promise<DesktopResult<null>>;
   createScheduledPost(input: CreateScheduledPostInput): Promise<DesktopResult<AppSnapshot>>;

AgentCompany-New/src/shared/domain.ts

@@ -398,6 +398,15 @@ export interface SkillRuntimeSnapshotRecord {
   createdAt: string;
 }
 
+export interface SkillTestRecord {
+  id: string;
+  revisionId: string;
+  testType: string;
+  expectedOutputRef: string;
+  lastResult: string | null;
+  lastRunAt: string | null;
+}
+
 export interface AuditEventRecord {
   id: string;
   orgId: string;
@@ -464,6 +473,7 @@ export interface AppSnapshot {
   skillRevisions: SkillRevisionRecord[];
   skillBindings: SkillBindingRecord[];
   skillRuntimeSnapshots: SkillRuntimeSnapshotRecord[];
+  skillTests: SkillTestRecord[];
   socialAccounts: SocialAccountRecord[];
   socialAccountGrants: SocialAccountGrantRecord[];
   browserSessionContainers: BrowserSessionContainerRecord[];

AgentCompany-New/src/shared/ipc.ts

@@ -29,6 +29,7 @@ export const IPC_CHANNELS = {
   importSkillDirectory: "agentcompany:import-skill-directory",
   bindSkill: "agentcompany:bind-skill",
   resolveAgentSkills: "agentcompany:resolve-agent-skills",
+  runSkillTests: "agentcompany:run-skill-tests",
   createSocialAccount: "agentcompany:create-social-account",
   launchSocialLogin: "agentcompany:launch-social-login",
   createScheduledPost: "agentcompany:create-scheduled-post",

AgentCompany-New/tests/e2e/desktop.spec.ts

@@ -1,5 +1,5 @@
 import { _electron as electron, expect, test, type Page } from "@playwright/test";
-import { mkdirSync } from "node:fs";
+import { mkdirSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
 import { randomUUID } from "node:crypto";
@@ -133,6 +133,14 @@ test("onboards, hires, and creates work in the Electron shell", async () => {
     await page.getByLabel("OpenAI monthly request limit").fill("25");
     await page.getByRole("button", { name: "Save OpenAI policy" }).click();
     await expect(page.getByText("0 / 25").first()).toBeVisible();
+    await page.getByRole("button", { name: "Skills" }).click();
+    await page.getByRole("button", { name: "Import" }).click();
+    await page.getByLabel("Directory path").fill(createSkillDirectory());
+    await page.locator(".modal").getByRole("button", { name: "Import" }).click();
+    await expect(page.getByText("Release Gate").first()).toBeVisible();
+    await expect(page.getByText("Never run").first()).toBeVisible();
+    await page.getByRole("button", { name: "Run tests" }).click();
+    await expect(page.getByText("1/1 passed").first()).toBeVisible();
     await page.getByRole("button", { name: "Compliance" }).click();
     await page.getByRole("button", { name: "Create" }).click();
     await expect(page.getByText("gdpr export · employee").first()).toBeVisible();
@@ -167,6 +175,26 @@ function createLegacyDatabase(): string {
   return legacyPath;
 }
 
+function createSkillDirectory(): string {
+  const root = join(tmpdir(), `agentcompany-skill-${randomUUID()}`);
+  mkdirSync(join(root, "checks"), { recursive: true });
+  writeFileSync(join(root, "SKILL.md"), "Review release evidence and produce a concise gate decision.\n", "utf8");
+  writeFileSync(join(root, "checks", "acceptance.txt"), "release evidence accepted\n", "utf8");
+  writeFileSync(join(root, "skill.manifest.json"), JSON.stringify({
+    name: "Release Gate",
+    version: "1.0.0",
+    entrypoint: "SKILL.md",
+    capabilities: ["release-review"],
+    required_tools: ["git"],
+    compatible_models: ["gpt-5.4"],
+    risk_level: "low",
+    tests: ["checks/acceptance.txt"],
+    inputs: {},
+    outputs: {},
+  }, null, 2), "utf8");
+  return root;
+}
+
 async function verifyPagesDoNotOverflow(page: Page): Promise<void> {
   for (const size of [{ width: 1280, height: 860 }, { width: 980, height: 700 }]) {
     await page.setViewportSize(size);

AgentCompany-New/tests/unit/domain-services.test.ts

@@ -145,6 +145,7 @@ describe("domain services", () => {
         .get(orgId, "review-routes") as { id: string };
       skills.bind({ orgId, agentId, skillId: skill.id, revisionId });
       rmSync(skillDir, { recursive: true, force: true });
+      skills.runTests({ orgId, revisionId });
       skills.resolveRuntimeSnapshot({ orgId, agentId, runId: "run-review-1" });
     });
 
@@ -153,8 +154,11 @@ describe("domain services", () => {
     expect(state.skillRevisions[0]?.versionSemver).toBe("1.0.0");
     expect(state.skillRevisions[0]?.immutableChecksum).toHaveLength(64);
     expect(state.skillRevisions[0]?.manifestJson).toContain("architecture-review");
+    expect(state.skillTests[0]?.lastResult).toContain("\"status\":\"passed\"");
+    expect(state.skillTests[0]?.lastRunAt).toBeTruthy();
     expect(state.skillRuntimeSnapshots[0]?.sourceRevisionIds).toContain(revisionId);
     expect(state.skillRuntimeSnapshots[0]?.compiledPrompt).toContain("# Review Routes");
+    expect(state.auditEvents.some((event) => event.entityType === "skill_revision" && event.action === "tests_run")).toBe(true);
     connection.close();
   });