Run skill revision tests

This commit is contained in:
ZacharyZhang-NY committed 2026-05-23 22:56:06 -04:00
1 parent 16855b696a
commit 216d485c58
12 files changed
+245 -7

No files matched your search

+29 -1
View File
@@ -1,5 +1,5 @@
import { _electron as electron, expect, test, type Page } from "@playwright/test";
import { mkdirSync } from "node:fs";
import { mkdirSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { randomUUID } from "node:crypto";
@@ -133,6 +133,14 @@ test("onboards, hires, and creates work in the Electron shell", async () => {
await page.getByLabel("OpenAI monthly request limit").fill("25");
await page.getByRole("button", { name: "Save OpenAI policy" }).click();
await expect(page.getByText("0 / 25").first()).toBeVisible();
await page.getByRole("button", { name: "Skills" }).click();
await page.getByRole("button", { name: "Import" }).click();
await page.getByLabel("Directory path").fill(createSkillDirectory());
await page.locator(".modal").getByRole("button", { name: "Import" }).click();
await expect(page.getByText("Release Gate").first()).toBeVisible();
await expect(page.getByText("Never run").first()).toBeVisible();
await page.getByRole("button", { name: "Run tests" }).click();
await expect(page.getByText("1/1 passed").first()).toBeVisible();
await page.getByRole("button", { name: "Compliance" }).click();
await page.getByRole("button", { name: "Create" }).click();
await expect(page.getByText("gdpr export · employee").first()).toBeVisible();
@@ -167,6 +175,26 @@ function createLegacyDatabase(): string {
return legacyPath;
}
function createSkillDirectory(): string {
const root = join(tmpdir(), `agentcompany-skill-${randomUUID()}`);
mkdirSync(join(root, "checks"), { recursive: true });
writeFileSync(join(root, "SKILL.md"), "Review release evidence and produce a concise gate decision.\n", "utf8");
writeFileSync(join(root, "checks", "acceptance.txt"), "release evidence accepted\n", "utf8");
writeFileSync(join(root, "skill.manifest.json"), JSON.stringify({
name: "Release Gate",
version: "1.0.0",
entrypoint: "SKILL.md",
capabilities: ["release-review"],
required_tools: ["git"],
compatible_models: ["gpt-5.4"],
risk_level: "low",
tests: ["checks/acceptance.txt"],
inputs: {},
outputs: {},
}, null, 2), "utf8");
return root;
}
async function verifyPagesDoNotOverflow(page: Page): Promise<void> {
for (const size of [{ width: 1280, height: 860 }, { width: 980, height: 700 }]) {
await page.setViewportSize(size);
@@ -145,6 +145,7 @@ describe("domain services", () => {
.get(orgId, "review-routes") as { id: string };
skills.bind({ orgId, agentId, skillId: skill.id, revisionId });
rmSync(skillDir, { recursive: true, force: true });
skills.runTests({ orgId, revisionId });
skills.resolveRuntimeSnapshot({ orgId, agentId, runId: "run-review-1" });
});
@@ -153,8 +154,11 @@ describe("domain services", () => {
expect(state.skillRevisions[0]?.versionSemver).toBe("1.0.0");
expect(state.skillRevisions[0]?.immutableChecksum).toHaveLength(64);
expect(state.skillRevisions[0]?.manifestJson).toContain("architecture-review");
expect(state.skillTests[0]?.lastResult).toContain("\"status\":\"passed\"");
expect(state.skillTests[0]?.lastRunAt).toBeTruthy();
expect(state.skillRuntimeSnapshots[0]?.sourceRevisionIds).toContain(revisionId);
expect(state.skillRuntimeSnapshots[0]?.compiledPrompt).toContain("# Review Routes");
expect(state.auditEvents.some((event) => event.entityType === "skill_revision" && event.action === "tests_run")).toBe(true);
connection.close();
});