Run skill revision tests
This commit is contained in:
1 parent
16855b696a
commit
216d485c58
12 files changed
+245
-7
No files matched your search
@@ -1,5 +1,5 @@
|
||||
import { _electron as electron, expect, test, type Page } from "@playwright/test";
|
||||
import { mkdirSync } from "node:fs";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { randomUUID } from "node:crypto";
|
||||
@@ -133,6 +133,14 @@ test("onboards, hires, and creates work in the Electron shell", async () => {
|
||||
await page.getByLabel("OpenAI monthly request limit").fill("25");
|
||||
await page.getByRole("button", { name: "Save OpenAI policy" }).click();
|
||||
await expect(page.getByText("0 / 25").first()).toBeVisible();
|
||||
await page.getByRole("button", { name: "Skills" }).click();
|
||||
await page.getByRole("button", { name: "Import" }).click();
|
||||
await page.getByLabel("Directory path").fill(createSkillDirectory());
|
||||
await page.locator(".modal").getByRole("button", { name: "Import" }).click();
|
||||
await expect(page.getByText("Release Gate").first()).toBeVisible();
|
||||
await expect(page.getByText("Never run").first()).toBeVisible();
|
||||
await page.getByRole("button", { name: "Run tests" }).click();
|
||||
await expect(page.getByText("1/1 passed").first()).toBeVisible();
|
||||
await page.getByRole("button", { name: "Compliance" }).click();
|
||||
await page.getByRole("button", { name: "Create" }).click();
|
||||
await expect(page.getByText("gdpr export · employee").first()).toBeVisible();
|
||||
@@ -167,6 +175,26 @@ function createLegacyDatabase(): string {
|
||||
return legacyPath;
|
||||
}
|
||||
|
||||
function createSkillDirectory(): string {
|
||||
const root = join(tmpdir(), `agentcompany-skill-${randomUUID()}`);
|
||||
mkdirSync(join(root, "checks"), { recursive: true });
|
||||
writeFileSync(join(root, "SKILL.md"), "Review release evidence and produce a concise gate decision.\n", "utf8");
|
||||
writeFileSync(join(root, "checks", "acceptance.txt"), "release evidence accepted\n", "utf8");
|
||||
writeFileSync(join(root, "skill.manifest.json"), JSON.stringify({
|
||||
name: "Release Gate",
|
||||
version: "1.0.0",
|
||||
entrypoint: "SKILL.md",
|
||||
capabilities: ["release-review"],
|
||||
required_tools: ["git"],
|
||||
compatible_models: ["gpt-5.4"],
|
||||
risk_level: "low",
|
||||
tests: ["checks/acceptance.txt"],
|
||||
inputs: {},
|
||||
outputs: {},
|
||||
}, null, 2), "utf8");
|
||||
return root;
|
||||
}
|
||||
|
||||
async function verifyPagesDoNotOverflow(page: Page): Promise<void> {
|
||||
for (const size of [{ width: 1280, height: 860 }, { width: 980, height: 700 }]) {
|
||||
await page.setViewportSize(size);
|
||||
|
||||
@@ -145,6 +145,7 @@ describe("domain services", () => {
|
||||
.get(orgId, "review-routes") as { id: string };
|
||||
skills.bind({ orgId, agentId, skillId: skill.id, revisionId });
|
||||
rmSync(skillDir, { recursive: true, force: true });
|
||||
skills.runTests({ orgId, revisionId });
|
||||
skills.resolveRuntimeSnapshot({ orgId, agentId, runId: "run-review-1" });
|
||||
});
|
||||
|
||||
@@ -153,8 +154,11 @@ describe("domain services", () => {
|
||||
expect(state.skillRevisions[0]?.versionSemver).toBe("1.0.0");
|
||||
expect(state.skillRevisions[0]?.immutableChecksum).toHaveLength(64);
|
||||
expect(state.skillRevisions[0]?.manifestJson).toContain("architecture-review");
|
||||
expect(state.skillTests[0]?.lastResult).toContain("\"status\":\"passed\"");
|
||||
expect(state.skillTests[0]?.lastRunAt).toBeTruthy();
|
||||
expect(state.skillRuntimeSnapshots[0]?.sourceRevisionIds).toContain(revisionId);
|
||||
expect(state.skillRuntimeSnapshots[0]?.compiledPrompt).toContain("# Review Routes");
|
||||
expect(state.auditEvents.some((event) => event.entityType === "skill_revision" && event.action === "tests_run")).toBe(true);
|
||||
connection.close();
|
||||
});
|
||||
|
||||
|
||||
Reference in new issue
Block a user