Run skill revision tests
This commit is contained in:
1 parent
16855b696a
commit
216d485c58
12 files changed
+245
-7
No files matched your search
@@ -171,6 +171,7 @@ export function registerIpcHandlers(deps: IpcDeps): void {
|
|||||||
ipcMain.handle(IPC_CHANNELS.importSkillDirectory, (_event, input) => mutate(() => skills.importDirectory(input)));
|
ipcMain.handle(IPC_CHANNELS.importSkillDirectory, (_event, input) => mutate(() => skills.importDirectory(input)));
|
||||||
ipcMain.handle(IPC_CHANNELS.bindSkill, (_event, input) => mutate(() => skills.bind(input)));
|
ipcMain.handle(IPC_CHANNELS.bindSkill, (_event, input) => mutate(() => skills.bind(input)));
|
||||||
ipcMain.handle(IPC_CHANNELS.resolveAgentSkills, (_event, input) => mutate(() => skills.resolveRuntimeSnapshot(input)));
|
ipcMain.handle(IPC_CHANNELS.resolveAgentSkills, (_event, input) => mutate(() => skills.resolveRuntimeSnapshot(input)));
|
||||||
|
ipcMain.handle(IPC_CHANNELS.runSkillTests, (_event, input) => mutate(() => skills.runTests(input)));
|
||||||
ipcMain.handle(IPC_CHANNELS.createSocialAccount, (_event, input) => mutate(() => social.createAccount(input)));
|
ipcMain.handle(IPC_CHANNELS.createSocialAccount, (_event, input) => mutate(() => social.createAccount(input)));
|
||||||
ipcMain.handle(IPC_CHANNELS.createScheduledPost, (_event, input) => mutate(() => social.createScheduledPost(input)));
|
ipcMain.handle(IPC_CHANNELS.createScheduledPost, (_event, input) => mutate(() => social.createScheduledPost(input)));
|
||||||
ipcMain.handle(IPC_CHANNELS.setScheduledPostState, (_event, input) => mutate(() => social.setScheduledPostState(input)));
|
ipcMain.handle(IPC_CHANNELS.setScheduledPostState, (_event, input) => mutate(() => social.setScheduledPostState(input)));
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
import { existsSync, readFileSync } from "node:fs";
|
import { existsSync, readFileSync } from "node:fs";
|
||||||
|
import { createHash } from "node:crypto";
|
||||||
|
import { normalize } from "node:path";
|
||||||
import type { DatabaseSync } from "node:sqlite";
|
import type { DatabaseSync } from "node:sqlite";
|
||||||
import type { BindSkillInput, ImportSkillDirectoryInput, ResolveAgentSkillsInput } from "../../shared/commands";
|
import type { BindSkillInput, ImportSkillDirectoryInput, ResolveAgentSkillsInput, RunSkillTestsInput } from "../../shared/commands";
|
||||||
import { SLO_SIGNAL_KEY } from "../../shared/domain";
|
import { SLO_SIGNAL_KEY } from "../../shared/domain";
|
||||||
import { createId } from "../../shared/ids";
|
import { createId } from "../../shared/ids";
|
||||||
import { AuditService } from "./audit-service";
|
import { AuditService } from "./audit-service";
|
||||||
@@ -181,6 +183,53 @@ export class SkillService {
|
|||||||
return snapshotId;
|
return snapshotId;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
runTests(input: RunSkillTestsInput): string {
|
||||||
|
const revision = this.db.prepare(`
|
||||||
|
select sr.id, sr.version_semver, sd.slug
|
||||||
|
from skill_revisions sr
|
||||||
|
join skill_definitions sd on sd.id = sr.skill_id
|
||||||
|
where sr.id = ? and sd.org_id = ?
|
||||||
|
`).get(input.revisionId, input.orgId) as SkillRevisionTestRow | undefined;
|
||||||
|
if (!revision) throw new Error("Skill revision was not found.");
|
||||||
|
|
||||||
|
const tests = this.db.prepare(`
|
||||||
|
select id, test_type, expected_output_ref
|
||||||
|
from skill_tests
|
||||||
|
where revision_id = ?
|
||||||
|
order by expected_output_ref asc
|
||||||
|
`).all(input.revisionId) as unknown as SkillTestRow[];
|
||||||
|
const artifacts = this.db.prepare(`
|
||||||
|
select artifact_type, object_key, sha256, size_bytes
|
||||||
|
from skill_artifacts
|
||||||
|
where revision_id = ?
|
||||||
|
order by artifact_type asc, object_key asc
|
||||||
|
`).all(input.revisionId) as unknown as SkillArtifactRow[];
|
||||||
|
const structureFailure = verifyRevisionStructure(artifacts);
|
||||||
|
if (structureFailure && tests.length === 0) throw new Error(structureFailure.failure ?? "Skill revision structure check failed.");
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
let passed = 0;
|
||||||
|
let failed = 0;
|
||||||
|
|
||||||
|
for (const test of tests) {
|
||||||
|
const result = structureFailure ?? runManifestArtifactTest(test, artifacts);
|
||||||
|
if (result.status === "passed") passed += 1;
|
||||||
|
if (result.status === "failed") failed += 1;
|
||||||
|
this.db.prepare("update skill_tests set last_result = ?, last_run_at = ? where id = ?")
|
||||||
|
.run(JSON.stringify(result), now, test.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.audit.record({
|
||||||
|
orgId: input.orgId,
|
||||||
|
entityType: "skill_revision",
|
||||||
|
entityId: revision.id,
|
||||||
|
actorType: "system",
|
||||||
|
actorId: null,
|
||||||
|
action: "tests_run",
|
||||||
|
payload: { slug: revision.slug, version: revision.version_semver, total: tests.length, passed, failed },
|
||||||
|
});
|
||||||
|
return revision.id;
|
||||||
|
}
|
||||||
|
|
||||||
private verifyBindingTargets(input: BindSkillInput): void {
|
private verifyBindingTargets(input: BindSkillInput): void {
|
||||||
const agent = this.db.prepare("select id from agents where id = ? and org_id = ?")
|
const agent = this.db.prepare("select id from agents where id = ? and org_id = ?")
|
||||||
.get(input.agentId, input.orgId) as { id: string } | undefined;
|
.get(input.agentId, input.orgId) as { id: string } | undefined;
|
||||||
@@ -207,3 +256,103 @@ interface RuntimeSkillRow {
|
|||||||
manifest_json: string;
|
manifest_json: string;
|
||||||
object_key: string;
|
object_key: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface SkillRevisionTestRow {
|
||||||
|
id: string;
|
||||||
|
version_semver: string;
|
||||||
|
slug: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SkillTestRow {
|
||||||
|
id: string;
|
||||||
|
test_type: string;
|
||||||
|
expected_output_ref: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SkillArtifactRow {
|
||||||
|
artifact_type: string;
|
||||||
|
object_key: string;
|
||||||
|
sha256: string;
|
||||||
|
size_bytes: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SkillTestResult {
|
||||||
|
status: "passed" | "failed";
|
||||||
|
testType: string;
|
||||||
|
expectedOutputRef: string;
|
||||||
|
checks: Array<{ name: string; status: "passed" | "failed"; detail: string }>;
|
||||||
|
artifactSha256?: string;
|
||||||
|
failure?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function verifyRevisionStructure(artifacts: SkillArtifactRow[]): SkillTestResult | null {
|
||||||
|
const requiredTypes = ["manifest", "entrypoint"];
|
||||||
|
for (const type of requiredTypes) {
|
||||||
|
if (!artifacts.some((artifact) => artifact.artifact_type === type)) {
|
||||||
|
return failedSkillTest("revision_structure", type, `Skill revision artifact is missing: ${type}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const artifact of artifacts) {
|
||||||
|
try {
|
||||||
|
verifyArtifactIntegrity(artifact);
|
||||||
|
} catch (error) {
|
||||||
|
return failedSkillTest("artifact_integrity", artifact.object_key, error instanceof Error ? error.message : String(error));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function runManifestArtifactTest(test: SkillTestRow, artifacts: SkillArtifactRow[]): SkillTestResult {
|
||||||
|
if (test.test_type !== "manifest") {
|
||||||
|
return failedSkillTest(test.test_type, test.expected_output_ref, `Unsupported skill test type: ${test.test_type}`);
|
||||||
|
}
|
||||||
|
const testArtifact = artifacts.find((artifact) => (
|
||||||
|
artifact.artifact_type === "test" && artifactMatchesExpectedRef(artifact.object_key, test.expected_output_ref)
|
||||||
|
));
|
||||||
|
if (!testArtifact) return failedSkillTest(test.test_type, test.expected_output_ref, "Skill test artifact was not found.");
|
||||||
|
|
||||||
|
try {
|
||||||
|
const content = verifyArtifactIntegrity(testArtifact).toString("utf8");
|
||||||
|
if (!content.trim()) throw new Error("Skill test artifact is empty.");
|
||||||
|
return {
|
||||||
|
status: "passed",
|
||||||
|
testType: test.test_type,
|
||||||
|
expectedOutputRef: test.expected_output_ref,
|
||||||
|
artifactSha256: testArtifact.sha256,
|
||||||
|
checks: [
|
||||||
|
{ name: "manifest_test_artifact", status: "passed", detail: "Artifact exists, checksum matches, and content is non-empty." },
|
||||||
|
],
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
return failedSkillTest(test.test_type, test.expected_output_ref, error instanceof Error ? error.message : String(error));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function verifyArtifactIntegrity(artifact: SkillArtifactRow): Buffer {
|
||||||
|
if (!existsSync(artifact.object_key)) throw new Error(`Skill artifact is missing: ${artifact.object_key}`);
|
||||||
|
const body = readFileSync(artifact.object_key);
|
||||||
|
const sha256 = createHash("sha256").update(body).digest("hex");
|
||||||
|
if (sha256 !== artifact.sha256) throw new Error(`Skill artifact checksum mismatch: ${artifact.object_key}`);
|
||||||
|
if (body.byteLength !== artifact.size_bytes) throw new Error(`Skill artifact size mismatch: ${artifact.object_key}`);
|
||||||
|
return body;
|
||||||
|
}
|
||||||
|
|
||||||
|
function artifactMatchesExpectedRef(objectKey: string, expectedRef: string): boolean {
|
||||||
|
const expected = normalizeSkillPath(expectedRef);
|
||||||
|
const actual = normalizeSkillPath(objectKey);
|
||||||
|
return actual === expected || actual.endsWith(`/${expected}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeSkillPath(value: string): string {
|
||||||
|
return normalize(value).replaceAll("\\", "/");
|
||||||
|
}
|
||||||
|
|
||||||
|
function failedSkillTest(testType: string, expectedOutputRef: string, failure: string): SkillTestResult {
|
||||||
|
return {
|
||||||
|
status: "failed",
|
||||||
|
testType,
|
||||||
|
expectedOutputRef,
|
||||||
|
checks: [{ name: "skill_test", status: "failed", detail: failure }],
|
||||||
|
failure,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -115,6 +115,7 @@ export class SnapshotService {
|
|||||||
skillRevisions: list(this.db, "skill_revisions", "published_at desc"),
|
skillRevisions: list(this.db, "skill_revisions", "published_at desc"),
|
||||||
skillBindings,
|
skillBindings,
|
||||||
skillRuntimeSnapshots: list(this.db, "skill_runtime_snapshots", "created_at desc"),
|
skillRuntimeSnapshots: list(this.db, "skill_runtime_snapshots", "created_at desc"),
|
||||||
|
skillTests: list(this.db, "skill_tests", "last_run_at desc, id asc"),
|
||||||
socialAccounts: list(this.db, "social_accounts", "created_at asc"),
|
socialAccounts: list(this.db, "social_accounts", "created_at asc"),
|
||||||
socialAccountGrants: list(this.db, "social_account_grants", "created_at asc"),
|
socialAccountGrants: list(this.db, "social_account_grants", "created_at asc"),
|
||||||
browserSessionContainers: list<AppSnapshot["browserSessionContainers"][number]>(this.db, "browser_session_containers", "created_at asc")
|
browserSessionContainers: list<AppSnapshot["browserSessionContainers"][number]>(this.db, "browser_session_containers", "created_at asc")
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ const api: DesktopApi = {
|
|||||||
importSkillDirectory: (input) => invoke(IPC_CHANNELS.importSkillDirectory, input),
|
importSkillDirectory: (input) => invoke(IPC_CHANNELS.importSkillDirectory, input),
|
||||||
bindSkill: (input) => invoke(IPC_CHANNELS.bindSkill, input),
|
bindSkill: (input) => invoke(IPC_CHANNELS.bindSkill, input),
|
||||||
resolveAgentSkills: (input) => invoke(IPC_CHANNELS.resolveAgentSkills, input),
|
resolveAgentSkills: (input) => invoke(IPC_CHANNELS.resolveAgentSkills, input),
|
||||||
|
runSkillTests: (input) => invoke(IPC_CHANNELS.runSkillTests, input),
|
||||||
createSocialAccount: (input) => invoke(IPC_CHANNELS.createSocialAccount, input),
|
createSocialAccount: (input) => invoke(IPC_CHANNELS.createSocialAccount, input),
|
||||||
launchSocialLogin: (input) => invoke(IPC_CHANNELS.launchSocialLogin, input),
|
launchSocialLogin: (input) => invoke(IPC_CHANNELS.launchSocialLogin, input),
|
||||||
createScheduledPost: (input) => invoke(IPC_CHANNELS.createScheduledPost, input),
|
createScheduledPost: (input) => invoke(IPC_CHANNELS.createScheduledPost, input),
|
||||||
|
|||||||
@@ -71,6 +71,10 @@ export function SettingsPage({ snapshot, actions }: PageProps) {
|
|||||||
const skillIds = new Set(skills.map((skill) => skill.id));
|
const skillIds = new Set(skills.map((skill) => skill.id));
|
||||||
return snapshot.skillRevisions.filter((revision) => skillIds.has(revision.skillId));
|
return snapshot.skillRevisions.filter((revision) => skillIds.has(revision.skillId));
|
||||||
}, [snapshot.skillRevisions, skills]);
|
}, [snapshot.skillRevisions, skills]);
|
||||||
|
const skillTests = useMemo(() => {
|
||||||
|
const revisionIds = new Set(skillRevisions.map((revision) => revision.id));
|
||||||
|
return snapshot.skillTests.filter((test) => revisionIds.has(test.revisionId));
|
||||||
|
}, [snapshot.skillTests, skillRevisions]);
|
||||||
const skillBindings = useMemo(() => snapshot.skillBindings.filter((item) => item.orgId === orgId), [snapshot.skillBindings, orgId]);
|
const skillBindings = useMemo(() => snapshot.skillBindings.filter((item) => item.orgId === orgId), [snapshot.skillBindings, orgId]);
|
||||||
const policies = useMemo(() => snapshot.policyRules.filter((item) => item.orgId === orgId), [snapshot.policyRules, orgId]);
|
const policies = useMemo(() => snapshot.policyRules.filter((item) => item.orgId === orgId), [snapshot.policyRules, orgId]);
|
||||||
const workspaces = useMemo(() => snapshot.workspaces.filter((item) => item.orgId === orgId), [snapshot.workspaces, orgId]);
|
const workspaces = useMemo(() => snapshot.workspaces.filter((item) => item.orgId === orgId), [snapshot.workspaces, orgId]);
|
||||||
@@ -147,6 +151,7 @@ export function SettingsPage({ snapshot, actions }: PageProps) {
|
|||||||
skills={skills}
|
skills={skills}
|
||||||
revisions={skillRevisions}
|
revisions={skillRevisions}
|
||||||
bindings={skillBindings}
|
bindings={skillBindings}
|
||||||
|
tests={skillTests}
|
||||||
agents={agents}
|
agents={agents}
|
||||||
actions={actions}
|
actions={actions}
|
||||||
/>
|
/>
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import type {
|
|||||||
SkillBindingRecord,
|
SkillBindingRecord,
|
||||||
SkillDefinitionRecord,
|
SkillDefinitionRecord,
|
||||||
SkillRevisionRecord,
|
SkillRevisionRecord,
|
||||||
|
SkillTestRecord,
|
||||||
} from "../../../../shared/domain";
|
} from "../../../../shared/domain";
|
||||||
import type { SnapshotActions } from "../../app/useSnapshot";
|
import type { SnapshotActions } from "../../app/useSnapshot";
|
||||||
|
|
||||||
@@ -14,11 +15,12 @@ interface SkillsPanelProps {
|
|||||||
skills: SkillDefinitionRecord[];
|
skills: SkillDefinitionRecord[];
|
||||||
revisions: SkillRevisionRecord[];
|
revisions: SkillRevisionRecord[];
|
||||||
bindings: SkillBindingRecord[];
|
bindings: SkillBindingRecord[];
|
||||||
|
tests: SkillTestRecord[];
|
||||||
agents: AgentRecord[];
|
agents: AgentRecord[];
|
||||||
actions: SnapshotActions;
|
actions: SnapshotActions;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function SkillsPanel({ orgId, skills, revisions, bindings, agents, actions }: SkillsPanelProps) {
|
export function SkillsPanel({ orgId, skills, revisions, bindings, tests, agents, actions }: SkillsPanelProps) {
|
||||||
const [importOpen, setImportOpen] = useState(false);
|
const [importOpen, setImportOpen] = useState(false);
|
||||||
|
|
||||||
const importSkill = (event: FormEvent<HTMLFormElement>) => {
|
const importSkill = (event: FormEvent<HTMLFormElement>) => {
|
||||||
@@ -37,6 +39,10 @@ export function SkillsPanel({ orgId, skills, revisions, bindings, agents, action
|
|||||||
void actions.run(() => window.agentCompany.bindSkill({ orgId, agentId, skillId: skill.id, revisionId: latest.id }));
|
void actions.run(() => window.agentCompany.bindSkill({ orgId, agentId, skillId: skill.id, revisionId: latest.id }));
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const runTests = (revisionId: string) => {
|
||||||
|
void actions.run(() => window.agentCompany.runSkillTests({ orgId, revisionId }));
|
||||||
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="page">
|
<div className="page">
|
||||||
<div className="row" style={{ marginBottom: 24 }}>
|
<div className="row" style={{ marginBottom: 24 }}>
|
||||||
@@ -54,31 +60,36 @@ export function SkillsPanel({ orgId, skills, revisions, bindings, agents, action
|
|||||||
<div className="card" style={{ padding: 0, overflow: "hidden" }}>
|
<div className="card" style={{ padding: 0, overflow: "hidden" }}>
|
||||||
<div
|
<div
|
||||||
className="table-row head"
|
className="table-row head"
|
||||||
style={{ gridTemplateColumns: "minmax(0, 1.6fr) minmax(0, 1fr) minmax(0, 1.4fr) minmax(0, 1.6fr)" }}
|
style={{ gridTemplateColumns: "minmax(0, 1.4fr) 92px 132px minmax(0, 1.2fr) minmax(220px, 1.4fr)" }}
|
||||||
>
|
>
|
||||||
<div>Skill</div><div>Version</div><div>Bindings</div><div>Bind to agent</div>
|
<div>Skill</div><div>Version</div><div>Tests</div><div>Bindings</div><div>Actions</div>
|
||||||
</div>
|
</div>
|
||||||
{skills.map((skill, index) => {
|
{skills.map((skill, index) => {
|
||||||
const last = index === skills.length - 1;
|
const last = index === skills.length - 1;
|
||||||
const latest = revisions.find((revision) => revision.skillId === skill.id);
|
const latest = revisions.find((revision) => revision.skillId === skill.id);
|
||||||
const skillBindings = bindings.filter((binding) => binding.skillId === skill.id && binding.enabled);
|
const skillBindings = bindings.filter((binding) => binding.skillId === skill.id && binding.enabled);
|
||||||
|
const revisionTests = latest ? tests.filter((test) => test.revisionId === latest.id) : [];
|
||||||
return (
|
return (
|
||||||
<div
|
<div
|
||||||
key={skill.id}
|
key={skill.id}
|
||||||
className="table-row"
|
className="table-row"
|
||||||
style={{ gridTemplateColumns: "minmax(0, 1.6fr) minmax(0, 1fr) minmax(0, 1.4fr) minmax(0, 1.6fr)", borderBottom: last ? "none" : "1px solid var(--border-soft)" }}
|
style={{ gridTemplateColumns: "minmax(0, 1.4fr) 92px 132px minmax(0, 1.2fr) minmax(220px, 1.4fr)", borderBottom: last ? "none" : "1px solid var(--border-soft)" }}
|
||||||
>
|
>
|
||||||
<div style={{ minWidth: 0 }}>
|
<div style={{ minWidth: 0 }}>
|
||||||
<div style={{ fontWeight: 540, overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap" }}>{skill.name}</div>
|
<div style={{ fontWeight: 540, overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap" }}>{skill.name}</div>
|
||||||
<div style={{ color: "var(--text-3)", fontSize: 11 }}>{skill.slug}</div>
|
<div style={{ color: "var(--text-3)", fontSize: 11 }}>{skill.slug}</div>
|
||||||
</div>
|
</div>
|
||||||
<Pill>{latest ? latest.versionSemver : "—"}</Pill>
|
<Pill>{latest ? latest.versionSemver : "—"}</Pill>
|
||||||
|
<Pill>{testSummary(revisionTests)}</Pill>
|
||||||
<div className="row" style={{ gap: 6, flexWrap: "wrap" }}>
|
<div className="row" style={{ gap: 6, flexWrap: "wrap" }}>
|
||||||
{skillBindings.map((binding) => (
|
{skillBindings.map((binding) => (
|
||||||
<Pill key={binding.id}>{agentName(binding.agentId, agents)}</Pill>
|
<Pill key={binding.id}>{agentName(binding.agentId, agents)}</Pill>
|
||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
<BindAgent skill={skill} agents={agents} onBind={(agentId) => bind(skill, agentId)} disabled={!latest} />
|
<div className="row" style={{ gap: 8, minWidth: 0 }}>
|
||||||
|
<Button size="small" onClick={() => { if (latest && revisionTests.length > 0) runTests(latest.id); }} disabled={!latest || revisionTests.length === 0}>Run tests</Button>
|
||||||
|
<BindAgent skill={skill} agents={agents} onBind={(agentId) => bind(skill, agentId)} disabled={!latest} />
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
})}
|
})}
|
||||||
@@ -124,3 +135,23 @@ function BindAgent({
|
|||||||
function agentName(agentId: string, agents: AgentRecord[]): string {
|
function agentName(agentId: string, agents: AgentRecord[]): string {
|
||||||
return agents.find((agent) => agent.id === agentId)?.name ?? agentId;
|
return agents.find((agent) => agent.id === agentId)?.name ?? agentId;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function testSummary(tests: SkillTestRecord[]): string {
|
||||||
|
if (tests.length === 0) return "No tests";
|
||||||
|
const completed = tests.filter((test) => test.lastResult);
|
||||||
|
if (completed.length === 0) return "Never run";
|
||||||
|
const passed = completed.filter((test) => testStatus(test) === "passed").length;
|
||||||
|
const failed = completed.length - passed;
|
||||||
|
if (failed > 0) return `${failed} failed`;
|
||||||
|
return `${passed}/${tests.length} passed`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function testStatus(test: SkillTestRecord): "passed" | "failed" {
|
||||||
|
if (!test.lastResult) return "failed";
|
||||||
|
try {
|
||||||
|
const result = JSON.parse(test.lastResult) as { status?: unknown };
|
||||||
|
return result.status === "passed" ? "passed" : "failed";
|
||||||
|
} catch {
|
||||||
|
return "failed";
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -223,6 +223,11 @@ export interface ResolveAgentSkillsInput {
|
|||||||
runId: string;
|
runId: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface RunSkillTestsInput {
|
||||||
|
orgId: string;
|
||||||
|
revisionId: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface CreateSocialAccountInput {
|
export interface CreateSocialAccountInput {
|
||||||
orgId: string;
|
orgId: string;
|
||||||
platform: SocialPlatform;
|
platform: SocialPlatform;
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ import type {
|
|||||||
RunChaosDrillInput,
|
RunChaosDrillInput,
|
||||||
RunEvalSuiteInput,
|
RunEvalSuiteInput,
|
||||||
RunSdlcReviewInput,
|
RunSdlcReviewInput,
|
||||||
|
RunSkillTestsInput,
|
||||||
RunTaskInput,
|
RunTaskInput,
|
||||||
RevokeProviderCredentialInput,
|
RevokeProviderCredentialInput,
|
||||||
SaveProviderCredentialInput,
|
SaveProviderCredentialInput,
|
||||||
@@ -90,6 +91,7 @@ export interface DesktopApi {
|
|||||||
importSkillDirectory(input: ImportSkillDirectoryInput): Promise<DesktopResult<AppSnapshot>>;
|
importSkillDirectory(input: ImportSkillDirectoryInput): Promise<DesktopResult<AppSnapshot>>;
|
||||||
bindSkill(input: BindSkillInput): Promise<DesktopResult<AppSnapshot>>;
|
bindSkill(input: BindSkillInput): Promise<DesktopResult<AppSnapshot>>;
|
||||||
resolveAgentSkills(input: ResolveAgentSkillsInput): Promise<DesktopResult<AppSnapshot>>;
|
resolveAgentSkills(input: ResolveAgentSkillsInput): Promise<DesktopResult<AppSnapshot>>;
|
||||||
|
runSkillTests(input: RunSkillTestsInput): Promise<DesktopResult<AppSnapshot>>;
|
||||||
createSocialAccount(input: CreateSocialAccountInput): Promise<DesktopResult<AppSnapshot>>;
|
createSocialAccount(input: CreateSocialAccountInput): Promise<DesktopResult<AppSnapshot>>;
|
||||||
launchSocialLogin(input: LaunchSocialLoginInput): Promise<DesktopResult<null>>;
|
launchSocialLogin(input: LaunchSocialLoginInput): Promise<DesktopResult<null>>;
|
||||||
createScheduledPost(input: CreateScheduledPostInput): Promise<DesktopResult<AppSnapshot>>;
|
createScheduledPost(input: CreateScheduledPostInput): Promise<DesktopResult<AppSnapshot>>;
|
||||||
|
|||||||
@@ -398,6 +398,15 @@ export interface SkillRuntimeSnapshotRecord {
|
|||||||
createdAt: string;
|
createdAt: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface SkillTestRecord {
|
||||||
|
id: string;
|
||||||
|
revisionId: string;
|
||||||
|
testType: string;
|
||||||
|
expectedOutputRef: string;
|
||||||
|
lastResult: string | null;
|
||||||
|
lastRunAt: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
export interface AuditEventRecord {
|
export interface AuditEventRecord {
|
||||||
id: string;
|
id: string;
|
||||||
orgId: string;
|
orgId: string;
|
||||||
@@ -464,6 +473,7 @@ export interface AppSnapshot {
|
|||||||
skillRevisions: SkillRevisionRecord[];
|
skillRevisions: SkillRevisionRecord[];
|
||||||
skillBindings: SkillBindingRecord[];
|
skillBindings: SkillBindingRecord[];
|
||||||
skillRuntimeSnapshots: SkillRuntimeSnapshotRecord[];
|
skillRuntimeSnapshots: SkillRuntimeSnapshotRecord[];
|
||||||
|
skillTests: SkillTestRecord[];
|
||||||
socialAccounts: SocialAccountRecord[];
|
socialAccounts: SocialAccountRecord[];
|
||||||
socialAccountGrants: SocialAccountGrantRecord[];
|
socialAccountGrants: SocialAccountGrantRecord[];
|
||||||
browserSessionContainers: BrowserSessionContainerRecord[];
|
browserSessionContainers: BrowserSessionContainerRecord[];
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ export const IPC_CHANNELS = {
|
|||||||
importSkillDirectory: "agentcompany:import-skill-directory",
|
importSkillDirectory: "agentcompany:import-skill-directory",
|
||||||
bindSkill: "agentcompany:bind-skill",
|
bindSkill: "agentcompany:bind-skill",
|
||||||
resolveAgentSkills: "agentcompany:resolve-agent-skills",
|
resolveAgentSkills: "agentcompany:resolve-agent-skills",
|
||||||
|
runSkillTests: "agentcompany:run-skill-tests",
|
||||||
createSocialAccount: "agentcompany:create-social-account",
|
createSocialAccount: "agentcompany:create-social-account",
|
||||||
launchSocialLogin: "agentcompany:launch-social-login",
|
launchSocialLogin: "agentcompany:launch-social-login",
|
||||||
createScheduledPost: "agentcompany:create-scheduled-post",
|
createScheduledPost: "agentcompany:create-scheduled-post",
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { _electron as electron, expect, test, type Page } from "@playwright/test";
|
import { _electron as electron, expect, test, type Page } from "@playwright/test";
|
||||||
import { mkdirSync } from "node:fs";
|
import { mkdirSync, writeFileSync } from "node:fs";
|
||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import { join, resolve } from "node:path";
|
import { join, resolve } from "node:path";
|
||||||
import { randomUUID } from "node:crypto";
|
import { randomUUID } from "node:crypto";
|
||||||
@@ -133,6 +133,14 @@ test("onboards, hires, and creates work in the Electron shell", async () => {
|
|||||||
await page.getByLabel("OpenAI monthly request limit").fill("25");
|
await page.getByLabel("OpenAI monthly request limit").fill("25");
|
||||||
await page.getByRole("button", { name: "Save OpenAI policy" }).click();
|
await page.getByRole("button", { name: "Save OpenAI policy" }).click();
|
||||||
await expect(page.getByText("0 / 25").first()).toBeVisible();
|
await expect(page.getByText("0 / 25").first()).toBeVisible();
|
||||||
|
await page.getByRole("button", { name: "Skills" }).click();
|
||||||
|
await page.getByRole("button", { name: "Import" }).click();
|
||||||
|
await page.getByLabel("Directory path").fill(createSkillDirectory());
|
||||||
|
await page.locator(".modal").getByRole("button", { name: "Import" }).click();
|
||||||
|
await expect(page.getByText("Release Gate").first()).toBeVisible();
|
||||||
|
await expect(page.getByText("Never run").first()).toBeVisible();
|
||||||
|
await page.getByRole("button", { name: "Run tests" }).click();
|
||||||
|
await expect(page.getByText("1/1 passed").first()).toBeVisible();
|
||||||
await page.getByRole("button", { name: "Compliance" }).click();
|
await page.getByRole("button", { name: "Compliance" }).click();
|
||||||
await page.getByRole("button", { name: "Create" }).click();
|
await page.getByRole("button", { name: "Create" }).click();
|
||||||
await expect(page.getByText("gdpr export · employee").first()).toBeVisible();
|
await expect(page.getByText("gdpr export · employee").first()).toBeVisible();
|
||||||
@@ -167,6 +175,26 @@ function createLegacyDatabase(): string {
|
|||||||
return legacyPath;
|
return legacyPath;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function createSkillDirectory(): string {
|
||||||
|
const root = join(tmpdir(), `agentcompany-skill-${randomUUID()}`);
|
||||||
|
mkdirSync(join(root, "checks"), { recursive: true });
|
||||||
|
writeFileSync(join(root, "SKILL.md"), "Review release evidence and produce a concise gate decision.\n", "utf8");
|
||||||
|
writeFileSync(join(root, "checks", "acceptance.txt"), "release evidence accepted\n", "utf8");
|
||||||
|
writeFileSync(join(root, "skill.manifest.json"), JSON.stringify({
|
||||||
|
name: "Release Gate",
|
||||||
|
version: "1.0.0",
|
||||||
|
entrypoint: "SKILL.md",
|
||||||
|
capabilities: ["release-review"],
|
||||||
|
required_tools: ["git"],
|
||||||
|
compatible_models: ["gpt-5.4"],
|
||||||
|
risk_level: "low",
|
||||||
|
tests: ["checks/acceptance.txt"],
|
||||||
|
inputs: {},
|
||||||
|
outputs: {},
|
||||||
|
}, null, 2), "utf8");
|
||||||
|
return root;
|
||||||
|
}
|
||||||
|
|
||||||
async function verifyPagesDoNotOverflow(page: Page): Promise<void> {
|
async function verifyPagesDoNotOverflow(page: Page): Promise<void> {
|
||||||
for (const size of [{ width: 1280, height: 860 }, { width: 980, height: 700 }]) {
|
for (const size of [{ width: 1280, height: 860 }, { width: 980, height: 700 }]) {
|
||||||
await page.setViewportSize(size);
|
await page.setViewportSize(size);
|
||||||
|
|||||||
@@ -145,6 +145,7 @@ describe("domain services", () => {
|
|||||||
.get(orgId, "review-routes") as { id: string };
|
.get(orgId, "review-routes") as { id: string };
|
||||||
skills.bind({ orgId, agentId, skillId: skill.id, revisionId });
|
skills.bind({ orgId, agentId, skillId: skill.id, revisionId });
|
||||||
rmSync(skillDir, { recursive: true, force: true });
|
rmSync(skillDir, { recursive: true, force: true });
|
||||||
|
skills.runTests({ orgId, revisionId });
|
||||||
skills.resolveRuntimeSnapshot({ orgId, agentId, runId: "run-review-1" });
|
skills.resolveRuntimeSnapshot({ orgId, agentId, runId: "run-review-1" });
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -153,8 +154,11 @@ describe("domain services", () => {
|
|||||||
expect(state.skillRevisions[0]?.versionSemver).toBe("1.0.0");
|
expect(state.skillRevisions[0]?.versionSemver).toBe("1.0.0");
|
||||||
expect(state.skillRevisions[0]?.immutableChecksum).toHaveLength(64);
|
expect(state.skillRevisions[0]?.immutableChecksum).toHaveLength(64);
|
||||||
expect(state.skillRevisions[0]?.manifestJson).toContain("architecture-review");
|
expect(state.skillRevisions[0]?.manifestJson).toContain("architecture-review");
|
||||||
|
expect(state.skillTests[0]?.lastResult).toContain("\"status\":\"passed\"");
|
||||||
|
expect(state.skillTests[0]?.lastRunAt).toBeTruthy();
|
||||||
expect(state.skillRuntimeSnapshots[0]?.sourceRevisionIds).toContain(revisionId);
|
expect(state.skillRuntimeSnapshots[0]?.sourceRevisionIds).toContain(revisionId);
|
||||||
expect(state.skillRuntimeSnapshots[0]?.compiledPrompt).toContain("# Review Routes");
|
expect(state.skillRuntimeSnapshots[0]?.compiledPrompt).toContain("# Review Routes");
|
||||||
|
expect(state.auditEvents.some((event) => event.entityType === "skill_revision" && event.action === "tests_run")).toBe(true);
|
||||||
connection.close();
|
connection.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user