Q39: runs load only model-compatible skills and state each skill's contract
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f7530ba65b
commit
645ce686bb
8 files changed
+142
-34
No files matched your search
@@ -0,0 +1,42 @@
|
|||||||
|
import type { SkillManifest } from "./skill-manifest";
|
||||||
|
|
||||||
|
/** The provider and model a run talks to. */
|
||||||
|
export interface RunModel {
|
||||||
|
providerKey: string;
|
||||||
|
model: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** True when a compatible_models pattern ("openai/*", "gpt-5.4") matches the run's model; no patterns means any model. */
|
||||||
|
export function modelCompatible(patterns: string[], run: RunModel): boolean {
|
||||||
|
if (patterns.length === 0) return true;
|
||||||
|
if (!run.model) return false;
|
||||||
|
const names = [`${run.providerKey}/${run.model}`, run.model];
|
||||||
|
return patterns.some((pattern) => {
|
||||||
|
const regex = new RegExp(`^${pattern.split("*").map(escapeRegex).join(".*")}$`, "i");
|
||||||
|
return names.some((name) => regex.test(name));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function runModelLabel(run: RunModel): string {
|
||||||
|
return run.model ? `${run.providerKey}/${run.model}` : `${run.providerKey} (no model set)`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One skill's prompt section, its declared risk and contract first. */
|
||||||
|
export function skillPart(manifest: SkillManifest, version: string, body: string): string {
|
||||||
|
const header = [`# Skill: ${manifest.name}`, `Version: ${version}`, `Risk: ${manifest.riskLevel}`];
|
||||||
|
const inputs = declaration(manifest.inputs);
|
||||||
|
const outputs = declaration(manifest.outputs);
|
||||||
|
if (inputs) header.push(`Expects as input: ${inputs}`);
|
||||||
|
if (outputs) header.push(`Must produce: ${outputs}`);
|
||||||
|
return [header.join("\n"), body].join("\n\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
function declaration(value: unknown): string | null {
|
||||||
|
if (typeof value === "string") return value.trim() || null;
|
||||||
|
if (typeof value === "object" && value !== null && Object.keys(value).length > 0) return JSON.stringify(value);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function escapeRegex(text: string): string {
|
||||||
|
return text.replace(/[.+?^${}()|[\]\\]/g, "\\$&");
|
||||||
|
}
|
||||||
@@ -10,6 +10,7 @@ import { loadSkillManifest, type SkillManifest } from "./skill-manifest";
|
|||||||
import { orderByRelevance } from "./relevance";
|
import { orderByRelevance } from "./relevance";
|
||||||
import { runManifestArtifactTest, verifyRevisionStructure, type SkillArtifactRow, type SkillTestRow } from "./skill-tests";
|
import { runManifestArtifactTest, verifyRevisionStructure, type SkillArtifactRow, type SkillTestRow } from "./skill-tests";
|
||||||
import { estimateTokens } from "./context-budget";
|
import { estimateTokens } from "./context-budget";
|
||||||
|
import { modelCompatible, runModelLabel, skillPart } from "./skill-runtime";
|
||||||
|
|
||||||
function slugify(value: string): string {
|
function slugify(value: string): string {
|
||||||
return value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
|
return value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
|
||||||
@@ -280,14 +281,6 @@ export class SkillService {
|
|||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
private carriesRetiredSkill(revisionIds: string[]): boolean {
|
|
||||||
if (revisionIds.length === 0) return false;
|
|
||||||
return Boolean(this.db.prepare(`
|
|
||||||
select 1 from skill_revisions sr join skill_definitions sd on sd.id = sr.skill_id
|
|
||||||
where sd.status <> 'active' and sr.id in (${revisionIds.map(() => "?").join(", ")}) limit 1
|
|
||||||
`).get(...revisionIds));
|
|
||||||
}
|
|
||||||
|
|
||||||
private appLanguage(): string {
|
private appLanguage(): string {
|
||||||
const row = this.db.prepare("select language from app_state where id = 1").get() as { language: string } | undefined;
|
const row = this.db.prepare("select language from app_state where id = 1").get() as { language: string } | undefined;
|
||||||
return row?.language === "zh" ? "zh" : "en";
|
return row?.language === "zh" ? "zh" : "en";
|
||||||
@@ -301,13 +294,6 @@ export class SkillService {
|
|||||||
* prompt instead of silently overflowing the model's context.
|
* prompt instead of silently overflowing the model's context.
|
||||||
*/
|
*/
|
||||||
resolveRuntimeSnapshot(input: ResolveAgentSkillsInput): string {
|
resolveRuntimeSnapshot(input: ResolveAgentSkillsInput): string {
|
||||||
// Idempotent per run: a re-queued run reuses its frozen skill set unless a skill in it was retired since.
|
|
||||||
const existingSnapshot = this.db.prepare("select id, source_revision_ids from skill_runtime_snapshots where agent_id = ? and run_id = ?")
|
|
||||||
.get(input.agentId, input.runId) as { id: string; source_revision_ids: string } | undefined;
|
|
||||||
if (existingSnapshot) {
|
|
||||||
if (!this.carriesRetiredSkill(JSON.parse(existingSnapshot.source_revision_ids) as string[])) return existingSnapshot.id;
|
|
||||||
this.db.prepare("delete from skill_runtime_snapshots where id = ?").run(existingSnapshot.id);
|
|
||||||
}
|
|
||||||
const forcedRows = this.forcedSkillRows(input.orgId);
|
const forcedRows = this.forcedSkillRows(input.orgId);
|
||||||
const boundRows = this.db.prepare(`
|
const boundRows = this.db.prepare(`
|
||||||
select
|
select
|
||||||
@@ -340,15 +326,16 @@ export class SkillService {
|
|||||||
const tools = new Set<string>();
|
const tools = new Set<string>();
|
||||||
const revisionIds: string[] = [];
|
const revisionIds: string[] = [];
|
||||||
const omitted: string[] = [];
|
const omitted: string[] = [];
|
||||||
|
const incompatible: string[] = [];
|
||||||
let compiledTokens = 0;
|
let compiledTokens = 0;
|
||||||
for (const row of rows) {
|
for (const row of rows) {
|
||||||
if (!existsSync(row.object_key)) throw new Error(`Skill artifact is missing: ${row.object_key}`);
|
if (!existsSync(row.object_key)) throw new Error(`Skill artifact is missing: ${row.object_key}`);
|
||||||
const manifest = JSON.parse(row.manifest_json) as SkillManifest;
|
const manifest = JSON.parse(row.manifest_json) as SkillManifest;
|
||||||
const part = [
|
if (!modelCompatible(manifest.compatibleModels, input.runModel)) {
|
||||||
`# Skill: ${manifest.name}`,
|
incompatible.push(`${manifest.name} (${manifest.compatibleModels.join(", ")})`);
|
||||||
`Version: ${row.version_semver}`,
|
continue;
|
||||||
readFileSync(row.object_key, "utf8").trim(),
|
}
|
||||||
].join("\n\n");
|
const part = skillPart(manifest, row.version_semver, readFileSync(row.object_key, "utf8").trim());
|
||||||
const partTokens = estimateTokens(part);
|
const partTokens = estimateTokens(part);
|
||||||
if (compiledTokens + partTokens > input.maxTokens) {
|
if (compiledTokens + partTokens > input.maxTokens) {
|
||||||
omitted.push(manifest.name);
|
omitted.push(manifest.name);
|
||||||
@@ -359,12 +346,24 @@ export class SkillService {
|
|||||||
compiledParts.push(part);
|
compiledParts.push(part);
|
||||||
compiledTokens += partTokens;
|
compiledTokens += partTokens;
|
||||||
}
|
}
|
||||||
|
if (incompatible.length > 0) {
|
||||||
|
console.warn(`[skills] run ${input.runId}: skipped skills incompatible with ${runModelLabel(input.runModel)}: ${incompatible.join("; ")}`);
|
||||||
|
compiledParts.push(
|
||||||
|
`# Skills not loaded\n\nThese granted skills do not support this run's model (${runModelLabel(input.runModel)}): ${incompatible.join("; ")}.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
if (omitted.length > 0) {
|
if (omitted.length > 0) {
|
||||||
compiledParts.push(
|
compiledParts.push(
|
||||||
`# Skills omitted for size\n\nThese granted skills did not fit this run's prompt budget: ${omitted.join(", ")}. Ask the CEO to trim grants if they are essential.`,
|
`# Skills omitted for size\n\nThese granted skills did not fit this run's prompt budget: ${omitted.join(", ")}. Ask the CEO to trim grants if they are essential.`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const compiledPrompt = compiledParts.join("\n\n---\n\n");
|
||||||
|
// Idempotent per run: a re-queued run keeps its snapshot while the resolved skill set is unchanged.
|
||||||
|
const existing = this.db.prepare("select id, compiled_prompt from skill_runtime_snapshots where agent_id = ? and run_id = ?")
|
||||||
|
.get(input.agentId, input.runId) as { id: string; compiled_prompt: string } | undefined;
|
||||||
|
if (existing?.compiled_prompt === compiledPrompt) return existing.id;
|
||||||
|
if (existing) this.db.prepare("delete from skill_runtime_snapshots where id = ?").run(existing.id);
|
||||||
const snapshotId = createId("srs");
|
const snapshotId = createId("srs");
|
||||||
this.db.prepare(`
|
this.db.prepare(`
|
||||||
insert into skill_runtime_snapshots(
|
insert into skill_runtime_snapshots(
|
||||||
@@ -375,7 +374,7 @@ export class SkillService {
|
|||||||
snapshotId,
|
snapshotId,
|
||||||
input.agentId,
|
input.agentId,
|
||||||
input.runId,
|
input.runId,
|
||||||
compiledParts.join("\n\n---\n\n"),
|
compiledPrompt,
|
||||||
JSON.stringify({ requiredTools: [...tools].sort() }),
|
JSON.stringify({ requiredTools: [...tools].sort() }),
|
||||||
JSON.stringify(revisionIds),
|
JSON.stringify(revisionIds),
|
||||||
new Date().toISOString(),
|
new Date().toISOString(),
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import { ChatService } from "./chat-service";
|
|||||||
import { deliverChatReplyIfMissing } from "./chat-agent-wake-service";
|
import { deliverChatReplyIfMissing } from "./chat-agent-wake-service";
|
||||||
import { SkillService } from "./skill-service";
|
import { SkillService } from "./skill-service";
|
||||||
import { runAgentLoop, runTools } from "./agent-run-loop";
|
import { runAgentLoop, runTools } from "./agent-run-loop";
|
||||||
|
import type { RunModel } from "./skill-runtime";
|
||||||
import type { AgentToolContext } from "./agent-tools";
|
import type { AgentToolContext } from "./agent-tools";
|
||||||
import {
|
import {
|
||||||
buildAgentInstructions,
|
buildAgentInstructions,
|
||||||
@@ -147,7 +148,7 @@ export class TaskExecutionService {
|
|||||||
let providerRequestId: string | null = null;
|
let providerRequestId: string | null = null;
|
||||||
let skillRuntimeSnapshot: SkillRuntimeSnapshotRow | null = null;
|
let skillRuntimeSnapshot: SkillRuntimeSnapshotRow | null = null;
|
||||||
try {
|
try {
|
||||||
skillRuntimeSnapshot = this.resolveSkillRuntimeSnapshot(orgId, agent.id, runId, skillBudgetTokens(provider.context_window_tokens), `${task.title}\n${task.description}`);
|
skillRuntimeSnapshot = this.resolveSkillRuntimeSnapshot(orgId, agent.id, runId, skillBudgetTokens(provider.context_window_tokens), `${task.title}\n${task.description}`, { providerKey: provider.provider_key, model: provider.model });
|
||||||
// Watermark for reply delivery and mid-run message polling.
|
// Watermark for reply delivery and mid-run message polling.
|
||||||
const runStartedAt = new Date().toISOString();
|
const runStartedAt = new Date().toISOString();
|
||||||
const chatContext = loadChatWakeContext(this.db, orgId, agent.id, isolation.secretScopeJson);
|
const chatContext = loadChatWakeContext(this.db, orgId, agent.id, isolation.secretScopeJson);
|
||||||
@@ -390,9 +391,9 @@ export class TaskExecutionService {
|
|||||||
`).run(TASK_RUN_STATE.FAILED, providerRequestId, new Date().toISOString(), errorMessage, runId, orgId);
|
`).run(TASK_RUN_STATE.FAILED, providerRequestId, new Date().toISOString(), errorMessage, runId, orgId);
|
||||||
}
|
}
|
||||||
|
|
||||||
private resolveSkillRuntimeSnapshot(orgId: string, agentId: string, runId: string, maxTokens: number, taskText: string): SkillRuntimeSnapshotRow | null {
|
private resolveSkillRuntimeSnapshot(orgId: string, agentId: string, runId: string, maxTokens: number, taskText: string, runModel: RunModel): SkillRuntimeSnapshotRow | null {
|
||||||
if (!this.skills.hasRuntimeSkills(orgId, agentId)) return null;
|
if (!this.skills.hasRuntimeSkills(orgId, agentId)) return null;
|
||||||
const snapshotId = this.skills.resolveRuntimeSnapshot({ orgId, agentId, runId, maxTokens, taskText });
|
const snapshotId = this.skills.resolveRuntimeSnapshot({ orgId, agentId, runId, maxTokens, taskText, runModel });
|
||||||
const snapshot = this.db.prepare(`
|
const snapshot = this.db.prepare(`
|
||||||
select id, compiled_prompt, tools_manifest, source_revision_ids
|
select id, compiled_prompt, tools_manifest, source_revision_ids
|
||||||
from skill_runtime_snapshots
|
from skill_runtime_snapshots
|
||||||
|
|||||||
@@ -23,6 +23,8 @@ export interface ResolveAgentSkillsInput {
|
|||||||
maxTokens: number;
|
maxTokens: number;
|
||||||
/** Task title and description, to rank bound skills by relevance. */
|
/** Task title and description, to rank bound skills by relevance. */
|
||||||
taskText: string;
|
taskText: string;
|
||||||
|
/** The run's provider and model, matched against each skill's compatible_models. */
|
||||||
|
runModel: { providerKey: string; model: string | null };
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RunSkillTestsInput {
|
export interface RunSkillTestsInput {
|
||||||
|
|||||||
@@ -98,7 +98,7 @@ describe("collaboration integration flow", () => {
|
|||||||
revisionId = skills.importDirectory({ orgId, directoryPath: createSkillDirectory(), versionSemver: "1.0.0" });
|
revisionId = skills.importDirectory({ orgId, directoryPath: createSkillDirectory(), versionSemver: "1.0.0" });
|
||||||
skillId = (connection.db.prepare("select skill_id from skill_revisions where id = ?").get(revisionId) as { skill_id: string }).skill_id;
|
skillId = (connection.db.prepare("select skill_id from skill_revisions where id = ?").get(revisionId) as { skill_id: string }).skill_id;
|
||||||
skills.bind({ orgId, agentId, skillId, revisionId });
|
skills.bind({ orgId, agentId, skillId, revisionId });
|
||||||
skills.resolveRuntimeSnapshot({ orgId, agentId, runId: `run_${randomUUID().replaceAll("-", "")}`, maxTokens: 44_800, taskText: "" });
|
skills.resolveRuntimeSnapshot({ orgId, agentId, runId: `run_${randomUUID().replaceAll("-", "")}`, maxTokens: 44_800, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" } });
|
||||||
suiteId = (connection.db.prepare("select id from eval_suites where org_id = ? and key = ?")
|
suiteId = (connection.db.prepare("select id from eval_suites where org_id = ? and key = ?")
|
||||||
.get(orgId, EVAL_SUITE_KEY.INTEGRATION) as { id: string }).id;
|
.get(orgId, EVAL_SUITE_KEY.INTEGRATION) as { id: string }).id;
|
||||||
evalRunId = sdlc.runEvalSuite({ orgId, suiteId });
|
evalRunId = sdlc.runEvalSuite({ orgId, suiteId });
|
||||||
|
|||||||
@@ -171,7 +171,7 @@ describe("built-in skills", () => {
|
|||||||
h.skills.bind({ orgId: h.orgId, agentId, skillId: skill.id, revisionId: skill.revision_id });
|
h.skills.bind({ orgId: h.orgId, agentId, skillId: skill.id, revisionId: skill.revision_id });
|
||||||
});
|
});
|
||||||
const prompt = (runId: string) => {
|
const prompt = (runId: string) => {
|
||||||
const id = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId, maxTokens: SKILL_TOKENS, taskText: "" }));
|
const id = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId, maxTokens: SKILL_TOKENS, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" } }));
|
||||||
return (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(id) as { compiled_prompt: string }).compiled_prompt;
|
return (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(id) as { compiled_prompt: string }).compiled_prompt;
|
||||||
};
|
};
|
||||||
expect(prompt("run_before")).toContain("# Skill: Backend Architect");
|
expect(prompt("run_before")).toContain("# Skill: Backend Architect");
|
||||||
@@ -265,7 +265,7 @@ describe("built-in skills", () => {
|
|||||||
expect(hired).toContain("Unknown skills skipped: no-such-skill");
|
expect(hired).toContain("Unknown skills skipped: no-such-skill");
|
||||||
|
|
||||||
const danaId = (h.connection.db.prepare("select id from agents where org_id = ? and name = 'Dana Lee'").get(h.orgId) as { id: string }).id;
|
const danaId = (h.connection.db.prepare("select id from agents where org_id = ? and name = 'Dana Lee'").get(h.orgId) as { id: string }).id;
|
||||||
const danaSnapshot = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId: danaId, runId: "run_dana", maxTokens: SKILL_TOKENS, taskText: "" }));
|
const danaSnapshot = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId: danaId, runId: "run_dana", maxTokens: SKILL_TOKENS, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" } }));
|
||||||
const danaPrompt = (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(danaSnapshot) as { compiled_prompt: string }).compiled_prompt;
|
const danaPrompt = (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(danaSnapshot) as { compiled_prompt: string }).compiled_prompt;
|
||||||
expect(danaPrompt.indexOf("# Skill: humanizer")).toBe(0);
|
expect(danaPrompt.indexOf("# Skill: humanizer")).toBe(0);
|
||||||
expect(danaPrompt).toContain("# Skill: Backend Architect");
|
expect(danaPrompt).toContain("# Skill: Backend Architect");
|
||||||
@@ -275,7 +275,7 @@ describe("built-in skills", () => {
|
|||||||
|
|
||||||
// The CEO carries humanizer too - forced by resolution, not by binding.
|
// The CEO carries humanizer too - forced by resolution, not by binding.
|
||||||
expect(h.skills.hasRuntimeSkills(h.orgId, ceoId)).toBe(true);
|
expect(h.skills.hasRuntimeSkills(h.orgId, ceoId)).toBe(true);
|
||||||
const ceoSnapshot = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId: ceoId, runId: "run_ceo", maxTokens: SKILL_TOKENS, taskText: "" }));
|
const ceoSnapshot = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId: ceoId, runId: "run_ceo", maxTokens: SKILL_TOKENS, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" } }));
|
||||||
const ceoPrompt = (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(ceoSnapshot) as { compiled_prompt: string }).compiled_prompt;
|
const ceoPrompt = (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(ceoSnapshot) as { compiled_prompt: string }).compiled_prompt;
|
||||||
expect(ceoPrompt).toContain("# Skill: humanizer");
|
expect(ceoPrompt).toContain("# Skill: humanizer");
|
||||||
expect(ceoPrompt).not.toContain("# Skill: Backend Architect");
|
expect(ceoPrompt).not.toContain("# Skill: Backend Architect");
|
||||||
@@ -300,7 +300,7 @@ describe("built-in skills", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
const promptFor = (runId: string) => {
|
const promptFor = (runId: string) => {
|
||||||
const snapshotId = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId, maxTokens: SKILL_TOKENS, taskText: "" }));
|
const snapshotId = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId, maxTokens: SKILL_TOKENS, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" } }));
|
||||||
return (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(snapshotId) as { compiled_prompt: string }).compiled_prompt;
|
return (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(snapshotId) as { compiled_prompt: string }).compiled_prompt;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -346,7 +346,7 @@ describe("built-in skills", () => {
|
|||||||
modelProviderId: null, modelId: null, budgetCents: 1000, managerAgentId: null,
|
modelProviderId: null, modelId: null, budgetCents: 1000, managerAgentId: null,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
const snapshotId = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId: "run_tiny", maxTokens: 10, taskText: "" }));
|
const snapshotId = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId: "run_tiny", maxTokens: 10, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" } }));
|
||||||
const prompt = (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(snapshotId) as { compiled_prompt: string }).compiled_prompt;
|
const prompt = (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(snapshotId) as { compiled_prompt: string }).compiled_prompt;
|
||||||
expect(prompt).not.toContain("# Skill:");
|
expect(prompt).not.toContain("# Skill:");
|
||||||
expect(prompt).toContain("# Skills omitted for size");
|
expect(prompt).toContain("# Skills omitted for size");
|
||||||
@@ -373,7 +373,7 @@ describe("built-in skills", () => {
|
|||||||
bindSkillSlugs({ db: h.connection.db, organization: h.organization, chat: new ChatService(h.connection.db) }, h.orgId, agentId, ["marketing-growth-hacker", "engineering-backend-architect"]);
|
bindSkillSlugs({ db: h.connection.db, organization: h.organization, chat: new ChatService(h.connection.db) }, h.orgId, agentId, ["marketing-growth-hacker", "engineering-backend-architect"]);
|
||||||
});
|
});
|
||||||
const compile = (runId: string, maxTokens: number, taskText: string) => {
|
const compile = (runId: string, maxTokens: number, taskText: string) => {
|
||||||
const snapshotId = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId, maxTokens, taskText }));
|
const snapshotId = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId, maxTokens, taskText, runModel: { providerKey: "openai", model: "gpt-5.4" } }));
|
||||||
return (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(snapshotId) as { compiled_prompt: string }).compiled_prompt;
|
return (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(snapshotId) as { compiled_prompt: string }).compiled_prompt;
|
||||||
};
|
};
|
||||||
const parts = compile("run_full", 1_000_000, "").split("\n\n---\n\n");
|
const parts = compile("run_full", 1_000_000, "").split("\n\n---\n\n");
|
||||||
|
|||||||
@@ -186,7 +186,7 @@ describe("domain services", () => {
|
|||||||
skills.bind({ orgId, agentId, skillId: skill.id, revisionId });
|
skills.bind({ orgId, agentId, skillId: skill.id, revisionId });
|
||||||
rmSync(skillDir, { recursive: true, force: true });
|
rmSync(skillDir, { recursive: true, force: true });
|
||||||
skills.runTests({ orgId, revisionId });
|
skills.runTests({ orgId, revisionId });
|
||||||
skills.resolveRuntimeSnapshot({ orgId, agentId, runId: "run-review-1", maxTokens: 44_800, taskText: "" });
|
skills.resolveRuntimeSnapshot({ orgId, agentId, runId: "run-review-1", maxTokens: 44_800, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" } });
|
||||||
});
|
});
|
||||||
|
|
||||||
// Catalogue metadata ships in the snapshot; skill internals stay in the
|
// Catalogue metadata ships in the snapshot; skill internals stay in the
|
||||||
@@ -247,10 +247,10 @@ describe("domain services", () => {
|
|||||||
.get(orgA, "review-routes") as { id: string }).id;
|
.get(orgA, "review-routes") as { id: string }).id;
|
||||||
expect(() => skills.bind({ orgId: orgA, agentId: agentB, skillId, revisionId })).toThrow("Agent was not found.");
|
expect(() => skills.bind({ orgId: orgA, agentId: agentB, skillId, revisionId })).toThrow("Agent was not found.");
|
||||||
skills.bind({ orgId: orgA, agentId: agentA, skillId, revisionId });
|
skills.bind({ orgId: orgA, agentId: agentA, skillId, revisionId });
|
||||||
const firstResolve = skills.resolveRuntimeSnapshot({ orgId: orgA, agentId: agentA, runId: "run-review-1", maxTokens: 44_800, taskText: "" });
|
const firstResolve = skills.resolveRuntimeSnapshot({ orgId: orgA, agentId: agentA, runId: "run-review-1", maxTokens: 44_800, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" } });
|
||||||
// Idempotent per run: a recovered run reuses its frozen snapshot
|
// Idempotent per run: a recovered run reuses its frozen snapshot
|
||||||
// instead of failing on the uniqueness guard.
|
// instead of failing on the uniqueness guard.
|
||||||
expect(skills.resolveRuntimeSnapshot({ orgId: orgA, agentId: agentA, runId: "run-review-1", maxTokens: 44_800, taskText: "" })).toBe(firstResolve);
|
expect(skills.resolveRuntimeSnapshot({ orgId: orgA, agentId: agentA, runId: "run-review-1", maxTokens: 44_800, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" } })).toBe(firstResolve);
|
||||||
expect(connection.db.prepare("select count(*) as n from skill_runtime_snapshots where run_id = 'run-review-1'").get())
|
expect(connection.db.prepare("select count(*) as n from skill_runtime_snapshots where run_id = 'run-review-1'").get())
|
||||||
.toEqual({ n: 1 });
|
.toEqual({ n: 1 });
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,64 @@
|
|||||||
|
import { randomUUID } from "node:crypto";
|
||||||
|
import { mkdirSync, writeFileSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { describe, expect, test } from "vitest";
|
||||||
|
import { DatabaseConnection } from "../../src/main/database/connection";
|
||||||
|
import { OrganizationService } from "../../src/main/services/organization-service";
|
||||||
|
import { SkillService } from "../../src/main/services/skill-service";
|
||||||
|
import { modelCompatible } from "../../src/main/services/skill-runtime";
|
||||||
|
|
||||||
|
describe("skill compatibility", () => {
|
||||||
|
test("model patterns match provider/model or the bare model name", () => {
|
||||||
|
expect(modelCompatible([], { providerKey: "openai", model: null })).toBe(true);
|
||||||
|
expect(modelCompatible(["openai/*"], { providerKey: "openai", model: "gpt-5.4" })).toBe(true);
|
||||||
|
expect(modelCompatible(["gpt-5.4"], { providerKey: "openai", model: "gpt-5.4" })).toBe(true);
|
||||||
|
expect(modelCompatible(["gpt-5.?"], { providerKey: "openai", model: "gpt-5.4" })).toBe(false);
|
||||||
|
expect(modelCompatible(["anthropic/*"], { providerKey: "openai", model: "gpt-5.4" })).toBe(false);
|
||||||
|
expect(modelCompatible(["openai/*"], { providerKey: "openai", model: null })).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a run loads only skills compatible with its model and states each skill's contract", () => {
|
||||||
|
const connection = new DatabaseConnection(join(tmpdir(), `ac-compat-${randomUUID()}.db`));
|
||||||
|
const organization = new OrganizationService(connection.db);
|
||||||
|
const skills = new SkillService(connection.db, join(tmpdir(), `ac-compat-art-${randomUUID()}`));
|
||||||
|
const skillDir = join(tmpdir(), `ac-compat-skill-${randomUUID()}`);
|
||||||
|
mkdirSync(skillDir, { recursive: true });
|
||||||
|
writeFileSync(join(skillDir, "SKILL.md"), "# Claude Review\n\nReview with care.\n");
|
||||||
|
writeFileSync(join(skillDir, "skill.manifest.json"), JSON.stringify({
|
||||||
|
name: "claude-review", version: "1.0.0", entrypoint: "SKILL.md", capabilities: [], required_tools: [],
|
||||||
|
compatible_models: ["anthropic/*"], risk_level: "high", inputs: "A patch diff", outputs: { verdict: "markdown" }, tests: [],
|
||||||
|
}));
|
||||||
|
let orgId = "";
|
||||||
|
let agentId = "";
|
||||||
|
connection.transaction(() => {
|
||||||
|
orgId = organization.createOrganization({ organizationName: "C", employeeName: "Y", mission: "m", goal: "g" });
|
||||||
|
agentId = organization.createAgent({
|
||||||
|
orgId, name: "Ada", role: "Reviewer", department: "Engineering",
|
||||||
|
modelProviderId: null, modelId: null, budgetCents: 1000, managerAgentId: null,
|
||||||
|
});
|
||||||
|
const revisionId = skills.importDirectory({ orgId, directoryPath: skillDir, versionSemver: "1.0.0" });
|
||||||
|
const skill = connection.db.prepare("select id from skill_definitions where org_id = ? and slug = 'claude-review'").get(orgId) as { id: string };
|
||||||
|
skills.bind({ orgId, agentId, skillId: skill.id, revisionId });
|
||||||
|
});
|
||||||
|
const prompt = (runId: string, providerKey: string, model: string) => {
|
||||||
|
const id = connection.transaction(() => skills.resolveRuntimeSnapshot({ orgId, agentId, runId, maxTokens: 40_000, taskText: "", runModel: { providerKey, model } }));
|
||||||
|
return connection.db.prepare("select compiled_prompt, source_revision_ids from skill_runtime_snapshots where id = ?").get(id) as { compiled_prompt: string; source_revision_ids: string };
|
||||||
|
};
|
||||||
|
|
||||||
|
const openai = prompt("run_openai", "openai", "gpt-5.4");
|
||||||
|
expect(openai.compiled_prompt).not.toContain("# Skill: claude-review");
|
||||||
|
expect(openai.compiled_prompt).toContain("# Skills not loaded\n\nThese granted skills do not support this run's model (openai/gpt-5.4): claude-review (anthropic/*).");
|
||||||
|
expect(JSON.parse(openai.source_revision_ids)).toEqual([]);
|
||||||
|
|
||||||
|
const anthropic = prompt("run_anthropic", "anthropic", "claude-sonnet-5-5");
|
||||||
|
expect(anthropic.compiled_prompt).toContain([
|
||||||
|
"# Skill: claude-review", "Version: 1.0.0", "Risk: high",
|
||||||
|
"Expects as input: A patch diff", 'Must produce: {"verdict":"markdown"}',
|
||||||
|
].join("\n"));
|
||||||
|
const recovered = prompt("run_anthropic", "openai", "gpt-5.4");
|
||||||
|
expect(recovered.compiled_prompt).not.toContain("# Skill: claude-review");
|
||||||
|
expect(recovered.compiled_prompt).toContain("# Skills not loaded");
|
||||||
|
connection.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in new issue
Block a user