From 645ce686bbec221a1044f4f4254d098ce572cb86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Thu, 8 Oct 2026 07:22:08 -0400 Subject: [PATCH] Q39: runs load only model-compatible skills and state each skill's contract Co-Authored-By: Claude Opus 5.5 --- src/main/services/skill-runtime.ts | 42 +++++++++++++ src/main/services/skill-service.ts | 41 ++++++------- src/main/services/task-execution-service.ts | 7 ++- src/shared/commands/skills.ts | 2 + tests/integration/collaboration-flow.test.ts | 2 +- tests/unit/builtin-skills.test.ts | 12 ++-- tests/unit/domain-services.test.ts | 6 +- tests/unit/skill-compatibility.test.ts | 64 ++++++++++++++++++++ 8 files changed, 142 insertions(+), 34 deletions(-) create mode 100644 src/main/services/skill-runtime.ts create mode 100644 tests/unit/skill-compatibility.test.ts diff --git a/src/main/services/skill-runtime.ts b/src/main/services/skill-runtime.ts new file mode 100644 index 00000000..74c31793 --- /dev/null +++ b/src/main/services/skill-runtime.ts @@ -0,0 +1,42 @@ +import type { SkillManifest } from "./skill-manifest"; + +/** The provider and model a run talks to. */ +export interface RunModel { + providerKey: string; + model: string | null; +} + +/** True when a compatible_models pattern ("openai/*", "gpt-5.4") matches the run's model; no patterns means any model. */ +export function modelCompatible(patterns: string[], run: RunModel): boolean { + if (patterns.length === 0) return true; + if (!run.model) return false; + const names = [`${run.providerKey}/${run.model}`, run.model]; + return patterns.some((pattern) => { + const regex = new RegExp(`^${pattern.split("*").map(escapeRegex).join(".*")}$`, "i"); + return names.some((name) => regex.test(name)); + }); +} + +export function runModelLabel(run: RunModel): string { + return run.model ? `${run.providerKey}/${run.model}` : `${run.providerKey} (no model set)`; +} + +/** One skill's prompt section, its declared risk and contract first. */ +export function skillPart(manifest: SkillManifest, version: string, body: string): string { + const header = [`# Skill: ${manifest.name}`, `Version: ${version}`, `Risk: ${manifest.riskLevel}`]; + const inputs = declaration(manifest.inputs); + const outputs = declaration(manifest.outputs); + if (inputs) header.push(`Expects as input: ${inputs}`); + if (outputs) header.push(`Must produce: ${outputs}`); + return [header.join("\n"), body].join("\n\n"); +} + +function declaration(value: unknown): string | null { + if (typeof value === "string") return value.trim() || null; + if (typeof value === "object" && value !== null && Object.keys(value).length > 0) return JSON.stringify(value); + return null; +} + +function escapeRegex(text: string): string { + return text.replace(/[.+?^${}()|[\]\\]/g, "\\$&"); +} diff --git a/src/main/services/skill-service.ts b/src/main/services/skill-service.ts index 50096de4..dc1b22aa 100644 --- a/src/main/services/skill-service.ts +++ b/src/main/services/skill-service.ts @@ -10,6 +10,7 @@ import { loadSkillManifest, type SkillManifest } from "./skill-manifest"; import { orderByRelevance } from "./relevance"; import { runManifestArtifactTest, verifyRevisionStructure, type SkillArtifactRow, type SkillTestRow } from "./skill-tests"; import { estimateTokens } from "./context-budget"; +import { modelCompatible, runModelLabel, skillPart } from "./skill-runtime"; function slugify(value: string): string { return value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, ""); @@ -280,14 +281,6 @@ export class SkillService { return []; } - private carriesRetiredSkill(revisionIds: string[]): boolean { - if (revisionIds.length === 0) return false; - return Boolean(this.db.prepare(` - select 1 from skill_revisions sr join skill_definitions sd on sd.id = sr.skill_id - where sd.status <> 'active' and sr.id in (${revisionIds.map(() => "?").join(", ")}) limit 1 - `).get(...revisionIds)); - } - private appLanguage(): string { const row = this.db.prepare("select language from app_state where id = 1").get() as { language: string } | undefined; return row?.language === "zh" ? "zh" : "en"; @@ -301,13 +294,6 @@ export class SkillService { * prompt instead of silently overflowing the model's context. */ resolveRuntimeSnapshot(input: ResolveAgentSkillsInput): string { - // Idempotent per run: a re-queued run reuses its frozen skill set unless a skill in it was retired since. - const existingSnapshot = this.db.prepare("select id, source_revision_ids from skill_runtime_snapshots where agent_id = ? and run_id = ?") - .get(input.agentId, input.runId) as { id: string; source_revision_ids: string } | undefined; - if (existingSnapshot) { - if (!this.carriesRetiredSkill(JSON.parse(existingSnapshot.source_revision_ids) as string[])) return existingSnapshot.id; - this.db.prepare("delete from skill_runtime_snapshots where id = ?").run(existingSnapshot.id); - } const forcedRows = this.forcedSkillRows(input.orgId); const boundRows = this.db.prepare(` select @@ -340,15 +326,16 @@ export class SkillService { const tools = new Set(); const revisionIds: string[] = []; const omitted: string[] = []; + const incompatible: string[] = []; let compiledTokens = 0; for (const row of rows) { if (!existsSync(row.object_key)) throw new Error(`Skill artifact is missing: ${row.object_key}`); const manifest = JSON.parse(row.manifest_json) as SkillManifest; - const part = [ - `# Skill: ${manifest.name}`, - `Version: ${row.version_semver}`, - readFileSync(row.object_key, "utf8").trim(), - ].join("\n\n"); + if (!modelCompatible(manifest.compatibleModels, input.runModel)) { + incompatible.push(`${manifest.name} (${manifest.compatibleModels.join(", ")})`); + continue; + } + const part = skillPart(manifest, row.version_semver, readFileSync(row.object_key, "utf8").trim()); const partTokens = estimateTokens(part); if (compiledTokens + partTokens > input.maxTokens) { omitted.push(manifest.name); @@ -359,12 +346,24 @@ export class SkillService { compiledParts.push(part); compiledTokens += partTokens; } + if (incompatible.length > 0) { + console.warn(`[skills] run ${input.runId}: skipped skills incompatible with ${runModelLabel(input.runModel)}: ${incompatible.join("; ")}`); + compiledParts.push( + `# Skills not loaded\n\nThese granted skills do not support this run's model (${runModelLabel(input.runModel)}): ${incompatible.join("; ")}.`, + ); + } if (omitted.length > 0) { compiledParts.push( `# Skills omitted for size\n\nThese granted skills did not fit this run's prompt budget: ${omitted.join(", ")}. Ask the CEO to trim grants if they are essential.`, ); } + const compiledPrompt = compiledParts.join("\n\n---\n\n"); + // Idempotent per run: a re-queued run keeps its snapshot while the resolved skill set is unchanged. + const existing = this.db.prepare("select id, compiled_prompt from skill_runtime_snapshots where agent_id = ? and run_id = ?") + .get(input.agentId, input.runId) as { id: string; compiled_prompt: string } | undefined; + if (existing?.compiled_prompt === compiledPrompt) return existing.id; + if (existing) this.db.prepare("delete from skill_runtime_snapshots where id = ?").run(existing.id); const snapshotId = createId("srs"); this.db.prepare(` insert into skill_runtime_snapshots( @@ -375,7 +374,7 @@ export class SkillService { snapshotId, input.agentId, input.runId, - compiledParts.join("\n\n---\n\n"), + compiledPrompt, JSON.stringify({ requiredTools: [...tools].sort() }), JSON.stringify(revisionIds), new Date().toISOString(), diff --git a/src/main/services/task-execution-service.ts b/src/main/services/task-execution-service.ts index 1a607c71..83853437 100644 --- a/src/main/services/task-execution-service.ts +++ b/src/main/services/task-execution-service.ts @@ -10,6 +10,7 @@ import { ChatService } from "./chat-service"; import { deliverChatReplyIfMissing } from "./chat-agent-wake-service"; import { SkillService } from "./skill-service"; import { runAgentLoop, runTools } from "./agent-run-loop"; +import type { RunModel } from "./skill-runtime"; import type { AgentToolContext } from "./agent-tools"; import { buildAgentInstructions, @@ -147,7 +148,7 @@ export class TaskExecutionService { let providerRequestId: string | null = null; let skillRuntimeSnapshot: SkillRuntimeSnapshotRow | null = null; try { - skillRuntimeSnapshot = this.resolveSkillRuntimeSnapshot(orgId, agent.id, runId, skillBudgetTokens(provider.context_window_tokens), `${task.title}\n${task.description}`); + skillRuntimeSnapshot = this.resolveSkillRuntimeSnapshot(orgId, agent.id, runId, skillBudgetTokens(provider.context_window_tokens), `${task.title}\n${task.description}`, { providerKey: provider.provider_key, model: provider.model }); // Watermark for reply delivery and mid-run message polling. const runStartedAt = new Date().toISOString(); const chatContext = loadChatWakeContext(this.db, orgId, agent.id, isolation.secretScopeJson); @@ -390,9 +391,9 @@ export class TaskExecutionService { `).run(TASK_RUN_STATE.FAILED, providerRequestId, new Date().toISOString(), errorMessage, runId, orgId); } - private resolveSkillRuntimeSnapshot(orgId: string, agentId: string, runId: string, maxTokens: number, taskText: string): SkillRuntimeSnapshotRow | null { + private resolveSkillRuntimeSnapshot(orgId: string, agentId: string, runId: string, maxTokens: number, taskText: string, runModel: RunModel): SkillRuntimeSnapshotRow | null { if (!this.skills.hasRuntimeSkills(orgId, agentId)) return null; - const snapshotId = this.skills.resolveRuntimeSnapshot({ orgId, agentId, runId, maxTokens, taskText }); + const snapshotId = this.skills.resolveRuntimeSnapshot({ orgId, agentId, runId, maxTokens, taskText, runModel }); const snapshot = this.db.prepare(` select id, compiled_prompt, tools_manifest, source_revision_ids from skill_runtime_snapshots diff --git a/src/shared/commands/skills.ts b/src/shared/commands/skills.ts index 16a20242..7acd7ce8 100644 --- a/src/shared/commands/skills.ts +++ b/src/shared/commands/skills.ts @@ -23,6 +23,8 @@ export interface ResolveAgentSkillsInput { maxTokens: number; /** Task title and description, to rank bound skills by relevance. */ taskText: string; + /** The run's provider and model, matched against each skill's compatible_models. */ + runModel: { providerKey: string; model: string | null }; } export interface RunSkillTestsInput { diff --git a/tests/integration/collaboration-flow.test.ts b/tests/integration/collaboration-flow.test.ts index 9cf8d667..571f9288 100644 --- a/tests/integration/collaboration-flow.test.ts +++ b/tests/integration/collaboration-flow.test.ts @@ -98,7 +98,7 @@ describe("collaboration integration flow", () => { revisionId = skills.importDirectory({ orgId, directoryPath: createSkillDirectory(), versionSemver: "1.0.0" }); skillId = (connection.db.prepare("select skill_id from skill_revisions where id = ?").get(revisionId) as { skill_id: string }).skill_id; skills.bind({ orgId, agentId, skillId, revisionId }); - skills.resolveRuntimeSnapshot({ orgId, agentId, runId: `run_${randomUUID().replaceAll("-", "")}`, maxTokens: 44_800, taskText: "" }); + skills.resolveRuntimeSnapshot({ orgId, agentId, runId: `run_${randomUUID().replaceAll("-", "")}`, maxTokens: 44_800, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" } }); suiteId = (connection.db.prepare("select id from eval_suites where org_id = ? and key = ?") .get(orgId, EVAL_SUITE_KEY.INTEGRATION) as { id: string }).id; evalRunId = sdlc.runEvalSuite({ orgId, suiteId }); diff --git a/tests/unit/builtin-skills.test.ts b/tests/unit/builtin-skills.test.ts index 5128373b..362b112e 100644 --- a/tests/unit/builtin-skills.test.ts +++ b/tests/unit/builtin-skills.test.ts @@ -171,7 +171,7 @@ describe("built-in skills", () => { h.skills.bind({ orgId: h.orgId, agentId, skillId: skill.id, revisionId: skill.revision_id }); }); const prompt = (runId: string) => { - const id = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId, maxTokens: SKILL_TOKENS, taskText: "" })); + const id = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId, maxTokens: SKILL_TOKENS, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" } })); return (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(id) as { compiled_prompt: string }).compiled_prompt; }; expect(prompt("run_before")).toContain("# Skill: Backend Architect"); @@ -265,7 +265,7 @@ describe("built-in skills", () => { expect(hired).toContain("Unknown skills skipped: no-such-skill"); const danaId = (h.connection.db.prepare("select id from agents where org_id = ? and name = 'Dana Lee'").get(h.orgId) as { id: string }).id; - const danaSnapshot = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId: danaId, runId: "run_dana", maxTokens: SKILL_TOKENS, taskText: "" })); + const danaSnapshot = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId: danaId, runId: "run_dana", maxTokens: SKILL_TOKENS, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" } })); const danaPrompt = (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(danaSnapshot) as { compiled_prompt: string }).compiled_prompt; expect(danaPrompt.indexOf("# Skill: humanizer")).toBe(0); expect(danaPrompt).toContain("# Skill: Backend Architect"); @@ -275,7 +275,7 @@ describe("built-in skills", () => { // The CEO carries humanizer too - forced by resolution, not by binding. expect(h.skills.hasRuntimeSkills(h.orgId, ceoId)).toBe(true); - const ceoSnapshot = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId: ceoId, runId: "run_ceo", maxTokens: SKILL_TOKENS, taskText: "" })); + const ceoSnapshot = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId: ceoId, runId: "run_ceo", maxTokens: SKILL_TOKENS, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" } })); const ceoPrompt = (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(ceoSnapshot) as { compiled_prompt: string }).compiled_prompt; expect(ceoPrompt).toContain("# Skill: humanizer"); expect(ceoPrompt).not.toContain("# Skill: Backend Architect"); @@ -300,7 +300,7 @@ describe("built-in skills", () => { }); }); const promptFor = (runId: string) => { - const snapshotId = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId, maxTokens: SKILL_TOKENS, taskText: "" })); + const snapshotId = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId, maxTokens: SKILL_TOKENS, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" } })); return (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(snapshotId) as { compiled_prompt: string }).compiled_prompt; }; @@ -346,7 +346,7 @@ describe("built-in skills", () => { modelProviderId: null, modelId: null, budgetCents: 1000, managerAgentId: null, }); }); - const snapshotId = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId: "run_tiny", maxTokens: 10, taskText: "" })); + const snapshotId = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId: "run_tiny", maxTokens: 10, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" } })); const prompt = (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(snapshotId) as { compiled_prompt: string }).compiled_prompt; expect(prompt).not.toContain("# Skill:"); expect(prompt).toContain("# Skills omitted for size"); @@ -373,7 +373,7 @@ describe("built-in skills", () => { bindSkillSlugs({ db: h.connection.db, organization: h.organization, chat: new ChatService(h.connection.db) }, h.orgId, agentId, ["marketing-growth-hacker", "engineering-backend-architect"]); }); const compile = (runId: string, maxTokens: number, taskText: string) => { - const snapshotId = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId, maxTokens, taskText })); + const snapshotId = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId, maxTokens, taskText, runModel: { providerKey: "openai", model: "gpt-5.4" } })); return (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(snapshotId) as { compiled_prompt: string }).compiled_prompt; }; const parts = compile("run_full", 1_000_000, "").split("\n\n---\n\n"); diff --git a/tests/unit/domain-services.test.ts b/tests/unit/domain-services.test.ts index 666cb81a..7c91d12e 100644 --- a/tests/unit/domain-services.test.ts +++ b/tests/unit/domain-services.test.ts @@ -186,7 +186,7 @@ describe("domain services", () => { skills.bind({ orgId, agentId, skillId: skill.id, revisionId }); rmSync(skillDir, { recursive: true, force: true }); skills.runTests({ orgId, revisionId }); - skills.resolveRuntimeSnapshot({ orgId, agentId, runId: "run-review-1", maxTokens: 44_800, taskText: "" }); + skills.resolveRuntimeSnapshot({ orgId, agentId, runId: "run-review-1", maxTokens: 44_800, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" } }); }); // Catalogue metadata ships in the snapshot; skill internals stay in the @@ -247,10 +247,10 @@ describe("domain services", () => { .get(orgA, "review-routes") as { id: string }).id; expect(() => skills.bind({ orgId: orgA, agentId: agentB, skillId, revisionId })).toThrow("Agent was not found."); skills.bind({ orgId: orgA, agentId: agentA, skillId, revisionId }); - const firstResolve = skills.resolveRuntimeSnapshot({ orgId: orgA, agentId: agentA, runId: "run-review-1", maxTokens: 44_800, taskText: "" }); + const firstResolve = skills.resolveRuntimeSnapshot({ orgId: orgA, agentId: agentA, runId: "run-review-1", maxTokens: 44_800, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" } }); // Idempotent per run: a recovered run reuses its frozen snapshot // instead of failing on the uniqueness guard. - expect(skills.resolveRuntimeSnapshot({ orgId: orgA, agentId: agentA, runId: "run-review-1", maxTokens: 44_800, taskText: "" })).toBe(firstResolve); + expect(skills.resolveRuntimeSnapshot({ orgId: orgA, agentId: agentA, runId: "run-review-1", maxTokens: 44_800, taskText: "", runModel: { providerKey: "openai", model: "gpt-5.4" } })).toBe(firstResolve); expect(connection.db.prepare("select count(*) as n from skill_runtime_snapshots where run_id = 'run-review-1'").get()) .toEqual({ n: 1 }); }); diff --git a/tests/unit/skill-compatibility.test.ts b/tests/unit/skill-compatibility.test.ts new file mode 100644 index 00000000..3ccd9844 --- /dev/null +++ b/tests/unit/skill-compatibility.test.ts @@ -0,0 +1,64 @@ +import { randomUUID } from "node:crypto"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, test } from "vitest"; +import { DatabaseConnection } from "../../src/main/database/connection"; +import { OrganizationService } from "../../src/main/services/organization-service"; +import { SkillService } from "../../src/main/services/skill-service"; +import { modelCompatible } from "../../src/main/services/skill-runtime"; + +describe("skill compatibility", () => { + test("model patterns match provider/model or the bare model name", () => { + expect(modelCompatible([], { providerKey: "openai", model: null })).toBe(true); + expect(modelCompatible(["openai/*"], { providerKey: "openai", model: "gpt-5.4" })).toBe(true); + expect(modelCompatible(["gpt-5.4"], { providerKey: "openai", model: "gpt-5.4" })).toBe(true); + expect(modelCompatible(["gpt-5.?"], { providerKey: "openai", model: "gpt-5.4" })).toBe(false); + expect(modelCompatible(["anthropic/*"], { providerKey: "openai", model: "gpt-5.4" })).toBe(false); + expect(modelCompatible(["openai/*"], { providerKey: "openai", model: null })).toBe(false); + }); + + test("a run loads only skills compatible with its model and states each skill's contract", () => { + const connection = new DatabaseConnection(join(tmpdir(), `ac-compat-${randomUUID()}.db`)); + const organization = new OrganizationService(connection.db); + const skills = new SkillService(connection.db, join(tmpdir(), `ac-compat-art-${randomUUID()}`)); + const skillDir = join(tmpdir(), `ac-compat-skill-${randomUUID()}`); + mkdirSync(skillDir, { recursive: true }); + writeFileSync(join(skillDir, "SKILL.md"), "# Claude Review\n\nReview with care.\n"); + writeFileSync(join(skillDir, "skill.manifest.json"), JSON.stringify({ + name: "claude-review", version: "1.0.0", entrypoint: "SKILL.md", capabilities: [], required_tools: [], + compatible_models: ["anthropic/*"], risk_level: "high", inputs: "A patch diff", outputs: { verdict: "markdown" }, tests: [], + })); + let orgId = ""; + let agentId = ""; + connection.transaction(() => { + orgId = organization.createOrganization({ organizationName: "C", employeeName: "Y", mission: "m", goal: "g" }); + agentId = organization.createAgent({ + orgId, name: "Ada", role: "Reviewer", department: "Engineering", + modelProviderId: null, modelId: null, budgetCents: 1000, managerAgentId: null, + }); + const revisionId = skills.importDirectory({ orgId, directoryPath: skillDir, versionSemver: "1.0.0" }); + const skill = connection.db.prepare("select id from skill_definitions where org_id = ? and slug = 'claude-review'").get(orgId) as { id: string }; + skills.bind({ orgId, agentId, skillId: skill.id, revisionId }); + }); + const prompt = (runId: string, providerKey: string, model: string) => { + const id = connection.transaction(() => skills.resolveRuntimeSnapshot({ orgId, agentId, runId, maxTokens: 40_000, taskText: "", runModel: { providerKey, model } })); + return connection.db.prepare("select compiled_prompt, source_revision_ids from skill_runtime_snapshots where id = ?").get(id) as { compiled_prompt: string; source_revision_ids: string }; + }; + + const openai = prompt("run_openai", "openai", "gpt-5.4"); + expect(openai.compiled_prompt).not.toContain("# Skill: claude-review"); + expect(openai.compiled_prompt).toContain("# Skills not loaded\n\nThese granted skills do not support this run's model (openai/gpt-5.4): claude-review (anthropic/*)."); + expect(JSON.parse(openai.source_revision_ids)).toEqual([]); + + const anthropic = prompt("run_anthropic", "anthropic", "claude-sonnet-5-5"); + expect(anthropic.compiled_prompt).toContain([ + "# Skill: claude-review", "Version: 1.0.0", "Risk: high", + "Expects as input: A patch diff", 'Must produce: {"verdict":"markdown"}', + ].join("\n")); + const recovered = prompt("run_anthropic", "openai", "gpt-5.4"); + expect(recovered.compiled_prompt).not.toContain("# Skill: claude-review"); + expect(recovered.compiled_prompt).toContain("# Skills not loaded"); + connection.close(); + }); +});