Q19: pack the skills most relevant to the task first

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/main/services/skill-relevance.ts

@@ -0,0 +1,25 @@
+const LATIN_WORD = /[a-z0-9]{3,}/g;
+const CJK_RUN = /[\u3400-\u9fff]+/g;
+
+/** Words (latin, 3+ chars) and character pairs (CJK) of a text. */
+function terms(text: string): Set<string> {
+  const lower = text.toLowerCase();
+  const found = new Set(lower.match(LATIN_WORD) ?? []);
+  for (const run of lower.match(CJK_RUN) ?? []) {
+    for (let index = 0; index < run.length - 1; index += 1) found.add(run.slice(index, index + 2));
+  }
+  return found;
+}
+
+/** Orders items by how many terms they share with the task; ties keep their order. */
+export function orderByRelevance<T>(items: T[], describe: (item: T) => string, taskText: string): T[] {
+  const taskTerms = terms(taskText);
+  return items
+    .map((item, index) => {
+      let score = 0;
+      for (const term of terms(describe(item))) if (taskTerms.has(term)) score += 1;
+      return { item, index, score };
+    })
+    .sort((a, b) => b.score - a.score || a.index - b.index)
+    .map((entry) => entry.item);
+}

src/main/services/skill-service.ts

@@ -1,7 +1,4 @@
-import { estimateTokens } from "./context-budget";
 import { existsSync, readFileSync } from "node:fs";
-import { createHash } from "node:crypto";
-import { normalize } from "node:path";
 import type { DatabaseSync } from "node:sqlite";
 import type { BindSkillInput, ImportSkillDirectoryInput, ResolveAgentSkillsInput, RunSkillTestsInput } from "../../shared/commands";
 import { SLO_SIGNAL_KEY } from "../../shared/domain";
@@ -10,6 +7,9 @@ import { AuditService } from "./audit-service";
 import { OperationsService } from "./operations-service";
 import { copySkillRevision, recursiveSkillChecksum } from "./skill-artifacts";
 import { loadSkillManifest, type SkillManifest } from "./skill-manifest";
+import { orderByRelevance } from "./skill-relevance";
+import { runManifestArtifactTest, verifyRevisionStructure, type SkillArtifactRow, type SkillTestRow } from "./skill-tests";
+import { estimateTokens } from "./context-budget";
 
 function slugify(value: string): string {
   return value.trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
@@ -236,7 +236,7 @@ export class SkillService {
   /**
    * Freezes the exact skill set this run carries: forced house-style skills
    * first (identical for everyone, CEO included), then the agent's enabled
-   * bindings in priority order, deduplicated per skill. Skills pack whole
+   * bindings, most relevant to the task first, deduplicated per skill. Skills pack whole
    * until input.maxTokens; whatever does not fit is named in the compiled
    * prompt instead of silently overflowing the model's context.
    */
@@ -263,7 +263,11 @@ export class SkillService {
     `).all(input.orgId, input.agentId) as unknown as RuntimeSkillRow[];
     const rows: RuntimeSkillRow[] = [];
     const seenSkillIds = new Set<string>();
-    for (const row of [...forcedRows, ...boundRows]) {
+    const relevantBound = orderByRelevance(boundRows, (row) => {
+      const manifest = JSON.parse(row.manifest_json) as SkillManifest;
+      return `${manifest.name} ${manifest.description ?? ""}`;
+    }, input.taskText);
+    for (const row of [...forcedRows, ...relevantBound]) {
       if (seenSkillIds.has(row.skill_id)) continue;
       seenSkillIds.add(row.skill_id);
       rows.push(row);
@@ -414,97 +418,3 @@ interface SkillRevisionTestRow {
   version_semver: string;
   slug: string;
 }
-
-interface SkillTestRow {
-  id: string;
-  test_type: string;
-  expected_output_ref: string;
-}
-
-interface SkillArtifactRow {
-  artifact_type: string;
-  object_key: string;
-  sha256: string;
-  size_bytes: number;
-}
-
-interface SkillTestResult {
-  status: "passed" | "failed";
-  testType: string;
-  expectedOutputRef: string;
-  checks: Array<{ name: string; status: "passed" | "failed"; detail: string }>;
-  artifactSha256?: string;
-  failure?: string;
-}
-
-function verifyRevisionStructure(artifacts: SkillArtifactRow[]): SkillTestResult | null {
-  const requiredTypes = ["manifest", "entrypoint"];
-  for (const type of requiredTypes) {
-    if (!artifacts.some((artifact) => artifact.artifact_type === type)) {
-      return failedSkillTest("revision_structure", type, `Skill revision artifact is missing: ${type}`);
-    }
-  }
-  for (const artifact of artifacts) {
-    try {
-      verifyArtifactIntegrity(artifact);
-    } catch (error) {
-      return failedSkillTest("artifact_integrity", artifact.object_key, error instanceof Error ? error.message : String(error));
-    }
-  }
-  return null;
-}
-
-function runManifestArtifactTest(test: SkillTestRow, artifacts: SkillArtifactRow[]): SkillTestResult {
-  if (test.test_type !== "manifest") {
-    return failedSkillTest(test.test_type, test.expected_output_ref, `Unsupported skill test type: ${test.test_type}`);
-  }
-  const testArtifact = artifacts.find((artifact) => (
-    artifact.artifact_type === "test" && artifactMatchesExpectedRef(artifact.object_key, test.expected_output_ref)
-  ));
-  if (!testArtifact) return failedSkillTest(test.test_type, test.expected_output_ref, "Skill test artifact was not found.");
-
-  try {
-    const content = verifyArtifactIntegrity(testArtifact).toString("utf8");
-    if (!content.trim()) throw new Error("Skill test artifact is empty.");
-    return {
-      status: "passed",
-      testType: test.test_type,
-      expectedOutputRef: test.expected_output_ref,
-      artifactSha256: testArtifact.sha256,
-      checks: [
-        { name: "manifest_test_artifact", status: "passed", detail: "Artifact exists, checksum matches, and content is non-empty." },
-      ],
-    };
-  } catch (error) {
-    return failedSkillTest(test.test_type, test.expected_output_ref, error instanceof Error ? error.message : String(error));
-  }
-}
-
-function verifyArtifactIntegrity(artifact: SkillArtifactRow): Buffer {
-  if (!existsSync(artifact.object_key)) throw new Error(`Skill artifact is missing: ${artifact.object_key}`);
-  const body = readFileSync(artifact.object_key);
-  const sha256 = createHash("sha256").update(body).digest("hex");
-  if (sha256 !== artifact.sha256) throw new Error(`Skill artifact checksum mismatch: ${artifact.object_key}`);
-  if (body.byteLength !== artifact.size_bytes) throw new Error(`Skill artifact size mismatch: ${artifact.object_key}`);
-  return body;
-}
-
-function artifactMatchesExpectedRef(objectKey: string, expectedRef: string): boolean {
-  const expected = normalizeSkillPath(expectedRef);
-  const actual = normalizeSkillPath(objectKey);
-  return actual === expected || actual.endsWith(`/${expected}`);
-}
-
-function normalizeSkillPath(value: string): string {
-  return normalize(value).replaceAll("\\", "/");
-}
-
-function failedSkillTest(testType: string, expectedOutputRef: string, failure: string): SkillTestResult {
-  return {
-    status: "failed",
-    testType,
-    expectedOutputRef,
-    checks: [{ name: "skill_test", status: "failed", detail: failure }],
-    failure,
-  };
-}

src/main/services/skill-tests.ts

@@ -0,0 +1,97 @@
+import { createHash } from "node:crypto";
+import { existsSync, readFileSync } from "node:fs";
+import { normalize } from "node:path";
+
+export interface SkillTestRow {
+  id: string;
+  test_type: string;
+  expected_output_ref: string;
+}
+
+export interface SkillArtifactRow {
+  artifact_type: string;
+  object_key: string;
+  sha256: string;
+  size_bytes: number;
+}
+
+export interface SkillTestResult {
+  status: "passed" | "failed";
+  testType: string;
+  expectedOutputRef: string;
+  checks: Array<{ name: string; status: "passed" | "failed"; detail: string }>;
+  artifactSha256?: string;
+  failure?: string;
+}
+
+export function verifyRevisionStructure(artifacts: SkillArtifactRow[]): SkillTestResult | null {
+  const requiredTypes = ["manifest", "entrypoint"];
+  for (const type of requiredTypes) {
+    if (!artifacts.some((artifact) => artifact.artifact_type === type)) {
+      return failedSkillTest("revision_structure", type, `Skill revision artifact is missing: ${type}`);
+    }
+  }
+  for (const artifact of artifacts) {
+    try {
+      verifyArtifactIntegrity(artifact);
+    } catch (error) {
+      return failedSkillTest("artifact_integrity", artifact.object_key, error instanceof Error ? error.message : String(error));
+    }
+  }
+  return null;
+}
+
+export function runManifestArtifactTest(test: SkillTestRow, artifacts: SkillArtifactRow[]): SkillTestResult {
+  if (test.test_type !== "manifest") {
+    return failedSkillTest(test.test_type, test.expected_output_ref, `Unsupported skill test type: ${test.test_type}`);
+  }
+  const testArtifact = artifacts.find((artifact) => (
+    artifact.artifact_type === "test" && artifactMatchesExpectedRef(artifact.object_key, test.expected_output_ref)
+  ));
+  if (!testArtifact) return failedSkillTest(test.test_type, test.expected_output_ref, "Skill test artifact was not found.");
+
+  try {
+    const content = verifyArtifactIntegrity(testArtifact).toString("utf8");
+    if (!content.trim()) throw new Error("Skill test artifact is empty.");
+    return {
+      status: "passed",
+      testType: test.test_type,
+      expectedOutputRef: test.expected_output_ref,
+      artifactSha256: testArtifact.sha256,
+      checks: [
+        { name: "manifest_test_artifact", status: "passed", detail: "Artifact exists, checksum matches, and content is non-empty." },
+      ],
+    };
+  } catch (error) {
+    return failedSkillTest(test.test_type, test.expected_output_ref, error instanceof Error ? error.message : String(error));
+  }
+}
+
+function verifyArtifactIntegrity(artifact: SkillArtifactRow): Buffer {
+  if (!existsSync(artifact.object_key)) throw new Error(`Skill artifact is missing: ${artifact.object_key}`);
+  const body = readFileSync(artifact.object_key);
+  const sha256 = createHash("sha256").update(body).digest("hex");
+  if (sha256 !== artifact.sha256) throw new Error(`Skill artifact checksum mismatch: ${artifact.object_key}`);
+  if (body.byteLength !== artifact.size_bytes) throw new Error(`Skill artifact size mismatch: ${artifact.object_key}`);
+  return body;
+}
+
+function artifactMatchesExpectedRef(objectKey: string, expectedRef: string): boolean {
+  const expected = normalizeSkillPath(expectedRef);
+  const actual = normalizeSkillPath(objectKey);
+  return actual === expected || actual.endsWith(`/${expected}`);
+}
+
+function normalizeSkillPath(value: string): string {
+  return normalize(value).replaceAll("\\", "/");
+}
+
+function failedSkillTest(testType: string, expectedOutputRef: string, failure: string): SkillTestResult {
+  return {
+    status: "failed",
+    testType,
+    expectedOutputRef,
+    checks: [{ name: "skill_test", status: "failed", detail: failure }],
+    failure,
+  };
+}

src/main/services/task-execution-service.ts

@@ -156,7 +156,7 @@ export class TaskExecutionService {
     let providerRequestId: string | null = null;
     let skillRuntimeSnapshot: SkillRuntimeSnapshotRow | null = null;
     try {
-      skillRuntimeSnapshot = this.resolveSkillRuntimeSnapshot(orgId, agent.id, runId, skillBudgetTokens(provider.context_window_tokens));
+      skillRuntimeSnapshot = this.resolveSkillRuntimeSnapshot(orgId, agent.id, runId, skillBudgetTokens(provider.context_window_tokens), `${task.title}\n${task.description}`);
       // Watermark for reply delivery and mid-run message polling.
       const runStartedAt = new Date().toISOString();
       const chatContext = loadChatWakeContext(this.db, orgId, agent.id, isolation.secretScopeJson);
@@ -616,9 +616,9 @@ export class TaskExecutionService {
     `).run(TASK_RUN_STATE.FAILED, providerRequestId, new Date().toISOString(), errorMessage, runId, orgId);
   }
 
-  private resolveSkillRuntimeSnapshot(orgId: string, agentId: string, runId: string, maxTokens: number): SkillRuntimeSnapshotRow | null {
+  private resolveSkillRuntimeSnapshot(orgId: string, agentId: string, runId: string, maxTokens: number, taskText: string): SkillRuntimeSnapshotRow | null {
     if (!this.skills.hasRuntimeSkills(orgId, agentId)) return null;
-    const snapshotId = this.skills.resolveRuntimeSnapshot({ orgId, agentId, runId, maxTokens });
+    const snapshotId = this.skills.resolveRuntimeSnapshot({ orgId, agentId, runId, maxTokens, taskText });
     const snapshot = this.db.prepare(`
       select id, compiled_prompt, tools_manifest, source_revision_ids
       from skill_runtime_snapshots

src/shared/commands/skills.ts

@@ -21,6 +21,8 @@ export interface ResolveAgentSkillsInput {
   runId: string;
   /** Token budget for the compiled skills. */
   maxTokens: number;
+  /** Task title and description, to rank bound skills by relevance. */
+  taskText: string;
 }
 
 export interface RunSkillTestsInput {

tests/integration/collaboration-flow.test.ts

@@ -98,7 +98,7 @@ describe("collaboration integration flow", () => {
       revisionId = skills.importDirectory({ orgId, directoryPath: createSkillDirectory(), versionSemver: "1.0.0" });
       skillId = (connection.db.prepare("select skill_id from skill_revisions where id = ?").get(revisionId) as { skill_id: string }).skill_id;
       skills.bind({ orgId, agentId, skillId, revisionId });
-      skills.resolveRuntimeSnapshot({ orgId, agentId, runId: `run_${randomUUID().replaceAll("-", "")}`, maxTokens: 44_800 });
+      skills.resolveRuntimeSnapshot({ orgId, agentId, runId: `run_${randomUUID().replaceAll("-", "")}`, maxTokens: 44_800, taskText: "" });
       suiteId = (connection.db.prepare("select id from eval_suites where org_id = ? and key = ?")
         .get(orgId, EVAL_SUITE_KEY.INTEGRATION) as { id: string }).id;
       evalRunId = sdlc.runEvalSuite({ orgId, suiteId });

tests/unit/builtin-skills.test.ts

@@ -7,11 +7,12 @@ import { DatabaseConnection } from "../../src/main/database/connection";
 import { OrganizationService } from "../../src/main/services/organization-service";
 import { ChatService } from "../../src/main/services/chat-service";
 import { SkillService } from "../../src/main/services/skill-service";
-import { skillBudgetTokens } from "../../src/main/services/context-budget";
+import { estimateTokens, skillBudgetTokens } from "../../src/main/services/context-budget";
 
 const SKILL_TOKENS = skillBudgetTokens(128_000);
 import { importStagedSkills, parseFrontmatter, stageBuiltinSkillsLibrary } from "../../src/main/services/builtin-skills";
 import { dispatchAgentTool, type AgentToolContext } from "../../src/main/services/agent-tools";
+import { bindSkillSlugs } from "../../src/main/services/agent-skill-tools";
 
 const HUMANIZER_MD = `---
 name: humanizer
@@ -178,7 +179,7 @@ describe("built-in skills", () => {
     expect(hired).toContain("Unknown skills skipped: no-such-skill");
 
     const danaId = (h.connection.db.prepare("select id from agents where org_id = ? and name = 'Dana Lee'").get(h.orgId) as { id: string }).id;
-    const danaSnapshot = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId: danaId, runId: "run_dana", maxTokens: SKILL_TOKENS }));
+    const danaSnapshot = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId: danaId, runId: "run_dana", maxTokens: SKILL_TOKENS, taskText: "" }));
     const danaPrompt = (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(danaSnapshot) as { compiled_prompt: string }).compiled_prompt;
     expect(danaPrompt.indexOf("# Skill: humanizer")).toBe(0);
     expect(danaPrompt).toContain("# Skill: Backend Architect");
@@ -188,7 +189,7 @@ describe("built-in skills", () => {
 
     // The CEO carries humanizer too - forced by resolution, not by binding.
     expect(h.skills.hasRuntimeSkills(h.orgId, ceoId)).toBe(true);
-    const ceoSnapshot = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId: ceoId, runId: "run_ceo", maxTokens: SKILL_TOKENS }));
+    const ceoSnapshot = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId: ceoId, runId: "run_ceo", maxTokens: SKILL_TOKENS, taskText: "" }));
     const ceoPrompt = (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(ceoSnapshot) as { compiled_prompt: string }).compiled_prompt;
     expect(ceoPrompt).toContain("# Skill: humanizer");
     expect(ceoPrompt).not.toContain("# Skill: Backend Architect");
@@ -213,7 +214,7 @@ describe("built-in skills", () => {
       });
     });
     const promptFor = (runId: string) => {
-      const snapshotId = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId, maxTokens: SKILL_TOKENS }));
+      const snapshotId = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId, maxTokens: SKILL_TOKENS, taskText: "" }));
       return (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(snapshotId) as { compiled_prompt: string }).compiled_prompt;
     };
 
@@ -259,11 +260,48 @@ describe("built-in skills", () => {
         modelProviderId: null, modelId: null, budgetCents: 1000, managerAgentId: null,
       });
     });
-    const snapshotId = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId: "run_tiny", maxTokens: 10 }));
+    const snapshotId = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId: "run_tiny", maxTokens: 10, taskText: "" }));
     const prompt = (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(snapshotId) as { compiled_prompt: string }).compiled_prompt;
     expect(prompt).not.toContain("# Skill:");
     expect(prompt).toContain("# Skills omitted for size");
     expect(prompt).toContain("humanizer");
     h.connection.close();
   });
+
+  test("under a tight budget the skills relevant to the task win over binding order", () => {
+    const h = harness();
+    mkdirSync(join(h.sourceRoot, "marketing"), { recursive: true });
+    writeFileSync(
+      join(h.sourceRoot, "marketing", "marketing-growth-hacker.md"),
+      `---\nname: Growth Hacker\ndescription: Viral loops and paid acquisition campaigns.\n---\n\n${"Run growth experiments. ".repeat(40)}\n`,
+      "utf8",
+    );
+    const staged = stageBuiltinSkillsLibrary(h.sourceRoot, h.libraryRoot);
+    h.connection.transaction(() => importStagedSkills(h.skills, h.orgId, staged));
+    let agentId = "";
+    h.connection.transaction(() => {
+      agentId = h.organization.createAgent({
+        orgId: h.orgId, name: "Ada", role: "Engineer", department: "Engineering",
+        modelProviderId: null, modelId: null, budgetCents: 1000, managerAgentId: null,
+      });
+      bindSkillSlugs({ db: h.connection.db, organization: h.organization, chat: new ChatService(h.connection.db) }, h.orgId, agentId, ["marketing-growth-hacker", "engineering-backend-architect"]);
+    });
+    const compile = (runId: string, maxTokens: number, taskText: string) => {
+      const snapshotId = h.connection.transaction(() => h.skills.resolveRuntimeSnapshot({ orgId: h.orgId, agentId, runId, maxTokens, taskText }));
+      return (h.connection.db.prepare("select compiled_prompt from skill_runtime_snapshots where id = ?").get(snapshotId) as { compiled_prompt: string }).compiled_prompt;
+    };
+    const parts = compile("run_full", 1_000_000, "").split("\n\n---\n\n");
+    expect(parts.map((part) => part.split("\n")[0])).toEqual(["# Skill: humanizer", "# Skill: Backend Architect", "# Skill: Growth Hacker"]);
+    const tight = estimateTokens(parts[0]) + Math.max(estimateTokens(parts[1]), estimateTokens(parts[2])) + 2;
+
+    const backendTask = compile("run_backend", tight, "Design the scalable backend system for orders");
+    expect(backendTask).toContain("# Skill: Backend Architect");
+    expect(backendTask).not.toContain("# Skill: Growth Hacker");
+    expect(backendTask).toContain("did not fit this run's prompt budget: Growth Hacker");
+
+    const growthTask = compile("run_growth", tight, "Plan paid acquisition campaigns");
+    expect(growthTask).toContain("# Skill: Growth Hacker");
+    expect(growthTask).not.toContain("# Skill: Backend Architect");
+    h.connection.close();
+  });
 });

tests/unit/domain-services.test.ts

@@ -186,7 +186,7 @@ describe("domain services", () => {
       skills.bind({ orgId, agentId, skillId: skill.id, revisionId });
       rmSync(skillDir, { recursive: true, force: true });
       skills.runTests({ orgId, revisionId });
-      skills.resolveRuntimeSnapshot({ orgId, agentId, runId: "run-review-1", maxTokens: 44_800 });
+      skills.resolveRuntimeSnapshot({ orgId, agentId, runId: "run-review-1", maxTokens: 44_800, taskText: "" });
     });
 
     // Catalogue metadata ships in the snapshot; skill internals stay in the
@@ -247,10 +247,10 @@ describe("domain services", () => {
         .get(orgA, "review-routes") as { id: string }).id;
       expect(() => skills.bind({ orgId: orgA, agentId: agentB, skillId, revisionId })).toThrow("Agent was not found.");
       skills.bind({ orgId: orgA, agentId: agentA, skillId, revisionId });
-      const firstResolve = skills.resolveRuntimeSnapshot({ orgId: orgA, agentId: agentA, runId: "run-review-1", maxTokens: 44_800 });
+      const firstResolve = skills.resolveRuntimeSnapshot({ orgId: orgA, agentId: agentA, runId: "run-review-1", maxTokens: 44_800, taskText: "" });
       // Idempotent per run: a recovered run reuses its frozen snapshot
       // instead of failing on the uniqueness guard.
-      expect(skills.resolveRuntimeSnapshot({ orgId: orgA, agentId: agentA, runId: "run-review-1", maxTokens: 44_800 })).toBe(firstResolve);
+      expect(skills.resolveRuntimeSnapshot({ orgId: orgA, agentId: agentA, runId: "run-review-1", maxTokens: 44_800, taskText: "" })).toBe(firstResolve);
       expect(connection.db.prepare("select count(*) as n from skill_runtime_snapshots where run_id = 'run-review-1'").get())
         .toEqual({ n: 1 });
     });

tests/unit/skill-relevance.test.ts

@@ -0,0 +1,11 @@
+import { describe, expect, test } from "vitest";
+import { orderByRelevance } from "../../src/main/services/skill-relevance";
+
+describe("skill relevance", () => {
+  test("ranks by shared words, keeps order on ties, and matches Chinese by character pairs", () => {
+    const skills = ["Growth Hacker: viral campaigns", "Backend Architect: scalable system design", "财务分析：预算与现金流"];
+    expect(orderByRelevance(skills, (item) => item, "Design a scalable backend system")).toEqual([skills[1], skills[0], skills[2]]);
+    expect(orderByRelevance(skills, (item) => item, "整理下季度的现金流预算")).toEqual([skills[2], skills[0], skills[1]]);
+    expect(orderByRelevance(skills, (item) => item, "")).toEqual(skills);
+  });
+});