fix(dispatch): fail explicitly when no enabled model qualifies

`pickModelForAgent` used to fall back to the CEO's pair when the
auto-strategy scoring returned no winner, and then silently route
the run on that pair. Three distinct problems rendered invisible by
that fallback: a missing capability row for a deployed model, a
mis-inferred task profile, a misconfigured model pool. Cost
attribution became meaningless — the agent ran on the CEO's model,
not its own tier — and operators had no signal to fix the root
cause.

The picker now returns a `ModelPickResult` tagged union. Success
yields the dispatch-ready assignment; failure yields a
`NO_QUALIFYING_CANDIDATE` reason with a human-readable message
naming the agent, the required domain, and the required tier.

Heartbeat maps the failure to a new `model_pick_failed` skip
reason, logs an activity row so it shows up in the company audit
trail, and defers. Task-runner returns `fail("MODEL_PICK_FAILED",
...)` so the UI surfaces the message directly. `getCeoAssignment`
drops out of `PickModelDeps` entirely — no caller needs a silent
CEO path any more, so the dep surface shrinks and the tests do
too. The test suite gains a regression assertion for the explicit
failure case.
This commit is contained in:
ZacharyZhang-NY committed 2026-04-19 01:06:50 -04:00
1 parent 76c2f085b6
commit cdf783c0b8
4 files changed
+111 -70

No files matched your search

+41 -18
View File
@@ -78,7 +78,6 @@ function makeConnector(id: string): ConnectorRecord {
function makeDeps(options: {
enabledModels: Array<{ connectorId: string; provider: string; modelId: string }>;
capabilities: ModelCapability[];
ceoAssignment?: { connectorId: string; modelId: string | null } | null;
}): PickModelDeps {
const connectorsById = new Map<string, ConnectorRecord>();
for (const entry of options.enabledModels) {
@@ -93,13 +92,6 @@ function makeDeps(options: {
return c;
},
getDisabledModels: () => [],
getCeoAssignment: () =>
options.ceoAssignment
? {
connectorId: options.ceoAssignment.connectorId as ConnectorId,
modelId: options.ceoAssignment.modelId,
}
: null,
listModelCapabilities: () => options.capabilities,
readConnectors: () => Array.from(connectorsById.values()),
};
@@ -155,8 +147,11 @@ describe("pickModelForAgent", () => {
],
capabilities: [claudeOpus],
});
const assignment = pickModelForAgent(agent, deps);
expect(assignment).toEqual({ connectorId: "anthropic-api", modelId: "claude-opus-4-7" });
const pick = pickModelForAgent(agent, deps);
expect(pick.ok).toBe(true);
if (pick.ok) {
expect(pick.assignment).toEqual({ connectorId: "anthropic-api", modelId: "claude-opus-4-7" });
}
});
it("CEO role always takes the user-picked pair regardless of modelStrategy", () => {
@@ -164,6 +159,7 @@ describe("pickModelForAgent", () => {
modelStrategy: "auto",
roleLevel: "c_suite",
role: "CEO",
reportsTo: "some-parent",
connectorId: "anthropic-api" as ConnectorId,
model: "claude-opus-4-7",
});
@@ -174,15 +170,12 @@ describe("pickModelForAgent", () => {
],
capabilities: [claudeOpus, llama],
});
const assignment = pickModelForAgent(agent, deps);
expect(assignment.modelId).toBe("claude-opus-4-7");
const pick = pickModelForAgent(agent, deps);
expect(pick.ok).toBe(true);
if (pick.ok) expect(pick.assignment.modelId).toBe("claude-opus-4-7");
});
it("CEO is identified by reportsTo === null even when roleLevel stays at the schema default 'ic'", () => {
// Regression test for the v39 migration gap: CEO agents created before
// the explicit role_level hookup have roleLevel = 'ic' in the DB. The
// dispatcher must still recognise them via the DB invariant used by
// getCeoForCompany: reports_to IS NULL.
const agent = makeAgent({
modelStrategy: "auto",
roleLevel: "ic",
@@ -198,8 +191,38 @@ describe("pickModelForAgent", () => {
],
capabilities: [claudeOpus, llama],
});
const assignment = pickModelForAgent(agent, deps);
expect(assignment.modelId).toBe("claude-opus-4-7");
const pick = pickModelForAgent(agent, deps);
expect(pick.ok).toBe(true);
if (pick.ok) expect(pick.assignment.modelId).toBe("claude-opus-4-7");
});
it("auto strategy with no qualifying candidate fails explicitly (no silent CEO fallback)", () => {
const agent = makeAgent({
modelStrategy: "auto",
roleLevel: "ic",
reportsTo: "manager-1",
role: "Senior Architect",
title: "Principal Architect",
connectorId: "fireworks" as ConnectorId,
model: "llama-70b",
});
// Two enabled models, both scoring too low for a hard reasoning agent.
const deps = makeDeps({
enabledModels: [
{ connectorId: "fireworks", provider: "fireworks", modelId: "llama-70b" },
{ connectorId: "fireworks-b", provider: "fireworks", modelId: "llama-7b" },
],
capabilities: [
llama,
{ provider: "fireworks", modelId: "llama-7b", reasoning: 2, coding: 2, writing: 2, tier: "trivial", costPerMtokOut: 0.5, updatedAt: "" },
],
});
const pick = pickModelForAgent(agent, deps);
expect(pick.ok).toBe(false);
if (!pick.ok) {
expect(pick.reason).toBe("NO_QUALIFYING_CANDIDATE");
expect(pick.message).toContain(agent.name);
}
});
});