fix(runtime): serialize Fire Pass calls per credential to stop boss-build 429s
Boss build seeds one Fire Pass key for every agent. Three concurrent agents -> three concurrent streamSimple calls against the Kimi K2.6 Turbo router -> three HTTP 429s on the overview dashboard. Add an in-memory async semaphore keyed by (provider, sha1(apiKey)) and acquire a slot inside `createSafePiStream` before invoking the source. The slot is released when the stream finishes, errors, or aborts. `fireworks-firepass` is capped at 1 concurrent call; every other provider stays uncapped (no fallback layer, no 429 retry — flow control prevents the burst at the source instead of recovering after). Same code path applies to public and boss builds: only the seeded key differs, the gate behaviour is identical.
This commit is contained in:
1 parent
a532120649
commit
fccb0d8586
6 files changed
+421
-7
No files matched your search
@@ -0,0 +1,106 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { InMemoryCredentialConcurrencyGate } from "../../src/main/worker/runtime/credential-concurrency-gate";
|
||||
|
||||
function makeGate(limit: number | null) {
|
||||
return new InMemoryCredentialConcurrencyGate(() => limit);
|
||||
}
|
||||
|
||||
describe("InMemoryCredentialConcurrencyGate", () => {
|
||||
it("returns a no-op release when the provider is uncapped", async () => {
|
||||
const gate = makeGate(null);
|
||||
const release = await gate.acquire("anthropic", "key-x");
|
||||
expect(typeof release).toBe("function");
|
||||
release();
|
||||
release();
|
||||
});
|
||||
|
||||
it("hands out slots up to the limit immediately", async () => {
|
||||
const gate = makeGate(2);
|
||||
const r1 = await gate.acquire("fireworks-firepass", "shared");
|
||||
const r2 = await gate.acquire("fireworks-firepass", "shared");
|
||||
expect(r1).not.toBe(r2);
|
||||
r1();
|
||||
r2();
|
||||
});
|
||||
|
||||
it("queues the third caller until a slot is released", async () => {
|
||||
const gate = makeGate(1);
|
||||
const release = await gate.acquire("fireworks-firepass", "shared");
|
||||
const order: string[] = [];
|
||||
|
||||
const second = (async () => {
|
||||
const r = await gate.acquire("fireworks-firepass", "shared");
|
||||
order.push("second");
|
||||
r();
|
||||
})();
|
||||
const third = (async () => {
|
||||
const r = await gate.acquire("fireworks-firepass", "shared");
|
||||
order.push("third");
|
||||
r();
|
||||
})();
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
expect(order).toEqual([]);
|
||||
|
||||
release();
|
||||
await second;
|
||||
expect(order).toEqual(["second"]);
|
||||
|
||||
await third;
|
||||
expect(order).toEqual(["second", "third"]);
|
||||
});
|
||||
|
||||
it("scopes pools by (provider, apiKey) so different keys do not block each other", async () => {
|
||||
const gate = makeGate(1);
|
||||
const r1 = await gate.acquire("fireworks-firepass", "key-a");
|
||||
// Different apiKey -> new pool, no waiting.
|
||||
const r2 = await gate.acquire("fireworks-firepass", "key-b");
|
||||
expect(typeof r1).toBe("function");
|
||||
expect(typeof r2).toBe("function");
|
||||
r1();
|
||||
r2();
|
||||
});
|
||||
|
||||
it("releases the pool when no callers remain", async () => {
|
||||
const gate = makeGate(1);
|
||||
const release = await gate.acquire("fireworks-firepass", "shared");
|
||||
release();
|
||||
// Re-acquire should not be queued.
|
||||
const next = await gate.acquire("fireworks-firepass", "shared");
|
||||
next();
|
||||
});
|
||||
|
||||
it("rejects when the abort signal fires while waiting", async () => {
|
||||
const gate = makeGate(1);
|
||||
const held = await gate.acquire("fireworks-firepass", "shared");
|
||||
|
||||
const controller = new AbortController();
|
||||
const pending = gate.acquire("fireworks-firepass", "shared", controller.signal);
|
||||
controller.abort();
|
||||
|
||||
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
|
||||
|
||||
// Slot must remain available after the aborted waiter is removed.
|
||||
held();
|
||||
const next = await gate.acquire("fireworks-firepass", "shared");
|
||||
next();
|
||||
});
|
||||
|
||||
it("rejects immediately when the abort signal is already aborted", async () => {
|
||||
const gate = makeGate(1);
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
await expect(
|
||||
gate.acquire("fireworks-firepass", "shared", controller.signal),
|
||||
).rejects.toMatchObject({ name: "AbortError" });
|
||||
});
|
||||
|
||||
it("treats double-release as a no-op", async () => {
|
||||
const gate = makeGate(1);
|
||||
const release = await gate.acquire("fireworks-firepass", "shared");
|
||||
release();
|
||||
release();
|
||||
const next = await gate.acquire("fireworks-firepass", "shared");
|
||||
next();
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
} from "@mariozechner/pi-ai";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createSafePiStream } from "../../src/main/worker/runtime/pi-safe-stream";
|
||||
import { InMemoryCredentialConcurrencyGate } from "../../src/main/worker/runtime/credential-concurrency-gate";
|
||||
|
||||
function createModel(): Model<"openai-responses"> {
|
||||
return {
|
||||
@@ -67,6 +68,102 @@ describe("pi safe stream", () => {
|
||||
await expect(stream.result()).resolves.toMatchObject({ stopReason: "stop" });
|
||||
});
|
||||
|
||||
it("serializes calls that share a gated credential", async () => {
|
||||
const gate = new InMemoryCredentialConcurrencyGate(() => 1);
|
||||
let activeCalls = 0;
|
||||
let peakActive = 0;
|
||||
|
||||
const safeStream = createSafePiStream((_model, _context) => {
|
||||
activeCalls += 1;
|
||||
peakActive = Math.max(peakActive, activeCalls);
|
||||
const stream = createAssistantMessageEventStream();
|
||||
setTimeout(() => {
|
||||
const message = createAssistantMessage();
|
||||
stream.push({ type: "done", reason: "stop", message });
|
||||
stream.end(message);
|
||||
activeCalls -= 1;
|
||||
}, 5);
|
||||
return stream;
|
||||
}, gate);
|
||||
|
||||
const drain = async (stream: ReturnType<typeof safeStream>) => {
|
||||
for await (const _event of stream) {
|
||||
// drain
|
||||
}
|
||||
await stream.result();
|
||||
};
|
||||
|
||||
const model = createModel();
|
||||
const a = safeStream(model, emptyContext, { apiKey: "shared" });
|
||||
const b = safeStream(model, emptyContext, { apiKey: "shared" });
|
||||
const c = safeStream(model, emptyContext, { apiKey: "shared" });
|
||||
|
||||
await Promise.all([drain(a), drain(b), drain(c)]);
|
||||
expect(peakActive).toBe(1);
|
||||
});
|
||||
|
||||
it("does not gate calls without an apiKey", async () => {
|
||||
const gate = new InMemoryCredentialConcurrencyGate(() => 1);
|
||||
let activeCalls = 0;
|
||||
let peakActive = 0;
|
||||
|
||||
const safeStream = createSafePiStream((_model, _context) => {
|
||||
activeCalls += 1;
|
||||
peakActive = Math.max(peakActive, activeCalls);
|
||||
const stream = createAssistantMessageEventStream();
|
||||
setTimeout(() => {
|
||||
const message = createAssistantMessage();
|
||||
stream.push({ type: "done", reason: "stop", message });
|
||||
stream.end(message);
|
||||
activeCalls -= 1;
|
||||
}, 5);
|
||||
return stream;
|
||||
}, gate);
|
||||
|
||||
const drain = async (stream: ReturnType<typeof safeStream>) => {
|
||||
for await (const _event of stream) {
|
||||
// drain
|
||||
}
|
||||
await stream.result();
|
||||
};
|
||||
|
||||
const model = createModel();
|
||||
await Promise.all([
|
||||
drain(safeStream(model, emptyContext)),
|
||||
drain(safeStream(model, emptyContext)),
|
||||
]);
|
||||
expect(peakActive).toBe(2);
|
||||
});
|
||||
|
||||
it("releases the slot when the upstream stream throws", async () => {
|
||||
const gate = new InMemoryCredentialConcurrencyGate(() => 1);
|
||||
let attempt = 0;
|
||||
|
||||
const safeStream = createSafePiStream(() => {
|
||||
attempt += 1;
|
||||
if (attempt === 1) throw new Error("first call boom");
|
||||
const stream = createAssistantMessageEventStream();
|
||||
queueMicrotask(() => {
|
||||
const message = createAssistantMessage();
|
||||
stream.push({ type: "done", reason: "stop", message });
|
||||
stream.end(message);
|
||||
});
|
||||
return stream;
|
||||
}, gate);
|
||||
|
||||
const failing = safeStream(createModel(), emptyContext, { apiKey: "shared" });
|
||||
for await (const _event of failing) {
|
||||
// drain
|
||||
}
|
||||
await failing.result();
|
||||
|
||||
const succeeding = safeStream(createModel(), emptyContext, { apiKey: "shared" });
|
||||
for await (const _event of succeeding) {
|
||||
// drain
|
||||
}
|
||||
await expect(succeeding.result()).resolves.toMatchObject({ stopReason: "stop" });
|
||||
});
|
||||
|
||||
it("turns rejected stream setup into an error event", async () => {
|
||||
const safeStream = createSafePiStream(async () => {
|
||||
throw new Error("stream setup failed");
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
PROVIDER_CONCURRENCY_LIMITS,
|
||||
getProviderConcurrencyLimit,
|
||||
} from "../../src/main/worker/runtime/provider-rate-limits";
|
||||
|
||||
describe("provider rate limits", () => {
|
||||
it("caps fireworks-firepass at one concurrent call", () => {
|
||||
expect(PROVIDER_CONCURRENCY_LIMITS["fireworks-firepass"]).toBe(1);
|
||||
expect(getProviderConcurrencyLimit("fireworks-firepass")).toBe(1);
|
||||
});
|
||||
|
||||
it("returns null for providers without an explicit cap", () => {
|
||||
expect(getProviderConcurrencyLimit("anthropic")).toBeNull();
|
||||
expect(getProviderConcurrencyLimit("openai")).toBeNull();
|
||||
expect(getProviderConcurrencyLimit("ollama")).toBeNull();
|
||||
expect(getProviderConcurrencyLimit("unknown-provider")).toBeNull();
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user