Resume parked work when a provider's monthly limit is raised
The heartbeat scheduler enqueued runs even while a provider's monthly usage window was full, so every task burned its 3-attempt retry budget on instant quota failures within seconds and stayed dead forever - raising the limit (or restarting) never revived anything. - Park tasks (skip enqueue) while their provider is at its monthly_request_limit instead of creating runs doomed to fail - Exclude quota-blocked failures from the 3-attempt retry budget, retroactively reviving tasks abandoned by past filled windows - Share the gate message as PROVIDER_MONTHLY_LIMIT_MESSAGE and log quota-block transitions once per state change for diagnosability - Ignore landing/ (self-contained Next.js sub-project) and built-in skills/ (prompt content) in eslint so verify:ci lints only application code
This commit is contained in:
1 parent
8e01317a46
commit
349547a0de
6 files changed
+222
-8
No files matched your search
@@ -9,7 +9,7 @@ import {
|
||||
type ProviderAuthType,
|
||||
type ProviderExecutionMode,
|
||||
} from "../../shared/domain";
|
||||
import { currentProviderUsageWindow, type ProviderObservedRateLimitState } from "../../shared/provider-runtime-domain";
|
||||
import { currentProviderUsageWindow, PROVIDER_MONTHLY_LIMIT_MESSAGE, type ProviderObservedRateLimitState } from "../../shared/provider-runtime-domain";
|
||||
import { AuditService } from "./audit-service";
|
||||
import { AuthBrokerService, type ProviderSecretVault } from "./auth-broker-service";
|
||||
import { OperationsService } from "./operations-service";
|
||||
@@ -238,7 +238,7 @@ export class ProviderBackgroundService {
|
||||
where org_id = ? and provider_id = ? and window_start = ?
|
||||
`).get(orgId, provider.id, window.start) as { request_count: number } | undefined;
|
||||
const requestCount = usage?.request_count ?? 0;
|
||||
if (requestCount >= provider.monthly_request_limit) throw new Error("Provider monthly request limit has been reached.");
|
||||
if (requestCount >= provider.monthly_request_limit) throw new Error(PROVIDER_MONTHLY_LIMIT_MESSAGE);
|
||||
}
|
||||
|
||||
private normalizeWorkspaceId(orgId: string, workspaceId: string | null): string | null {
|
||||
|
||||
@@ -2,7 +2,7 @@ import type { DatabaseSync } from "node:sqlite";
|
||||
import type { CancelProviderResponseInput, CreateProviderResponseInput, StreamProviderResponseInput } from "../../shared/commands";
|
||||
import { FEATURE_FLAG_KEY, PROVIDER_AUTH_TYPE, PROVIDER_DATA_POLICY, PROVIDER_EXECUTION_MODE, PROVIDER_REQUEST_STATE, type ProviderExecutionMode } from "../../shared/domain";
|
||||
import type { ProviderStreamRendererEvent } from "../../shared/provider-request-domain";
|
||||
import { currentProviderUsageWindow } from "../../shared/provider-runtime-domain";
|
||||
import { currentProviderUsageWindow, PROVIDER_MONTHLY_LIMIT_MESSAGE } from "../../shared/provider-runtime-domain";
|
||||
import type { AuditService } from "./audit-service";
|
||||
import type { AuthBrokerService } from "./auth-broker-service";
|
||||
import type { OperationsService } from "./operations-service";
|
||||
@@ -331,7 +331,7 @@ export class ProviderResponseRunner {
|
||||
where org_id = ? and provider_id = ? and window_start = ?
|
||||
`).get(orgId, provider.id, window.start) as { request_count: number } | undefined;
|
||||
const requestCount = usage?.request_count ?? 0;
|
||||
if (requestCount >= provider.monthly_request_limit) throw new Error("Provider monthly request limit has been reached.");
|
||||
if (requestCount >= provider.monthly_request_limit) throw new Error(PROVIDER_MONTHLY_LIMIT_MESSAGE);
|
||||
}
|
||||
|
||||
private normalizeWorkspaceId(orgId: string, workspaceId: string | null): string | null {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import type { RunTaskInput } from "../../shared/commands";
|
||||
import { APPROVAL_STATE, RISK_LEVEL } from "../../shared/domain";
|
||||
import { currentProviderUsageWindow, PROVIDER_MONTHLY_LIMIT_MESSAGE } from "../../shared/provider-runtime-domain";
|
||||
import { createId } from "../../shared/ids";
|
||||
import { AuditService } from "./audit-service";
|
||||
import { ProviderService } from "./provider-service";
|
||||
@@ -53,6 +54,8 @@ export class TaskExecutionService {
|
||||
private readonly audit: AuditService;
|
||||
private readonly organization: OrganizationService;
|
||||
private readonly chat: ChatService;
|
||||
/** Providers currently parked on their monthly quota, for transition logging. */
|
||||
private quotaBlockedProviders = new Map<string, string>();
|
||||
|
||||
constructor(
|
||||
private readonly db: DatabaseSync,
|
||||
@@ -296,20 +299,31 @@ export class TaskExecutionService {
|
||||
* heartbeat-enabled agent with an enabled provider that has not run yet, so
|
||||
* CEO-delegated work starts on its own. Each task is enqueued at most once
|
||||
* (guarded by existing runs), preventing runaway re-execution.
|
||||
*
|
||||
* Two quota rules keep the loop self-healing when a provider's monthly
|
||||
* window fills up: tasks on an exhausted provider are parked (not enqueued
|
||||
* into certain failure), and past quota-blocked failures never count toward
|
||||
* the 3-attempt retry budget. Raising the limit therefore resumes work on
|
||||
* the next tick — no restart or manual unpausing.
|
||||
*/
|
||||
enqueueHeartbeatRuns(): string[] {
|
||||
const window = currentProviderUsageWindow();
|
||||
const rows = this.db.prepare(`
|
||||
select t.id as task_id, t.org_id as org_id
|
||||
from tasks t
|
||||
join agents a on a.id = t.assignee_agent_id and a.org_id = t.org_id
|
||||
join providers p on p.id = a.model_provider_id and p.org_id = t.org_id
|
||||
left join provider_usage_windows u
|
||||
on u.org_id = p.org_id and u.provider_id = p.id and u.window_start = ?
|
||||
where t.status in ('backlog', 'in_progress')
|
||||
and a.heartbeat_enabled = 1
|
||||
and a.lifecycle = 'active'
|
||||
and p.enabled = 1
|
||||
and coalesce(u.request_count, 0) < p.monthly_request_limit
|
||||
and not exists (select 1 from task_runs r where r.task_id = t.id and r.state in ('queued', 'running', 'completed'))
|
||||
and (select count(*) from task_runs r where r.task_id = t.id) < 3
|
||||
`).all() as Array<{ task_id: string; org_id: string }>;
|
||||
and (select count(*) from task_runs r where r.task_id = t.id and coalesce(r.error_message, '') <> ?) < 3
|
||||
`).all(window.start, PROVIDER_MONTHLY_LIMIT_MESSAGE) as Array<{ task_id: string; org_id: string }>;
|
||||
this.observeQuotaBlocks(window.start);
|
||||
const created: string[] = [];
|
||||
for (const row of rows) {
|
||||
try {
|
||||
@@ -321,6 +335,33 @@ export class TaskExecutionService {
|
||||
return created;
|
||||
}
|
||||
|
||||
/**
|
||||
* Logs monthly-quota transitions (blocked <-> unblocked) exactly once per
|
||||
* state change so a parked queue is diagnosable from the main-process log
|
||||
* without spamming a line every 5s tick.
|
||||
*/
|
||||
private observeQuotaBlocks(windowStart: string): void {
|
||||
const rows = this.db.prepare(`
|
||||
select p.id as id, p.label as label
|
||||
from providers p
|
||||
join provider_usage_windows u
|
||||
on u.org_id = p.org_id and u.provider_id = p.id and u.window_start = ?
|
||||
where p.enabled = 1 and u.request_count >= p.monthly_request_limit
|
||||
`).all(windowStart) as Array<{ id: string; label: string }>;
|
||||
const current = new Map(rows.map((row) => [row.id, row.label]));
|
||||
for (const [id, label] of current) {
|
||||
if (!this.quotaBlockedProviders.has(id)) {
|
||||
console.warn(`[task-queue] provider "${label}" hit its monthly request limit; its tasks are parked until the limit is raised or the window resets.`);
|
||||
}
|
||||
}
|
||||
for (const [id, label] of this.quotaBlockedProviders) {
|
||||
if (!current.has(id)) {
|
||||
console.info(`[task-queue] provider "${label}" is under its monthly request limit again; parked tasks resume.`);
|
||||
}
|
||||
}
|
||||
this.quotaBlockedProviders = current;
|
||||
}
|
||||
|
||||
private enqueueRun(orgId: string, taskId: string): string {
|
||||
const task = this.requireTask(orgId, taskId);
|
||||
// Autonomous execution honours the same gate as a manual run: a high-risk or
|
||||
|
||||
@@ -29,6 +29,15 @@ export interface ProviderRateLimitObservationRecord extends ProviderObservedRate
|
||||
observedAt: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Exact message recorded on runs blocked by a provider's monthly usage window.
|
||||
* The heartbeat scheduler matches task_runs.error_message against this string
|
||||
* so quota-blocked attempts never consume a task's retry budget — the text is
|
||||
* a cross-module contract and also matches rows written by earlier builds
|
||||
* that inlined it.
|
||||
*/
|
||||
export const PROVIDER_MONTHLY_LIMIT_MESSAGE = "Provider monthly request limit has been reached.";
|
||||
|
||||
export function currentProviderUsageWindow(now = new Date()): { start: string; end: string } {
|
||||
const start = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
|
||||
const end = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + 1, 1));
|
||||
|
||||
Reference in new issue
Block a user