fix(startup): thread-exhaustion resilience — cap workers, never panic
Port of upstream grok-build 0.2.113/0.2.114 fixes, extended to Kigi's startup order. - kigi-tty-utils runtime module: worker-cap policy (8 max) — tokio's per-core default pins 100+ thread slots on many-core shared hosts against RLIMIT_NPROC / pids.max, and later spawns die with EAGAIN. - kigi-bin builds the main runtime with capped workers and reports an out-of-threads host with a diagnostic + exit(1) instead of a panic. - The pre-runtime optimization threads (HTTP warmup, early model prefetch) spawn recoverably and degrade — their panic would have preempted the runtime's own diagnostic. - The per-session actor thread and its runtime report spawn/build failure through the existing acp::Error / AgentBuildError boundaries (new RuntimeBuild variant) instead of panicking the agent host. - fsnotify's watcher thread spawn maps into FsNotifyError::WatcherStart so callers degrade instead of crashing. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01KSy9S31pZNWXtRZNVJ1sDf
This commit is contained in:
1 parent
bf93091416
commit
97de661554
8 files changed
+144
-17
No files matched your search
@@ -0,0 +1,55 @@
|
||||
//! Worker-thread policy for multi-thread tokio runtimes.
|
||||
//!
|
||||
//! Tokio defaults to one worker per core. On many-core shared hosts (100+-core
|
||||
//! HPC login nodes) that pins 100+ thread slots per kigi process against
|
||||
//! per-user ceilings — systemd user-slice `pids.max` (commonly 1000) or
|
||||
//! `RLIMIT_NPROC` — and later thread spawns die with EAGAIN. Kigi's runtimes
|
||||
//! are I/O-bound, so throughput does not scale with workers past a small
|
||||
//! count.
|
||||
//!
|
||||
//! This is the single home for the cap policy; every multi-thread runtime in
|
||||
//! the workspace derives its worker count from here so the policy cannot
|
||||
//! drift across crates.
|
||||
|
||||
use std::num::NonZeroUsize;
|
||||
|
||||
/// Maximum runtime worker threads for any kigi process.
|
||||
pub const MAX_WORKER_THREADS: NonZeroUsize = NonZeroUsize::new(8).unwrap();
|
||||
|
||||
/// Pure, testable: `min(cores, MAX_WORKER_THREADS)`.
|
||||
pub fn cap_worker_threads(cores: NonZeroUsize) -> NonZeroUsize {
|
||||
cores.min(MAX_WORKER_THREADS)
|
||||
}
|
||||
|
||||
/// Reads the host: `min(available_parallelism, MAX_WORKER_THREADS)`.
|
||||
pub fn capped_worker_threads() -> NonZeroUsize {
|
||||
cap_worker_threads(std::thread::available_parallelism().unwrap_or(NonZeroUsize::MIN))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn nz(n: usize) -> NonZeroUsize {
|
||||
NonZeroUsize::new(n).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cap_is_identity_at_or_below_max() {
|
||||
assert_eq!(cap_worker_threads(nz(1)), nz(1));
|
||||
assert_eq!(cap_worker_threads(nz(4)), nz(4));
|
||||
assert_eq!(cap_worker_threads(nz(8)), nz(8));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cap_clamps_many_core_hosts() {
|
||||
assert_eq!(cap_worker_threads(nz(9)), nz(8));
|
||||
assert_eq!(cap_worker_threads(nz(360)), nz(8));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capped_worker_threads_stays_in_bounds() {
|
||||
let n = capped_worker_threads();
|
||||
assert!(n <= MAX_WORKER_THREADS, "got {n}");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user