ZacharyZhang-NY/Kigi-CLI
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity
Files
a02b555e66100c95d5a4422b961d41bfe60af3fd
Kigi-CLI/crates/codegen/kigi-tools/src/computer/local/terminal.rs
T
ZacharyZhang-NY a02b555e66 docs(comments): rewrite comments across all crates to the guidelines 
Sweep every first-party crate source (1956 .rs files) to the project comment
guidelines: delete redundant restatements, decorative banners, change
narration, and end-of-line comments; keep and tighten the crucial ones
(invariants, bug rationale, SAFETY blocks, ported-source attribution).

No functional code changed. Every edit is proven comment-only against the
prior tree by a comment-stripping lexer (string/char/raw-string aware) plus a
separate doctest-fence check. Where removing a comment made rustfmt or clippy
want to re-lay-out adjacent code, the minimal triggering comment is restored so
code tokens stay byte-identical.

Gates green: cargo fmt --all --check (0 diffs), cargo check and cargo clippy
--workspace --all-targets (0 warnings).

Adds scripts/check_codegen_comment_guidelines.py — the enforcement gate for
these guidelines (flags banners, end-of-line comments, change narration, and
commented-out code).
2026-07-23 16:55:39 -04:00

4617 lines
176 KiB
Rust
Raw Blame History

//! Actor-based terminal implementation with support for foreground and background execution.
//!
//! This module implements a terminal backend using the actor pattern:
//! - `LocalTerminalBackend` is a handle that sends commands to the actor via channels
//! - `LocalTerminalActor` runs in a spawned task and owns all mutable state
//! - No mutex locks are needed - all state access is through message passing
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Stdio;
use std::time::{Duration, Instant};
use tokio::fs::File;
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
use tokio::sync::{mpsc, oneshot};
use tokio_util::sync::CancellationToken;
use crate::computer::local::cgroup::{
CgroupGuard, CgroupMemoryConfig, MemoryMonitor, PROCESS_OOM_EXIT_CODE,
};
use crate::computer::types::{
BackgroundHandle, ComputerError, KillOutcome, TaskSnapshot, TerminalBackend,
TerminalRunRequest, TerminalRunResult,
};
use crate::notification::types::{BashNotificationBase, BashOutputChunk, ToolNotificationHandle};
use super::SearchShadowConfig;
#[cfg(unix)]
use super::shell_state;
struct SpawnResult {
child: tokio::process::Child,
process_group: crate::util::ProcessGroup,
/// Handle for reading the state dump from fd 4 (persistent shell only).
state_dump_handle: Option<tokio::task::JoinHandle<std::io::Result<String>>>,
}
const READ_BUFFER_SIZE: usize = 8192;
const DEFAULT_NOTIFICATION_INTERVAL_MS: u64 = 100;
const COMMAND_CHANNEL_SIZE: usize = 32;
/// How long to keep completed background tasks in memory before eviction.
/// The output file on disk persists for the session lifetime.
const COMPLETED_TASK_TTL: Duration = Duration::from_secs(300);
/// SIGTERM → SIGKILL grace period.
const SIGTERM_GRACE: Duration = Duration::from_secs(1);
/// Maximum lifetime for a background task; the actor kills it once exceeded.
/// 10 hours, so long background monitor and bash runs survive.
const BACKGROUND_MAX_RUNTIME: Duration = Duration::from_secs(36_000);
/// Max time an *auto-backgroundable* foreground command blocks the turn before
/// it's moved to the background (kept running, never killed), independent of its
/// requested `timeout`. Env override: `KIGI_FOREGROUND_BLOCK_BUDGET_MS`.
const FOREGROUND_BLOCK_BUDGET: Duration = Duration::from_secs(15);
fn foreground_block_budget_from_env() -> Duration {
std::env::var("KIGI_FOREGROUND_BLOCK_BUDGET_MS")
.ok()
.and_then(|s| s.parse::<u64>().ok())
.map(Duration::from_millis)
.unwrap_or(FOREGROUND_BLOCK_BUDGET)
}
/// Max bytes a command's output file may reach before the actor kills it — the
/// size analogue of [`BACKGROUND_MAX_RUNTIME`], stopping an unbounded writer
/// (`yes`, a runaway log) from filling the disk. Env override:
/// `KIGI_MAX_OUTPUT_FILE_BYTES`.
const MAX_OUTPUT_FILE_BYTES: u64 = 5 * 1024 * 1024 * 1024;
fn output_file_cap_from_env() -> u64 {
std::env::var("KIGI_MAX_OUTPUT_FILE_BYTES")
.ok()
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(MAX_OUTPUT_FILE_BYTES)
}
/// Max time to drain stdout/stderr after process exit. Prevents `cmd &`
/// (inherited pipe, no redirect) from blocking the actor loop forever.
const DRAIN_TIMEOUT: Duration = Duration::from_secs(2);
/// Max bytes retained in the output file after process exit. Truncated
/// so `to_task_snapshot` / `read_file` don't materialize huge strings.
const MAX_RETAINED_OUTPUT_FILE_BYTES: u64 = 64 * 1024 * 1024;
/// Maximum number of completed-task tombstones to keep; the oldest entries are
/// evicted past this. Each tombstone is metadata only (no output), ~10 KB total.
const MAX_COMPLETED_TASK_SNAPSHOTS: usize = 100;
fn notification_interval() -> Duration {
Duration::from_millis(DEFAULT_NOTIFICATION_INTERVAL_MS)
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ExitStatus {
pub exit_code: Option<i32>,
pub signal: Option<String>,
}
enum TerminalCommand {
/// Foreground: spawn process, block until exit or timeout, reply with result.
Run {
request: TerminalRunRequest,
reply: oneshot::Sender<Result<TerminalRunResult, ComputerError>>,
},
/// Background: spawn process, register under a generated task_id,
/// reply immediately with BackgroundHandle. Process keeps running.
RunBackground {
request: TerminalRunRequest,
reply: oneshot::Sender<Result<BackgroundHandle, ComputerError>>,
},
/// Get snapshot of a background task (by task_id from RunBackground).
GetTask {
task_id: String,
reply: oneshot::Sender<Option<TaskSnapshot>>,
},
Kill {
task_id: String,
reply: oneshot::Sender<KillOutcome>,
},
/// Kill foregrounded processes. Called on turn cancellation.
KillForegroundCommands,
/// Unblocks the completion waiter with signal="backgrounded".
BackgroundForeground {
tool_call_id: String,
reply: oneshot::Sender<bool>,
},
WaitForCompletion {
task_id: String,
timeout: Option<Duration>,
reply: oneshot::Sender<Option<TaskSnapshot>>,
},
ListTasks {
reply: oneshot::Sender<Vec<TaskSnapshot>>,
},
GetShellCwd {
reply: oneshot::Sender<Option<PathBuf>>,
},
WarmShell {
cwd: PathBuf,
},
KillForegroundCommandsByOwner {
owner_session_id: String,
},
/// Kills only the session's *background* tasks.
KillTasksByOwner {
owner_session_id: String,
reply: oneshot::Sender<()>,
},
/// Reparent notification handles for all tasks owned by a session.
/// Swaps the old notification handle with a new one so events from
/// surviving processes route to the parent session. Also re-spawns
/// monitor pipelines so monitor events continue streaming.
ReparentNotifications {
old_owner_session_id: String,
new_owner_session_id: String,
new_handle: crate::notification::types::ToolNotificationHandle,
/// Weak (not strong) backend handle for re-spawning monitor pipelines,
/// so a reparented monitor doesn't pin the backend. See `run_monitor_pipeline`.
backend_weak: std::sync::Weak<dyn crate::computer::types::TerminalBackend>,
reply: oneshot::Sender<()>,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum BackgroundStatus {
Foreground { auto_bg_on_timeout: bool },
Backgrounded { reason: BackgroundReason },
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum BackgroundReason {
/// Model requested `is_background=true`.
Explicit,
/// User pressed Ctrl+G.
UserSignal,
/// Foreground command exceeded default timeout.
ForegroundTimeout,
}
impl BackgroundStatus {
fn is_backgrounded(self) -> bool {
matches!(self, Self::Backgrounded { .. })
}
}
impl BackgroundReason {
fn as_signal(&self) -> &'static str {
match self {
Self::Explicit | Self::UserSignal => "backgrounded",
Self::ForegroundTimeout => "auto_backgrounded",
}
}
}
struct ProcessState {
child: tokio::process::Child,
/// Process-tree teardown handle, shared (`Arc`) with the process-global
/// `ProcessScope` so the TUI exit paths can reap it if this actor never runs
/// its own teardown. On Unix this stores the leader pid for `killpg`; on
/// Windows it owns a Job Object that terminates every descendant when killed
/// or dropped.
///
/// On Unix this auto-drops to `None` the tick the child is reaped (`child.id()`
/// is `None`) — the poll sweep (and the explicit-kill path) release the `Arc`
/// so the scope's `Weak` dies at reap. A completed task lingers here for
/// `completed_task_ttl`; holding the `Arc` that long would let `kill_all`
/// `killpg` a pid the OS may have recycled. On Windows it stays `Some` until
/// the `ProcessState` is removed (the JobObject HANDLE has no recyclable pid,
/// so dropping early at reap is unnecessary).
process_group: Option<std::sync::Arc<crate::util::ProcessGroup>>,
/// Accumulated output buffer — tail portion (may be truncated)
output_buffer: Vec<u8>,
/// Front portion of output, captured before truncation kicks in.
/// Once the total char count exceeds the limit, the first half of the
/// budget is frozen here and only the tail is kept in `output_buffer`.
front_buffer: Option<Vec<u8>>,
truncated: bool,
/// Total bytes written to file, before any truncation.
total_bytes: usize,
exit_status: Option<ExitStatus>,
bg_status: BackgroundStatus,
/// Foreground only.
completion_waiters: Vec<oneshot::Sender<Result<TerminalRunResult, ComputerError>>>,
output_byte_limit: usize,
timeout: Duration,
/// When auto_bg_on_timeout: max FG block before auto-bg (per-request or backend default).
foreground_block_budget: Duration,
start_time: Instant,
/// Every byte of output lands here, even for foreground commands.
output_file: PathBuf,
file_handle: Option<File>,
/// The command as executed (may be isolation-wrapped).
command: String,
/// Original user command before isolation wrapping, for display.
display_command: Option<String>,
cwd: String,
/// Wall-clock counterpart to `start_time`, for `TaskSnapshot`.
start_wall_time: std::time::SystemTime,
/// Drives TTL-based eviction of background tasks.
completed_at: Option<Instant>,
/// Wall-clock counterpart to `completed_at`, for `TaskSnapshot` durations.
end_wall_time: Option<std::time::SystemTime>,
notification_handle: ToolNotificationHandle,
tool_call_id: String,
kind: crate::computer::types::TaskKind,
/// Monotonic `total_bytes` at the time of the last chunk notification.
/// Used to detect "new output since last tick" — only send a chunk
/// when total_bytes > last_notified_total. Keyed off the monotonic
/// byte counter rather than `output_buffer.len()` because the buffer
/// is a truncated tail that *shrinks* once `maybe_truncate` fires; a
/// length-based gate would go (and stay) false after truncation.
last_notified_total: usize,
/// Whether stdout/stderr have already been drained after exit.
/// Prevents repeated 2s drain timeouts on every poll tick when
/// orphaned children hold pipes open.
drained: bool,
/// Set when a `block=true` waiter consumed this task's result.
block_waited: bool,
/// Set when the model explicitly killed this task via the kill tool,
/// or during `kill_all_background_tasks` teardown.
explicitly_killed: bool,
/// Join handle for reading the state dump from fd 4 (persistent shell only).
/// When present, the actor collects the dump on process exit and updates
/// the canonical `ShellState`.
state_dump_handle: Option<tokio::task::JoinHandle<std::io::Result<String>>>,
/// Session that owns this process. Used to scope kill operations so
/// subagent teardown only kills the subagent's own tasks.
owner_session_id: Option<String>,
}
impl ProcessState {
fn to_result(&self) -> TerminalRunResult {
let combined_output = if let Some(ref front) = self.front_buffer {
let front_str = String::from_utf8_lossy(front);
let back_str = String::from_utf8_lossy(&self.output_buffer);
format!(
"{}\n\n... (output truncated) ...\n\n{}",
front_str.trim_end(),
back_str.trim_start()
)
} else {
String::from_utf8_lossy(&self.output_buffer).into_owned()
};
TerminalRunResult {
combined_output,
exit_code: self.exit_status.as_ref().and_then(|s| s.exit_code),
truncated: self.truncated,
signal: match self.bg_status {
BackgroundStatus::Backgrounded { reason } => Some(reason.as_signal().to_string()),
_ => self.exit_status.as_ref().and_then(|s| s.signal.clone()),
},
timed_out: self
.exit_status
.as_ref()
.map(|s| s.signal.as_deref() == Some("timeout"))
.unwrap_or(false),
output_file: self.output_file.clone(),
total_bytes: self.total_bytes,
pid: self.child.id(),
}
}
fn notify_waiters(&mut self, result: Result<TerminalRunResult, ComputerError>) {
for waiter in self.completion_waiters.drain(..) {
let _ = waiter.send(result.clone());
}
}
/// Front-and-back truncation using character counts.
///
/// When the total char count of `output_buffer` exceeds `output_char_limit`:
/// 1. Freeze the first `half` chars into `front_buffer` (done once).
/// 2. Keep only the last `half` chars in `output_buffer`.
///
/// The two halves are re-joined by `to_result()` with a separator.
fn maybe_truncate(&mut self) {
let s = String::from_utf8_lossy(&self.output_buffer);
let char_count = s.chars().count();
if char_count <= self.output_byte_limit {
return;
}
let half = self.output_byte_limit / 2;
if self.front_buffer.is_none() {
let front_end = s
.char_indices()
.nth(half)
.map(|(i, _)| i)
.unwrap_or(s.len());
self.front_buffer = Some(s[..front_end].as_bytes().to_vec());
}
let tail_start_char = char_count.saturating_sub(half);
let tail_start_byte = s
.char_indices()
.nth(tail_start_char)
.map(|(i, _)| i)
.unwrap_or(s.len());
self.output_buffer = s[tail_start_byte..].as_bytes().to_vec();
self.truncated = true;
}
/// Flush and truncate the output file to [`MAX_RETAINED_OUTPUT_FILE_BYTES`].
async fn flush_and_truncate_output_file(&mut self) {
if let Some(ref mut file) = self.file_handle {
let _ = file.flush().await;
if self.total_bytes as u64 > MAX_RETAINED_OUTPUT_FILE_BYTES {
let _ = file.set_len(MAX_RETAINED_OUTPUT_FILE_BYTES).await;
// Seek to new end so post-exit drain appends correctly.
let _ = file.seek(std::io::SeekFrom::End(0)).await;
}
}
}
fn is_timed_out(&self) -> bool {
self.start_time.elapsed() > self.timeout
}
async fn to_task_snapshot(&self, task_id: &str) -> TaskSnapshot {
// For completed background tasks, the in-memory buffer is cleared to free
// memory. Fall back to reading from the output file (non-blocking).
let output = if self.output_buffer.is_empty() && self.exit_status.is_some() {
tokio::fs::read_to_string(&self.output_file)
.await
.unwrap_or_default()
} else if let Some(ref front) = self.front_buffer {
let front_str = String::from_utf8_lossy(front);
let back_str = String::from_utf8_lossy(&self.output_buffer);
format!(
"{}\n\n... (output truncated) ...\n\n{}",
front_str.trim_end(),
back_str.trim_start()
)
} else {
String::from_utf8_lossy(&self.output_buffer).into_owned()
};
TaskSnapshot {
task_id: task_id.to_string(),
command: self.command.clone(),
display_command: self.display_command.clone(),
cwd: self.cwd.clone(),
start_time: self.start_wall_time,
end_time: if self.exit_status.is_some() {
Some(
self.end_wall_time
.unwrap_or_else(std::time::SystemTime::now),
)
} else {
None
},
output,
output_file: self.output_file.clone(),
truncated: self.truncated,
exit_code: self.exit_status.as_ref().and_then(|s| s.exit_code),
signal: self.exit_status.as_ref().and_then(|s| s.signal.clone()),
completed: self.exit_status.is_some(),
block_waited: self.block_waited,
explicitly_killed: self.explicitly_killed,
kind: self.kind,
owner_session_id: self.owner_session_id.clone(),
}
}
}
/// Waiter registered by WaitForCompletion commands.
/// Instead of blocking the actor loop, we store the reply sender and deadline,
/// then check on each poll tick whether to fire it.
struct CompletionWaiter {
reply: oneshot::Sender<Option<TaskSnapshot>>,
deadline: Instant,
}
struct LocalTerminalActor {
cmd_rx: mpsc::Receiver<TerminalCommand>,
cancel_token: CancellationToken,
/// Reaper for spawned child trees. Each spawned process enrolls its
/// `ProcessGroup` here (this actor keeps the owning `Arc`) so the TUI exit
/// paths can `kill_all()` `setsid`-detached children that would otherwise
/// outlive the process. Defaults to the process-global scope; tests inject
/// their own to avoid latching the global.
scope: crate::util::ProcessScope,
/// task_id -> live process state.
processes: HashMap<String, ProcessState>,
/// task_id -> waiters registered by WaitForCompletion commands.
completion_waiters: HashMap<String, Vec<CompletionWaiter>>,
/// Lightweight snapshots of completed background tasks that were evicted
/// from `processes`. Prevents "Task not found" when the model queries
/// a completed task after the 5-minute process eviction TTL.
/// These are small (no child process, no file handles) and retained
/// for the session lifetime.
completed_task_snapshots: HashMap<String, TaskSnapshot>,
/// How long to keep completed background tasks in `processes` before
/// evicting them to `completed_task_snapshots`.
completed_task_ttl: Duration,
/// Foreground turn-blocking budget (on the actor so tests can shorten it).
/// See [`FOREGROUND_BLOCK_BUDGET`].
foreground_block_budget: Duration,
/// Per-command output-file size cap (on the actor so tests can shrink it).
/// See [`MAX_OUTPUT_FILE_BYTES`].
output_file_cap: u64,
/// Cgroup guard — owns the child cgroup's lifecycle. Spawned processes
/// are moved into this cgroup so their memory is bounded.
_cgroup_guard: CgroupGuard,
/// Polls for memory pressure events from the cgroup.
memory_monitor: MemoryMonitor,
persistent_shell: bool,
/// Per-backend `find`→`bfs` / `grep`→`ugrep` shadow enable state, resolved
/// once by the host and baked in at construction. Passed to
/// `search_injection` per command rather than read from a process-global, so
/// a subagent reusing this backend can't clobber the parent's shadows.
search_shadows: SearchShadowConfig,
/// Persistent shell state (env vars, cwd, functions, aliases).
/// Lazily initialized on first command when `persistent_shell` is true.
#[cfg(unix)]
shell_state: Option<shell_state::ShellState>,
/// Captured login-shell PATH for the non-persistent path.
/// Lazily initialized on first command when `persistent_shell` is false.
/// Ensures CLI tools from rc files are discoverable even without a full
/// shell snapshot.
#[cfg(unix)]
login_path_env: Option<HashMap<String, String>>,
}
impl LocalTerminalActor {
fn new(
cmd_rx: mpsc::Receiver<TerminalCommand>,
cancel_token: CancellationToken,
cgroup_guard: CgroupGuard,
memory_monitor: MemoryMonitor,
persistent_shell: bool,
search_shadows: SearchShadowConfig,
completed_task_ttl: Duration,
foreground_block_budget: Duration,
output_file_cap: u64,
scope: crate::util::ProcessScope,
) -> Self {
Self {
cmd_rx,
cancel_token,
scope,
processes: HashMap::new(),
completion_waiters: HashMap::new(),
completed_task_snapshots: HashMap::new(),
completed_task_ttl,
foreground_block_budget,
output_file_cap,
_cgroup_guard: cgroup_guard,
memory_monitor,
persistent_shell,
search_shadows,
#[cfg(unix)]
shell_state: None,
#[cfg(unix)]
login_path_env: None,
}
}
/// Spawn a child process, using persistent shell wrapping when enabled.
///
/// Returns the spawned child and an optional handle for reading the state dump
/// from fd 4 (only present when persistent shell is active).
async fn spawn_command(
&mut self,
command: &str,
cwd: &std::path::Path,
env: &HashMap<String, String>,
) -> Result<SpawnResult, ComputerError> {
#[cfg(unix)]
if self.persistent_shell {
return self.spawn_persistent_command(command, cwd, env).await;
}
// Lazy-init: capture the user's login-shell PATH on first command so
// CLI tools from rc files (.bashrc, .zshrc, virtualenvs) are visible.
#[cfg(unix)]
if self.login_path_env.is_none() {
self.login_path_env = Some(capture_login_path().await);
}
#[cfg(unix)]
let login_env = self.login_path_env.as_ref();
#[cfg(not(unix))]
let login_env: Option<&HashMap<String, String>> = None;
let (child, process_group) =
spawn_shell_command(command, cwd, env, login_env, self.search_shadows)?;
Ok(SpawnResult {
child,
process_group,
state_dump_handle: None,
})
}
#[cfg(unix)]
async fn ensure_persistent_shell_initialized(&mut self, cwd: &std::path::Path) {
if self.shell_state.is_some() {
return;
}
let shell = shell_state::ShellKind::detect();
match shell_state::ShellState::init(shell, cwd).await {
Ok(state) => self.shell_state = Some(state),
Err(e) => {
tracing::warn!("persistent shell init failed, using empty state: {e}");
self.shell_state = Some(shell_state::ShellState {
cwd: cwd.to_path_buf(),
snapshot: String::new(),
shell,
});
}
}
}
/// Spawn a command with persistent shell state: restore the prior snapshot
/// via fd 3, run the user command, dump the new state to fd 4.
#[cfg(unix)]
async fn spawn_persistent_command(
&mut self,
command: &str,
cwd: &std::path::Path,
env: &HashMap<String, String>,
) -> Result<SpawnResult, ComputerError> {
use command_fds::CommandFdExt;
self.ensure_persistent_shell_initialized(cwd).await;
let shell_state = self.shell_state.as_ref().unwrap();
// The shell's own tracked cwd always wins, so there is never an override.
// The bash tool populates `request.working_directory` with the workspace's
// resolved Cwd even when no per-call override is intended; honoring that as
// an explicit override resets the model's `cd` on every Shell call.
//
// Genuine per-call overrides arrive through the shell adapter, which
// prefixes a subshell `(cd <wd> && …)` to the command string — local to one
// call, and it does not mutate the parent shell's `$PWD`.
let cwd_override: Option<&std::path::Path> = None;
// `cwd` is unused on this path; the non-persistent path in `spawn_command`
// still needs it, so it stays in the signature.
let _ = cwd;
let prep = shell_state
.prepare_command(command, cwd_override, self.search_shadows)
.map_err(|e| ComputerError::io(format!("prepare persistent command: {e}")))?;
let mut cmd = tokio::process::Command::new(&prep.binary);
cmd.args(&prep.args)
.current_dir(&prep.cwd)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true);
// Overrides (TERM=dumb, NO_COLOR, KIGI_AGENT=1, …), then request env, then
// pager env. The agent marker is applied last so request env cannot clear it.
cmd.envs(shell_state::shell_env_overrides());
for (key, value) in env {
cmd.env(key, value);
}
cmd.envs(crate::util::pager_env());
crate::util::apply_kigi_agent_marker(&mut cmd);
cmd.fd_mappings(prep.fd_mappings)
.map_err(|e| ComputerError::io(format!("fd mapping: {e}")))?;
unsafe {
cmd.pre_exec(crate::util::detach_from_tty);
}
#[cfg(target_os = "linux")]
if kigi_sandbox::should_restrict_child_network() {
unsafe {
cmd.pre_exec(|| kigi_sandbox::child_net::install_child_network_filter());
}
}
let child = cmd.spawn().map_err(ComputerError::from)?;
// Drop cmd to release the FdMapping OwnedFds held in its pre_exec closure.
// Without this, the parent keeps the write-end of the state-out pipe open,
// preventing the dump reader from seeing EOF.
drop(cmd);
let mut process_group = crate::util::ProcessGroup::new()
.map_err(|e| ComputerError::io(format!("ProcessGroup::new: {e}")))?;
if let Err(e) = process_group.attach(&child) {
tracing::debug!("Failed to attach persistent-shell child to ProcessGroup: {e}");
}
// fd 3 is the state input pipe.
let snapshot = shell_state.snapshot.clone();
tokio::spawn(async move {
if let Err(e) =
shell_state::write_snapshot_to_pipe(&snapshot, prep.state_in_write).await
{
tracing::debug!("failed to write shell snapshot to pipe: {e}");
}
});
// fd 4 is the state output pipe.
let dump_handle =
tokio::spawn(
async move { shell_state::read_dump_from_pipe(prep.state_out_read).await },
);
Ok(SpawnResult {
child,
process_group,
state_dump_handle: Some(dump_handle),
})
}
async fn run(mut self) {
let mut ticker = tokio::time::interval(notification_interval());
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
tokio::select! {
// Bias towards commands (cancel, kill) over periodic ticking
// so that kill_foreground_commands is handled promptly even
// when poll_all_processes was slow (e.g. drain timeouts).
biased;
_ = self.cancel_token.cancelled() => {
self.shutdown_all().await;
break;
}
cmd = self.cmd_rx.recv() => {
match cmd {
Some(cmd) => self.handle_command(cmd).await,
None => {
self.shutdown_all().await;
break;
}
}
}
// Gated on live processes: the actor exists for the whole
// session lifetime, and an idle session must not wake 10x/sec
// to poll an empty map (one actor per open session/tab adds
// up). With the arm disabled the interval isn't polled, so no
// timer is registered at all; the first command that spawns a
// process re-enables it on the next loop iteration.
_ = ticker.tick(), if !self.processes.is_empty() => {
self.poll_all_processes().await;
}
}
}
}
async fn handle_command(&mut self, cmd: TerminalCommand) {
match cmd {
TerminalCommand::Run { request, reply } => {
self.handle_run(request, reply).await;
}
TerminalCommand::RunBackground { request, reply } => {
self.handle_run_background(request, reply).await;
}
TerminalCommand::GetTask { task_id, reply } => {
let snapshot = match self.processes.get(&task_id) {
Some(p) => Some(p.to_task_snapshot(&task_id).await),
None => self.completed_task_snapshots.get(&task_id).cloned(),
};
let _ = reply.send(snapshot);
}
TerminalCommand::Kill { task_id, reply } => {
let outcome = self.handle_kill(&task_id).await;
let _ = reply.send(outcome);
}
TerminalCommand::WaitForCompletion {
task_id,
timeout,
reply,
} => {
self.handle_wait_for_completion(task_id, timeout, reply)
.await;
}
TerminalCommand::ListTasks { reply } => {
let mut snapshots =
Vec::with_capacity(self.processes.len() + self.completed_task_snapshots.len());
for (id, p) in &self.processes {
snapshots.push(p.to_task_snapshot(id).await);
}
for snap in self.completed_task_snapshots.values() {
snapshots.push(snap.clone());
}
let _ = reply.send(snapshots);
}
TerminalCommand::GetShellCwd { reply } => {
#[cfg(unix)]
let cwd = self.shell_state.as_ref().map(|s| s.cwd.clone());
#[cfg(not(unix))]
let cwd = None;
let _ = reply.send(cwd);
}
TerminalCommand::WarmShell { cwd } => {
#[cfg(unix)]
if self.persistent_shell {
self.ensure_persistent_shell_initialized(&cwd).await;
}
#[cfg(not(unix))]
let _ = cwd;
}
TerminalCommand::KillForegroundCommands => {
self.kill_foreground_commands().await;
}
TerminalCommand::BackgroundForeground {
tool_call_id,
reply,
} => {
let found = self.handle_background_foreground(&tool_call_id);
let _ = reply.send(found);
}
TerminalCommand::KillForegroundCommandsByOwner { owner_session_id } => {
self.kill_foreground_commands_by_owner(&owner_session_id)
.await;
}
TerminalCommand::KillTasksByOwner {
owner_session_id,
reply,
} => {
self.kill_tasks_by_owner(&owner_session_id).await;
let _ = reply.send(());
}
TerminalCommand::ReparentNotifications {
old_owner_session_id,
new_owner_session_id,
new_handle,
backend_weak,
reply,
} => {
self.reparent_notifications(
&old_owner_session_id,
&new_owner_session_id,
new_handle,
backend_weak,
);
let _ = reply.send(());
}
}
}
/// Wrap a freshly-spawned child's process group in an `Arc`, enroll a `Weak`
/// into the `ProcessScope` so the TUI exit paths can reap this
/// setsid-detached tree if the actor's own teardown never runs, and return
/// the owning `Arc` for the `ProcessState` to hold. The actor keeps the only
/// strong ref, so a clean reap leaves a dead `Weak` (PID-reuse-safe).
fn enroll_spawned(
&self,
group: crate::util::ProcessGroup,
) -> std::sync::Arc<crate::util::ProcessGroup> {
let group = std::sync::Arc::new(group);
self.scope.register(&group);
group
}
async fn handle_run(
&mut self,
request: TerminalRunRequest,
reply: oneshot::Sender<Result<TerminalRunResult, ComputerError>>,
) {
// Foreground callers never see this id; the reply goes back on the oneshot.
let internal_id = uuid::Uuid::now_v7().to_string();
let SpawnResult {
child,
process_group,
state_dump_handle,
} = match self
.spawn_command(&request.command, &request.working_directory, &request.env)
.await
{
Ok(r) => r,
Err(e) => {
let _ = reply.send(Err(e));
return;
}
};
// Move the child process into the memory-limited cgroup (best-effort).
if let Some(pid) = child.id()
&& let Err(e) = self._cgroup_guard.add_process(pid).await
{
tracing::debug!("Failed to add pid {pid} to cgroup (non-fatal): {e}");
}
let file_handle = match open_output_file(&request.output_file).await {
Ok(file) => Some(file),
Err(e) => {
tracing::warn!(
"Failed to open output file {}: {}",
request.output_file.display(),
e
);
None
}
};
let process_state = ProcessState {
child,
process_group: Some(self.enroll_spawned(process_group)),
output_buffer: Vec::new(),
front_buffer: None,
truncated: false,
total_bytes: 0,
exit_status: None,
bg_status: BackgroundStatus::Foreground {
auto_bg_on_timeout: request.auto_background_on_timeout,
},
completion_waiters: vec![reply],
output_byte_limit: request.output_byte_limit,
timeout: request.timeout,
foreground_block_budget: request
.foreground_block_budget
.unwrap_or(self.foreground_block_budget),
start_time: Instant::now(),
output_file: request.output_file,
file_handle,
command: request.command.clone(),
display_command: request.display_command.clone(),
cwd: request.working_directory.display().to_string(),
start_wall_time: std::time::SystemTime::now(),
completed_at: None,
end_wall_time: None,
notification_handle: request.notification_handle.clone(),
tool_call_id: request.tool_call_id.clone(),
kind: request.kind,
last_notified_total: 0,
drained: false,
block_waited: false,
explicitly_killed: false,
state_dump_handle,
owner_session_id: request.owner_session_id.clone(),
};
// Send an initial empty notification so the TUI shows the execution
// timer immediately, before any stdout/stderr output arrives.
request
.notification_handle
.send_output_chunk(BashOutputChunk {
base: BashNotificationBase {
tool_call_id: request.tool_call_id.clone(),
command: request.command.clone(),
output: Vec::new(),
total_bytes: 0,
truncated: false,
cwd: request.working_directory.clone(),
},
});
self.processes.insert(internal_id, process_state);
}
async fn handle_kill(&mut self, terminal_id: &str) -> KillOutcome {
let Some(process) = self.processes.get_mut(terminal_id) else {
return KillOutcome::NotFound;
};
if process.exit_status.is_some() {
return KillOutcome::AlreadyExited;
}
// Mark as explicitly killed BEFORE sending the kill signal so the
// exit watcher's snapshot carries the flag.
process.explicitly_killed = true;
let outcome = kill_and_finalize(process).await;
// Resolve waiters here so callers blocked on wait_for_completion() unblock
// without waiting for the next poll tick.
if let Some(waiters) = self.completion_waiters.remove(terminal_id) {
let snapshot = match self.processes.get(terminal_id) {
Some(p) => Some(p.to_task_snapshot(terminal_id).await),
None => None,
};
for waiter in waiters {
let _ = waiter.reply.send(snapshot.clone());
}
}
outcome
}
async fn handle_run_background(
&mut self,
request: TerminalRunRequest,
reply: oneshot::Sender<Result<BackgroundHandle, ComputerError>>,
) {
// Background commands fork the current shell state but don't update it on exit.
let SpawnResult {
child,
process_group,
state_dump_handle,
} = match self
.spawn_command(&request.command, &request.working_directory, &request.env)
.await
{
Ok(r) => r,
Err(e) => {
let _ = reply.send(Err(e));
return;
}
};
// Move the child process into the memory-limited cgroup (best-effort).
if let Some(pid) = child.id()
&& let Err(e) = self._cgroup_guard.add_process(pid).await
{
tracing::debug!("Failed to add pid {pid} to cgroup (non-fatal): {e}");
}
let file_handle = match open_output_file(&request.output_file).await {
Ok(file) => Some(file),
Err(e) => {
tracing::warn!(
"Failed to open output file {}: {}",
request.output_file.display(),
e
);
None
}
};
// The actor owns task identity; callers only ever receive it.
let task_id = uuid::Uuid::now_v7().to_string();
let process_state = ProcessState {
child,
process_group: Some(self.enroll_spawned(process_group)),
output_buffer: Vec::new(),
front_buffer: None,
truncated: false,
total_bytes: 0,
exit_status: None,
bg_status: BackgroundStatus::Backgrounded {
reason: BackgroundReason::Explicit,
},
completion_waiters: vec![],
output_byte_limit: request.output_byte_limit,
timeout: request.timeout,
// Unused for already-backgrounded tasks; keep a defined value.
foreground_block_budget: request
.foreground_block_budget
.unwrap_or(self.foreground_block_budget),
start_time: Instant::now(),
output_file: request.output_file.clone(),
file_handle,
command: request.command.clone(),
display_command: request.display_command.clone(),
cwd: request.working_directory.display().to_string(),
start_wall_time: std::time::SystemTime::now(),
completed_at: None,
end_wall_time: None,
notification_handle: request.notification_handle.clone(),
tool_call_id: request.tool_call_id.clone(),
kind: request.kind,
last_notified_total: 0,
drained: false,
block_waited: false,
explicitly_killed: false,
// Background commands don't update the canonical shell state —
// they may run for hours and their env mutations shouldn't leak.
// Detach the dump reader so its result is discarded (the task
// continues independently until EOF or DUMP_READ_TIMEOUT).
state_dump_handle: if self.persistent_shell {
// The spawn still carries the state wrapping so bg commands
// inherit the session env; only the dump reader is discarded.
drop(state_dump_handle);
None
} else {
None
},
owner_session_id: request.owner_session_id.clone(),
};
let pid = process_state.child.id();
self.processes.insert(task_id.clone(), process_state);
let _ = reply.send(Ok(BackgroundHandle {
task_id,
output_file: request.output_file,
pid,
}));
}
/// Register a completion waiter and return immediately.
/// The polling loop will notify this waiter when the process exits or the deadline passes.
async fn handle_wait_for_completion(
&mut self,
task_id: String,
timeout: Option<Duration>,
reply: oneshot::Sender<Option<TaskSnapshot>>,
) {
let Some(process) = self.processes.get_mut(&task_id) else {
// The task is already evicted from `processes`, so serve the tombstone
// and imprint `block_waited=true` on it in place, so other readers
// (list_tasks, get_task) also see that the model awaited the result.
// Imprint only when the waiter actually receives the reply — a dropped
// receiver (cancelled turn) means the model never saw it.
let snapshot = self.completed_task_snapshots.get(&task_id).map(|s| {
let mut s = s.clone();
s.block_waited = true;
s
});
let found = snapshot.is_some();
let delivered = reply.send(snapshot).is_ok();
if found
&& delivered
&& let Some(s) = self.completed_task_snapshots.get_mut(&task_id)
{
s.block_waited = true;
}
return;
};
// Mark so the notification bridge skips auto-wake for this task.
// If the waiter is cancelled before receiving the result, this flag
// is cleared again (timeout expiry in `poll_all_processes` step 2,
// or undelivered completion in step 1) so auto-wake still fires.
let prev_block_waited = process.block_waited;
process.block_waited = true;
if process.exit_status.is_some() {
let snapshot = process.to_task_snapshot(&task_id).await;
if reply.send(Some(snapshot)).is_err() {
// Receiver dropped (e.g. the awaiting turn was cancelled):
// the model never received the result, so don't let this
// registration suppress auto-wake bookkeeping downstream.
process.block_waited = prev_block_waited;
}
return;
}
let timeout = timeout.unwrap_or(Duration::from_secs(30));
self.completion_waiters
.entry(task_id)
.or_default()
.push(CompletionWaiter {
reply,
deadline: Instant::now() + timeout,
});
}
async fn poll_all_processes(&mut self) {
// 0a. On a memory.high breach, kill the newest running process — the
// most recent command is the likeliest culprit.
if let Some(event) = self.memory_monitor.try_recv() {
tracing::warn!(
memory_current = event.memory_current,
memory_high = event.memory_high_threshold,
"Memory high threshold breached — killing newest running process"
);
let newest_id = self
.processes
.iter()
.filter(|(_, p)| p.exit_status.is_none())
.max_by_key(|(_, p)| p.start_time)
.map(|(id, _)| id.clone());
if let Some(id) = newest_id
&& let Some(process) = self.processes.get_mut(&id)
{
send_sigkill_to_group(process);
drain_remaining_output(process).await;
process.exit_status = Some(ExitStatus {
exit_code: Some(PROCESS_OOM_EXIT_CODE),
signal: Some("oom".to_owned()),
});
process.end_wall_time = Some(std::time::SystemTime::now());
process.flush_and_truncate_output_file().await;
let result = Ok(process.to_result());
process.notify_waiters(result);
}
}
// 0b. Kill backgrounded tasks that exceeded BACKGROUND_MAX_RUNTIME
let bg_expired: Vec<String> = self
.processes
.iter()
.filter(|(_, p)| {
p.bg_status.is_backgrounded()
&& p.exit_status.is_none()
&& p.start_time.elapsed() > BACKGROUND_MAX_RUNTIME
})
.map(|(id, _)| id.clone())
.collect();
for task_id in &bg_expired {
if let Some(process) = self.processes.get_mut(task_id) {
tracing::warn!(task_id, "Background task exceeded max runtime, killing");
// Fire-and-forget SIGTERM — poll loop escalates to SIGKILL
// on the next tick if the process doesn't exit.
send_sigterm_to_group(process);
process.exit_status = Some(ExitStatus {
exit_code: None,
signal: Some("max_runtime".to_owned()),
});
process.end_wall_time = Some(std::time::SystemTime::now());
process.flush_and_truncate_output_file().await;
}
}
// 0b (size). Kill any running task whose output file passed the cap
// (a detached writer has no timeout watching its disk use).
let output_cap = self.output_file_cap;
let size_exceeded: Vec<String> = self
.processes
.iter()
.filter(|(_, p)| p.exit_status.is_none() && p.total_bytes as u64 > output_cap)
.map(|(id, _)| id.clone())
.collect();
for task_id in &size_exceeded {
if let Some(process) = self.processes.get_mut(task_id) {
tracing::warn!(
task_id,
total_bytes = process.total_bytes,
cap = output_cap,
"Task exceeded output size cap, killing"
);
// Fire-and-forget SIGTERM — poll loop escalates to SIGKILL
// on the next tick if the process doesn't exit.
send_sigterm_to_group(process);
process.exit_status = Some(ExitStatus {
exit_code: None,
signal: Some("output_limit".to_owned()),
});
process.end_wall_time = Some(std::time::SystemTime::now());
process.flush_and_truncate_output_file().await;
// Notify any waiter immediately (may be a still-foreground
// command, unlike the bg-only max-runtime sweep; mirrors OOM).
let result = Ok(process.to_result());
process.notify_waiters(result);
}
}
let task_ids: Vec<String> = self.processes.keys().cloned().collect();
for task_id in &task_ids {
self.poll_process(task_id).await;
}
// Once a child is reaped (`child.id()` is `None`), drop the actor's
// strong `Arc<ProcessGroup>` so the `ProcessScope`'s `Weak` dies at reap.
// A completed task lingers in `self.processes` for the TTL; keeping the
// Arc that long would let `kill_all` upgrade the `Weak` and `killpg` a
// pid the OS may have recycled. Runs after the poll loop so it catches a
// reap from any path (normal/kill/oom/timeout) within one tick.
//
// Unix-only: only `killpg` can hit a recycled pid. On Windows the group
// is a JobObject HANDLE (no recyclable pid), so an early drop buys
// nothing — the Arc is released when the `ProcessState` is removed.
#[cfg(unix)]
for process in self.processes.values_mut() {
if process.process_group.is_some() && process.child.id().is_none() {
process.process_group = None;
}
}
// 0c. Collect state dumps from completed foreground processes and update
// the canonical shell state (persistent shell only).
// Uses a scoped borrow on self.processes so self.shell_state can be
// updated afterwards.
#[cfg(unix)]
if self.persistent_shell {
for task_id in &task_ids {
let handle = {
let Some(process) = self.processes.get_mut(task_id) else {
continue;
};
// Only foreground processes update the canonical state.
if process.exit_status.is_none() || process.bg_status.is_backgrounded() {
continue;
}
process.state_dump_handle.take()
};
// Borrow on self.processes is released — safe to access self.shell_state.
if let Some(handle) = handle {
match handle.await {
Ok(Ok(dump)) => {
if let Some(ref mut state) = self.shell_state {
state.update_from_dump(&dump);
}
}
Ok(Err(e)) => {
tracing::debug!("failed to read shell state dump: {e}");
}
Err(e) => {
tracing::debug!("shell state dump task panicked: {e}");
}
}
}
}
}
// 1. Notify completion waiters for processes that have exited
let waiter_task_ids: Vec<String> = self.completion_waiters.keys().cloned().collect();
for task_id in waiter_task_ids {
let completed = self
.processes
.get(&task_id)
.map(|p| p.exit_status.is_some())
// A process that is gone counts as completed.
.unwrap_or(true);
if completed && let Some(waiters) = self.completion_waiters.remove(&task_id) {
let snapshot = match self.processes.get(&task_id) {
Some(p) => Some(p.to_task_snapshot(&task_id).await),
None => None,
};
let mut any_delivered = false;
for waiter in waiters {
if waiter.reply.send(snapshot.clone()).is_ok() {
any_delivered = true;
}
}
// If no waiter actually received the completion — every
// oneshot receiver was dropped because the awaiting turn(s)
// were cancelled (e.g. Ctrl+C mid `get_task_output`) — the
// model never saw the result. Clear `block_waited` so the
// TaskCompleted auto-wake in the notification bridge is NOT
// suppressed (this runs before step 3 emits the completion
// notification, so the cleared flag is what gets snapshotted).
if !any_delivered && let Some(process) = self.processes.get_mut(&task_id) {
process.block_waited = false;
}
}
}
// 2. Expire timed-out waiters (process still running but deadline passed)
let now = Instant::now();
let waiter_keys: Vec<String> = self.completion_waiters.keys().cloned().collect();
let mut timed_out_tasks: Vec<String> = Vec::new();
for task_id in waiter_keys {
let snapshot = match self.processes.get(&task_id) {
Some(p) => Some(p.to_task_snapshot(&task_id).await),
None => None,
};
if let Some(waiters) = self.completion_waiters.get_mut(&task_id) {
let mut i = 0;
while i < waiters.len() {
if now >= waiters[i].deadline {
let waiter = waiters.swap_remove(i);
let _ = waiter.reply.send(snapshot.clone());
timed_out_tasks.push(task_id.clone());
} else {
i += 1;
}
}
}
}
self.completion_waiters.retain(|_, v| !v.is_empty());
// Clear block_waited for tasks where all waiters timed out without
// receiving the completion. Without this, auto-wake is permanently
// suppressed even though the agent never saw the result.
for task_id in timed_out_tasks {
if !self.completion_waiters.contains_key(&task_id)
&& let Some(process) = self.processes.get_mut(&task_id)
{
process.block_waited = false;
}
}
// 3. Set completed_at and clear output buffer for completed background tasks.
let mut newly_completed: Vec<String> = Vec::new();
for (task_id, process) in self.processes.iter_mut() {
if process.exit_status.is_some()
&& process.bg_status.is_backgrounded()
&& process.completed_at.is_none()
{
process.completed_at = Some(Instant::now());
if process.end_wall_time.is_none() {
process.end_wall_time = Some(std::time::SystemTime::now());
}
// Drop in-memory buffer — output file on disk has the full content
process.output_buffer.clear();
newly_completed.push(task_id.clone());
}
}
// The `block_waited` gate that suppresses the redundant auto-wake
// synthetic prompt for awaited tasks lives in
// `tools/notification_bridge.rs` (the `TaskCompleted` arm checks
// `task_snapshot.block_waited` before the auto-wake injection
// branch — see the comment there). This loop must still fire
// `send_task_complete` unconditionally for newly-completed
// background tasks so the pager UI, persistence, and
// `AutoWakeDeliveredIds` bookkeeping all still get the snapshot.
for task_id in newly_completed {
if let Some(process) = self.processes.get(&task_id) {
let snapshot = process.to_task_snapshot(&task_id).await;
process.notification_handle.send_task_complete(snapshot);
}
}
// 4. Evict processes based on completion state and TTL.
// Before evicting completed background tasks, save a lightweight
// TaskSnapshot so get_task can still return status after eviction.
let evict_ids: Vec<String> = self
.processes
.iter()
.filter(|(_, p)| {
if p.exit_status.is_none() {
return false;
}
// Foreground processes have already replied to their caller.
if !p.bg_status.is_backgrounded() {
return true;
}
// Backgrounded + completed: evict after TTL
matches!(p.completed_at, Some(t) if t.elapsed() >= self.completed_task_ttl)
})
.map(|(id, _)| id.clone())
.collect();
for id in &evict_ids {
if let Some(p) = self.processes.get(id)
&& p.bg_status.is_backgrounded()
{
// Save metadata-only snapshot (no output) before eviction so
// get_task still returns status/exit_code. Output is on disk
// at `output_file` — reading it into memory here would leak
// unbounded data for long-running tasks.
let snapshot = TaskSnapshot {
task_id: id.clone(),
command: p.command.clone(),
display_command: p.display_command.clone(),
cwd: p.cwd.clone(),
start_time: p.start_wall_time,
end_time: p.end_wall_time,
output: String::new(),
output_file: p.output_file.clone(),
truncated: p.truncated,
exit_code: p.exit_status.as_ref().and_then(|s| s.exit_code),
signal: p.exit_status.as_ref().and_then(|s| s.signal.clone()),
completed: true,
kind: p.kind,
block_waited: p.block_waited,
explicitly_killed: p.explicitly_killed,
owner_session_id: p.owner_session_id.clone(),
};
self.completed_task_snapshots.insert(id.clone(), snapshot);
}
self.processes.remove(id);
}
// Cap the tombstone map by evicting the oldest entries.
while self.completed_task_snapshots.len() > MAX_COMPLETED_TASK_SNAPSHOTS {
let oldest = self
.completed_task_snapshots
.iter()
.min_by_key(|(_, s)| s.start_time)
.map(|(id, _)| id.clone());
if let Some(id) = oldest {
self.completed_task_snapshots.remove(&id);
} else {
break;
}
}
}
async fn poll_process(&mut self, terminal_id: &str) {
let Some(process) = self.processes.get_mut(terminal_id) else {
return;
};
// If exit_status is already set (e.g., by timeout handler or external signal),
// the process may still be running. Escalate to SIGKILL if needed, and drain
// output once it exits.
if process.exit_status.is_some() {
if process.drained {
return;
}
match process.child.try_wait() {
Ok(None) => {
send_sigkill_to_group(process);
}
Ok(Some(_)) => {
drain_remaining_output(process).await;
process.flush_and_truncate_output_file().await;
process.drained = true;
}
Err(_) => {
drain_remaining_output(process).await;
process.flush_and_truncate_output_file().await;
process.drained = true;
}
}
return;
}
// Both streams are read with non-blocking `poll_read`, taking only what is
// available right now. A timeout-per-stream read costs that timeout per
// process on every tick even when idle, which for N processes overruns the
// 100 ms tick interval and delays the output-file writes.
let mut new_bytes: Vec<u8> = Vec::new();
let mut stdout_eof = false;
if let Some(stdout) = process.child.stdout.as_mut() {
loop {
let mut buf = [0u8; READ_BUFFER_SIZE];
match try_read_nonblocking(stdout, &mut buf) {
Some(Ok(0)) => {
stdout_eof = true;
break;
}
Some(Ok(n)) => {
new_bytes.extend_from_slice(&buf[..n]);
}
Some(Err(_)) => {
stdout_eof = true;
break;
}
None => break,
}
}
}
let mut stderr_eof = false;
if let Some(stderr) = process.child.stderr.as_mut() {
loop {
let mut buf = [0u8; READ_BUFFER_SIZE];
match try_read_nonblocking(stderr, &mut buf) {
Some(Ok(0)) => {
stderr_eof = true;
break;
}
Some(Ok(n)) => {
new_bytes.extend_from_slice(&buf[..n]);
}
Some(Err(_)) => {
stderr_eof = true;
break;
}
None => break,
}
}
}
// Batch write to file + flush so the output is visible to readers
// (e.g. `read_file` on the output_file from get_task_output).
if !new_bytes.is_empty() {
process.output_buffer.extend_from_slice(&new_bytes);
process.total_bytes += new_bytes.len();
if let Some(ref mut file) = process.file_handle {
let _ = file.write_all(&new_bytes).await;
let _ = file.flush().await;
}
}
// Truncate in-memory buffer if needed (file has full output)
process.maybe_truncate();
// Keyed off the monotonic `total_bytes` (not `output_buffer.len()`):
// after `maybe_truncate` freezes the front half and keeps only the
// shrinking tail, a length-based gate would go false and stay false,
// starving the stream of chunks for the rest of a long-output command.
if process.total_bytes > process.last_notified_total {
process
.notification_handle
.send_output_chunk(BashOutputChunk {
base: BashNotificationBase {
tool_call_id: process.tool_call_id.clone(),
command: process.command.clone(),
output: process.output_buffer.clone(),
total_bytes: process.total_bytes,
truncated: process.truncated,
cwd: process.cwd.clone().into(),
},
});
process.last_notified_total = process.total_bytes;
}
// Foreground budget: auto-backgroundable commands stop blocking the
// turn after the per-process budget (independent of `timeout`) — this
// second timer only backgrounds, never kills. Default is 15s; sessions
// can override via BashParams.foreground_block_budget_ms (0 = disable
// short budget so only `timeout` auto-bgs). The `timeout` check below
// also auto-bgs when auto_bg is on, or kills when it is off.
if process.exit_status.is_none()
&& matches!(
process.bg_status,
BackgroundStatus::Foreground {
auto_bg_on_timeout: true
}
)
&& process.start_time.elapsed() > process.foreground_block_budget
{
self.transition_to_background(terminal_id, BackgroundReason::ForegroundTimeout);
return;
}
if process.is_timed_out() && process.exit_status.is_none() {
if matches!(
process.bg_status,
BackgroundStatus::Foreground {
auto_bg_on_timeout: true
}
) {
self.transition_to_background(terminal_id, BackgroundReason::ForegroundTimeout);
return;
}
send_sigterm_to_group(process);
process.exit_status = Some(ExitStatus {
exit_code: None,
signal: Some("timeout".to_owned()),
});
process.end_wall_time = Some(std::time::SystemTime::now());
process.flush_and_truncate_output_file().await;
let result = Ok(process.to_result());
process.notify_waiters(result);
return;
}
let process_done = stdout_eof && stderr_eof;
match process.child.try_wait() {
Ok(Some(status)) => {
// Drain what arrived after the reads above: fast commands (e.g.
// `python3 -c "print('x')"`) exit before their pipe buffers are
// read, and without this drain their output comes back empty.
drain_remaining_output(process).await;
process.exit_status = Some(extract_exit_status(status));
process.end_wall_time = Some(std::time::SystemTime::now());
process.flush_and_truncate_output_file().await;
let result = Ok(process.to_result());
process.notify_waiters(result);
}
Ok(None) if process_done => {
// Streams closed but process hasn't exited yet - wait a bit
}
Ok(None) => {
// Still running
}
Err(e) => {
process.exit_status = Some(ExitStatus {
exit_code: None,
signal: Some(format!("error: {}", e)),
});
process.end_wall_time = Some(std::time::SystemTime::now());
process.flush_and_truncate_output_file().await;
let result = Ok(process.to_result());
process.notify_waiters(result);
}
}
}
async fn shutdown_all(&mut self) {
for process in self.processes.values_mut() {
send_sigkill_to_group(process);
// Abort the state dump reader so its spawn_blocking thread
// doesn't outlive the actor.
if let Some(handle) = process.state_dump_handle.take() {
handle.abort();
}
}
self.processes.clear();
}
/// Transition a foreground process to background (shared by auto-timeout
/// and user Ctrl+G). Re-keys from `old_key` to `tool_call_id`.
fn transition_to_background(&mut self, old_key: &str, reason: BackgroundReason) -> bool {
let Some(mut process) = self.processes.remove(old_key) else {
return false;
};
process.bg_status = BackgroundStatus::Backgrounded { reason };
process.timeout = BACKGROUND_MAX_RUNTIME;
let result = Ok(process.to_result());
process.notify_waiters(result);
let tool_call_id = process.tool_call_id.clone();
tracing::info!(
tool_call_id = %tool_call_id,
?reason,
"Foreground command transitioned to background"
);
self.processes.insert(tool_call_id, process);
true
}
/// Background a foreground command by `tool_call_id` (user Ctrl+G).
fn handle_background_foreground(&mut self, tool_call_id: &str) -> bool {
let internal_id = self
.processes
.iter()
.find(|(_, p)| p.tool_call_id == tool_call_id && !p.bg_status.is_backgrounded())
.map(|(id, _)| id.clone());
let Some(internal_id) = internal_id else {
return false;
};
self.transition_to_background(&internal_id, BackgroundReason::UserSignal)
}
/// Kill all non-backgrounded (foreground) processes and notify their waiters.
/// Backgrounded processes are left untouched. The actor stays alive for reuse.
///
/// After sending SIGKILL we **wait for each child to actually exit** (with a
/// 5 s timeout) so the kernel reclaims its memory pages before the next tool
/// call can allocate. Without this wait, a rapid OOM → recover → OOM cycle
/// can hit memory.max because the previous child's RSS hasn't been freed yet.
async fn kill_foreground_commands(&mut self) {
let fg_ids: Vec<String> = self
.processes
.iter()
.filter(|(_, p)| !p.bg_status.is_backgrounded() && p.exit_status.is_none())
.map(|(id, _)| id.clone())
.collect();
for id in &fg_ids {
if let Some(process) = self.processes.get_mut(id) {
send_sigkill_to_group(process);
// Bounded at 5 s; SIGKILL is unconditional so this resolves
// almost instantly in practice.
let _ =
tokio::time::timeout(std::time::Duration::from_secs(5), process.child.wait())
.await;
// Abort the state dump reader task so its `spawn_blocking`
// thread doesn't leak. Without this, a grandchild that
// inherited fd 4 and escaped the process group keeps the
// pipe's write end open, and the blocking `read_to_string`
// hangs indefinitely (the timeout future was abandoned when
// the JoinHandle was dropped).
if let Some(handle) = process.state_dump_handle.take() {
handle.abort();
}
process.exit_status = Some(ExitStatus {
exit_code: None,
signal: Some("cancelled".to_owned()),
});
process.end_wall_time = Some(std::time::SystemTime::now());
process.flush_and_truncate_output_file().await;
let result = Ok(process.to_result());
process.notify_waiters(result);
}
}
for id in &fg_ids {
self.processes.remove(id);
}
}
/// Kill all running foreground processes owned by a specific session.
/// Processes owned by other sessions are left untouched.
async fn kill_foreground_commands_by_owner(&mut self, owner_session_id: &str) {
let fg_ids: Vec<String> = self
.processes
.iter()
.filter(|(_, p)| {
p.owner_session_id.as_deref() == Some(owner_session_id)
&& !p.bg_status.is_backgrounded()
&& p.exit_status.is_none()
})
.map(|(id, _)| id.clone())
.collect();
for id in &fg_ids {
if let Some(process) = self.processes.get_mut(id) {
send_sigkill_to_group(process);
let _ =
tokio::time::timeout(std::time::Duration::from_secs(5), process.child.wait())
.await;
if let Some(handle) = process.state_dump_handle.take() {
handle.abort();
}
process.exit_status = Some(ExitStatus {
exit_code: None,
signal: Some("cancelled".to_owned()),
});
process.end_wall_time = Some(std::time::SystemTime::now());
process.flush_and_truncate_output_file().await;
let result = Ok(process.to_result());
process.notify_waiters(result);
}
}
for id in &fg_ids {
self.processes.remove(id);
}
}
/// Kill all running background tasks owned by a specific session.
/// Foreground tasks and tasks owned by other sessions are left untouched.
async fn kill_tasks_by_owner(&mut self, owner_session_id: &str) {
let owned_ids: Vec<String> = self
.processes
.iter()
.filter(|(_, p)| {
p.owner_session_id.as_deref() == Some(owner_session_id)
&& p.exit_status.is_none()
&& p.bg_status.is_backgrounded()
})
.map(|(id, _)| id.clone())
.collect();
for id in owned_ids {
if let Some(process) = self.processes.get_mut(&id) {
process.explicitly_killed = true;
}
self.handle_kill(&id).await;
}
}
/// Reparent notification handles for all tasks owned by `old_owner_session_id`.
/// Swaps the notification handle to `new_handle` so events from surviving
/// processes (monitors, background tasks) route to the parent session.
///
/// Also sends synthetic `BashExecutionBackgrounded` notifications through
/// the new handle for each reparented background task, so the parent's TUI
/// creates a `bg_tasks` entry and subsequent `MonitorEvent`/`TaskCompleted`
/// notifications have a target to attach to.
///
/// For monitors, re-spawns the output pipeline so events continue streaming.
fn reparent_notifications(
&mut self,
old_owner_session_id: &str,
new_owner_session_id: &str,
new_handle: crate::notification::types::ToolNotificationHandle,
backend_weak: std::sync::Weak<dyn crate::computer::types::TerminalBackend>,
) {
for (task_id, process) in self.processes.iter_mut() {
if process.owner_session_id.as_deref() == Some(old_owner_session_id)
&& process.bg_status.is_backgrounded()
&& process.exit_status.is_none()
{
// Only reparent backgrounded, still-running tasks. Foreground
// processes keep the child's owner_session_id so the subsequent
// kill_foreground_commands_by_owner call can reap them.
process.owner_session_id = Some(new_owner_session_id.to_string());
process.notification_handle = new_handle.clone();
// Send a synthetic backgrounded notification so the parent's
// TUI registers this task in its bg_tasks map. For a reparented
// monitor, recover the human description from the baked
// "[monitor] <desc>" display command and forward the real
// command + `monitor_description`, so the pager renders a proper
// "Monitor" row (matching the original-spawn path) rather than a
// bash-highlighted "[monitor] …".
let is_monitor = process.kind == crate::computer::types::TaskKind::Monitor;
let monitor_description = if is_monitor {
process
.display_command
.as_deref()
.and_then(|d| d.strip_prefix("[monitor] "))
.map(str::to_string)
} else {
None
};
let reparent_command = if is_monitor {
process.command.clone()
} else {
process
.display_command
.clone()
.unwrap_or_else(|| process.command.clone())
};
new_handle.send_backgrounded(crate::notification::BashExecutionBackgrounded {
base: crate::notification::BashNotificationBase {
tool_call_id: process.tool_call_id.clone(),
command: reparent_command,
output: Vec::new(),
total_bytes: 0,
truncated: false,
cwd: std::path::PathBuf::from(&process.cwd),
},
output_file: process.output_file.clone(),
task_id: task_id.clone(),
monitor_description,
// Reparent path has no model tool description; monitors use
// `monitor_description` above.
description: None,
});
// Re-spawn the monitor pipeline so events continue streaming.
// The old pipeline died with the child's runtime.
if process.kind == crate::computer::types::TaskKind::Monitor {
let pipeline_task_id = task_id.clone();
let pipeline_description = process
.display_command
.clone()
.unwrap_or_else(|| process.command.clone());
// Weak so the reparented monitor doesn't pin the backend.
let pipeline_terminal = backend_weak.clone();
let pipeline_notif = new_handle.clone();
let pipeline_output_file = process.output_file.clone();
// Start from current file size so we don't re-emit
// events already delivered by the old pipeline.
let start_offset = std::fs::metadata(&pipeline_output_file)
.map(|m| m.len())
.unwrap_or(0);
tokio::spawn(async move {
crate::implementations::kigi::monitor::tool::run_monitor_pipeline(
&pipeline_task_id,
&pipeline_description,
pipeline_terminal,
&pipeline_notif,
&pipeline_output_file,
Some("kill_command_or_subagent".to_string()),
start_offset,
)
.await;
});
}
}
}
}
}
/// Handle to the terminal actor: the public `TerminalBackend` implementation,
/// which reaches the actor's state only by sending commands over a channel.
#[derive(Clone)]
pub struct LocalTerminalBackend {
cmd_tx: mpsc::Sender<TerminalCommand>,
cancel_token: CancellationToken,
}
impl LocalTerminalBackend {
/// Spawns the actor task that owns all terminal state.
pub fn new() -> Self {
Self::new_inner(None, false, false, SearchShadowConfig::default())
}
/// Environment variables, working directory, functions, aliases, and shell
/// options persist across command invocations. The user's login shell (bash or
/// zsh) is detected and its rc files are loaded once on the first command.
pub fn with_persistent_shell() -> Self {
Self::new_inner(None, false, true, SearchShadowConfig::default())
}
/// See [`CgroupMemoryConfig`] for details on the soft/hard limit model.
pub fn with_memory_limit(config: CgroupMemoryConfig) -> Self {
Self::new_inner(Some(config), false, false, SearchShadowConfig::default())
}
pub fn with_memory_limit_and_persistent_shell(config: CgroupMemoryConfig) -> Self {
Self::new_inner(Some(config), false, true, SearchShadowConfig::default())
}
/// Runs the actor on `spawn_local`, for single-threaded runtimes.
///
/// `search_shadows` is the host-resolved `find`→`bfs` / `grep`→`ugrep` enable
/// state, baked into this backend (see [`SearchShadowConfig`]).
pub fn new_local(search_shadows: SearchShadowConfig) -> Self {
Self::new_inner(None, true, false, search_shadows)
}
/// `search_shadows` is the host-resolved `find`→`bfs` / `grep`→`ugrep` enable
/// state, baked into this backend (see [`SearchShadowConfig`]).
pub fn new_local_with_persistent_shell(search_shadows: SearchShadowConfig) -> Self {
Self::new_inner(None, true, true, search_shadows)
}
pub fn new_local_with_memory_limit(config: CgroupMemoryConfig) -> Self {
Self::new_inner(Some(config), true, false, SearchShadowConfig::default())
}
/// Test-only: a spawn_local backend that enrolls spawned children into
/// `scope` instead of the process-global one, so a test can `kill_all()` in
/// isolation without latching the global scope shared by other tests.
#[cfg(test)]
pub(crate) fn new_local_with_scope(
search_shadows: SearchShadowConfig,
scope: crate::util::ProcessScope,
) -> Self {
Self::new_with_ttl(
None,
true,
false,
search_shadows,
COMPLETED_TASK_TTL,
FOREGROUND_BLOCK_BUDGET,
MAX_OUTPUT_FILE_BYTES,
scope,
)
}
#[cfg(test)]
pub(crate) fn new_with_completed_task_ttl(ttl: Duration) -> Self {
Self::new_with_ttl(
None,
false,
false,
SearchShadowConfig::default(),
ttl,
FOREGROUND_BLOCK_BUDGET,
MAX_OUTPUT_FILE_BYTES,
crate::util::global_process_scope().clone(),
)
}
#[cfg(test)]
pub(crate) fn new_with_foreground_budget(budget: Duration) -> Self {
Self::new_with_ttl(
None,
false,
false,
SearchShadowConfig::default(),
COMPLETED_TASK_TTL,
budget,
MAX_OUTPUT_FILE_BYTES,
crate::util::global_process_scope().clone(),
)
}
#[cfg(test)]
pub(crate) fn new_with_output_cap(output_file_cap: u64) -> Self {
Self::new_with_ttl(
None,
false,
false,
SearchShadowConfig::default(),
COMPLETED_TASK_TTL,
FOREGROUND_BLOCK_BUDGET,
output_file_cap,
crate::util::global_process_scope().clone(),
)
}
fn new_inner(
memory_config: Option<CgroupMemoryConfig>,
use_spawn_local: bool,
persistent_shell: bool,
search_shadows: SearchShadowConfig,
) -> Self {
Self::new_with_ttl(
memory_config,
use_spawn_local,
persistent_shell,
search_shadows,
COMPLETED_TASK_TTL,
foreground_block_budget_from_env(),
output_file_cap_from_env(),
crate::util::global_process_scope().clone(),
)
}
fn new_with_ttl(
memory_config: Option<CgroupMemoryConfig>,
use_spawn_local: bool,
persistent_shell: bool,
search_shadows: SearchShadowConfig,
completed_task_ttl: Duration,
foreground_block_budget: Duration,
output_file_cap: u64,
scope: crate::util::ProcessScope,
) -> Self {
let (cmd_tx, cmd_rx) = mpsc::channel(COMMAND_CHANNEL_SIZE);
let cancel_token = CancellationToken::new();
let cancel_token_clone = cancel_token.clone();
let actor_fut = async move {
let (cgroup_guard, memory_monitor) = match memory_config {
Some(config) => {
let guard = CgroupGuard::try_create(&config).await;
let monitor = MemoryMonitor::start(&guard, &config, use_spawn_local).await;
(guard, monitor)
}
None => (CgroupGuard::noop(), MemoryMonitor::noop()),
};
let actor = LocalTerminalActor::new(
cmd_rx,
cancel_token_clone,
cgroup_guard,
memory_monitor,
persistent_shell,
search_shadows,
completed_task_ttl,
foreground_block_budget,
output_file_cap,
scope,
);
actor.run().await;
};
if use_spawn_local {
tokio::task::spawn_local(actor_fut);
} else {
tokio::spawn(actor_fut);
}
Self {
cmd_tx,
cancel_token,
}
}
pub fn cancel(&self) {
self.cancel_token.cancel();
}
}
impl Default for LocalTerminalBackend {
fn default() -> Self {
Self::new()
}
}
#[async_trait::async_trait]
impl TerminalBackend for LocalTerminalBackend {
async fn run(&self, request: TerminalRunRequest) -> Result<TerminalRunResult, ComputerError> {
let (reply_tx, reply_rx) = oneshot::channel();
self.cmd_tx
.send(TerminalCommand::Run {
request,
reply: reply_tx,
})
.await
.map_err(|_| ComputerError::io("terminal actor shut down"))?;
reply_rx
.await
.map_err(|_| ComputerError::io("terminal actor dropped reply channel"))?
}
async fn run_background(
&self,
request: TerminalRunRequest,
) -> Result<BackgroundHandle, ComputerError> {
let (reply_tx, reply_rx) = oneshot::channel();
self.cmd_tx
.send(TerminalCommand::RunBackground {
request,
reply: reply_tx,
})
.await
.map_err(|_| ComputerError::io("terminal actor shut down"))?;
reply_rx
.await
.map_err(|_| ComputerError::io("terminal actor dropped reply channel"))?
}
async fn get_task(&self, task_id: &str) -> Option<TaskSnapshot> {
let (reply_tx, reply_rx) = oneshot::channel();
self.cmd_tx
.send(TerminalCommand::GetTask {
task_id: task_id.to_string(),
reply: reply_tx,
})
.await
.ok()?;
reply_rx.await.ok().flatten()
}
async fn kill_task(&self, task_id: &str) -> KillOutcome {
let (reply_tx, reply_rx) = oneshot::channel();
if self
.cmd_tx
.send(TerminalCommand::Kill {
task_id: task_id.to_string(),
reply: reply_tx,
})
.await
.is_err()
{
return KillOutcome::NotFound;
}
reply_rx.await.unwrap_or(KillOutcome::NotFound)
}
async fn wait_for_completion(
&self,
task_id: &str,
timeout: Option<Duration>,
) -> Option<TaskSnapshot> {
let (reply_tx, reply_rx) = oneshot::channel();
self.cmd_tx
.send(TerminalCommand::WaitForCompletion {
task_id: task_id.to_string(),
timeout,
reply: reply_tx,
})
.await
.ok()?;
reply_rx.await.ok().flatten()
}
async fn list_tasks(&self) -> Vec<TaskSnapshot> {
let (reply_tx, reply_rx) = oneshot::channel();
if self
.cmd_tx
.send(TerminalCommand::ListTasks { reply: reply_tx })
.await
.is_err()
{
return vec![];
}
reply_rx.await.unwrap_or_default()
}
async fn get_shell_cwd(&self) -> Option<PathBuf> {
let (reply_tx, reply_rx) = oneshot::channel();
self.cmd_tx
.send(TerminalCommand::GetShellCwd { reply: reply_tx })
.await
.ok()?;
reply_rx.await.ok().flatten()
}
async fn warm_persistent_shell(&self, cwd: &std::path::Path) {
let _ = self
.cmd_tx
.send(TerminalCommand::WarmShell {
cwd: cwd.to_path_buf(),
})
.await;
}
async fn kill_foreground_commands(&self) {
let _ = self
.cmd_tx
.send(TerminalCommand::KillForegroundCommands)
.await;
}
async fn kill_all_background_tasks(&self) {
let tasks = self.list_tasks().await;
for task in tasks {
if task.exit_code.is_none() && task.signal.is_none() {
self.kill_task(&task.task_id).await;
}
}
}
async fn kill_foreground_commands_by_owner(&self, owner_session_id: &str) {
let _ = self
.cmd_tx
.send(TerminalCommand::KillForegroundCommandsByOwner {
owner_session_id: owner_session_id.to_string(),
})
.await;
}
async fn kill_all_background_tasks_by_owner(&self, owner_session_id: &str) {
let (reply_tx, reply_rx) = oneshot::channel();
if self
.cmd_tx
.send(TerminalCommand::KillTasksByOwner {
owner_session_id: owner_session_id.to_string(),
reply: reply_tx,
})
.await
.is_err()
{
return;
}
let _ = reply_rx.await;
}
async fn reparent_notifications(
&self,
old_owner_session_id: &str,
new_owner_session_id: &str,
new_handle: crate::notification::types::ToolNotificationHandle,
backend_weak: std::sync::Weak<dyn crate::computer::types::TerminalBackend>,
) {
let (reply_tx, reply_rx) = oneshot::channel();
// `backend_weak` (anchored by the parent's `Arc`) drives re-spawned
// pipelines without keeping the backend alive.
if self
.cmd_tx
.send(TerminalCommand::ReparentNotifications {
old_owner_session_id: old_owner_session_id.to_string(),
new_owner_session_id: new_owner_session_id.to_string(),
new_handle,
backend_weak,
reply: reply_tx,
})
.await
.is_err()
{
return;
}
// Wait for the actor to process the reparent before returning,
// so the caller can safely shut down the old session without
// risking notification loss.
let _ = reply_rx.await;
}
async fn background_foreground_command(&self, tool_call_id: &str) -> bool {
let (reply_tx, reply_rx) = oneshot::channel();
if self
.cmd_tx
.send(TerminalCommand::BackgroundForeground {
tool_call_id: tool_call_id.to_string(),
reply: reply_tx,
})
.await
.is_err()
{
return false;
}
reply_rx.await.unwrap_or(false)
}
}
/// Non-blocking read: returns `Some(Ok(n))` if data is available,
/// `Some(Err(e))` on I/O error, `Some(Ok(0))` on EOF, or `None` if
/// no data is ready right now.
///
/// Uses `Waker::noop()` — safe because the actor runs a periodic
/// polling loop and doesn't need wake-up notifications from the pipe.
fn try_read_nonblocking(
reader: &mut (impl tokio::io::AsyncRead + Unpin),
buf: &mut [u8],
) -> Option<std::io::Result<usize>> {
use std::pin::Pin;
use std::task::{Context, Poll, Waker};
use tokio::io::ReadBuf;
let waker = Waker::noop();
let mut cx = Context::from_waker(waker);
let mut read_buf = ReadBuf::new(buf);
match Pin::new(reader).poll_read(&mut cx, &mut read_buf) {
Poll::Ready(Ok(())) => Some(Ok(read_buf.filled().len())),
Poll::Ready(Err(e)) => Some(Err(e)),
Poll::Pending => None,
}
}
/// Soft-kill the process tree. Non-blocking, fire-and-forget.
/// The actor's poll loop will reap the process on the next tick.
fn send_sigterm_to_group(process: &ProcessState) {
// Unix: skip if child already reaped (avoid pid-reuse race on the
// stored leader_pid). Windows JobObjects don't have this issue.
#[cfg(unix)]
if process.child.id().is_none() {
return;
}
if let Some(pg) = process.process_group.as_ref() {
let _ = pg.terminate();
}
}
/// Hard-kill the process tree, plus `start_kill` on the immediate child
/// as a fallback when group teardown is degraded.
fn send_sigkill_to_group(process: &mut ProcessState) {
// Unix: skip if child already reaped (see send_sigterm_to_group).
#[cfg(unix)]
if process.child.id().is_none() {
let _ = process.child.start_kill();
return;
}
if let Some(pg) = process.process_group.as_ref() {
let _ = pg.kill();
}
let _ = process.child.start_kill();
}
/// Drain remaining stdout/stderr into the output buffer and file.
/// Bounded by `DRAIN_TIMEOUT` so a backgrounded child holding the
/// pipe open cannot block the actor loop indefinitely.
async fn drain_remaining_output(process: &mut ProcessState) {
let timed_out = tokio::time::timeout(DRAIN_TIMEOUT, async {
if let Some(stdout) = process.child.stdout.as_mut() {
let mut buf = [0u8; READ_BUFFER_SIZE];
loop {
match stdout.read(&mut buf).await {
Ok(0) | Err(_) => break,
Ok(n) => {
process.output_buffer.extend_from_slice(&buf[..n]);
process.total_bytes += n;
if let Some(ref mut file) = process.file_handle {
let _ = file.write_all(&buf[..n]).await;
}
}
}
}
}
if let Some(stderr) = process.child.stderr.as_mut() {
let mut buf = [0u8; READ_BUFFER_SIZE];
loop {
match stderr.read(&mut buf).await {
Ok(0) | Err(_) => break,
Ok(n) => {
process.output_buffer.extend_from_slice(&buf[..n]);
process.total_bytes += n;
if let Some(ref mut file) = process.file_handle {
let _ = file.write_all(&buf[..n]).await;
}
}
}
}
}
})
.await;
if timed_out.is_err() {
tracing::debug!(
command = %process.command,
"drain timed out after {:?}, a backgrounded child may be holding the pipe open",
DRAIN_TIMEOUT,
);
}
// Drop the pipe handles so orphaned children holding them open cannot
// cause repeated drain timeouts on subsequent poll ticks.
process.child.stdout.take();
process.child.stderr.take();
process.maybe_truncate();
}
/// Two-phase kill that synchronously waits for the process to exit.
/// Used ONLY by `kill_and_finalize` (the explicit kill_task API) where
/// the caller expects the process to be dead when the call returns.
///
/// Every `.await` is bounded by a timeout so the actor loop is never
/// blocked indefinitely.
async fn graceful_kill_and_wait(process: &mut ProcessState) {
send_sigterm_to_group(process);
if tokio::time::timeout(SIGTERM_GRACE, process.child.wait())
.await
.is_ok()
{
drain_remaining_output(process).await;
return;
}
send_sigkill_to_group(process);
// Wait for reap — bounded at 5s. SIGKILL is unconditional so this
// almost always returns instantly. The cap protects against D-state
// (uninterruptible kernel I/O, e.g. NFS hang). If it times out,
// abandon — poll_process will pick it up later.
const SIGKILL_REAP_TIMEOUT: Duration = Duration::from_secs(5);
if tokio::time::timeout(SIGKILL_REAP_TIMEOUT, process.child.wait())
.await
.is_err()
{
tracing::warn!(
pid = ?process.child.id(),
"Process did not exit after SIGKILL within {:?}, \
abandoning reap — poll loop will pick it up",
SIGKILL_REAP_TIMEOUT
);
}
drain_remaining_output(process).await;
}
#[tracing::instrument(
name = "terminal.kill_and_finalize",
skip_all,
fields(pid = process.child.id())
)]
async fn kill_and_finalize(process: &mut ProcessState) -> KillOutcome {
// Already reaped between the caller's check and here (race with poll_process)
if process.exit_status.is_some() {
return KillOutcome::AlreadyExited;
}
// Fast path: process already exited on its own.
match process.child.try_wait() {
Ok(Some(status)) => {
drain_remaining_output(process).await;
finalize_process(process, Some(status)).await;
return KillOutcome::AlreadyExited;
}
Err(_) => {
drain_remaining_output(process).await;
finalize_process(process, None).await;
return KillOutcome::AlreadyExited;
}
// Still running, proceed to kill.
Ok(None) => {}
}
graceful_kill_and_wait(process).await;
// The child is reaped now, so drop the scope's reaping handle immediately
// rather than waiting for the next poll sweep — closes the window where a
// racing kill_all() could killpg the (now reused) pid. Guarded like the
// sweep: if a D-state reap was abandoned (child.id() still Some), keep the
// Arc so the poll loop can still reap it later.
#[cfg(unix)]
if process.child.id().is_none() {
process.process_group = None;
}
// Abort the state dump reader so its spawn_blocking thread doesn't
// leak (same rationale as kill_foreground_commands).
if let Some(handle) = process.state_dump_handle.take() {
handle.abort();
}
finalize_process(process, None).await;
KillOutcome::Killed
}
async fn finalize_process(process: &mut ProcessState, status: Option<std::process::ExitStatus>) {
if process.exit_status.is_some() {
return;
}
process.exit_status = Some(match status {
Some(s) => extract_exit_status(s),
None => ExitStatus {
exit_code: None,
signal: Some("killed".to_owned()),
},
});
if process.end_wall_time.is_none() {
process.end_wall_time = Some(std::time::SystemTime::now());
}
process.flush_and_truncate_output_file().await;
let result = Ok(process.to_result());
process.notify_waiters(result);
}
/// Open an output file for writing, creating parent directories if needed.
#[tracing::instrument(name = "fs.open_output_file", skip_all)]
async fn open_output_file(path: &std::path::Path) -> std::io::Result<File> {
if let Some(parent) = path.parent() {
tokio::fs::create_dir_all(parent).await?;
}
tokio::fs::OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.open(path)
.await
}
/// Capture the user's login-shell PATH so CLI tools from rc files are discoverable.
///
/// Non-interactive shells (`/bin/bash -c`) don't source rc files, so tools
/// installed via `.bashrc`/`.zshrc`/virtualenvs are invisible. This runs the
/// detected shell with `-lc` plus an explicit `source` of the rc file, extracts
/// PATH using SOH byte markers, and merges it with the current process PATH.
///
/// Returns a `HashMap` with a single `PATH` key, or an empty map on failure.
/// A 5-second timeout kills the child if rc files hang (conda init, nvm, etc.).
#[cfg(unix)]
async fn capture_login_path() -> HashMap<String, String> {
use tokio::io::AsyncReadExt;
let shell = shell_state::ShellKind::detect();
let rc_file = shell.rc_file_name();
// Use $HOME inside the script (not interpolated from Rust) to avoid
// shell injection if HOME contains special characters.
let script = format!("source \"$HOME/{rc_file}\" 2>/dev/null; printf '\\x01%s\\x01' \"$PATH\"");
let result = tokio::time::timeout(Duration::from_secs(5), async {
let mut cmd = tokio::process::Command::new(shell.binary_path());
cmd.args(["-lc", &script])
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.kill_on_drop(true);
crate::util::detach_command(&mut cmd);
cmd.envs(crate::util::pager_env());
let mut child = cmd.spawn().ok()?;
let mut stdout_buf = Vec::new();
if let Some(ref mut stdout) = child.stdout {
stdout.read_to_end(&mut stdout_buf).await.ok();
}
let status = child.wait().await.ok()?;
if !status.success() {
return None;
}
let stdout = String::from_utf8_lossy(&stdout_buf);
let parts: Vec<&str> = stdout.split('\x01').collect();
let login_path = (parts.len() >= 3).then(|| parts[1].trim())?;
if login_path.is_empty() {
return None;
}
// Merge: login PATH first, then current-process entries not already present.
let current_path = std::env::var("PATH").unwrap_or_default();
let mut seen = std::collections::HashSet::new();
let merged: Vec<&str> = login_path
.split(':')
.chain(current_path.split(':'))
.filter(|e| !e.is_empty() && seen.insert(*e))
.collect();
Some(merged.join(":"))
})
.await;
match result {
Ok(Some(path)) => HashMap::from([("PATH".to_string(), path)]),
Ok(None) => HashMap::new(),
Err(_) => {
tracing::warn!("login-shell PATH capture timed out after 5s");
HashMap::new()
}
}
}
/// Spawn the shell command and attach the child to a [`ProcessGroup`].
///
/// The returned `ProcessGroup` is what the teardown helpers
/// ([`send_sigterm_to_group`], [`send_sigkill_to_group`]) dispatch to:
/// `killpg` on Unix; `TerminateJobObject` on Windows. This gives
/// grandchild teardown for fan-out workloads (npm install, git clone,
/// cargo build) on both platforms.
fn spawn_shell_command(
command: &str,
cwd: &std::path::Path,
env: &HashMap<String, String>,
login_env: Option<&HashMap<String, String>>,
search_shadows: SearchShadowConfig,
) -> std::io::Result<(tokio::process::Child, crate::util::ProcessGroup)> {
// `login_env` and `search_shadows` are only consumed by the `#[cfg(unix)]`
// shell wrapper below; keep them live on Windows to avoid unused-arg warnings.
#[cfg(not(unix))]
let _ = (&login_env, &search_shadows);
#[cfg(unix)]
let mut cmd = {
let shell = shell_state::ShellKind::detect();
let wrapped_command = {
let inject = super::embedded_search_tools::search_injection(search_shadows);
if inject.is_empty() {
command.to_string()
} else {
format!("{inject}{command}")
}
};
let mut cmd = tokio::process::Command::new(shell.binary_path());
// Non-interactive zsh still defaults to NOMATCH; pass via argv like init's -o extendedglob.
if matches!(shell, shell_state::ShellKind::Zsh) {
cmd.arg("-o").arg("nonomatch");
}
cmd.arg("-c")
.arg(&wrapped_command)
.current_dir(cwd)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
// NOTE: do NOT set .process_group(0) here — std runs setpgid()
// BEFORE pre_exec hooks, which would make the child a process
// group leader and cause setsid() to fail with EPERM.
// detach_from_tty() handles both session and process group creation.
.kill_on_drop(true);
// Request env carries .envrc, color vars, ACP-provided vars.
cmd.envs(shell_state::shell_env_overrides());
for (key, value) in env {
cmd.env(key, value);
}
cmd.envs(crate::util::pager_env());
// Inject the user's login-shell PATH LAST so tools installed via rc
// files (.bashrc, .zshrc, virtualenvs) are always discoverable. The
// request env often carries a copy of the parent process's PATH which
// doesn't include rc-file additions — applying login PATH after the
// request env ensures those additions aren't clobbered.
if let Some(login) = login_env {
cmd.envs(login);
}
// Agent marker must win over request/login env.
crate::util::apply_kigi_agent_marker(&mut cmd);
// Detach from the controlling terminal so subprocesses cannot open
// /dev/tty and compete with the TUI for terminal input.
crate::util::detach_command(&mut cmd);
// If the sandbox profile restricts network, install a seccomp BPF
// filter on the child that blocks connect/bind/sendto/listen/accept.
// The parent (kigi) process retains network for the LLM API.
// Filesystem restrictions are already inherited from the process-level
// Landlock/Seatbelt sandbox — no action needed here for FS.
#[cfg(target_os = "linux")]
if kigi_sandbox::should_restrict_child_network() {
unsafe {
cmd.pre_exec(|| kigi_sandbox::child_net::install_child_network_filter());
}
}
cmd
};
#[cfg(not(unix))]
let mut build_cmd = |with_breakaway: bool| {
use windows::Win32::System::Threading::{
CREATE_BREAKAWAY_FROM_JOB, CREATE_NEW_PROCESS_GROUP, CREATE_NO_WINDOW,
};
let inv = kigi_config::shell::shell_command_argv(command);
let mut cmd = tokio::process::Command::new(&inv.program);
cmd.args(&inv.args)
.envs(inv.env)
.current_dir(cwd)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true);
for (key, value) in env {
cmd.env(key, value);
}
cmd.envs(crate::util::pager_env());
// Agent marker must win over request env.
crate::util::apply_kigi_agent_marker(&mut cmd);
// Set creation flags inline rather than via crate::util::detach_command
// + new_process_group: tokio's creation_flags is a SET, not OR, so
// the helpers don't compose.
// - CREATE_NO_WINDOW: no console window pops up.
// - CREATE_NEW_PROCESS_GROUP: child can receive its own Ctrl+Break.
// - CREATE_BREAKAWAY_FROM_JOB: lets the child escape any inherited
// job so we can assign it to our own ProcessGroup. Per Microsoft
// docs this *fails CreateProcess with ERROR_ACCESS_DENIED* (os
// error 5) when the parent process is in a job that does not
// have JOB_OBJECT_LIMIT_BREAKAWAY_OK set — common when kigi-agent
// is launched under a Windows service, scheduled task, or some
// ACP host wrappers. The caller below retries without this flag
// on os error 5.
let mut flags = CREATE_NO_WINDOW | CREATE_NEW_PROCESS_GROUP;
if with_breakaway {
flags |= CREATE_BREAKAWAY_FROM_JOB;
}
cmd.creation_flags(flags.0);
cmd
};
#[cfg(unix)]
let mut group = crate::util::ProcessGroup::new()?;
#[cfg(unix)]
let child = cmd.spawn()?;
#[cfg(not(unix))]
let (child, mut group) = {
let group = crate::util::ProcessGroup::new()?;
let mut cmd = build_cmd(true);
match cmd.spawn() {
Ok(child) => (child, group),
Err(e) if e.raw_os_error() == Some(5) => {
// Parent's containing Job Object does not allow breakaway.
// Retry without CREATE_BREAKAWAY_FROM_JOB. We lose the
// ability to assign the child to our own job (so the
// attach() below will also fail), but the command runs.
// kill_on_drop + child.kill() still terminate the immediate
// child via TerminateProcess.
tracing::debug!(
"spawn with CREATE_BREAKAWAY_FROM_JOB returned ERROR_ACCESS_DENIED; \
retrying without breakaway (process-tree teardown disabled for this child)"
);
drop(cmd);
let mut cmd = build_cmd(false);
let child = cmd.spawn()?;
(child, group)
}
Err(e) => return Err(e),
}
};
if let Err(e) = group.attach(&child) {
tracing::debug!("Failed to attach child to ProcessGroup: {e}");
}
Ok((child, group))
}
fn extract_exit_status(status: std::process::ExitStatus) -> ExitStatus {
let exit_code = status.code();
#[cfg(unix)]
let signal = {
use std::os::unix::process::ExitStatusExt;
status.signal().map(|s| format!("signal {}", s))
};
#[cfg(not(unix))]
let signal: Option<String> = None;
ExitStatus { exit_code, signal }
}
#[cfg(test)]
mod tests {
use super::*;
use crate::computer::types::TaskKind;
use std::path::PathBuf;
fn make_request(command: &str) -> TerminalRunRequest {
let output_file = std::env::temp_dir().join(format!(
"terminal-test-{}-{}.out",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
TerminalRunRequest {
command: command.to_string(),
working_directory: PathBuf::from("/tmp"),
env: HashMap::new(),
timeout: Duration::from_secs(30),
output_byte_limit: 10000,
output_file,
notification_handle: ToolNotificationHandle::noop(),
tool_call_id: "test".to_string(),
display_command: None,
auto_background_on_timeout: false,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
}
}
/// Poll `get_task` every 25ms until the task reports `completed`, returning
/// `false` if `timeout` elapses first. Lets callers keep a bespoke assert
/// message while sharing the poll-until-reaped boilerplate.
async fn poll_until_task_completed(
backend: &LocalTerminalBackend,
task_id: &str,
timeout: Duration,
) -> bool {
let deadline = std::time::Instant::now() + timeout;
loop {
if backend
.get_task(task_id)
.await
.map(|s| s.completed)
.unwrap_or(false)
{
return true;
}
if std::time::Instant::now() >= deadline {
return false;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
}
#[tokio::test]
#[ignore = "flaky: combined_output is sometimes empty in CI"]
async fn test_simple_command() {
let backend = LocalTerminalBackend::new();
let request = make_request("echo hello");
let output_file = request.output_file.clone();
let result = backend.run(request).await.unwrap();
assert_eq!(result.combined_output.trim(), "hello");
assert_eq!(result.exit_code, Some(0));
assert!(!result.timed_out);
// Verify output was written to file
let file_content = tokio::fs::read_to_string(&output_file).await.unwrap();
assert_eq!(file_content.trim(), "hello");
// Cleanup
let _ = tokio::fs::remove_file(&output_file).await;
}
#[tokio::test]
async fn test_command_with_exit_code() {
let backend = LocalTerminalBackend::new();
let result = backend.run(make_request("exit 42")).await.unwrap();
assert_eq!(result.exit_code, Some(42));
}
#[tokio::test]
async fn test_timeout() {
let backend = LocalTerminalBackend::new();
let output_file =
std::env::temp_dir().join(format!("terminal-test-timeout-{}.out", std::process::id()));
let request = TerminalRunRequest {
command: "sleep 60".to_string(),
working_directory: PathBuf::from("/tmp"),
env: HashMap::new(),
timeout: Duration::from_millis(200),
output_byte_limit: 10000,
output_file: output_file.clone(),
notification_handle: ToolNotificationHandle::noop(),
tool_call_id: "test-timeout".to_string(),
display_command: None,
auto_background_on_timeout: false,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
};
let result = backend.run(request).await.unwrap();
assert!(result.timed_out);
// Cleanup
let _ = tokio::fs::remove_file(&output_file).await;
}
#[tokio::test]
async fn test_auto_background_on_timeout() {
let backend = LocalTerminalBackend::new();
let output_file =
std::env::temp_dir().join(format!("terminal-test-auto-bg-{}.out", std::process::id()));
let tool_call_id = "test-auto-bg";
let request = TerminalRunRequest {
command: "sleep 60".to_string(),
working_directory: PathBuf::from("/tmp"),
env: HashMap::new(),
timeout: Duration::from_millis(500),
output_byte_limit: 10000,
output_file: output_file.clone(),
notification_handle: ToolNotificationHandle::noop(),
tool_call_id: tool_call_id.to_string(),
display_command: None,
auto_background_on_timeout: true,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
};
let result = backend.run(request).await.unwrap();
// Should NOT be marked timed_out — it was auto-backgrounded instead.
assert!(
!result.timed_out,
"auto-backgrounded result must not be timed_out"
);
assert_eq!(
result.signal.as_deref(),
Some("auto_backgrounded"),
"signal should be auto_backgrounded, got {:?}",
result.signal
);
// The process should be accessible via get_task under the tool_call_id.
let snapshot = backend
.get_task(tool_call_id)
.await
.expect("auto-backgrounded task should be queryable by tool_call_id");
assert!(
!snapshot.completed,
"auto-backgrounded process should still be running"
);
// Cleanup: kill the background task so the test doesn't leak.
let outcome = backend.kill_task(tool_call_id).await;
assert!(
matches!(outcome, KillOutcome::Killed | KillOutcome::AlreadyExited),
"kill after auto-bg should succeed: {outcome:?}"
);
let _ = tokio::fs::remove_file(&output_file).await;
}
// Huge timeout (1h) + tiny budget: the budget, not the timeout, backgrounds
// an auto-backgroundable command.
#[tokio::test]
async fn test_foreground_block_budget_backgrounds_before_timeout() {
// Serialize flag-asserting tests; opt into the guards for this one.
let backend = LocalTerminalBackend::new_with_foreground_budget(Duration::from_millis(300));
let output_file = std::env::temp_dir().join(format!(
"terminal-test-fg-budget-{}.out",
std::process::id()
));
let tool_call_id = "test-fg-budget";
let request = TerminalRunRequest {
command: "sleep 60".to_string(),
working_directory: PathBuf::from("/tmp"),
env: HashMap::new(),
// 1h timeout, far longer than the 300ms budget.
timeout: Duration::from_secs(3600),
output_byte_limit: 10000,
output_file: output_file.clone(),
notification_handle: ToolNotificationHandle::noop(),
tool_call_id: tool_call_id.to_string(),
display_command: None,
auto_background_on_timeout: true,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
};
let start = Instant::now();
let result = backend.run(request).await.unwrap();
let elapsed = start.elapsed();
// Backgrounded by the budget, not killed by the timeout.
assert!(
!result.timed_out,
"budget-backgrounded result must not be timed_out"
);
assert_eq!(
result.signal.as_deref(),
Some("auto_backgrounded"),
"signal should be auto_backgrounded, got {:?}",
result.signal
);
// It returned on the budget (~300ms), nowhere near the 1h timeout.
assert!(
elapsed < Duration::from_secs(10),
"should background on the budget, not block on the timeout (took {elapsed:?})"
);
// Still running in the background under the tool_call_id.
let snapshot = backend
.get_task(tool_call_id)
.await
.expect("budget-backgrounded task should be queryable by tool_call_id");
assert!(
!snapshot.completed,
"budget-backgrounded process should still be running"
);
// Cleanup: kill the background task so the test doesn't leak.
let outcome = backend.kill_task(tool_call_id).await;
assert!(
matches!(outcome, KillOutcome::Killed | KillOutcome::AlreadyExited),
"kill after budget-bg should succeed: {outcome:?}"
);
let _ = tokio::fs::remove_file(&output_file).await;
}
// A non-backgroundable command must NOT be affected by the budget — it
// keeps its requested `timeout` as the sole (kill) deadline.
#[tokio::test]
async fn test_foreground_block_budget_skips_non_backgroundable() {
// Serialize flag-asserting tests; opt in so the "skip" is attributable
// to the command being non-backgroundable, not to the master switch.
let backend = LocalTerminalBackend::new_with_foreground_budget(Duration::from_millis(300));
let output_file = std::env::temp_dir().join(format!(
"terminal-test-fg-budget-skip-{}.out",
std::process::id()
));
let request = TerminalRunRequest {
command: "sleep 60".to_string(),
working_directory: PathBuf::from("/tmp"),
env: HashMap::new(),
// Short timeout so the test is fast; auto_bg is OFF so the budget
// must be ignored and the command killed on the timeout instead.
timeout: Duration::from_millis(500),
output_byte_limit: 10000,
output_file: output_file.clone(),
notification_handle: ToolNotificationHandle::noop(),
tool_call_id: "test-fg-budget-skip".to_string(),
display_command: None,
auto_background_on_timeout: false,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
};
let result = backend.run(request).await.unwrap();
// Killed by the timeout — NOT backgrounded by the budget.
assert!(
result.timed_out,
"non-backgroundable command should time out, not background"
);
assert_ne!(
result.signal.as_deref(),
Some("auto_backgrounded"),
"non-backgroundable command must not be auto-backgrounded by the budget"
);
let _ = tokio::fs::remove_file(&output_file).await;
}
// Per-request FG budget overrides the backend default (and can disable the
// short budget via Duration::MAX so only `timeout` auto-bgs).
#[tokio::test]
async fn test_per_request_foreground_block_budget_overrides_backend() {
// Backend default is 10s — request overrides to 300ms.
let backend = LocalTerminalBackend::new_with_foreground_budget(Duration::from_secs(10));
let output_file = std::env::temp_dir().join(format!(
"terminal-test-per-req-budget-{}.out",
std::process::id()
));
let tool_call_id = "test-per-req-budget";
let request = TerminalRunRequest {
command: "sleep 60".to_string(),
working_directory: PathBuf::from("/tmp"),
env: HashMap::new(),
timeout: Duration::from_secs(3600),
output_byte_limit: 10000,
output_file: output_file.clone(),
notification_handle: ToolNotificationHandle::noop(),
tool_call_id: tool_call_id.to_string(),
display_command: None,
auto_background_on_timeout: true,
// Per-request 300ms budget wins over backend's 10s.
foreground_block_budget: Some(Duration::from_millis(300)),
kind: TaskKind::Bash,
owner_session_id: None,
};
let start = Instant::now();
let result = backend.run(request).await.unwrap();
let elapsed = start.elapsed();
assert!(
!result.timed_out,
"per-request budget should auto-bg, not kill"
);
assert_eq!(result.signal.as_deref(), Some("auto_backgrounded"));
assert!(
elapsed < Duration::from_secs(5),
"should fire on per-request 300ms budget, not backend 10s (took {elapsed:?})"
);
let outcome = backend.kill_task(tool_call_id).await;
assert!(matches!(
outcome,
KillOutcome::Killed | KillOutcome::AlreadyExited
));
let _ = tokio::fs::remove_file(&output_file).await;
}
// `foreground_block_budget: Some(Duration::MAX)` disables the short budget:
// auto-bg only when the request timeout elapses.
#[tokio::test]
async fn test_duration_max_budget_waits_for_timeout_only() {
let backend = LocalTerminalBackend::new_with_foreground_budget(Duration::from_millis(100));
let output_file = std::env::temp_dir().join(format!(
"terminal-test-max-budget-{}.out",
std::process::id()
));
let tool_call_id = "test-max-budget";
let request = TerminalRunRequest {
command: "sleep 60".to_string(),
working_directory: PathBuf::from("/tmp"),
env: HashMap::new(),
// ~800ms kill/auto-bg timeout — short budget is disabled.
timeout: Duration::from_millis(800),
output_byte_limit: 10000,
output_file: output_file.clone(),
notification_handle: ToolNotificationHandle::noop(),
tool_call_id: tool_call_id.to_string(),
display_command: None,
auto_background_on_timeout: true,
foreground_block_budget: Some(Duration::MAX),
kind: TaskKind::Bash,
owner_session_id: None,
};
let start = Instant::now();
let result = backend.run(request).await.unwrap();
let elapsed = start.elapsed();
assert_eq!(result.signal.as_deref(), Some("auto_backgrounded"));
// Must wait past the short backend budget (100ms) and near the timeout.
assert!(
elapsed >= Duration::from_millis(500),
"Duration::MAX budget must not short-circuit at 100ms backend default (took {elapsed:?})"
);
assert!(
elapsed < Duration::from_secs(10),
"should auto-bg around the 800ms timeout (took {elapsed:?})"
);
let outcome = backend.kill_task(tool_call_id).await;
assert!(matches!(
outcome,
KillOutcome::Killed | KillOutcome::AlreadyExited
));
let _ = tokio::fs::remove_file(&output_file).await;
}
// Runaway-output guard: a stdout flood is killed once the output file
// passes the size cap, independent of its timeout.
#[tokio::test]
async fn test_output_size_guard_kills_runaway() {
// Serialize flag-asserting tests; opt into the guards for this one.
// Tiny cap so `yes` trips it within a tick or two.
let backend = LocalTerminalBackend::new_with_output_cap(2_000);
let output_file =
std::env::temp_dir().join(format!("terminal-test-size-{}.out", std::process::id()));
let request = TerminalRunRequest {
// `yes` floods stdout forever.
command: "yes".to_string(),
working_directory: PathBuf::from("/tmp"),
env: HashMap::new(),
// Long timeout: the SIZE guard, not the timeout, must fire.
timeout: Duration::from_secs(30),
output_byte_limit: 10_000,
output_file: output_file.clone(),
notification_handle: ToolNotificationHandle::noop(),
tool_call_id: "test-size".to_string(),
display_command: None,
auto_background_on_timeout: false,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
};
let result = backend.run(request).await.unwrap();
assert_eq!(
result.signal.as_deref(),
Some("output_limit"),
"runaway output should be killed by the size guard, got {:?}",
result.signal
);
assert!(!result.timed_out, "size kill is not a timeout");
let _ = tokio::fs::remove_file(&output_file).await;
}
#[tokio::test]
async fn test_stderr_captured() {
let backend = LocalTerminalBackend::new();
let result = backend.run(make_request("echo error >&2")).await.unwrap();
assert!(result.combined_output.contains("error"));
assert_eq!(result.exit_code, Some(0));
}
#[tokio::test]
#[ignore = "flaky: combined_output is sometimes empty in CI"]
async fn test_multiple_commands() {
let backend = LocalTerminalBackend::new();
let result1 = backend.run(make_request("echo first")).await.unwrap();
let result2 = backend.run(make_request("echo second")).await.unwrap();
assert_eq!(result1.combined_output.trim(), "first");
assert_eq!(result2.combined_output.trim(), "second");
}
#[tokio::test]
#[ignore = "flaky: output file content sometimes not flushed in CI"]
async fn test_output_file_written() {
let backend = LocalTerminalBackend::new();
let output_file =
std::env::temp_dir().join(format!("terminal-test-file-{}.out", std::process::id()));
let request = TerminalRunRequest {
command: "echo 'line1'; echo 'line2'; echo 'line3'".to_string(),
working_directory: PathBuf::from("/tmp"),
env: HashMap::new(),
timeout: Duration::from_secs(30),
output_byte_limit: 10000,
output_file: output_file.clone(),
notification_handle: ToolNotificationHandle::noop(),
tool_call_id: "test-output-file".to_string(),
display_command: None,
auto_background_on_timeout: false,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
};
let result = backend.run(request).await.unwrap();
assert_eq!(result.exit_code, Some(0));
// Verify file contains all output
let file_content = tokio::fs::read_to_string(&output_file).await.unwrap();
assert!(file_content.contains("line1"));
assert!(file_content.contains("line2"));
assert!(file_content.contains("line3"));
// Cleanup
let _ = tokio::fs::remove_file(&output_file).await;
}
#[tokio::test]
async fn test_run_background_and_get_task() {
let backend = LocalTerminalBackend::new();
let output_file =
std::env::temp_dir().join(format!("terminal-test-bg-{}.out", std::process::id()));
let request = TerminalRunRequest {
command: "echo background_test && sleep 0.1".to_string(),
working_directory: PathBuf::from("/tmp"),
env: HashMap::new(),
timeout: Duration::from_secs(30),
output_byte_limit: 10000,
output_file: output_file.clone(),
notification_handle: ToolNotificationHandle::noop(),
tool_call_id: "test-bg".to_string(),
display_command: None,
auto_background_on_timeout: false,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
};
// Start background task
let handle = backend.run_background(request).await.unwrap();
assert!(!handle.task_id.is_empty());
// Wait for completion
let snapshot = backend
.wait_for_completion(&handle.task_id, Some(Duration::from_secs(5)))
.await;
assert!(snapshot.is_some());
let snapshot = snapshot.unwrap();
assert!(snapshot.completed);
assert_eq!(snapshot.exit_code, Some(0));
// Cleanup
let _ = tokio::fs::remove_file(&output_file).await;
}
#[tokio::test]
async fn test_kill_background_task() {
let backend = LocalTerminalBackend::new();
let output_file =
std::env::temp_dir().join(format!("terminal-test-kill-{}.out", std::process::id()));
let request = TerminalRunRequest {
command: "sleep 60".to_string(),
working_directory: PathBuf::from("/tmp"),
env: HashMap::new(),
timeout: Duration::from_secs(300),
output_byte_limit: 10000,
output_file: output_file.clone(),
notification_handle: ToolNotificationHandle::noop(),
tool_call_id: "test-kill".to_string(),
display_command: None,
auto_background_on_timeout: false,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
};
let handle = backend.run_background(request).await.unwrap();
// Kill it
let outcome = backend.kill_task(&handle.task_id).await;
assert_eq!(outcome, KillOutcome::Killed);
// Cleanup
let _ = tokio::fs::remove_file(&output_file).await;
}
#[tokio::test]
async fn chunk_notifications_sent_during_execution() {
// Create a real notification channel (not noop)
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
let handle = ToolNotificationHandle::from_sender(tx);
let backend = LocalTerminalBackend::new();
let tmp = tempfile::TempDir::new().unwrap();
let request = TerminalRunRequest {
// Command that produces output over time (not all at once)
command: "for i in 1 2 3; do echo chunk_$i; sleep 0.15; done".to_string(),
working_directory: tmp.path().to_path_buf(),
env: HashMap::new(),
timeout: Duration::from_secs(5),
output_byte_limit: 1024 * 1024,
output_file: tmp.path().join("output.log"),
notification_handle: handle,
tool_call_id: "test-call-123".to_string(),
display_command: None,
auto_background_on_timeout: false,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
};
let result = backend.run(request).await.unwrap();
assert_eq!(result.exit_code, Some(0));
// Drain the notification channel
let mut chunks = vec![];
while let Ok(notification) = rx.try_recv() {
if let crate::notification::types::ToolNotification::BashOutputChunk(chunk) =
notification
{
chunks.push(chunk);
}
}
// Should have received at least 2 chunks: the initial empty notification
// plus at least one with output (command runs ~450ms with 100ms ticks)
assert!(
chunks.len() >= 2,
"Expected at least 2 BashOutputChunk notifications (initial + output), got {}",
chunks.len()
);
// The first chunk is the initial empty notification (sent before any output)
let initial = &chunks[0];
assert_eq!(initial.base.tool_call_id, "test-call-123");
assert!(!initial.base.command.is_empty());
assert!(
initial.base.output.is_empty(),
"Initial chunk should have empty output"
);
// Subsequent chunks should have output
let first_with_output = &chunks[1];
assert!(!first_with_output.base.output.is_empty());
// Later chunks should have more output than earlier ones
assert!(
chunks.last().unwrap().base.output.len() >= first_with_output.base.output.len(),
"Output should accumulate across chunks"
);
// The final output should contain all chunks
assert!(result.combined_output.contains("chunk_1"));
assert!(result.combined_output.contains("chunk_3"));
}
#[tokio::test]
async fn chunk_notifications_keep_flowing_after_truncation() {
// Regression guard for the emission gate fix: the gate is keyed off the
// monotonic `total_bytes`, not `output_buffer.len()`. Before the fix the
// length-based gate went false once `maybe_truncate` started shrinking
// the tail, starving the stream of chunks for the rest of a long command.
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
let handle = ToolNotificationHandle::from_sender(tx);
let backend = LocalTerminalBackend::new();
let tmp = tempfile::TempDir::new().unwrap();
let request = TerminalRunRequest {
// ~1.8 KB of ASCII over ~1.8s; far exceeds the 200-char limit so
// truncation fires early and keeps firing on the shrinking tail.
command: "for i in $(seq 1 60); do printf 'LINE%03d-XXXXXXXXXXXXXXXXXXXX\\n' \"$i\"; sleep 0.03; done".to_string(),
working_directory: tmp.path().to_path_buf(),
env: HashMap::new(),
timeout: Duration::from_secs(10),
output_byte_limit: 200,
output_file: tmp.path().join("output.log"),
notification_handle: handle,
tool_call_id: "trunc-call".to_string(),
display_command: None,
auto_background_on_timeout: false,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
};
let result = backend.run(request).await.unwrap();
assert_eq!(result.exit_code, Some(0));
assert!(result.truncated, "expected output to be truncated");
let mut chunks = vec![];
while let Ok(notification) = rx.try_recv() {
if let crate::notification::types::ToolNotification::BashOutputChunk(chunk) =
notification
{
chunks.push(chunk);
}
}
// total_bytes is monotonically non-decreasing across all chunks.
for w in chunks.windows(2) {
assert!(
w[1].base.total_bytes >= w[0].base.total_bytes,
"total_bytes regressed: {} < {}",
w[1].base.total_bytes,
w[0].base.total_bytes
);
}
// Truncation must surface, and chunks must keep arriving afterwards.
let first_truncated = chunks
.iter()
.position(|c| c.base.truncated)
.expect("expected at least one truncated chunk past the byte limit");
assert!(
first_truncated < chunks.len() - 1,
"chunks stopped after truncation (first_truncated={first_truncated}, total={})",
chunks.len()
);
// And the buffer length oscillates around the limit while total_bytes
// keeps climbing — proving the gate is total-based, not length-based.
assert!(
chunks.last().unwrap().base.total_bytes > 200,
"expected total_bytes to exceed the byte limit"
);
}
/// Size guard must kill runaway writers and bound the output file.
#[tokio::test]
async fn output_file_capped_by_size_guard() {
let cap: u64 = 5000;
let output_amount = cap * 1000;
let backend = LocalTerminalBackend::new_with_output_cap(cap);
let tmp = tempfile::TempDir::new().unwrap();
let output_file = tmp.path().join("output.log");
let request = TerminalRunRequest {
command: format!("head -c {output_amount} /dev/zero | tr '\\0' 'x'"),
working_directory: tmp.path().to_path_buf(),
env: HashMap::new(),
timeout: Duration::from_secs(30),
output_byte_limit: 1024,
output_file: output_file.clone(),
notification_handle: ToolNotificationHandle::noop(),
tool_call_id: "cap-test".to_string(),
display_command: None,
auto_background_on_timeout: false,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
};
let result = backend.run(request).await.unwrap();
assert_eq!(result.signal.as_deref(), Some("output_limit"));
let file_size = tokio::fs::metadata(&output_file).await.unwrap().len();
// Guard fires on 100ms tick — some overshoot expected (up to ~512 KB
// on arm64 CI). Without the guard the file would be `output_amount`.
assert!(
file_size < output_amount / 2,
"output file should be bounded by size guard, got {file_size} bytes (cap={cap})"
);
}
/// Verify retention cap constant and that small output is not truncated.
#[tokio::test]
async fn output_file_truncated_after_exit() {
let backend = LocalTerminalBackend::new();
let tmp = tempfile::TempDir::new().unwrap();
let output_file = tmp.path().join("output.log");
let request = TerminalRunRequest {
command: "head -c 200000 /dev/zero | tr '\\0' 'x'".to_string(),
working_directory: tmp.path().to_path_buf(),
env: HashMap::new(),
timeout: Duration::from_secs(10),
output_byte_limit: 1024,
output_file: output_file.clone(),
notification_handle: ToolNotificationHandle::noop(),
tool_call_id: "trunc-exit-test".to_string(),
display_command: None,
auto_background_on_timeout: false,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
};
let result = backend.run(request).await.unwrap();
assert_eq!(result.exit_code, Some(0));
// Output is under 64 MB, so file should have full content (no truncation).
let file_size = tokio::fs::metadata(&output_file).await.unwrap().len();
assert!(
file_size >= 190_000,
"output file should have full ~200 KB, got {file_size}"
);
// Verify the retention cap constant is reasonable.
assert_eq!(MAX_RETAINED_OUTPUT_FILE_BYTES, 64 * 1024 * 1024);
}
#[tokio::test]
async fn no_chunks_when_noop_handle() {
let backend = LocalTerminalBackend::new();
let tmp = tempfile::TempDir::new().unwrap();
let request = TerminalRunRequest {
command: "echo hello".to_string(),
working_directory: tmp.path().to_path_buf(),
env: HashMap::new(),
timeout: Duration::from_secs(5),
output_byte_limit: 1024 * 1024,
output_file: tmp.path().join("output.log"),
notification_handle: ToolNotificationHandle::noop(),
tool_call_id: "test-call".to_string(),
display_command: None,
auto_background_on_timeout: false,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
};
let result = backend.run(request).await.unwrap();
assert_eq!(result.exit_code, Some(0));
// noop() drops the receiver — send() is a silent no-op.
// This confirms no panic/crash when nobody is listening.
}
#[tokio::test]
async fn chunk_tool_call_id_matches_request() {
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
let handle = ToolNotificationHandle::from_sender(tx);
let backend = LocalTerminalBackend::new();
let tmp = tempfile::TempDir::new().unwrap();
let request = TerminalRunRequest {
command: "echo test; sleep 0.15; echo done".to_string(),
working_directory: tmp.path().to_path_buf(),
env: HashMap::new(),
timeout: Duration::from_secs(5),
output_byte_limit: 1024 * 1024,
output_file: tmp.path().join("output.log"),
notification_handle: handle,
tool_call_id: "unique-id-abc".to_string(),
display_command: None,
auto_background_on_timeout: false,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
};
backend.run(request).await.unwrap();
while let Ok(notification) = rx.try_recv() {
if let crate::notification::types::ToolNotification::BashOutputChunk(chunk) =
notification
{
assert_eq!(
chunk.base.tool_call_id, "unique-id-abc",
"Every chunk must carry the request's tool_call_id"
);
}
}
}
#[tokio::test]
async fn no_chunk_sent_when_no_new_output() {
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
let handle = ToolNotificationHandle::from_sender(tx);
let backend = LocalTerminalBackend::new();
let tmp = tempfile::TempDir::new().unwrap();
// Command that outputs once then sleeps (3 idle ticks with no new output)
let request = TerminalRunRequest {
command: "echo once; sleep 0.5".to_string(),
working_directory: tmp.path().to_path_buf(),
env: HashMap::new(),
timeout: Duration::from_secs(5),
output_byte_limit: 1024 * 1024,
output_file: tmp.path().join("output.log"),
notification_handle: handle,
tool_call_id: "test-idle".to_string(),
display_command: None,
auto_background_on_timeout: false,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
};
backend.run(request).await.unwrap();
let mut chunks = vec![];
while let Ok(notification) = rx.try_recv() {
if let crate::notification::types::ToolNotification::BashOutputChunk(chunk) =
notification
{
chunks.push(chunk);
}
}
// Should have chunks, but NOT one per tick — idle ticks should be skipped.
// The command outputs "once\n" early then sleeps 500ms (5 ticks).
// We should see 1-2 chunks, not 5.
assert!(
chunks.len() <= 3,
"Expected at most 3 chunks (not one per idle tick), got {}",
chunks.len()
);
}
#[tokio::test]
async fn test_timeout_uses_sigterm_then_sigkill() {
// Run a command with a short timeout. Verify the result has timed_out == true
// and the process is dead.
let backend = LocalTerminalBackend::new();
let tmp = tempfile::TempDir::new().unwrap();
let request = TerminalRunRequest {
command: "sleep 60".to_string(),
working_directory: PathBuf::from("/tmp"),
env: HashMap::new(),
timeout: Duration::from_millis(200),
output_byte_limit: 10000,
output_file: tmp.path().join("timeout-test.out"),
notification_handle: ToolNotificationHandle::noop(),
tool_call_id: "test-timeout-graceful".to_string(),
display_command: None,
auto_background_on_timeout: false,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
};
let result = backend.run(request).await.unwrap();
assert!(result.timed_out);
assert_eq!(result.signal.as_deref(), Some("timeout"));
}
#[tokio::test]
#[ignore = "flaky: output buffer sometimes not flushed before kill completes"]
async fn test_output_preserved_on_kill() {
// Spawn a command that produces output then sleeps.
// Kill it and verify the output before the kill is preserved.
let backend = LocalTerminalBackend::new();
let tmp = tempfile::TempDir::new().unwrap();
let request = TerminalRunRequest {
command: "echo before_kill; sleep 60".to_string(),
working_directory: PathBuf::from("/tmp"),
env: HashMap::new(),
timeout: Duration::from_secs(30),
output_byte_limit: 10000,
output_file: tmp.path().join("kill-output.out"),
notification_handle: ToolNotificationHandle::noop(),
tool_call_id: "test-kill-output".to_string(),
display_command: None,
auto_background_on_timeout: false,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
};
let handle = backend.run_background(request).await.unwrap();
// Give it time to produce output
tokio::time::sleep(Duration::from_millis(500)).await;
// Kill it
let outcome = backend.kill_task(&handle.task_id).await;
assert_eq!(outcome, KillOutcome::Killed);
// Check the snapshot has the output
let snapshot = backend.get_task(&handle.task_id).await;
if let Some(snapshot) = snapshot {
assert!(
snapshot.output.contains("before_kill"),
"Output should contain 'before_kill', got: {:?}",
snapshot.output
);
}
}
#[tokio::test]
async fn test_output_preserved_on_timeout() {
// Verify output captured before timeout is preserved in the result.
// Uses 2s timeout (not 500ms) so the poll loop has enough ticks to
// read the echo before the timeout handler snapshots the buffer.
let backend = LocalTerminalBackend::new();
let tmp = tempfile::TempDir::new().unwrap();
let request = TerminalRunRequest {
command: "echo before_timeout; sleep 60".to_string(),
working_directory: PathBuf::from("/tmp"),
env: HashMap::new(),
timeout: Duration::from_secs(2),
output_byte_limit: 10000,
output_file: tmp.path().join("timeout-output.out"),
notification_handle: ToolNotificationHandle::noop(),
tool_call_id: "test-timeout-output".to_string(),
display_command: None,
auto_background_on_timeout: false,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
};
let result = backend.run(request).await.unwrap();
assert!(result.timed_out);
assert!(
result.combined_output.contains("before_timeout"),
"Timed-out output should contain 'before_timeout', got: {:?}",
result.combined_output
);
}
#[tokio::test]
async fn test_background_child_with_inherited_pipe_does_not_block() {
// `sleep 300 &` inherits the pipe — without drain timeout this blocks forever.
let backend = LocalTerminalBackend::new();
let request = TerminalRunRequest {
command: "sleep 300 &\nsleep 1\necho done".to_string(),
working_directory: PathBuf::from("/tmp"),
env: HashMap::new(),
timeout: Duration::from_secs(30),
output_byte_limit: 10000,
output_file: std::env::temp_dir().join(format!(
"terminal-test-drain-timeout-{}.out",
std::process::id()
)),
notification_handle: ToolNotificationHandle::noop(),
tool_call_id: "test-drain-timeout".to_string(),
display_command: None,
auto_background_on_timeout: false,
foreground_block_budget: None,
kind: TaskKind::Bash,
owner_session_id: None,
};
let start = Instant::now();
let result = backend.run(request).await.unwrap();
let elapsed = start.elapsed();
assert!(
elapsed < Duration::from_secs(10),
"Command should complete in ~3s (1s sleep + 2s drain timeout), got {:?}",
elapsed
);
assert_eq!(result.exit_code, Some(0));
assert!(
result.combined_output.contains("done"),
"Output should contain 'done', got: {:?}",
result.combined_output
);
}
#[tokio::test]
async fn test_drain_timeout_constant() {
assert_eq!(DRAIN_TIMEOUT, Duration::from_secs(2));
}
#[tokio::test]
async fn test_sigterm_grace_is_one_second() {
// Static assertion: guard against someone changing the constant
assert_eq!(SIGTERM_GRACE, Duration::from_secs(1));
}
#[tokio::test]
async fn test_background_max_runtime_constant() {
// Static assertion: guard against accidental changes
assert_eq!(BACKGROUND_MAX_RUNTIME, Duration::from_secs(36_000));
}
#[tokio::test]
#[ignore = "flaky: pgrep sees sleep processes from other sandbox co-tenants"]
async fn test_kill_kills_child_processes() {
// Spawn a background command that creates multiple child processes.
// Kill it, then verify no sleep processes remain.
let backend = LocalTerminalBackend::new();
let request = make_request("bash -c 'sleep 60 & sleep 60 & wait'");
// Run in background
let handle = backend.run_background(request).await.unwrap();
// Give it a moment to spawn children
tokio::time::sleep(Duration::from_millis(200)).await;
// Kill the background task
let _ = backend.kill_task(&handle.task_id).await;
// Wait for processes to be reaped
tokio::time::sleep(Duration::from_millis(500)).await;
// Verify no sleep processes remain owned by this test process
#[cfg(unix)]
{
let pgrep = std::process::Command::new("pgrep")
.arg("-P")
.arg(std::process::id().to_string())
.arg("sleep")
.output()
.expect("pgrep should run");
// No sleep processes should be found
assert!(
pgrep.stdout.is_empty(),
"Expected no sleep processes to remain, but pgrep found: {:?}",
String::from_utf8_lossy(&pgrep.stdout)
);
}
}
#[test]
fn new_local_runs_on_current_thread_runtime() {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let local = tokio::task::LocalSet::new();
local.block_on(&rt, async {
let backend = LocalTerminalBackend::new_local(SearchShadowConfig::default());
let result = backend
.run(make_request("echo hello"))
.await
.expect("command should succeed");
assert_eq!(result.exit_code, Some(0));
assert!(result.combined_output.contains("hello"));
});
}
#[test]
fn new_local_sequential_commands_dont_stall() {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let local = tokio::task::LocalSet::new();
local.block_on(&rt, async {
let backend = LocalTerminalBackend::new_local(SearchShadowConfig::default());
for i in 0..5 {
let result = backend
.run(make_request(&format!("echo run_{i}")))
.await
.expect("command should succeed");
assert_eq!(result.exit_code, Some(0));
assert!(result.combined_output.contains(&format!("run_{i}")));
}
});
}
#[test]
fn new_local_background_task_lifecycle() {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let local = tokio::task::LocalSet::new();
local.block_on(&rt, async {
let backend = LocalTerminalBackend::new_local(SearchShadowConfig::default());
let mut bg_req = make_request("sleep 60");
bg_req.tool_call_id = "bg-1".to_string();
let bg = backend
.run_background(bg_req)
.await
.expect("background spawn should succeed");
let snap = backend
.get_task(&bg.task_id)
.await
.expect("task should exist");
assert!(!snap.completed);
let outcome = backend.kill_task(&bg.task_id).await;
assert!(
matches!(outcome, KillOutcome::Killed | KillOutcome::AlreadyExited),
"kill should succeed: {outcome:?}"
);
});
}
/// A background command must be reaped by the `ProcessScope` the backend
/// enrolled it into -- the same handle the TUI exit paths `kill_all()` on
/// process exit. The kill is driven through the scope (not the actor's
/// `kill_task`), so this fails if the spawn site stops enrolling children.
/// An injected scope keeps the process-global one un-latched for other tests.
#[test]
fn background_child_is_reaped_via_process_scope() {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let local = tokio::task::LocalSet::new();
local.block_on(&rt, async {
let scope = crate::util::ProcessScope::new();
let backend = LocalTerminalBackend::new_local_with_scope(
SearchShadowConfig::default(),
scope.clone(),
);
let mut bg_req = make_request("sleep 120");
bg_req.tool_call_id = "bg-scope-1".to_string();
let bg = backend
.run_background(bg_req)
.await
.expect("background spawn should succeed");
assert!(
!backend
.get_task(&bg.task_id)
.await
.expect("task should exist")
.completed,
"task should be running before kill_all"
);
// Reap via the scope, exactly like the TUI exit handlers do.
scope.kill_all();
// The actor observes the externally-killed child and marks the task
// complete. If enrollment regressed, the `sleep 120` would outlive
// the scope kill and this would time out.
assert!(
poll_until_task_completed(&backend, &bg.task_id, Duration::from_secs(10)).await,
"kill_all did not reap the enrolled background child"
);
});
}
/// Once a background child is reaped, the actor must drop its
/// `Arc<ProcessGroup>` so the scope's `Weak` dies. The completed task lingers
/// in `self.processes` for `COMPLETED_TASK_TTL`; if the actor kept the `Arc`
/// that long, a `kill_all()` on exit could `killpg` a pid the OS recycled.
/// Asserts the injected scope's live-group count goes 1 -> 0 across the reap.
#[test]
fn reaped_background_child_leaves_scope_empty() {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let local = tokio::task::LocalSet::new();
local.block_on(&rt, async {
let scope = crate::util::ProcessScope::new();
let backend = LocalTerminalBackend::new_local_with_scope(
SearchShadowConfig::default(),
scope.clone(),
);
// A brief sleep (not `true`): it must still be running when we read
// live_count below so we can observe the `1` end of the transition.
// The actor's first poll tick fires right after spawn, and `true`
// could already be reaped by then, making the `== 1` check racy.
let mut bg_req = make_request("sleep 1");
bg_req.tool_call_id = "bg-reap-1".to_string();
let bg = backend
.run_background(bg_req)
.await
.expect("background spawn should succeed");
// Enrollment is synchronous with the reply and the still-running
// child keeps the Arc alive, so exactly one group is live here.
assert_eq!(
scope.live_count(),
1,
"spawn must enroll exactly one live group"
);
// Drive the actor until it observes the exit and reaps the child.
assert!(
poll_until_task_completed(&backend, &bg.task_id, Duration::from_secs(10)).await,
"background `sleep 1` was never reaped"
);
// The reap sweep runs in the same poll tick that sets exit_status, so
// by the time get_task reports completed the Arc is already dropped —
// kill_all() would be a no-op and can't killpg a reused pid.
assert_eq!(
scope.live_count(),
0,
"a reaped child must leave no live group enrolled in the scope"
);
});
}
#[tokio::test]
async fn test_persistent_shell_cd_persists() {
let backend = LocalTerminalBackend::with_persistent_shell();
// cd to /tmp
let result = backend.run(make_request("cd /tmp")).await.unwrap();
assert_eq!(result.exit_code, Some(0));
// pwd should return /tmp (macOS resolves to /private/tmp)
let result = backend.run(make_request("pwd")).await.unwrap();
assert_eq!(result.exit_code, Some(0));
let pwd = result.combined_output.trim();
assert!(
pwd == "/tmp" || pwd == "/private/tmp",
"cwd should persist across commands, got: {pwd}"
);
}
#[tokio::test]
async fn test_persistent_shell_env_var_persists() {
let backend = LocalTerminalBackend::with_persistent_shell();
let result = backend
.run(make_request("export KIGI_PERSIST_TEST=hello123"))
.await
.unwrap();
assert_eq!(result.exit_code, Some(0));
let result = backend
.run(make_request("echo $KIGI_PERSIST_TEST"))
.await
.unwrap();
assert_eq!(result.exit_code, Some(0));
assert_eq!(
result.combined_output.trim(),
"hello123",
"env var should persist across commands"
);
}
#[tokio::test]
async fn test_persistent_shell_clears_gpg_tty() {
// GPG_TTY must be forced empty on the live path even when supplied via the request env.
let backend = LocalTerminalBackend::with_persistent_shell();
let mut req = make_request("echo \"[$GPG_TTY]\"");
req.env
.insert("GPG_TTY".to_string(), "/kigi-sentinel-tty".to_string());
let result = backend.run(req).await.unwrap();
assert_eq!(result.exit_code, Some(0));
assert_eq!(
result.combined_output.trim(),
"[]",
"GPG_TTY must be empty on the live tool path, got: {:?}",
result.combined_output
);
}
#[tokio::test]
async fn test_persistent_shell_function_persists() {
let backend = LocalTerminalBackend::with_persistent_shell();
let result = backend
.run(make_request("myfunc() { echo \"called with $1\"; }"))
.await
.unwrap();
assert_eq!(result.exit_code, Some(0));
let result = backend.run(make_request("myfunc test_arg")).await.unwrap();
assert_eq!(result.exit_code, Some(0));
assert_eq!(
result.combined_output.trim(),
"called with test_arg",
"function should persist across commands"
);
}
#[tokio::test]
async fn test_persistent_shell_variable_capture() {
let backend = LocalTerminalBackend::with_persistent_shell();
// Set a variable via command substitution
let result = backend
.run(make_request("export CAPTURED=$(echo captured_value)"))
.await
.unwrap();
assert_eq!(result.exit_code, Some(0));
// Read it back
let result = backend.run(make_request("echo $CAPTURED")).await.unwrap();
assert_eq!(result.exit_code, Some(0));
assert_eq!(
result.combined_output.trim(),
"captured_value",
"variable from command substitution should persist"
);
}
#[tokio::test]
async fn test_non_persistent_shell_no_state() {
// Verify the default (non-persistent) mode doesn't carry state.
let backend = LocalTerminalBackend::new();
let result = backend
.run(make_request("export SHOULD_NOT_PERSIST=yes"))
.await
.unwrap();
assert_eq!(result.exit_code, Some(0));
let result = backend
.run(make_request("echo ${SHOULD_NOT_PERSIST:-empty}"))
.await
.unwrap();
assert_eq!(result.exit_code, Some(0));
assert_eq!(
result.combined_output.trim(),
"empty",
"non-persistent mode should not carry state"
);
}
/// End-to-end test: completed background task remains queryable after
/// its ProcessState is evicted from the actor's process map.
///
/// Uses a 200ms TTL so we don't have to wait 5 minutes.
#[tokio::test]
async fn completed_bg_task_queryable_after_eviction() {
let ttl = Duration::from_millis(200);
let backend = LocalTerminalBackend::new_with_completed_task_ttl(ttl);
// 1. Start a fast background command.
let mut req = make_request("echo eviction_test_output");
req.tool_call_id = "evict-test".to_string();
let bg = backend
.run_background(req)
.await
.expect("background spawn should succeed");
// 2. Wait for completion.
let snap = backend
.wait_for_completion(&bg.task_id, Some(Duration::from_secs(5)))
.await
.expect("task should complete");
assert!(snap.completed, "task should be completed");
assert_eq!(snap.exit_code, Some(0));
// 3. Sleep past the TTL so the actor evicts the ProcessState.
tokio::time::sleep(ttl + Duration::from_millis(200)).await;
// 4. get_task should still return the snapshot (from the tombstone map).
let snap_after = backend
.get_task(&bg.task_id)
.await
.expect("task should still be queryable after eviction");
assert!(snap_after.completed);
assert_eq!(snap_after.exit_code, Some(0));
assert_eq!(snap_after.task_id, bg.task_id);
// 5. list_tasks should include the evicted task.
let all = backend.list_tasks().await;
assert!(
all.iter().any(|t| t.task_id == bg.task_id),
"evicted task should appear in list_tasks"
);
// 6. wait_for_completion should also return it (already done).
let snap_wait = backend
.wait_for_completion(&bg.task_id, Some(Duration::from_millis(100)))
.await
.expect("wait_for_completion should return evicted task");
assert!(snap_wait.completed);
}
/// Fix 3 — when wait_for_completion is called AFTER the task was
/// evicted from `processes` (snapshot-only branch), the returned
/// snapshot must reflect `block_waited=true` AND the in-place
/// tombstone in `completed_task_snapshots` must also be updated.
///
/// IMPORTANT: this test must NOT call `wait_for_completion` before
/// eviction, because that would set `process.block_waited=true` on
/// the live process and the eviction snapshot at
/// `poll_all_processes` step 4 would copy `block_waited: true` into
/// the tombstone (so the late-wait assertion would trivially pass
/// even with Fix 3 reverted). Sequence below keeps the tombstone
/// born with `block_waited=false` so the test actually exercises
/// the in-place mutation in `handle_wait_for_completion`'s
/// already-evicted branch.
#[tokio::test]
async fn wait_after_eviction_returns_snapshot_with_block_waited_true() {
let ttl = Duration::from_millis(100);
let backend = LocalTerminalBackend::new_with_completed_task_ttl(ttl);
// 1. Spawn a fast-exit background task. DO NOT call wait_for_completion
// so process.block_waited stays false.
let mut req = make_request("echo evict_and_wait");
req.tool_call_id = "evict-wait".to_string();
let bg = backend
.run_background(req)
.await
.expect("background spawn should succeed");
// 2. Poll `get_task` until the task completes.
let deadline = Instant::now() + Duration::from_secs(5);
let mut completed = false;
while Instant::now() < deadline {
if let Some(snap) = backend.get_task(&bg.task_id).await
&& snap.completed
{
completed = true;
break;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
assert!(completed, "task should complete within deadline");
// 3. Sleep past the TTL so eviction triggers.
tokio::time::sleep(ttl + Duration::from_millis(250)).await;
// 4. Assert the tombstone was born with `block_waited == false`.
// This pins the precondition: the tombstone has NOT inherited
// block_waited from the process, so a passing assertion below
// can only come from Fix 3's in-place mutation.
let snap_pre_wait = backend
.get_task(&bg.task_id)
.await
.expect("evicted task should still be queryable via tombstone");
assert!(
snap_pre_wait.completed,
"tombstone should reflect completion"
);
assert!(
!snap_pre_wait.block_waited,
"tombstone must be born with block_waited=false (no wait_for_completion \
was called before eviction); got block_waited=true which would make \
the late-wait assertion below trivial"
);
// 5. Late wait_for_completion against the tombstone.
let snap_after = backend
.wait_for_completion(&bg.task_id, Some(Duration::from_millis(100)))
.await
.expect("late wait should still return evicted snapshot");
assert!(snap_after.completed);
assert!(
snap_after.block_waited,
"late wait must set block_waited=true on the returned snapshot"
);
// 6. A subsequent get_task must see the imprinted flag too,
// confirming the mutation is in-place and observable to
// other readers of the tombstone (Fix 3's reason for using
// get_mut over get + clone).
let snap_via_get = backend
.get_task(&bg.task_id)
.await
.expect("get_task should still return after eviction");
assert!(
snap_via_get.block_waited,
"get_task after late wait must see the persisted block_waited flag \
— Fix 3's in-place mutation must be observable to other readers"
);
}
/// A blocking wait whose receiver is dropped before the task completes
/// (the awaiting turn was cancelled, e.g. Ctrl+C mid
/// `get_command_or_subagent_output`) must NOT leave `block_waited=true`
/// imprinted on the task. The model never received the result, so the
/// completion must still auto-wake it.
///
/// Regression test for the cancelled-wait race:
/// wait(timeout_ms=180000) → Ctrl+C → task exits before the wait
/// deadline → completion delivered to a dead oneshot → `block_waited`
/// stayed true → auto-wake suppressed → agent slept until the user
/// manually typed "continue".
#[tokio::test]
async fn cancelled_wait_does_not_suppress_auto_wake_on_completion() {
let backend = LocalTerminalBackend::new();
// 1. Background task that outlives the cancelled waiter below.
let mut req = make_request("sleep 1; echo done");
req.tool_call_id = "cancelled-wait".to_string();
let bg = backend
.run_background(req)
.await
.expect("background spawn should succeed");
// 2. Register a blocking wait with a deadline far beyond the task's
// runtime, then cancel it (drop the future — and with it the
// oneshot receiver) long before the task exits. This mirrors a
// turn abort while `get_task_output` blocks.
let cancelled = tokio::time::timeout(
Duration::from_millis(100),
backend.wait_for_completion(&bg.task_id, Some(Duration::from_secs(120))),
)
.await;
assert!(
cancelled.is_err(),
"wait must still be pending when the caller is cancelled"
);
// 3. Let the task complete and its completion be processed.
assert!(
poll_until_task_completed(&backend, &bg.task_id, Duration::from_secs(10)).await,
"task should complete within deadline"
);
// 4. The completion was never delivered to any waiter, so
// block_waited must be false — otherwise the notification
// bridge suppresses the TaskCompleted auto-wake.
let snap = backend
.get_task(&bg.task_id)
.await
.expect("completed task should be queryable");
assert!(
!snap.block_waited,
"a cancelled (never-delivered) blocking wait must not leave \
block_waited=true — that suppresses the completion auto-wake"
);
}
/// If one waiter is cancelled but another live waiter consumes the
/// completion, `block_waited` must stay true: the model received the
/// result through the surviving call, so the auto-wake would be
/// redundant noise.
#[tokio::test]
async fn cancelled_wait_alongside_live_waiter_keeps_block_waited() {
let backend = LocalTerminalBackend::new();
let mut req = make_request("sleep 1; echo done");
req.tool_call_id = "mixed-waiters".to_string();
let bg = backend
.run_background(req)
.await
.expect("background spawn should succeed");
// Dead waiter: cancelled before completion.
let cancelled = tokio::time::timeout(
Duration::from_millis(100),
backend.wait_for_completion(&bg.task_id, Some(Duration::from_secs(120))),
)
.await;
assert!(cancelled.is_err(), "first wait should be cancelled");
// Live waiter: rides until completion.
let snap = backend
.wait_for_completion(&bg.task_id, Some(Duration::from_secs(30)))
.await
.expect("live wait should return the completed snapshot");
assert!(snap.completed, "live wait should observe completion");
assert!(
snap.block_waited,
"delivered wait must keep block_waited=true on the returned snapshot"
);
let snap_via_get = backend
.get_task(&bg.task_id)
.await
.expect("completed task should be queryable");
assert!(
snap_via_get.block_waited,
"block_waited must remain true when at least one waiter received \
the completion — auto-wake would be redundant"
);
}
/// Helper: create a request owned by a specific session.
fn make_owned_request(command: &str, owner: &str) -> TerminalRunRequest {
let mut req = make_request(command);
req.owner_session_id = Some(owner.to_string());
req
}
#[tokio::test]
async fn kill_foreground_by_owner_only_kills_matching_session() {
let backend = LocalTerminalBackend::new();
// Spawn two foreground long-running commands owned by different sessions.
// Use auto_background_on_timeout with a long timeout so they stay foreground.
let mut req_a = make_owned_request("sleep 60", "session-a");
req_a.tool_call_id = "fg-a".to_string();
req_a.timeout = Duration::from_secs(300);
let mut req_b = make_owned_request("sleep 60", "session-b");
req_b.tool_call_id = "fg-b".to_string();
req_b.timeout = Duration::from_secs(300);
// Run both as background tasks first (so they don't block this test),
// then we'll test the scoped kill via the backend trait method.
let handle_a = backend.run_background(req_a).await.unwrap();
let handle_b = backend.run_background(req_b).await.unwrap();
// Both should be running
let snap_a = backend.get_task(&handle_a.task_id).await;
let snap_b = backend.get_task(&handle_b.task_id).await;
assert!(snap_a.is_some(), "task A should exist");
assert!(snap_b.is_some(), "task B should exist");
// Kill only session-a's tasks
backend
.kill_all_background_tasks_by_owner("session-a")
.await;
// Give the actor a moment to process
tokio::time::sleep(Duration::from_millis(100)).await;
// Task A should be killed (completed with explicitly_killed)
let snap_a = backend.get_task(&handle_a.task_id).await;
assert!(
snap_a.is_some_and(|s| s.completed && s.explicitly_killed),
"task A should be killed"
);
// Task B should still be running
let snap_b = backend.get_task(&handle_b.task_id).await;
assert!(
snap_b.is_some_and(|s| !s.completed),
"task B should still be running"
);
// Cleanup
backend.kill_task(&handle_b.task_id).await;
}
#[tokio::test]
async fn kill_by_owner_ignores_unowned_tasks() {
let backend = LocalTerminalBackend::new();
// Spawn a task with no owner (None)
let mut req = make_request("sleep 60");
req.tool_call_id = "fg-none".to_string();
let handle = backend.run_background(req).await.unwrap();
// Killing by a specific owner should NOT affect unowned tasks
backend
.kill_all_background_tasks_by_owner("some-session")
.await;
tokio::time::sleep(Duration::from_millis(100)).await;
let snap = backend.get_task(&handle.task_id).await;
assert!(
snap.is_some_and(|s| !s.completed),
"unowned task should NOT be killed by owner-scoped kill"
);
// Cleanup
backend.kill_task(&handle.task_id).await;
}
#[tokio::test]
async fn reparent_notifications_changes_owner_and_sends_synthetic() {
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
let new_handle = ToolNotificationHandle::from_sender(tx);
let backend: std::sync::Arc<dyn crate::computer::types::TerminalBackend> =
std::sync::Arc::new(LocalTerminalBackend::new());
// Spawn a background task owned by "child-session"
let mut req = make_owned_request("sleep 60", "child-session");
req.tool_call_id = "reparent-test".to_string();
let bg = backend.run_background(req).await.unwrap();
// Verify it's running and owned by child-session
let snap = backend.get_task(&bg.task_id).await.unwrap();
assert!(!snap.completed);
assert_eq!(snap.owner_session_id.as_deref(), Some("child-session"));
// Reparent from child-session to parent-session
backend
.reparent_notifications(
"child-session",
"parent-session",
new_handle,
std::sync::Arc::downgrade(&backend),
)
.await;
// Verify owner changed
let snap = backend.get_task(&bg.task_id).await.unwrap();
assert_eq!(
snap.owner_session_id.as_deref(),
Some("parent-session"),
"owner should be reparented to parent-session"
);
// Verify synthetic BashExecutionBackgrounded notification was sent
let mut found_backgrounded = false;
while let Ok(notification) = rx.try_recv() {
if let crate::notification::types::ToolNotification::BashExecutionBackgrounded(
bg_notif,
) = notification
{
assert_eq!(bg_notif.task_id, bg.task_id);
found_backgrounded = true;
}
}
assert!(
found_backgrounded,
"reparent should send a synthetic BashExecutionBackgrounded notification"
);
// Now killing by parent-session should kill the reparented task
backend
.kill_all_background_tasks_by_owner("parent-session")
.await;
tokio::time::sleep(Duration::from_millis(100)).await;
let snap = backend.get_task(&bg.task_id).await;
assert!(
snap.is_some_and(|s| s.completed),
"reparented task should be killed when parent-session tasks are killed"
);
}
#[tokio::test]
async fn reparent_skips_tasks_owned_by_other_sessions() {
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
let new_handle = ToolNotificationHandle::from_sender(tx);
let backend: std::sync::Arc<dyn crate::computer::types::TerminalBackend> =
std::sync::Arc::new(LocalTerminalBackend::new());
// Spawn tasks owned by different sessions
let mut req_child = make_owned_request("sleep 60", "child-session");
req_child.tool_call_id = "reparent-child".to_string();
let bg_child = backend.run_background(req_child).await.unwrap();
let mut req_sibling = make_owned_request("sleep 60", "sibling-session");
req_sibling.tool_call_id = "reparent-sibling".to_string();
let bg_sibling = backend.run_background(req_sibling).await.unwrap();
// Reparent only child-session
backend
.reparent_notifications(
"child-session",
"parent-session",
new_handle,
std::sync::Arc::downgrade(&backend),
)
.await;
// Child's owner should change
let snap_child = backend.get_task(&bg_child.task_id).await.unwrap();
assert_eq!(
snap_child.owner_session_id.as_deref(),
Some("parent-session")
);
// Sibling's owner should NOT change
let snap_sibling = backend.get_task(&bg_sibling.task_id).await.unwrap();
assert_eq!(
snap_sibling.owner_session_id.as_deref(),
Some("sibling-session"),
"sibling task should not be reparented"
);
// Only one synthetic notification (for the child, not the sibling)
let mut bg_count = 0;
while let Ok(notification) = rx.try_recv() {
if matches!(
notification,
crate::notification::types::ToolNotification::BashExecutionBackgrounded(_)
) {
bg_count += 1;
}
}
assert_eq!(
bg_count, 1,
"only the child task should produce a synthetic notification"
);
// Cleanup
backend.kill_task(&bg_child.task_id).await;
backend.kill_task(&bg_sibling.task_id).await;
}
#[tokio::test]
async fn owner_session_id_propagated_through_run_and_snapshot() {
let backend = LocalTerminalBackend::new();
let mut req = make_owned_request("echo owned", "test-owner");
req.tool_call_id = "owned-test".to_string();
let bg = backend.run_background(req).await.unwrap();
// Wait for completion
let snap = backend
.wait_for_completion(&bg.task_id, Some(Duration::from_secs(5)))
.await
.expect("task should complete");
assert!(snap.completed);
assert_eq!(
snap.owner_session_id.as_deref(),
Some("test-owner"),
"owner_session_id should propagate from request to snapshot"
);
}
}
Reference in New Issue View Git Blame Copy Permalink