//! Multi-client leader cluster: one shared leader, N pager clients, plus //! inspection of the durable session log so reattach tests can assert on //! what actually persisted. //! //! One [`ContentController`] gives one shared `$HOME` (hence one elected //! leader) plus a fixed leader socket beneath its `KIGI_SHARE_DIR`; clients //! spawn with the `--leader`/`--leader-socket` flags so they all attach to the //! SAME leader. use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; use anyhow::{Context, Result}; use serde_json::Value; use crate::{ContentController, PtyHarness, pager_binary}; /// One shared mock-backed leader plus the pager clients attached to it. pub struct LeaderCluster { content: ContentController, binary: PathBuf, socket: PathBuf, rows: u16, cols: u16, } impl LeaderCluster { pub async fn start(rows: u16, cols: u16) -> Result { let content = ContentController::start() .await .context("start content controller")?; // One shared KIGI_SHARE_DIR => one leader; the socket lives beneath it so // every client (sharing the same env) elects/attaches to the same one. let kigi_home = content.home().join(".kigi"); std::fs::create_dir_all(&kigi_home).context("create kigi home")?; let socket = kigi_home.join("leader-e2e.sock"); let binary = pager_binary().context("resolve pager binary")?; Ok(Self { content, binary, socket, rows, cols, }) } /// Spawn the client that starts a fresh session and brings up the leader. pub fn spawn_leader(&self, extra_args: &[&str]) -> Result { self.spawn_client(&[], extra_args) } /// Attach another client that resumes the shared session through the SAME /// leader. pub fn attach(&self, extra_args: &[&str]) -> Result { self.spawn_client(&["--resume"], extra_args) } /// `mode_args` carries the per-role flag (`--resume` for attachers); /// `extra_args` is the caller's. fn spawn_client(&self, mode_args: &[&str], extra_args: &[&str]) -> Result { let socket = self.socket.to_str().context("socket path is utf-8")?; let mut args: Vec<&str> = vec!["--leader", "--leader-socket", socket]; args.extend_from_slice(mode_args); args.extend_from_slice(extra_args); PtyHarness::spawn_with_content(&self.binary, self.rows, self.cols, &self.content, &args) .context("spawn pager client on shared leader") } pub fn content(&self) -> &ContentController { &self.content } /// Root of `sessions///updates.jsonl`. fn sessions_dir(&self) -> PathBuf { self.content.home().join(".kigi").join("sessions") } /// The `params.update` payload of every record across every /// `updates.jsonl` under the cluster, so a caller can match on the /// `sessionUpdate` tag directly. /// /// Infallible by design: a file that vanishes mid-walk, or whose appended /// tail tore across a multi-byte UTF-8 boundary (so `read_to_string` /// fails), is skipped for this call and picked up on the next one. pub fn session_updates(&self) -> Vec { let mut files = Vec::new(); collect_updates_files(&self.sessions_dir(), &mut files); let mut out = Vec::new(); for file in files { if let Ok(text) = std::fs::read_to_string(&file) { out.extend(parse_update_payloads(&text)); } } out } /// Poll [`session_updates`](Self::session_updates) until a /// `turn_completed` record appears, or error on timeout. pub fn wait_for_turn_completed(&self, timeout: Duration) -> Result { let deadline = Instant::now() + timeout; loop { let updates = self.session_updates(); if let Some(rec) = updates.iter().find(|u| is_turn_completed(u)) { return Ok(rec.clone()); } if Instant::now() >= deadline { // Surface what WAS persisted so "zero records / env problem" is // distinguishable from "records present but no turn_completed / // producer regression". let tags: std::collections::BTreeSet<&str> = updates .iter() .filter_map(|u| u.get("sessionUpdate").and_then(Value::as_str)) .collect(); anyhow::bail!( "timed out after {timeout:?} waiting for a turn_completed record under {}; \ saw {} update record(s) with sessionUpdate tags {tags:?}", self.sessions_dir().display(), updates.len(), ); } // Sync FS poll mirrors the harness's blocking wait_for_text; a stat // every 150ms is cheap and fine on a multi_thread runtime worker. std::thread::sleep(Duration::from_millis(150)); } } } fn is_turn_completed(update: &Value) -> bool { update.get("sessionUpdate").and_then(Value::as_str) == Some("turn_completed") } /// Parse the `params.update` payload out of each line of an `updates.jsonl` /// body, whose on-disk shape is `{..,"params":{"update":{..}}}`. A line that is /// blank, fails to parse (a torn trailing line that is still valid UTF-8), or /// carries no `params.update` is skipped rather than failing the batch. fn parse_update_payloads(text: &str) -> Vec { text.lines() .filter_map(|line| { let line = line.trim(); if line.is_empty() { return None; } let envelope: Value = serde_json::from_str(line).ok()?; envelope.get("params")?.get("update").cloned() }) .collect() } /// Recursively collect every `updates.jsonl` beneath `dir` (a manual walk to /// avoid a new crate dep). A missing/unreadable dir yields nothing — sessions /// may not exist yet, and the walk is re-run on every poll. fn collect_updates_files(dir: &Path, out: &mut Vec) { let Ok(entries) = std::fs::read_dir(dir) else { return; }; for entry in entries.flatten() { // No-follow file type: a symlinked directory has `is_dir() == false`, // so a symlink cycle can never recurse forever here. let Ok(file_type) = entry.file_type() else { continue; }; let path = entry.path(); if file_type.is_dir() { collect_updates_files(&path, out); } else if path.file_name().and_then(|n| n.to_str()) == Some("updates.jsonl") { out.push(path); } } } #[cfg(test)] mod tests { use super::*; /// Wrap a session update as the envelope stored in `updates.jsonl`. fn envelope(update_json: &str) -> String { format!( r#"{{"timestamp":1,"method":"_kigi/session/update","params":{{"sessionId":"s","update":{update_json}}}}}"# ) } #[test] fn parse_update_payloads_unwraps_and_tolerates_torn_trailing_line() { let body = format!( "{}\n{}\n{{\"timestamp\":2,\"method\":\"_kigi/sess", envelope( r#"{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"hi"}}"# ), envelope( r#"{"sessionUpdate":"turn_completed","prompt_id":"p-1","stop_reason":"end_turn"}"# ), ); let updates = parse_update_payloads(&body); // The two complete lines parse; the torn final line is dropped. assert_eq!(updates.len(), 2); let completed = updates .iter() .find(|u| is_turn_completed(u)) .expect("turn_completed payload is unwrapped from params.update"); assert_eq!(completed["stop_reason"], "end_turn"); assert_eq!(completed["prompt_id"], "p-1"); } #[test] fn parse_update_payloads_skips_blank_and_payloadless_lines() { let body = format!( "\n \n{}\n{{\"timestamp\":3,\"method\":\"x\",\"params\":{{\"sessionId\":\"s\"}}}}\n", envelope( r#"{"sessionUpdate":"turn_completed","prompt_id":"p","stop_reason":"cancelled"}"# ), ); let updates = parse_update_payloads(&body); // Only the one envelope carrying params.update survives. assert_eq!(updates.len(), 1); assert!(is_turn_completed(&updates[0])); assert_eq!(updates[0]["stop_reason"], "cancelled"); } }