ZacharyZhang-NY
a02b555e66
Sweep every first-party crate source (1956 .rs files) to the project comment guidelines: delete redundant restatements, decorative banners, change narration, and end-of-line comments; keep and tighten the crucial ones (invariants, bug rationale, SAFETY blocks, ported-source attribution). No functional code changed. Every edit is proven comment-only against the prior tree by a comment-stripping lexer (string/char/raw-string aware) plus a separate doctest-fence check. Where removing a comment made rustfmt or clippy want to re-lay-out adjacent code, the minimal triggering comment is restored so code tokens stay byte-identical. Gates green: cargo fmt --all --check (0 diffs), cargo check and cargo clippy --workspace --all-targets (0 warnings). Adds scripts/check_codegen_comment_guidelines.py — the enforcement gate for these guidelines (flags banners, end-of-line comments, change narration, and commented-out code).
334 lines
14 KiB
Rust
334 lines
14 KiB
Rust
//! Repro + regression test: leader process dies → connected clients must
|
|
//! re-elect a leader and transparently restore their sessions.
|
|
//!
|
|
//! Scenario (mirrors the field report "clients see `unknown session id` after
|
|
//! the leader dies"):
|
|
//!
|
|
//! 1. Two stdio clients (`kigi agent --leader stdio`) share one leader.
|
|
//! 2. Each creates its own session and completes a prompt round-trip.
|
|
//! 3. The leader is killed with SIGKILL (crash, no graceful shutdown).
|
|
//! 4. Each client's bridge must reconnect (re-electing / spawning a fresh
|
|
//! leader), replay `initialize` + `session/load`, and then prompts against
|
|
//! the ORIGINAL session IDs must succeed again.
|
|
//!
|
|
//! Tests are `#[ignore]`d by default — they require a pre-built binary:
|
|
//!
|
|
//! ```bash
|
|
//! cargo test -p kigi-shell --test test_leader_death_repro -- --ignored --nocapture
|
|
//! ```
|
|
|
|
#![cfg(unix)]
|
|
|
|
use std::time::Duration;
|
|
|
|
use agent_client_protocol::{self as acp, Agent as _};
|
|
|
|
use kigi_test_support::leader::{
|
|
LeaderStdioClient, leader_log, wait_for_live_leader, wait_for_new_leader,
|
|
wait_for_replay_notifications,
|
|
};
|
|
use kigi_test_support::*;
|
|
|
|
/// THE repro. Kill the shared leader with SIGKILL while two clients are
|
|
/// connected; both must recover their sessions on the re-elected leader.
|
|
#[tokio::test]
|
|
// requires pre-built binary; run with --ignored
|
|
#[ignore]
|
|
async fn test_leader_sigkill_clients_recover_sessions() {
|
|
tokio::task::LocalSet::new()
|
|
.run_until(async {
|
|
let server = MockInferenceServer::start().await.unwrap();
|
|
let workdir = git_workdir();
|
|
let home = tempfile::tempdir().unwrap();
|
|
std::fs::create_dir_all(home.path().join(".kigi")).unwrap();
|
|
|
|
// Phase 1: two clients, one leader, two sessions.
|
|
let client_a = LeaderStdioClient::spawn(&server, workdir.path(), home.path()).await;
|
|
client_a.initialize().await;
|
|
let session_a = client_a.create_session(workdir.path()).await;
|
|
let r = client_a.prompt(&session_a, "hello from A").await;
|
|
assert!(
|
|
r.is_ok(),
|
|
"pre-crash prompt A failed: {:?}\nstderr:\n{}\nleader log:\n{}",
|
|
r.err(),
|
|
client_a.stderr_text(),
|
|
leader_log(home.path()),
|
|
);
|
|
|
|
let client_b = LeaderStdioClient::spawn(&server, workdir.path(), home.path()).await;
|
|
client_b.initialize().await;
|
|
let session_b = client_b.create_session(workdir.path()).await;
|
|
let r = client_b.prompt(&session_b, "hello from B").await;
|
|
assert!(
|
|
r.is_ok(),
|
|
"pre-crash prompt B failed: {:?}\nstderr:\n{}\nleader log:\n{}",
|
|
r.err(),
|
|
client_b.stderr_text(),
|
|
leader_log(home.path()),
|
|
);
|
|
|
|
let leader_pid = wait_for_live_leader(home.path(), Duration::from_secs(5))
|
|
.await
|
|
.expect("no live leader PID in lock file");
|
|
assert_ne!(leader_pid, client_a.child.id().unwrap_or(0));
|
|
assert_ne!(leader_pid, client_b.child.id().unwrap_or(0));
|
|
|
|
// Phase 2: SIGKILL the leader (simulated crash).
|
|
let base_a = client_a.notification_count();
|
|
let base_b = client_b.notification_count();
|
|
eprintln!("killing leader pid {leader_pid}");
|
|
unsafe {
|
|
libc::kill(leader_pid as i32, libc::SIGKILL);
|
|
}
|
|
|
|
// Phase 3: clients must re-elect a leader and reconnect.
|
|
let new_pid = wait_for_new_leader(home.path(), leader_pid, Duration::from_secs(60))
|
|
.await
|
|
.unwrap_or_else(|| {
|
|
panic!(
|
|
"no new leader was elected after SIGKILL\n\
|
|
client A stderr:\n{}\nclient B stderr:\n{}\nleader log:\n{}",
|
|
client_a.stderr_text(),
|
|
client_b.stderr_text(),
|
|
leader_log(home.path()),
|
|
)
|
|
});
|
|
eprintln!("new leader elected: pid {new_pid}");
|
|
|
|
let a_reconnected =
|
|
wait_for_replay_notifications(&client_a, base_a, Duration::from_secs(60)).await;
|
|
let b_reconnected =
|
|
wait_for_replay_notifications(&client_b, base_b, Duration::from_secs(60)).await;
|
|
eprintln!("replay evidence: A={a_reconnected} B={b_reconnected}");
|
|
|
|
// Phase 4: prompts on the ORIGINAL session IDs must work.
|
|
let res_a = client_a.prompt(&session_a, "after crash A").await;
|
|
let res_b = client_b.prompt(&session_b, "after crash B").await;
|
|
|
|
assert!(
|
|
res_a.is_ok(),
|
|
"client A prompt after leader crash failed: {:?}\n\
|
|
stderr:\n{}\nleader log:\n{}",
|
|
res_a.err(),
|
|
client_a.stderr_text(),
|
|
leader_log(home.path()),
|
|
);
|
|
assert!(
|
|
res_b.is_ok(),
|
|
"client B prompt after leader crash failed: {:?}\n\
|
|
stderr:\n{}\nleader log:\n{}",
|
|
res_b.err(),
|
|
client_b.stderr_text(),
|
|
leader_log(home.path()),
|
|
);
|
|
})
|
|
.await;
|
|
}
|
|
|
|
/// Single-client variant: kill -9 the leader, the lone client must re-elect
|
|
/// and restore. Narrower failure surface than the two-client test.
|
|
#[tokio::test]
|
|
// requires pre-built binary; run with --ignored
|
|
#[ignore]
|
|
async fn test_leader_sigkill_single_client_recovers() {
|
|
tokio::task::LocalSet::new()
|
|
.run_until(async {
|
|
let server = MockInferenceServer::start().await.unwrap();
|
|
let workdir = git_workdir();
|
|
let home = tempfile::tempdir().unwrap();
|
|
std::fs::create_dir_all(home.path().join(".kigi")).unwrap();
|
|
|
|
let client = LeaderStdioClient::spawn(&server, workdir.path(), home.path()).await;
|
|
client.initialize().await;
|
|
let session = client.create_session(workdir.path()).await;
|
|
client
|
|
.prompt(&session, "hello")
|
|
.await
|
|
.expect("pre-crash prompt failed");
|
|
|
|
let leader_pid = wait_for_live_leader(home.path(), Duration::from_secs(5))
|
|
.await
|
|
.expect("no live leader PID in lock file");
|
|
let base = client.notification_count();
|
|
eprintln!("killing leader pid {leader_pid}");
|
|
unsafe {
|
|
libc::kill(leader_pid as i32, libc::SIGKILL);
|
|
}
|
|
|
|
let new_pid = wait_for_new_leader(home.path(), leader_pid, Duration::from_secs(60))
|
|
.await
|
|
.unwrap_or_else(|| {
|
|
panic!(
|
|
"no new leader was elected after SIGKILL\nstderr:\n{}\nleader log:\n{}",
|
|
client.stderr_text(),
|
|
leader_log(home.path()),
|
|
)
|
|
});
|
|
eprintln!("new leader elected: pid {new_pid}");
|
|
|
|
let reconnected =
|
|
wait_for_replay_notifications(&client, base, Duration::from_secs(60)).await;
|
|
eprintln!("replay evidence: {reconnected}");
|
|
|
|
let res = client.prompt(&session, "after crash").await;
|
|
assert!(
|
|
res.is_ok(),
|
|
"prompt after leader crash failed: {:?}\nstderr:\n{}\nleader log:\n{}",
|
|
res.err(),
|
|
client.stderr_text(),
|
|
leader_log(home.path()),
|
|
);
|
|
})
|
|
.await;
|
|
}
|
|
|
|
/// One client driving TWO sessions over a single stdio bridge (the IDE
|
|
/// shape). After a leader SIGKILL, BOTH sessions must be replayed onto the
|
|
/// re-elected leader — restoring only the most recent one left the other
|
|
/// failing with "unknown session id".
|
|
#[tokio::test]
|
|
// requires pre-built binary; run with --ignored
|
|
#[ignore]
|
|
async fn test_leader_sigkill_multi_session_client_recovers_all_sessions() {
|
|
tokio::task::LocalSet::new()
|
|
.run_until(async {
|
|
let server = MockInferenceServer::start().await.unwrap();
|
|
let workdir = git_workdir();
|
|
let home = tempfile::tempdir().unwrap();
|
|
std::fs::create_dir_all(home.path().join(".kigi")).unwrap();
|
|
|
|
let client = LeaderStdioClient::spawn(&server, workdir.path(), home.path()).await;
|
|
client.initialize().await;
|
|
let session_one = client.create_session(workdir.path()).await;
|
|
client
|
|
.prompt(&session_one, "hello one")
|
|
.await
|
|
.expect("pre-crash prompt on session one failed");
|
|
let session_two = client.create_session(workdir.path()).await;
|
|
client
|
|
.prompt(&session_two, "hello two")
|
|
.await
|
|
.expect("pre-crash prompt on session two failed");
|
|
assert_ne!(session_one.0, session_two.0);
|
|
|
|
let leader_pid = wait_for_live_leader(home.path(), Duration::from_secs(5))
|
|
.await
|
|
.expect("no live leader PID in lock file");
|
|
let base = client.notification_count();
|
|
eprintln!("killing leader pid {leader_pid}");
|
|
unsafe {
|
|
libc::kill(leader_pid as i32, libc::SIGKILL);
|
|
}
|
|
|
|
wait_for_new_leader(home.path(), leader_pid, Duration::from_secs(60))
|
|
.await
|
|
.unwrap_or_else(|| {
|
|
panic!(
|
|
"no new leader was elected after SIGKILL\nstderr:\n{}\nleader log:\n{}",
|
|
client.stderr_text(),
|
|
leader_log(home.path()),
|
|
)
|
|
});
|
|
wait_for_replay_notifications(&client, base, Duration::from_secs(60)).await;
|
|
|
|
let res_one = client.prompt(&session_one, "after crash one").await;
|
|
let res_two = client.prompt(&session_two, "after crash two").await;
|
|
assert!(
|
|
res_one.is_ok(),
|
|
"session one prompt after crash failed: {:?}\nstderr:\n{}\nleader log:\n{}",
|
|
res_one.err(),
|
|
client.stderr_text(),
|
|
leader_log(home.path()),
|
|
);
|
|
assert!(
|
|
res_two.is_ok(),
|
|
"session two prompt after crash failed: {:?}\nstderr:\n{}\nleader log:\n{}",
|
|
res_two.err(),
|
|
client.stderr_text(),
|
|
leader_log(home.path()),
|
|
);
|
|
})
|
|
.await;
|
|
}
|
|
|
|
/// Prompt sent DURING the outage (after the bridge noticed the dead leader
|
|
/// but before the new one is ready). The stdio bridge must hold and deliver
|
|
/// it once the session is restored — not silently drop it (which left the
|
|
/// client's request hanging forever).
|
|
#[tokio::test]
|
|
// requires pre-built binary; run with --ignored
|
|
#[ignore]
|
|
async fn test_prompt_sent_during_outage_is_delivered_after_recovery() {
|
|
tokio::task::LocalSet::new()
|
|
.run_until(async {
|
|
let server = MockInferenceServer::start().await.unwrap();
|
|
let workdir = git_workdir();
|
|
let home = tempfile::tempdir().unwrap();
|
|
std::fs::create_dir_all(home.path().join(".kigi")).unwrap();
|
|
|
|
let client = LeaderStdioClient::spawn(&server, workdir.path(), home.path()).await;
|
|
client.initialize().await;
|
|
let session = client.create_session(workdir.path()).await;
|
|
client
|
|
.prompt(&session, "hello")
|
|
.await
|
|
.expect("pre-crash prompt failed");
|
|
|
|
let leader_pid = wait_for_live_leader(home.path(), Duration::from_secs(5))
|
|
.await
|
|
.expect("no live leader PID in lock file");
|
|
eprintln!("killing leader pid {leader_pid}");
|
|
unsafe {
|
|
libc::kill(leader_pid as i32, libc::SIGKILL);
|
|
}
|
|
// Give the bridge a moment to observe the dead socket (its send
|
|
// channel closes), then prompt mid-outage: re-election + session
|
|
// restore are still seconds away.
|
|
tokio::time::sleep(Duration::from_millis(300)).await;
|
|
|
|
let res = tokio::time::timeout(
|
|
Duration::from_secs(90),
|
|
client.conn.prompt(acp::PromptRequest::new(session.clone(), vec![acp::ContentBlock::Text(acp::TextContent::new("sent during outage".to_string()))])),
|
|
)
|
|
.await
|
|
.unwrap_or_else(|_| {
|
|
panic!(
|
|
"prompt sent during outage never completed (dropped by bridge?)\n\
|
|
stderr:\n{}\nleader log:\n{}",
|
|
client.stderr_text(),
|
|
leader_log(home.path()),
|
|
)
|
|
});
|
|
assert!(
|
|
res.is_ok(),
|
|
"prompt sent during outage failed: {:?}\nstderr:\n{}\nleader log:\n{}",
|
|
res.err(),
|
|
client.stderr_text(),
|
|
leader_log(home.path()),
|
|
);
|
|
|
|
// A session-scoped request other than prompt (model switch) must
|
|
// also survive — same "unknown session id" class.
|
|
let set_model = tokio::time::timeout(
|
|
Duration::from_secs(30),
|
|
client.conn.set_session_model(acp::SetSessionModelRequest::new(session.clone(), acp::ModelId::new("test-model"))),
|
|
)
|
|
.await
|
|
.unwrap_or_else(|_| {
|
|
panic!(
|
|
"set_session_model after recovery never completed\nstderr:\n{}\nleader log:\n{}",
|
|
client.stderr_text(),
|
|
leader_log(home.path()),
|
|
)
|
|
});
|
|
assert!(
|
|
set_model.is_ok(),
|
|
"set_session_model after recovery failed: {:?}\nstderr:\n{}\nleader log:\n{}",
|
|
set_model.err(),
|
|
client.stderr_text(),
|
|
leader_log(home.path()),
|
|
);
|
|
})
|
|
.await;
|
|
}
|