ZacharyZhang-NY
ea0ce9d15f
Sampler / inference (PRD F3):
- kimi_compat.rs: single adaptation point for the Kimi chat/completions
dialect (thinking-field mapping, model_id stripping, empty-content
tool-call message fix, stream_options.include_usage), with kimi-cli
source citations
- Rate-limit handling reworked for Kimi/Moonshot semantics; UA kigi/{version}
- /models replaces the xAI models-v2 endpoint everywhere; idle model
refresh carries X-Msh-* device headers only (X-XAI-Token-Auth and
x-grok-client-mode/CLIENT_MODE_HEADER machinery deleted)
Cloud-surface excision (PRD §5, zero-egress):
- remote/ conversations lane, cli-chat-proxy-types crate, prod/ dir,
share command, credit bar: deleted (single local session lane;
paginate() replaces merge_and_paginate)
- Subscription/tier gate stack deleted end-to-end: AppView
gate/tier/team/ZDR fields, app/subscription.rs watch loop,
dispatch/billing.rs paywall + SuperGrok upsell, free-usage-exhausted
chain, tier-restricted commands, GateInfo, RemoteSettings gate fields,
SettingsUpdateNotification gate fields
- /privacy + coding-data-sharing setting deleted (backed by a dead xAI
RPC; Kigi is zero-egress — nothing to share or retain remotely)
Auth UX correctness (user-reported):
- Device-flow fixtures now mirror the live Kimi payload shape
(https://www.kimi.com/code/authorize_device?user_code=..., verified
against auth.kimi.com); the fabricated auth.kimi.com/device?code=...
URLs are gone
- open_browser_detached is a no-op under cfg(test): unit tests drove
wiremock fixture URLs into the real browser (root cause of the
"garbage mock link" ABCD-1234 tabs)
- Welcome/pager-minimal rebrand: Grok Build -> Kigi, grok.com ->
kimi.com, "Sign in to Grok" -> "Sign in to Kimi"
1642 lines
57 KiB
Rust
1642 lines
57 KiB
Rust
//! Built-binary end-to-end tests for the grok (kigi-tui) binary.
|
|
//!
|
|
//! These tests verify that the built grok binary works end-to-end against a mock
|
|
//! inference server. They catch dynamic linking failures (libgit2/OpenSSL),
|
|
//! session initialization crashes, and protocol regressions.
|
|
//!
|
|
//! The tests exercise:
|
|
//! - **Smoke** (`grok --version`): binary loads without crashing
|
|
//! - **ACP stdio** (`grok agent stdio`): full protocol lifecycle via ClientSideConnection
|
|
//!
|
|
//! Tests are `#[ignore]`d by default — they require a pre-built binary.
|
|
//!
|
|
//! Run locally (auto-builds the binary if not already present):
|
|
//! ```bash
|
|
//! cargo test -p kigi-shell --test test_built_binary_e2e -- --ignored
|
|
//! ```
|
|
//!
|
|
//! In CI, set `KIGI_BINARY` to point at the release artifact:
|
|
//! ```bash
|
|
//! KIGI_BINARY=./artifacts/grok-0.1.159-linux-x86_64 \
|
|
//! cargo test -p kigi-shell --test test_built_binary_e2e -- --ignored
|
|
//! ```
|
|
|
|
use std::future::Future;
|
|
use std::path::Path;
|
|
use std::process::Command;
|
|
use std::time::Duration;
|
|
|
|
use kigi_test_support::env::test_env_cmd_tokio;
|
|
use kigi_test_support::*;
|
|
use serde_json::Value;
|
|
|
|
/// Run an async test body inside a `LocalSet` (required by ACP's `!Send` futures).
|
|
/// Eliminates the `let local = LocalSet::new(); local.run_until(async { ... }).await`
|
|
/// boilerplate from every stdio test.
|
|
async fn with_local_set<F, Fut>(f: F)
|
|
where
|
|
F: FnOnce() -> Fut,
|
|
Fut: Future<Output = ()>,
|
|
{
|
|
tokio::task::LocalSet::new().run_until(f()).await;
|
|
}
|
|
|
|
/// Start a mock server with one model named `model` on the given API backend.
|
|
async fn single_model_server(model: &str, backend: &str) -> MockInferenceServer {
|
|
MockInferenceServer::start_with_models(vec![
|
|
MockModelEntry::new(model).with_api_backend(backend),
|
|
])
|
|
.await
|
|
.expect("start mock server")
|
|
}
|
|
|
|
async fn grok_build_server() -> MockInferenceServer {
|
|
MockInferenceServer::start_with_models(vec![
|
|
MockModelEntry::with_agent_type("grok-4.5", "grok-build")
|
|
.with_api_backend("responses")
|
|
.with_supports_backend_search(true),
|
|
])
|
|
.await
|
|
.expect("start mock server")
|
|
}
|
|
|
|
/// Parse a headless run's stdout as a single JSON object.
|
|
fn parse_stdout_json(result: &HeadlessResult) -> serde_json::Value {
|
|
serde_json::from_str(result.stdout.trim())
|
|
.unwrap_or_else(|e| panic!("stdout not valid JSON: {e}\n{}", result.stdout))
|
|
}
|
|
|
|
fn request_tool_name(tool: &Value) -> Option<&str> {
|
|
tool.pointer("/function/name")
|
|
.or_else(|| tool.get("name"))
|
|
.and_then(Value::as_str)
|
|
.or_else(|| {
|
|
tool.get("type")
|
|
.and_then(Value::as_str)
|
|
.and_then(|kind| kind.starts_with("web_search").then_some("web_search"))
|
|
})
|
|
}
|
|
|
|
fn inference_request(server: &MockInferenceServer) -> Value {
|
|
server
|
|
.request_bodies()
|
|
.into_iter()
|
|
.find(|body| {
|
|
body.get("tools")
|
|
.and_then(Value::as_array)
|
|
.is_some_and(|tools| {
|
|
!tools.is_empty()
|
|
&& !tools
|
|
.iter()
|
|
.any(|tool| request_tool_name(tool) == Some("session_title"))
|
|
})
|
|
})
|
|
.expect("mock server should receive a main inference request with tools")
|
|
}
|
|
|
|
fn inference_tool_names(server: &MockInferenceServer) -> Vec<String> {
|
|
let request = inference_request(server);
|
|
request["tools"]
|
|
.as_array()
|
|
.expect("inference request tools should be an array")
|
|
.iter()
|
|
.filter_map(request_tool_name)
|
|
.map(str::to_owned)
|
|
.collect()
|
|
}
|
|
|
|
async fn run_headless_with_env(
|
|
server: &MockInferenceServer,
|
|
args: &[&str],
|
|
cwd: &Path,
|
|
env: &[(&str, &str)],
|
|
) -> HeadlessResult {
|
|
let home = tempfile::TempDir::new().expect("create temp home");
|
|
let mut cmd = tokio::process::Command::new(grok_binary());
|
|
cmd.args(args)
|
|
.current_dir(cwd)
|
|
.stdin(std::process::Stdio::null())
|
|
.stdout(std::process::Stdio::piped())
|
|
.stderr(std::process::Stdio::piped())
|
|
.kill_on_drop(true)
|
|
.envs(env.iter().copied());
|
|
test_env_cmd_tokio(&mut cmd, &server.url(), home.path());
|
|
run_headless_with_cmd(cmd).await
|
|
}
|
|
|
|
// ============================================================================
|
|
// Smoke tests
|
|
// ============================================================================
|
|
|
|
/// Smoke test: the binary loads and exits without crashing.
|
|
/// This does NOT require the mock server — it's the absolute minimum bar.
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_version_exits_zero() {
|
|
let binary = grok_binary();
|
|
let output = Command::new(&binary)
|
|
.arg("--version")
|
|
.output()
|
|
.unwrap_or_else(|e| panic!("failed to run {}: {e}", binary.display()));
|
|
|
|
assert!(
|
|
output.status.success(),
|
|
"grok --version failed (exit {:?}):\n{}",
|
|
output.status.code(),
|
|
String::from_utf8_lossy(&output.stderr)
|
|
);
|
|
}
|
|
|
|
/// Verify the crash handler installs without interfering with normal startup.
|
|
/// Exercises install() (sigaction, sigaltstack, mmap, ucontext struct layouts)
|
|
/// on every platform the binary is built for.
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_version_with_crash_handler_exits_zero() {
|
|
let binary = grok_binary();
|
|
let output = Command::new(&binary)
|
|
.arg("--version")
|
|
.env("KIGI_CRASH_HANDLER", "1")
|
|
.output()
|
|
.unwrap_or_else(|e| panic!("failed to run {}: {e}", binary.display()));
|
|
|
|
assert!(
|
|
output.status.success(),
|
|
"grok --version with KIGI_CRASH_HANDLER=1 failed (exit {:?}):\n{}",
|
|
output.status.code(),
|
|
String::from_utf8_lossy(&output.stderr)
|
|
);
|
|
}
|
|
|
|
/// THE critical test. Exercises the full session lifecycle in a git repo:
|
|
/// binary start → agent init → libgit2 init → fs watchers → session create →
|
|
/// model resolve → inference request to mock server → SSE parse → response render → exit.
|
|
///
|
|
/// This catches the recurring libgit2/OpenSSL dynamic linking bug that has
|
|
/// caused ~5 broken releases.
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_headless_session_in_git_repo() {
|
|
let server = MockInferenceServer::start()
|
|
.await
|
|
.expect("start mock server");
|
|
let workdir = git_workdir();
|
|
let result = run_headless(&server, &["-p", "say hello", "--yolo"], workdir.path()).await;
|
|
|
|
assert_headless_success(&result, "grok -p in git repo", Some(&server));
|
|
assert_no_crashes(&result.stderr);
|
|
assert!(
|
|
server.request_count() > 0,
|
|
"mock server received no inference requests\nrequest log:\n{}",
|
|
server.request_log_summary()
|
|
);
|
|
assert!(
|
|
server.has_chat_completion_request() || server.has_responses_request(),
|
|
"headless mode should hit /v1/chat/completions or /v1/responses\nrequest log:\n{}",
|
|
server.request_log_summary()
|
|
);
|
|
}
|
|
|
|
/// Verify grok works in a non-git directory (exercises the fallback codepath
|
|
/// where libgit2 discovers there's no repo instead of initializing one).
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_headless_session_in_non_git_dir() {
|
|
let server = MockInferenceServer::start()
|
|
.await
|
|
.expect("start mock server");
|
|
let workdir = tempfile::tempdir().unwrap();
|
|
std::fs::write(workdir.path().join("test.txt"), "test\n").unwrap();
|
|
|
|
let result = run_headless(&server, &["-p", "say hello", "--yolo"], workdir.path()).await;
|
|
|
|
assert_headless_success(&result, "grok -p in non-git dir", Some(&server));
|
|
assert_no_crashes(&result.stderr);
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_headless_tools_allowlist_keeps_enabled_web_tools() {
|
|
let server = grok_build_server().await;
|
|
server.preset_allow_access();
|
|
let workdir = git_workdir();
|
|
|
|
let result = run_headless_with_env(
|
|
&server,
|
|
&[
|
|
"-p",
|
|
"say hello",
|
|
"--yolo",
|
|
"--tools",
|
|
"read_file,grep,list_dir,web_search,web_fetch",
|
|
],
|
|
workdir.path(),
|
|
&[("KIGI_WEB_FETCH", "1")],
|
|
)
|
|
.await;
|
|
|
|
assert_headless_success(&result, "grok -p --tools with web tools", Some(&server));
|
|
assert_no_crashes(&result.stderr);
|
|
let names = inference_tool_names(&server);
|
|
for expected in ["read_file", "grep", "list_dir", "web_search", "web_fetch"] {
|
|
assert!(names.iter().any(|name| name == expected), "got: {names:?}");
|
|
}
|
|
for excluded in ["run_terminal_command", "search_replace"] {
|
|
assert!(!names.iter().any(|name| name == excluded), "got: {names:?}");
|
|
}
|
|
let request = inference_request(&server);
|
|
let tools = request["tools"]
|
|
.as_array()
|
|
.expect("inference request tools should be an array");
|
|
assert!(
|
|
tools.iter().any(|tool| {
|
|
tool.get("type")
|
|
.and_then(Value::as_str)
|
|
.is_some_and(|kind| kind.starts_with("web_search"))
|
|
}),
|
|
"backend-capable model should receive hosted web search: {tools:?}"
|
|
);
|
|
assert!(
|
|
!tools.iter().any(|tool| {
|
|
tool.get("type").and_then(Value::as_str) == Some("function")
|
|
&& tool.get("name").and_then(Value::as_str) == Some("web_search")
|
|
}),
|
|
"backend-capable model must not receive function web_search: {tools:?}"
|
|
);
|
|
assert!(
|
|
tools.iter().any(|tool| {
|
|
tool.get("type").and_then(Value::as_str) == Some("function")
|
|
&& tool.get("name").and_then(Value::as_str) == Some("web_fetch")
|
|
}),
|
|
"web_fetch should remain a function tool: {tools:?}"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_headless_tools_allowlist_does_not_fail_open_for_disabled_web_fetch() {
|
|
let server = grok_build_server().await;
|
|
server.set_settings(serde_json::json!({
|
|
"allow_access": true,
|
|
"web_fetch_enabled": false,
|
|
}));
|
|
let workdir = git_workdir();
|
|
|
|
let result = run_headless_with_env(
|
|
&server,
|
|
&[
|
|
"-p",
|
|
"say hello",
|
|
"--yolo",
|
|
"--tools",
|
|
"read_file,web_fetch",
|
|
],
|
|
workdir.path(),
|
|
&[("KIGI_WEB_FETCH", "0")],
|
|
)
|
|
.await;
|
|
|
|
assert_headless_success(
|
|
&result,
|
|
"grok -p --tools with disabled web_fetch",
|
|
Some(&server),
|
|
);
|
|
assert_no_crashes(&result.stderr);
|
|
let names = inference_tool_names(&server);
|
|
assert!(
|
|
names.iter().any(|name| name == "read_file"),
|
|
"got: {names:?}"
|
|
);
|
|
for excluded in ["web_fetch", "run_terminal_command", "search_replace"] {
|
|
assert!(!names.iter().any(|name| name == excluded), "got: {names:?}");
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_headless_terminal_only_allowlist_is_foreground_only() {
|
|
let server = grok_build_server().await;
|
|
let workdir = git_workdir();
|
|
|
|
let result = run_headless(
|
|
&server,
|
|
&["-p", "say hello", "--yolo", "--tools", "run_terminal_cmd"],
|
|
workdir.path(),
|
|
)
|
|
.await;
|
|
|
|
assert_headless_success(&result, "grok -p --tools run_terminal_cmd", Some(&server));
|
|
assert_no_crashes(&result.stderr);
|
|
let request = inference_request(&server);
|
|
let terminal = request["tools"]
|
|
.as_array()
|
|
.expect("inference request tools should be an array")
|
|
.iter()
|
|
.find(|tool| request_tool_name(tool) == Some("run_terminal_command"))
|
|
.expect("terminal tool should remain in the allowlist");
|
|
let properties = terminal
|
|
.pointer("/function/parameters/properties")
|
|
.or_else(|| terminal.pointer("/parameters/properties"))
|
|
.and_then(Value::as_object)
|
|
.expect("terminal tool should have an input schema");
|
|
assert!(
|
|
!properties.contains_key("is_background"),
|
|
"foreground-only terminal schema must omit is_background: {terminal}"
|
|
);
|
|
}
|
|
|
|
/// Free-usage paywall in headless mode: 429s whose flat body carries the
|
|
/// `subscription:free-usage-exhausted` well-known code must surface the
|
|
/// pager's free-usage message instead of the generic rate-limit one. The
|
|
/// code reaches the pager embedded in the flattened error text (no
|
|
/// structured plumbing), so this exercises the whole detection path.
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_headless_free_usage_exhausted_prints_paywall_message() {
|
|
let server = MockInferenceServer::start()
|
|
.await
|
|
.expect("start mock server");
|
|
let free_usage = || {
|
|
ScriptedResponse::json(
|
|
429,
|
|
serde_json::json!({
|
|
"code": "subscription:free-usage-exhausted",
|
|
"error": "You have used all your free usage."
|
|
}),
|
|
)
|
|
};
|
|
// The binary may target either backend, generic-429 handling retries
|
|
// once before going fatal, and the background session-title generation
|
|
// may consume a script on the same path — queue three per path (any
|
|
// leftovers are simply unused).
|
|
for path in ["/v1/chat/completions", "/v1/responses"] {
|
|
for _ in 0..3 {
|
|
server.enqueue_response(path, free_usage());
|
|
}
|
|
}
|
|
let workdir = git_workdir();
|
|
|
|
let result = run_headless(&server, &["-p", "say hello", "--yolo"], workdir.path()).await;
|
|
|
|
assert!(
|
|
!result.timed_out && !result.status.success(),
|
|
"a free-usage-exhausted turn must finish and exit non-zero\nstderr tail:\n{}",
|
|
stderr_tail(&result.stderr, 500)
|
|
);
|
|
assert_no_crashes(&result.stderr);
|
|
let combined = format!("{}\n{}", result.stdout, result.stderr);
|
|
assert!(
|
|
combined.contains("reached your free Grok Build usage limit"),
|
|
"expected the free-usage paywall message\nstdout:\n{}\nstderr tail:\n{}",
|
|
result.stdout,
|
|
stderr_tail(&result.stderr, 1000)
|
|
);
|
|
assert!(
|
|
!combined.contains("hit the rate limit for your plan"),
|
|
"generic rate-limit message must be replaced by the paywall text"
|
|
);
|
|
}
|
|
|
|
/// Verify the streaming JSON output format works end-to-end.
|
|
/// This is the format used by programmatic integrations (`--output-format streaming-json`).
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_headless_streaming_json_output() {
|
|
let server = MockInferenceServer::start()
|
|
.await
|
|
.expect("start mock server");
|
|
let workdir = git_workdir();
|
|
let result = run_headless(
|
|
&server,
|
|
&[
|
|
"-p",
|
|
"say hello",
|
|
"--yolo",
|
|
"--output-format",
|
|
"streaming-json",
|
|
],
|
|
workdir.path(),
|
|
)
|
|
.await;
|
|
|
|
assert_headless_success(
|
|
&result,
|
|
"grok -p --output-format streaming-json",
|
|
Some(&server),
|
|
);
|
|
assert_no_crashes(&result.stderr);
|
|
|
|
let events: Vec<serde_json::Value> = result
|
|
.stdout
|
|
.lines()
|
|
.map(|line| {
|
|
serde_json::from_str::<serde_json::Value>(line)
|
|
.unwrap_or_else(|e| panic!("invalid streaming-json line `{line}`: {e}"))
|
|
})
|
|
.collect();
|
|
assert!(
|
|
!events.is_empty(),
|
|
"streaming-json stdout should not be empty"
|
|
);
|
|
assert!(
|
|
!events
|
|
.iter()
|
|
.any(|event| event.get("type").and_then(serde_json::Value::as_str) == Some("error")),
|
|
"streaming-json emitted an error event: {:?}",
|
|
events
|
|
);
|
|
assert!(
|
|
events
|
|
.iter()
|
|
.any(|event| event.get("type").and_then(serde_json::Value::as_str) == Some("text")),
|
|
"streaming-json output should include at least one text event: {:?}",
|
|
events
|
|
);
|
|
assert_eq!(
|
|
events
|
|
.last()
|
|
.and_then(|event| event.get("type"))
|
|
.and_then(serde_json::Value::as_str),
|
|
Some("end"),
|
|
"streaming-json output should end with an `end` event: {:?}",
|
|
events
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_headless_json_reports_server_cost() {
|
|
use kigi_test_support::scripted::SseEvent;
|
|
|
|
let server = single_model_server("grok-4.5", "chat_completions").await;
|
|
let chunk = |body: serde_json::Value| SseEvent::data(body.to_string());
|
|
server.enqueue_response(
|
|
"/v1/chat/completions",
|
|
kigi_test_support::scripted::ScriptedResponse::sse(vec![
|
|
chunk(serde_json::json!({
|
|
"id": "chatcmpl-cost", "object": "chat.completion.chunk", "created": 0,
|
|
"model": "grok-4.5",
|
|
"choices": [{ "index": 0, "delta": { "content": "4" }, "finish_reason": "stop" }]
|
|
})),
|
|
chunk(serde_json::json!({
|
|
"id": "chatcmpl-cost", "object": "chat.completion.chunk", "created": 0,
|
|
"model": "grok-4.5", "choices": [],
|
|
"usage": {
|
|
"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15,
|
|
"cost_in_usd_ticks": 1_234_500_000_i64
|
|
}
|
|
})),
|
|
SseEvent::data("[DONE]"),
|
|
]),
|
|
);
|
|
|
|
let workdir = git_workdir();
|
|
let result = run_headless(
|
|
&server,
|
|
&[
|
|
"-p",
|
|
"what is 2+2",
|
|
"--yolo",
|
|
"--model",
|
|
"grok-4.5",
|
|
"--max-turns",
|
|
"1",
|
|
"--output-format",
|
|
"json",
|
|
],
|
|
workdir.path(),
|
|
)
|
|
.await;
|
|
|
|
assert_headless_success(&result, "grok -p (scripted cost)", Some(&server));
|
|
let output = parse_stdout_json(&result);
|
|
assert_eq!(output["total_cost_usd"], 0.12345);
|
|
assert_eq!(output["total_cost_usd_ticks"], 1_234_500_000_i64);
|
|
assert!(output.get("cost_is_partial").is_none());
|
|
assert!(output["usage"]["input_tokens"].as_u64().unwrap() >= 10);
|
|
assert_eq!(output["num_turns"], 1);
|
|
let (_, model) = output["modelUsage"]
|
|
.as_object()
|
|
.expect("modelUsage")
|
|
.iter()
|
|
.next()
|
|
.expect("one model");
|
|
assert_eq!(model["costUSD"], 0.12345);
|
|
assert!(model["modelCalls"].as_u64().unwrap() >= 1);
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_headless_json_reports_usage_on_max_turns() {
|
|
let server = single_model_server("grok-4.5", "chat_completions").await;
|
|
server.enqueue_response(
|
|
"/v1/chat/completions",
|
|
kigi_test_support::scripted::ScriptedResponse::sse(
|
|
kigi_test_support::sse::chat_completions_reasoning_then_tool_call_events(
|
|
"let me look",
|
|
"call-1",
|
|
"read_file",
|
|
r#"{"path":"README.md"}"#,
|
|
"grok-4.5",
|
|
),
|
|
),
|
|
);
|
|
|
|
let workdir = git_workdir();
|
|
let result = run_headless(
|
|
&server,
|
|
&[
|
|
"-p",
|
|
"read the readme",
|
|
"--yolo",
|
|
"--model",
|
|
"grok-4.5",
|
|
"--max-turns",
|
|
"1",
|
|
"--output-format",
|
|
"json",
|
|
],
|
|
workdir.path(),
|
|
)
|
|
.await;
|
|
|
|
assert!(!result.status.success());
|
|
let output = parse_stdout_json(&result);
|
|
assert!(output["usage"]["input_tokens"].as_u64().unwrap() >= 10);
|
|
assert!(output["num_turns"].as_u64().unwrap() >= 1);
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_headless_streaming_json_usage() {
|
|
let server = single_model_server("grok-4.5", "chat_completions").await;
|
|
let workdir = git_workdir();
|
|
let result = run_headless(
|
|
&server,
|
|
&[
|
|
"-p",
|
|
"say hello",
|
|
"--yolo",
|
|
"--model",
|
|
"grok-4.5",
|
|
"--output-format",
|
|
"streaming-json",
|
|
],
|
|
workdir.path(),
|
|
)
|
|
.await;
|
|
|
|
assert_headless_success(&result, "streaming-json usage", Some(&server));
|
|
let events: Vec<serde_json::Value> = result
|
|
.stdout
|
|
.lines()
|
|
.map(|line| serde_json::from_str(line).expect("valid NDJSON line"))
|
|
.collect();
|
|
let end = events.last().unwrap();
|
|
assert_eq!(end["type"], "end");
|
|
assert!(end["usage"]["input_tokens"].as_u64().unwrap() >= 10);
|
|
assert!(end["num_turns"].as_u64().unwrap() >= 1);
|
|
}
|
|
|
|
/// Chat Completions backend: the schema is enforced natively via
|
|
/// `response_format`, and the model's final JSON answer surfaces as
|
|
/// `structuredOutput`. The StructuredOutput tool is NOT used.
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn headless_json_schema_chat_completions_uses_response_format() {
|
|
let server = single_model_server("grok-4.5", "chat_completions").await;
|
|
server.set_response(r#"{"name":"Alice","age":30}"#);
|
|
|
|
let workdir = git_workdir();
|
|
let result = run_headless(
|
|
&server,
|
|
&[
|
|
"-p",
|
|
"extract name and age",
|
|
"--yolo",
|
|
"--model",
|
|
"grok-4.5",
|
|
"--json-schema",
|
|
r#"{"type":"object","properties":{"name":{"type":"string"},"age":{"type":"integer"}},"required":["name","age"],"additionalProperties":false}"#,
|
|
"--max-turns",
|
|
"1",
|
|
],
|
|
workdir.path(),
|
|
)
|
|
.await;
|
|
|
|
assert_headless_success(
|
|
&result,
|
|
"grok -p --json-schema (chat_completions)",
|
|
Some(&server),
|
|
);
|
|
assert_no_crashes(&result.stderr);
|
|
|
|
let output = parse_stdout_json(&result);
|
|
assert_eq!(output["structuredOutput"]["name"], "Alice");
|
|
assert_eq!(output["structuredOutput"]["age"], 30);
|
|
assert!(output.get("structuredOutputError").is_none());
|
|
|
|
// Native path: the schema rides response_format; the StructuredOutput tool
|
|
// is never advertised.
|
|
let response_format_on_wire = server.requests().iter().any(|r| {
|
|
r.body.as_ref().is_some_and(|body| {
|
|
body.pointer("/response_format/type")
|
|
.and_then(|v| v.as_str())
|
|
== Some("json_schema")
|
|
})
|
|
});
|
|
assert!(
|
|
response_format_on_wire,
|
|
"response_format json_schema must reach the wire\n{}",
|
|
server.request_log_summary()
|
|
);
|
|
assert!(
|
|
!any_request_advertises_structured_output_tool(&server),
|
|
"native path must NOT advertise the StructuredOutput tool\n{}",
|
|
server.request_log_summary()
|
|
);
|
|
}
|
|
|
|
/// Responses backend: native schema rides `text.format` (not the tool), and the
|
|
/// final JSON answer surfaces as `structuredOutput`.
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn headless_json_schema_responses_uses_text_format() {
|
|
let server = single_model_server("grok-4.5", "responses").await;
|
|
server.set_response(r#"{"name":"Alice","age":30}"#);
|
|
|
|
let workdir = git_workdir();
|
|
let result = run_headless(
|
|
&server,
|
|
&[
|
|
"-p",
|
|
"extract name and age",
|
|
"--yolo",
|
|
"--model",
|
|
"grok-4.5",
|
|
"--json-schema",
|
|
NAME_AGE_SCHEMA,
|
|
"--max-turns",
|
|
"1",
|
|
],
|
|
workdir.path(),
|
|
)
|
|
.await;
|
|
|
|
assert_headless_success(&result, "grok -p --json-schema (responses)", Some(&server));
|
|
assert_no_crashes(&result.stderr);
|
|
|
|
let output = parse_stdout_json(&result);
|
|
assert_eq!(output["structuredOutput"]["name"], "Alice");
|
|
assert_eq!(output["structuredOutput"]["age"], 30);
|
|
assert!(output.get("structuredOutputError").is_none());
|
|
|
|
let text_format_on_wire = server.requests().iter().any(|r| {
|
|
r.body.as_ref().is_some_and(|body| {
|
|
body.pointer("/text/format/type").and_then(|v| v.as_str()) == Some("json_schema")
|
|
})
|
|
});
|
|
assert!(
|
|
text_format_on_wire,
|
|
"text.format json_schema must reach the wire\n{}",
|
|
server.request_log_summary()
|
|
);
|
|
assert!(
|
|
!any_request_advertises_structured_output_tool(&server),
|
|
"native path must NOT advertise the StructuredOutput tool\n{}",
|
|
server.request_log_summary()
|
|
);
|
|
}
|
|
|
|
/// Messages backend (Anthropic-style) can't constrain output natively, so the
|
|
/// model returns its answer by calling the synthetic `StructuredOutput` tool.
|
|
/// Verifies the tool reaches the wire and its validated args surface as
|
|
/// `structuredOutput`.
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn headless_json_schema_messages_backend_uses_structured_output_tool() {
|
|
let server = single_model_server("messages-compatible-model", "messages").await;
|
|
server.enqueue_response(
|
|
"/v1/messages",
|
|
structured_output_tool_call_sse("messages-compatible-model", r#"{"name":"Bob","age":42}"#),
|
|
);
|
|
|
|
let workdir = git_workdir();
|
|
let result = run_headless(
|
|
&server,
|
|
&[
|
|
"-p",
|
|
"extract name and age",
|
|
"--yolo",
|
|
"--model",
|
|
"messages-compatible-model",
|
|
"--json-schema",
|
|
r#"{"type":"object","properties":{"name":{"type":"string"},"age":{"type":"integer"}},"required":["name","age"],"additionalProperties":false}"#,
|
|
"--max-turns",
|
|
"2",
|
|
],
|
|
workdir.path(),
|
|
)
|
|
.await;
|
|
|
|
assert_headless_success(&result, "grok -p --json-schema (messages)", Some(&server));
|
|
assert_no_crashes(&result.stderr);
|
|
|
|
let output = parse_stdout_json(&result);
|
|
assert_eq!(output["structuredOutput"]["name"], "Bob");
|
|
assert_eq!(output["structuredOutput"]["age"], 42);
|
|
assert!(output.get("structuredOutputError").is_none());
|
|
|
|
// The schema is advertised as the StructuredOutput tool, not response_format.
|
|
assert!(
|
|
any_request_advertises_structured_output_tool(&server),
|
|
"StructuredOutput tool must reach the wire\n{}",
|
|
server.request_log_summary()
|
|
);
|
|
}
|
|
|
|
/// Whether any request advertised a tool named `StructuredOutput` in `tools[]`.
|
|
fn any_request_advertises_structured_output_tool(server: &MockInferenceServer) -> bool {
|
|
server.requests().iter().any(|r| {
|
|
r.body.as_ref().is_some_and(|body| {
|
|
body.pointer("/tools")
|
|
.and_then(|t| t.as_array())
|
|
.is_some_and(|tools| {
|
|
tools
|
|
.iter()
|
|
.any(|t| t.get("name").and_then(|n| n.as_str()) == Some("StructuredOutput"))
|
|
})
|
|
})
|
|
})
|
|
}
|
|
|
|
/// Anthropic Messages API SSE that streams a single `StructuredOutput` tool call
|
|
/// whose input is `args_json`.
|
|
fn structured_output_tool_call_sse(model: &str, args_json: &str) -> ScriptedResponse {
|
|
use serde_json::json;
|
|
ScriptedResponse::sse(vec![
|
|
SseEvent::data(
|
|
json!({"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"model":model,"stop_reason":null,"usage":{"input_tokens":10,"output_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":0}}}).to_string(),
|
|
),
|
|
SseEvent::data(
|
|
json!({"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_1","name":"StructuredOutput","input":{}}}).to_string(),
|
|
),
|
|
SseEvent::data(
|
|
json!({"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":args_json}}).to_string(),
|
|
),
|
|
SseEvent::data(json!({"type":"content_block_stop","index":0}).to_string()),
|
|
SseEvent::data(
|
|
json!({"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":5,"input_tokens":10}}).to_string(),
|
|
),
|
|
SseEvent::data(json!({"type":"message_stop"}).to_string()),
|
|
])
|
|
}
|
|
|
|
const NAME_AGE_SCHEMA: &str = r#"{"type":"object","properties":{"name":{"type":"string"},"age":{"type":"integer"}},"required":["name","age"],"additionalProperties":false}"#;
|
|
|
|
/// Messages backend, model ignores the StructuredOutput tool and answers as
|
|
/// prose: the turn-end fallback still validates the text against the schema and
|
|
/// surfaces `structuredOutput` (closes the "unvalidated fallback" gap).
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn headless_json_schema_messages_validates_text_when_tool_not_called() {
|
|
let server = single_model_server("messages-compatible-model", "messages").await;
|
|
server.set_response(r#"{"name":"Cara","age":7}"#);
|
|
|
|
let workdir = git_workdir();
|
|
let result = run_headless(
|
|
&server,
|
|
&[
|
|
"-p",
|
|
"extract name and age",
|
|
"--yolo",
|
|
"--model",
|
|
"messages-compatible-model",
|
|
"--json-schema",
|
|
NAME_AGE_SCHEMA,
|
|
"--max-turns",
|
|
"1",
|
|
],
|
|
workdir.path(),
|
|
)
|
|
.await;
|
|
|
|
assert_headless_success(
|
|
&result,
|
|
"grok -p --json-schema (messages, text)",
|
|
Some(&server),
|
|
);
|
|
assert_no_crashes(&result.stderr);
|
|
|
|
let output = parse_stdout_json(&result);
|
|
assert_eq!(output["structuredOutput"]["name"], "Cara");
|
|
assert_eq!(output["structuredOutput"]["age"], 7);
|
|
assert!(output.get("structuredOutputError").is_none());
|
|
}
|
|
|
|
/// Messages backend, first StructuredOutput call violates the schema (no `age`):
|
|
/// the agent feeds the error back and the model's retry conforms. Exercises the
|
|
/// validation + bounded-retry path.
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn headless_json_schema_messages_retries_on_schema_violation() {
|
|
let server = single_model_server("messages-compatible-model", "messages").await;
|
|
server.enqueue_response(
|
|
"/v1/messages",
|
|
structured_output_tool_call_sse("messages-compatible-model", r#"{"name":"Dan"}"#),
|
|
);
|
|
server.enqueue_response(
|
|
"/v1/messages",
|
|
structured_output_tool_call_sse("messages-compatible-model", r#"{"name":"Dan","age":9}"#),
|
|
);
|
|
|
|
let workdir = git_workdir();
|
|
let result = run_headless(
|
|
&server,
|
|
&[
|
|
"-p",
|
|
"extract name and age",
|
|
"--yolo",
|
|
"--model",
|
|
"messages-compatible-model",
|
|
"--json-schema",
|
|
NAME_AGE_SCHEMA,
|
|
"--max-turns",
|
|
"3",
|
|
],
|
|
workdir.path(),
|
|
)
|
|
.await;
|
|
|
|
assert_headless_success(
|
|
&result,
|
|
"grok -p --json-schema (messages, retry)",
|
|
Some(&server),
|
|
);
|
|
assert_no_crashes(&result.stderr);
|
|
|
|
let output = parse_stdout_json(&result);
|
|
assert_eq!(output["structuredOutput"]["name"], "Dan");
|
|
assert_eq!(output["structuredOutput"]["age"], 9);
|
|
assert!(output.get("structuredOutputError").is_none());
|
|
}
|
|
|
|
/// An invalid `--json-schema` (valid JSON object, but fails schema compilation)
|
|
/// disables both structured-output paths and surfaces the compile error.
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn invalid_json_schema_disables_structured_output_and_surfaces_error() {
|
|
let server = single_model_server("grok-4.5", "chat_completions").await;
|
|
server.set_response(r#"{"name":"Alice","age":30}"#);
|
|
|
|
let workdir = git_workdir();
|
|
let result = run_headless(
|
|
&server,
|
|
&[
|
|
"-p",
|
|
"extract name and age",
|
|
"--yolo",
|
|
"--model",
|
|
"grok-4.5",
|
|
// Valid JSON object, but `pattern` is an invalid regex → schema
|
|
// compilation (`jsonschema::validator_for`) fails.
|
|
"--json-schema",
|
|
r#"{"type":"string","pattern":"["}"#,
|
|
"--max-turns",
|
|
"1",
|
|
],
|
|
workdir.path(),
|
|
)
|
|
.await;
|
|
|
|
assert_headless_success(
|
|
&result,
|
|
"grok -p --json-schema (invalid schema)",
|
|
Some(&server),
|
|
);
|
|
assert_no_crashes(&result.stderr);
|
|
|
|
let output = parse_stdout_json(&result);
|
|
assert!(
|
|
output["structuredOutput"].is_null(),
|
|
"invalid schema must not produce a value\n{}",
|
|
result.stdout
|
|
);
|
|
assert!(
|
|
output["structuredOutputError"]
|
|
.as_str()
|
|
.is_some_and(|e| e.contains("invalid output schema")),
|
|
"invalid schema must surface structuredOutputError\n{}",
|
|
result.stdout
|
|
);
|
|
|
|
// Structured output disabled: no native response_format, no tool.
|
|
let response_format_on_wire = server.requests().iter().any(|r| {
|
|
r.body
|
|
.as_ref()
|
|
.is_some_and(|body| body.pointer("/response_format/type").is_some())
|
|
});
|
|
assert!(
|
|
!response_format_on_wire,
|
|
"invalid schema must NOT send response_format\n{}",
|
|
server.request_log_summary()
|
|
);
|
|
assert!(
|
|
!any_request_advertises_structured_output_tool(&server),
|
|
"invalid schema must NOT advertise the StructuredOutput tool\n{}",
|
|
server.request_log_summary()
|
|
);
|
|
}
|
|
|
|
// ============================================================================
|
|
// ACP stdio tests (grok agent stdio)
|
|
//
|
|
// These test the agent as a server: spawn `grok agent stdio`, speak the full
|
|
// ACP protocol over pipes, verify the lifecycle works end-to-end.
|
|
// ============================================================================
|
|
|
|
/// Full ACP lifecycle: initialize → authenticate → create session → prompt.
|
|
/// Verifies the agent boots, authenticates with a test API key, creates a
|
|
/// session (libgit2 init), and completes a prompt round-trip to the mock server.
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_stdio_full_session_lifecycle() {
|
|
with_local_set(|| async {
|
|
let server = MockInferenceServer::start().await.expect("start mock server");
|
|
let workdir = git_workdir();
|
|
let client = GrokStdioClient::spawn(&server, workdir.path()).await;
|
|
|
|
// Initialize and authenticate
|
|
let init_resp = client.initialize_with_timeout().await;
|
|
assert!(
|
|
!init_resp.auth_methods.is_empty(),
|
|
"agent should return at least one auth method"
|
|
);
|
|
|
|
// Create session (triggers libgit2 init)
|
|
let session_id = client.create_session_with_timeout(workdir.path()).await;
|
|
assert!(!session_id.0.is_empty(), "session ID should be non-empty");
|
|
|
|
// Send prompt — triggers inference to mock server
|
|
let result = client.prompt_with_timeout(&session_id, "say hello").await;
|
|
assert!(
|
|
result.is_ok(),
|
|
"prompt failed: {:?}\nrequest log:\n{}\ncaptured text: {:?}\nnotifications: {}\nstderr:\n{}",
|
|
result.err(),
|
|
server.request_log_summary(),
|
|
client.captured_text(),
|
|
client.notification_count(),
|
|
stderr_tail(&client.stderr(), 1200)
|
|
);
|
|
|
|
// Verify the mock server received at least one inference request
|
|
assert!(
|
|
server.request_count() > 0,
|
|
"mock server received no inference requests\nrequest log:\n{}\nstderr:\n{}",
|
|
server.request_log_summary(),
|
|
stderr_tail(&client.stderr(), 1200)
|
|
);
|
|
})
|
|
.await;
|
|
}
|
|
|
|
/// Verify that x.ai/session/close frees the session.
|
|
/// Creates a session, closes it via ext_method, then verifies session/info
|
|
/// returns an empty response (session no longer exists).
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_stdio_session_close() {
|
|
with_local_set(|| async {
|
|
let server = MockInferenceServer::start()
|
|
.await
|
|
.expect("start mock server");
|
|
let workdir = git_workdir();
|
|
let client = GrokStdioClient::spawn(&server, workdir.path()).await;
|
|
|
|
client.initialize_with_timeout().await;
|
|
let session_id = client.create_session_with_timeout(workdir.path()).await;
|
|
|
|
// Session should be alive — session/info returns data with sessionId
|
|
let info_resp = client
|
|
.ext_method(
|
|
"x.ai/session/info",
|
|
serde_json::json!({ "sessionId": session_id.0.as_ref() }),
|
|
)
|
|
.await;
|
|
assert!(
|
|
info_resp.is_ok(),
|
|
"session/info should succeed before close"
|
|
);
|
|
let info: serde_json::Value =
|
|
serde_json::from_str(info_resp.unwrap().0.get()).expect("parse info");
|
|
assert_eq!(
|
|
info["result"]["sessionId"].as_str(),
|
|
Some(session_id.0.as_ref()),
|
|
"session/info should return the session we created, got: {info}"
|
|
);
|
|
|
|
// Close the session
|
|
let close_resp = client
|
|
.ext_method(
|
|
"x.ai/session/close",
|
|
serde_json::json!({ "sessionId": session_id.0.as_ref() }),
|
|
)
|
|
.await;
|
|
assert!(
|
|
close_resp.is_ok(),
|
|
"session/close failed: {:?}\nstderr:\n{}",
|
|
close_resp.err(),
|
|
stderr_tail(&client.stderr(), 1200)
|
|
);
|
|
|
|
// Session should be gone — session/info returns empty result (no sessionId)
|
|
let info_after = client
|
|
.ext_method(
|
|
"x.ai/session/info",
|
|
serde_json::json!({ "sessionId": session_id.0.as_ref() }),
|
|
)
|
|
.await;
|
|
assert!(info_after.is_ok(), "session/info should still succeed");
|
|
let info_val: serde_json::Value =
|
|
serde_json::from_str(info_after.unwrap().0.get()).expect("parse info after close");
|
|
assert!(
|
|
info_val["result"].get("sessionId").is_none(),
|
|
"session/info should not contain sessionId after close, got: {info_val}"
|
|
);
|
|
})
|
|
.await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_stdio_prompt_then_immediate_load_session() {
|
|
with_local_set(|| async {
|
|
let server = MockInferenceServer::start().await.expect("start mock server");
|
|
let workdir = git_workdir();
|
|
let mut writer = GrokStdioClient::spawn(&server, workdir.path()).await;
|
|
|
|
let init_resp = writer.initialize_with_timeout().await;
|
|
assert!(
|
|
!init_resp.auth_methods.is_empty(),
|
|
"agent should return at least one auth method"
|
|
);
|
|
|
|
let session_id = writer.create_session_with_timeout(workdir.path()).await;
|
|
let result = writer.prompt_with_timeout(&session_id, "say hello").await;
|
|
assert!(
|
|
result.is_ok(),
|
|
"prompt failed before load_session: {:?}\nrequest log:\n{}\nstderr:\n{}",
|
|
result.err(),
|
|
server.request_log_summary(),
|
|
stderr_tail(&writer.stderr(), 1200)
|
|
);
|
|
|
|
let shared_home = writer.take_home();
|
|
drop(writer);
|
|
|
|
let reader = GrokStdioClient::spawn_with_home(&server, workdir.path(), shared_home).await;
|
|
reader.initialize_with_timeout().await;
|
|
let _ = reader
|
|
.load_session_with_timeout(&session_id, workdir.path())
|
|
.await;
|
|
assert!(
|
|
reader.notification_count() > 0,
|
|
"reloaded session should emit replay notifications\nstderr:\n{}",
|
|
stderr_tail(&reader.stderr(), 1200)
|
|
);
|
|
assert!(
|
|
reader.captured_text().contains("Echo:") && reader.captured_text().contains("say hello"),
|
|
"reloaded session should replay the expected assistant text\ncaptured:\n{}\nstderr:\n{}",
|
|
reader.captured_text(),
|
|
stderr_tail(&reader.stderr(), 1200)
|
|
);
|
|
})
|
|
.await;
|
|
}
|
|
|
|
// ── Raw-wire stdio driving (Xcode / Foundation shape) ───────────────────────
|
|
|
|
/// Serialize `req` compactly, then rewrite its method to the Foundation-escaped
|
|
/// form (`"session/new"` → `"session\/new"`) by string surgery, asserting the
|
|
/// escape really is in the wire bytes — so a serde_json formatting change can
|
|
/// never silently downgrade this test to the unescaped path.
|
|
fn line_with_escaped_method(req: &serde_json::Value, method: &str) -> String {
|
|
let plain = format!(r#""method":"{method}""#);
|
|
let escaped = format!(r#""method":"{}""#, method.replace('/', r"\/"));
|
|
let line = req.to_string().replacen(&plain, &escaped, 1);
|
|
assert!(
|
|
line.contains(&escaped),
|
|
"escaped method must be on the wire: {line}"
|
|
);
|
|
// One replacement only: a params value carrying the same substring must
|
|
// fail here rather than get silently double-mangled.
|
|
assert!(
|
|
!line.contains(&plain),
|
|
"plain method form must be gone from the wire: {line}"
|
|
);
|
|
line
|
|
}
|
|
|
|
/// Xcode 27 beta's ACP client (Swift/Foundation `JSONEncoder`) escapes forward
|
|
/// slashes in the JSON-RPC `method` field (`"session\/new"` — spec-legal JSON)
|
|
/// and uses uppercase string UUID request ids. acp 0.6 parses `method` as a
|
|
/// borrowed str, so an escaped method used to fail the envelope parse and the
|
|
/// line was silently dropped: `initialize` (no slash) worked, every `session/*`
|
|
/// request hung forever. Drives the built binary with the raw wire bytes and
|
|
/// asserts every escaped-method request gets a response.
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_stdio_xcode_escaped_slash_methods_get_responses() {
|
|
let server = MockInferenceServer::start()
|
|
.await
|
|
.expect("start mock server");
|
|
let workdir = git_workdir();
|
|
let mut agent = RawStdioClient::spawn(&server, workdir.path()).await;
|
|
|
|
// initialize/authenticate carry no slash (they work from Xcode too), but
|
|
// ride string UUID ids and minimal capabilities like Xcode's client.
|
|
let init_id = "9B25E574-2F0C-4C8A-8C7E-2E9B3A4A0F01";
|
|
agent
|
|
.send_line(&format!(
|
|
r#"{{"jsonrpc":"2.0","id":"{init_id}","method":"initialize","params":{{"protocolVersion":1,"clientCapabilities":{{"fs":{{"readTextFile":false,"writeTextFile":false}},"terminal":false}},"_meta":{{"startupHints":{{"nonInteractive":true,"skipGitStatus":true,"skipProjectLayout":true}},"clientType":"xcode-test","clientVersion":"27.0"}}}}}}"#
|
|
))
|
|
.await;
|
|
let init_resp = agent
|
|
.response_for_id(init_id, "initialize", Duration::from_secs(20))
|
|
.await;
|
|
assert!(
|
|
init_resp.get("result").is_some(),
|
|
"initialize must respond with a result, got: {init_resp}"
|
|
);
|
|
|
|
let auth_id = "3C41A7D9-6B58-4E2F-A0D3-5F8C1B7E0A02";
|
|
agent
|
|
.send_line(&format!(
|
|
r#"{{"jsonrpc":"2.0","id":"{auth_id}","method":"authenticate","params":{{"methodId":"xai.api_key","_meta":{{"headless":true}}}}}}"#
|
|
))
|
|
.await;
|
|
let auth_resp = agent
|
|
.response_for_id(auth_id, "authenticate", Duration::from_secs(20))
|
|
.await;
|
|
assert!(
|
|
auth_resp.get("error").is_none(),
|
|
"authenticate failed: {auth_resp}\nstderr:\n{}",
|
|
stderr_tail(&agent.stderr(), 1200)
|
|
);
|
|
|
|
// session/new with the escaped method literally on the wire.
|
|
let new_id = "5DE7EA60-0B0C-4A43-9650-2B72CDF6A44B";
|
|
let line = line_with_escaped_method(
|
|
&serde_json::json!({
|
|
"jsonrpc": "2.0",
|
|
"id": new_id,
|
|
"method": "session/new",
|
|
"params": { "cwd": workdir.path(), "mcpServers": [] },
|
|
}),
|
|
"session/new",
|
|
);
|
|
agent.send_line(&line).await;
|
|
// Returning at all asserts the exact-string-UUID id echo: response_for_id
|
|
// only matches on it.
|
|
let new_resp = agent
|
|
.response_for_id(new_id, "escaped session/new", Duration::from_secs(20))
|
|
.await;
|
|
let session_id = new_resp["result"]["sessionId"]
|
|
.as_str()
|
|
.unwrap_or_else(|| {
|
|
panic!(
|
|
"escaped session/new must return a sessionId, got: {new_resp}\nstderr:\n{}",
|
|
stderr_tail(&agent.stderr(), 1200)
|
|
)
|
|
})
|
|
.to_string();
|
|
|
|
// session/prompt with the escaped method: must produce a response (result
|
|
// or error) rather than silence; against the echo mock it completes.
|
|
let prompt_id = "A1F3C9B2-7D64-4E85-B9A0-8C2D5E6F1A03";
|
|
let line = line_with_escaped_method(
|
|
&serde_json::json!({
|
|
"jsonrpc": "2.0",
|
|
"id": prompt_id,
|
|
"method": "session/prompt",
|
|
"params": {
|
|
"sessionId": session_id,
|
|
"prompt": [{ "type": "text", "text": "say hello" }],
|
|
},
|
|
}),
|
|
"session/prompt",
|
|
);
|
|
agent.send_line(&line).await;
|
|
let prompt_resp = agent
|
|
.response_for_id(prompt_id, "escaped session/prompt", Duration::from_secs(30))
|
|
.await;
|
|
assert!(
|
|
prompt_resp.get("error").is_none(),
|
|
"escaped session/prompt must complete: {prompt_resp}\nrequest log:\n{}\nstderr:\n{}",
|
|
server.request_log_summary(),
|
|
stderr_tail(&agent.stderr(), 1200)
|
|
);
|
|
assert!(
|
|
prompt_resp["result"]["stopReason"].is_string(),
|
|
"prompt response should carry a stopReason, got: {prompt_resp}"
|
|
);
|
|
assert!(
|
|
server.request_count() > 0,
|
|
"mock server received no inference requests\nrequest log:\n{}\nstderr:\n{}",
|
|
server.request_log_summary(),
|
|
stderr_tail(&agent.stderr(), 1200)
|
|
);
|
|
}
|
|
|
|
// ── Config test harness ─────────────────────────────────────────────────────
|
|
|
|
/// Isolated headless run with a custom `~/.kigi/`. Clean env (no leaked
|
|
/// host credentials). Write config files into `grok_dir()` before `run()`.
|
|
struct ConfigTestHarness {
|
|
home: tempfile::TempDir,
|
|
workdir: tempfile::TempDir,
|
|
env: Vec<(String, String)>,
|
|
}
|
|
|
|
impl ConfigTestHarness {
|
|
fn new(server: &MockInferenceServer) -> Self {
|
|
let home = tempfile::tempdir().unwrap();
|
|
std::fs::create_dir_all(home.path().join(".kigi")).unwrap();
|
|
Self {
|
|
home,
|
|
workdir: git_workdir(),
|
|
env: vec![
|
|
("KIGI_CODE_BASE_URL".into(), server.url()),
|
|
("KIGI_TELEMETRY_ENABLED".into(), "false".into()),
|
|
("KIGI_FEEDBACK_ENABLED".into(), "false".into()),
|
|
("KIGI_TRACE_UPLOAD".into(), "false".into()),
|
|
("KIGI_INSTRUMENTATION".into(), "disabled".into()),
|
|
("KIGI_DISABLE_AUTOUPDATER".into(), "1".into()),
|
|
],
|
|
}
|
|
}
|
|
|
|
fn grok_dir(&self) -> std::path::PathBuf {
|
|
self.home.path().join(".kigi")
|
|
}
|
|
|
|
fn env(&mut self, key: &str, value: &str) -> &mut Self {
|
|
self.env.push((key.into(), value.into()));
|
|
self
|
|
}
|
|
|
|
async fn run(&self) -> HeadlessResult {
|
|
let mut cmd = tokio::process::Command::new(grok_binary());
|
|
cmd.args(["-p", "say hello", "--yolo"])
|
|
.current_dir(self.workdir.path())
|
|
.stdin(std::process::Stdio::null())
|
|
.stdout(std::process::Stdio::piped())
|
|
.stderr(std::process::Stdio::piped())
|
|
.kill_on_drop(true)
|
|
.env_clear()
|
|
.env("HOME", self.home.path())
|
|
// Windows resolves `~` via USERPROFILE, not HOME — pin the grok
|
|
// home explicitly so the sandbox holds on all platforms (see
|
|
// `test_env_cmd_tokio`).
|
|
.env("KIGI_SHARE_DIR", self.grok_dir())
|
|
.env("PATH", std::env::var("PATH").unwrap_or_default());
|
|
for (k, v) in &self.env {
|
|
cmd.env(k, v);
|
|
}
|
|
run_headless_with_cmd(cmd).await
|
|
}
|
|
}
|
|
|
|
// ── Enterprise managed config tests ────────────────────────────────────────
|
|
|
|
/// Enterprise BYOK: managed_config.toml overrides grok-build with a custom
|
|
/// endpoint + env_key. Mock rejects unauthenticated requests with 401.
|
|
/// Regression guard for the 0.1.220 authentication regression.
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_headless_managed_config_byok_sends_authorized_requests() {
|
|
let server = MockInferenceServer::start_with_required_auth(
|
|
vec![MockModelEntry::new("grok-4.5")],
|
|
"test-byok-secret-token",
|
|
)
|
|
.await
|
|
.expect("start mock server");
|
|
|
|
let mut h = ConfigTestHarness::new(&server);
|
|
std::fs::write(
|
|
h.grok_dir().join("managed_config.toml"),
|
|
format!(
|
|
r#"
|
|
[endpoints]
|
|
deployment_key = "test-deployment-key"
|
|
xai_api_base_url = "{url}"
|
|
|
|
[model.kigi-build]
|
|
api_backend = "responses"
|
|
base_url = "{url}"
|
|
context_window = 500000
|
|
env_key = "KIGI_TEST_BYOK_TOKEN"
|
|
model = "grok-4.5"
|
|
|
|
[models]
|
|
default = "grok-4.5"
|
|
"#,
|
|
url = server.url()
|
|
),
|
|
)
|
|
.unwrap();
|
|
h.env("KIGI_TEST_BYOK_TOKEN", "test-byok-secret-token");
|
|
|
|
let result = h.run().await;
|
|
assert_headless_success(&result, "managed config BYOK", Some(&server));
|
|
assert_no_crashes(&result.stderr);
|
|
assert!(
|
|
server.has_responses_request(),
|
|
"mock server received no /v1/responses request\n{}",
|
|
server.request_log_summary()
|
|
);
|
|
}
|
|
|
|
/// New-server payload — a `reasoning_efforts` menu plus the legacy
|
|
/// `supportsReasoningEffort`/`reasoningEffort` (exactly what CCP emits) — parses
|
|
/// without error and the legacy effort scalar still rides the wire. Proves the
|
|
/// backwards-compat contract end-to-end through the built binary: the unknown
|
|
/// `reasoningEfforts` field never breaks the `/v1/models` parse. On the headless
|
|
/// path the wire effort comes from the legacy scalar, not from the list; the
|
|
/// list→default derivation is unit-tested in `acp_model_meta_*`.
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn headless_reasoning_efforts_payload_parses_and_legacy_effort_rides_wire() {
|
|
let server = MockInferenceServer::start_with_models(vec![
|
|
MockModelEntry::new("grok-4.5")
|
|
.with_api_backend("chat_completions")
|
|
.with_supports_reasoning_effort(true)
|
|
.with_reasoning_effort("xhigh")
|
|
.with_reasoning_efforts(vec![
|
|
serde_json::json!({ "id": "deep", "value": "xhigh", "label": "Deep", "default": true }),
|
|
serde_json::json!({ "id": "balanced", "value": "medium", "label": "Balanced" }),
|
|
]),
|
|
])
|
|
.await
|
|
.expect("start mock server");
|
|
server.set_response("done");
|
|
|
|
let workdir = git_workdir();
|
|
let result = run_headless(
|
|
&server,
|
|
&[
|
|
"-p",
|
|
"hi",
|
|
"--yolo",
|
|
"--model",
|
|
"grok-4.5",
|
|
"--max-turns",
|
|
"1",
|
|
],
|
|
workdir.path(),
|
|
)
|
|
.await;
|
|
|
|
assert_headless_success(&result, "grok -p reasoning_efforts list", Some(&server));
|
|
assert_no_crashes(&result.stderr);
|
|
|
|
// The legacy effort scalar rides the chat-completions request unchanged.
|
|
let effort_on_wire = server.requests().iter().any(|r| {
|
|
r.body.as_ref().is_some_and(|body| {
|
|
body.pointer("/reasoning_effort").and_then(|v| v.as_str()) == Some("xhigh")
|
|
})
|
|
});
|
|
assert!(
|
|
effort_on_wire,
|
|
"legacy reasoning_effort=xhigh must reach the wire\n{}",
|
|
server.request_log_summary()
|
|
);
|
|
}
|
|
|
|
// ============================================================================
|
|
// Background-task reaping at headless exit
|
|
// ============================================================================
|
|
|
|
#[cfg(unix)]
|
|
use kigi_test_support::sse::{
|
|
chat_completions_reasoning_then_tool_call_events, responses_api_reasoning_then_tool_call_events,
|
|
};
|
|
|
|
/// Poll `kill -0 <pid>` until the process is gone or the deadline passes.
|
|
#[cfg(unix)]
|
|
fn process_dead_within(pid: u32, deadline: Duration) -> bool {
|
|
let start = std::time::Instant::now();
|
|
loop {
|
|
let alive = Command::new("kill")
|
|
.args(["-0", &pid.to_string()])
|
|
.status()
|
|
.map(|s| s.success())
|
|
.unwrap_or(false);
|
|
if !alive {
|
|
return true;
|
|
}
|
|
if start.elapsed() > deadline {
|
|
return false;
|
|
}
|
|
std::thread::sleep(Duration::from_millis(200));
|
|
}
|
|
}
|
|
|
|
/// Read the PID a scripted background task wrote to `pid_file`, waiting for
|
|
/// the file to exist (the task writes it as its first action).
|
|
#[cfg(unix)]
|
|
fn read_task_pid(pid_file: &std::path::Path) -> u32 {
|
|
let start = std::time::Instant::now();
|
|
while !pid_file.exists() && start.elapsed() < Duration::from_secs(2) {
|
|
std::thread::sleep(Duration::from_millis(100));
|
|
}
|
|
let contents = std::fs::read_to_string(pid_file).unwrap_or_else(|e| {
|
|
panic!(
|
|
"background task never ran: pid file {} unreadable: {e}",
|
|
pid_file.display()
|
|
)
|
|
});
|
|
contents
|
|
.trim()
|
|
.parse()
|
|
.unwrap_or_else(|e| panic!("pid file {} held {contents:?}: {e}", pid_file.display()))
|
|
}
|
|
|
|
/// Script one turn that starts an `is_background: true` shell task recording
|
|
/// its PID and sleeping far longer than any timeout, followed by a plain-text
|
|
/// answer for the post-tool turn.
|
|
#[cfg(unix)]
|
|
fn enqueue_background_task_turn(server: &MockInferenceServer, pid_file: &std::path::Path) {
|
|
let command = format!("echo $$ > {} && exec /bin/sleep 300", pid_file.display());
|
|
let args = serde_json::json!({
|
|
"command": command,
|
|
"description": "start long-lived background process",
|
|
"is_background": true,
|
|
})
|
|
.to_string();
|
|
server.enqueue_response(
|
|
"/v1/responses",
|
|
ScriptedResponse::sse(responses_api_reasoning_then_tool_call_events(
|
|
"starting a background task",
|
|
"call_bg",
|
|
"run_terminal_command",
|
|
&args,
|
|
"test-model",
|
|
)),
|
|
);
|
|
server.enqueue_response(
|
|
"/v1/chat/completions",
|
|
ScriptedResponse::sse(chat_completions_reasoning_then_tool_call_events(
|
|
"starting a background task",
|
|
"call_bg",
|
|
"run_terminal_command",
|
|
&args,
|
|
"test-model",
|
|
)),
|
|
);
|
|
server.set_response("done");
|
|
}
|
|
|
|
/// Timeout path: a background task outlives `--background-wait-timeout`, so
|
|
/// headless exits via the timeout valve — and must kill the task instead of
|
|
/// orphaning it.
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_headless_timeout_exit_kills_pending_background_task() {
|
|
let server = MockInferenceServer::start()
|
|
.await
|
|
.expect("start mock server");
|
|
let workdir = git_workdir();
|
|
let pid_file = workdir.path().join("task_pid.txt");
|
|
enqueue_background_task_turn(&server, &pid_file);
|
|
|
|
let result = run_headless(
|
|
&server,
|
|
&[
|
|
"-p",
|
|
"start the server",
|
|
"--yolo",
|
|
"--background-wait-timeout",
|
|
"1",
|
|
],
|
|
workdir.path(),
|
|
)
|
|
.await;
|
|
|
|
assert_headless_success(
|
|
&result,
|
|
"grok -p with pending background task",
|
|
Some(&server),
|
|
);
|
|
assert_no_crashes(&result.stderr);
|
|
|
|
let pid = read_task_pid(&pid_file);
|
|
assert!(
|
|
process_dead_within(pid, Duration::from_secs(5)),
|
|
"background task (pid {pid}) survived headless exit on the timeout path\nstderr:\n{}",
|
|
stderr_tail(&result.stderr, 2000)
|
|
);
|
|
}
|
|
|
|
/// `--no-wait-for-background` path: exit is immediate after the turn, and the
|
|
/// task — tracked despite the flag — must still be killed, not leaked.
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_headless_no_wait_exit_kills_background_task() {
|
|
let server = MockInferenceServer::start()
|
|
.await
|
|
.expect("start mock server");
|
|
let workdir = git_workdir();
|
|
let pid_file = workdir.path().join("task_pid.txt");
|
|
enqueue_background_task_turn(&server, &pid_file);
|
|
|
|
let result = run_headless(
|
|
&server,
|
|
&[
|
|
"-p",
|
|
"start the server",
|
|
"--yolo",
|
|
"--no-wait-for-background",
|
|
],
|
|
workdir.path(),
|
|
)
|
|
.await;
|
|
|
|
assert_headless_success(&result, "grok -p --no-wait-for-background", Some(&server));
|
|
assert_no_crashes(&result.stderr);
|
|
|
|
let pid = read_task_pid(&pid_file);
|
|
assert!(
|
|
process_dead_within(pid, Duration::from_secs(5)),
|
|
"background task (pid {pid}) survived --no-wait-for-background exit\nstderr:\n{}",
|
|
stderr_tail(&result.stderr, 2000)
|
|
);
|
|
}
|
|
|
|
/// Quiescent path regression guard: a background task that completes on its
|
|
/// own is waited for (intended behavior) and the run exits cleanly with
|
|
/// nothing reaped.
|
|
#[cfg(unix)]
|
|
#[tokio::test]
|
|
#[ignore] // requires pre-built binary; run with --ignored
|
|
async fn test_headless_waits_for_short_background_task_and_exits_clean() {
|
|
let server = MockInferenceServer::start()
|
|
.await
|
|
.expect("start mock server");
|
|
let workdir = git_workdir();
|
|
let marker = workdir.path().join("finished.txt");
|
|
let command = format!("/bin/sleep 1 && echo ok > {}", marker.display());
|
|
let args = serde_json::json!({
|
|
"command": command,
|
|
"description": "short background task",
|
|
"is_background": true,
|
|
})
|
|
.to_string();
|
|
server.enqueue_response(
|
|
"/v1/responses",
|
|
ScriptedResponse::sse(responses_api_reasoning_then_tool_call_events(
|
|
"starting a short background task",
|
|
"call_bg_short",
|
|
"run_terminal_command",
|
|
&args,
|
|
"test-model",
|
|
)),
|
|
);
|
|
server.enqueue_response(
|
|
"/v1/chat/completions",
|
|
ScriptedResponse::sse(chat_completions_reasoning_then_tool_call_events(
|
|
"starting a short background task",
|
|
"call_bg_short",
|
|
"run_terminal_command",
|
|
&args,
|
|
"test-model",
|
|
)),
|
|
);
|
|
server.set_response("done");
|
|
|
|
let result = run_headless(
|
|
&server,
|
|
&[
|
|
"-p",
|
|
"start it",
|
|
"--yolo",
|
|
"--background-wait-timeout",
|
|
"30",
|
|
],
|
|
workdir.path(),
|
|
)
|
|
.await;
|
|
|
|
assert_headless_success(&result, "grok -p with short background task", Some(&server));
|
|
assert_no_crashes(&result.stderr);
|
|
assert!(
|
|
marker.exists(),
|
|
"short background task did not finish before exit — the intended wait \
|
|
was skipped\nstderr:\n{}",
|
|
stderr_tail(&result.stderr, 2000)
|
|
);
|
|
}
|