ZacharyZhang-NY/Kigi-CLI
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

docs(comments): rewrite comments across all crates to the guidelines

Browse Source 
Sweep every first-party crate source (1956 .rs files) to the project comment
guidelines: delete redundant restatements, decorative banners, change
narration, and end-of-line comments; keep and tighten the crucial ones
(invariants, bug rationale, SAFETY blocks, ported-source attribution).

No functional code changed. Every edit is proven comment-only against the
prior tree by a comment-stripping lexer (string/char/raw-string aware) plus a
separate doctest-fence check. Where removing a comment made rustfmt or clippy
want to re-lay-out adjacent code, the minimal triggering comment is restored so
code tokens stay byte-identical.

Gates green: cargo fmt --all --check (0 diffs), cargo check and cargo clippy
--workspace --all-targets (0 warnings).

Adds scripts/check_codegen_comment_guidelines.py — the enforcement gate for
these guidelines (flags banners, end-of-line comments, change narration, and
commented-out code).
This commit is contained in:
ZacharyZhang-NY committed 2026-07-23 16:55:39 -04:00
1 parent ff0fb56c67
commit a02b555e66
1458 files changed
+10729 -21750
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/codegen/kigi-tools/src/util/base64_images.rs
+2 -3
View File
@@ -564,7 +564,8 @@ mod tests {
#[test]
fn strip_pdf_size_calculation() {
let pdf_b64 = payload(12288); // 12288 * 3/4 / 1024 = 9 KB
// 12288 * 3/4 / 1024 = 9 KB
let pdf_b64 = payload(12288);
let input = format!("data:application/pdf;base64,{pdf_b64} end");
let result = strip_pdf_data_uris(&input).unwrap();
assert!(result.contains("[PDF attachment removed \u{2014} 9 KB]"));
@@ -764,8 +765,6 @@ mod tests {
);
}
// ─── try_extract_base64_images tests ──────────────────────────────
#[test]
fn try_extract_no_images_returns_none() {
assert!(try_extract_base64_images("Plain text with no images.").is_none());
crates/codegen/kigi-tools/src/util/binary.rs
-2
View File
@@ -26,12 +26,10 @@ pub fn is_binary(extension: &str, bytes: &[u8]) -> bool {
let sample = &bytes[..bytes.len().min(SAMPLE_SIZE)];
// Any null byte → binary.
if sample.contains(&0x00) {
return true;
}
// High ratio of non-printable bytes → binary.
// Bytes 0-8 and 14-31 are control characters (excluding tab, newline, CR, etc.)
let non_printable = sample
.iter()
crates/codegen/kigi-tools/src/util/env.rs
+3 -4
View File
@@ -1,9 +1,8 @@
//! Environment variable helpers and process isolation for terminal execution.
//!
//! All implementations now live in the lightweight [`kigi_tty_utils`] crate
//! so that every crate in the workspace can use them without pulling in the
//! heavyweight `kigi-tools` dependency. This module re-exports the public
//! API for backward compatibility.
//! The implementations live in the lightweight [`kigi_tty_utils`] crate so
//! every workspace crate can use them without depending on `kigi-tools`; this
//! module re-exports the public API.
pub use kigi_tty_utils::{detach_from_tty, pager_env};
crates/codegen/kigi-tools/src/util/fs.rs
+4 -7
View File
@@ -67,8 +67,10 @@ pub(crate) async fn try_canonicalize(path: &Path) -> std::io::Result<PathBuf> {
/// legitimate. This map targets OS-generated filenames where the model can
/// never produce the exact character.
const FILENAME_SPECIAL_CHARACTER_MAP: &[(char, char)] = &[
('\u{202F}', ' '), // narrow no-break space (macOS screenshot/recording filenames)
('\u{00A0}', ' '), // no-break space
// narrow no-break space (macOS screenshot/recording filenames)
('\u{202F}', ' '),
// no-break space
('\u{00A0}', ' '),
];
fn normalize_filename(s: &str) -> String {
@@ -205,17 +207,13 @@ mod tests {
assert_eq!(result, path);
}
// ── try_resolve_unicode_filename ───────────────────────────────────
#[tokio::test]
async fn unicode_fallback_resolves_nnbsp_filename() {
let dir = tempfile::tempdir().unwrap();
// Create a file with U+202F (narrow no-break space) before "PM"
let actual_name = "Screenshot 2026-03-20 at 12.37.23\u{202F}PM.png";
let actual_path = dir.path().join(actual_name);
tokio::fs::write(&actual_path, b"img").await.unwrap();
// Model provides the same name with regular space
let model_name = "Screenshot 2026-03-20 at 12.37.23 PM.png";
let model_path = dir.path().join(model_name);
@@ -268,7 +266,6 @@ mod tests {
#[tokio::test]
async fn unicode_fallback_returns_none_for_ambiguous() {
let dir = tempfile::tempdir().unwrap();
// Two files that normalize to the same ASCII name
let a = dir.path().join("file\u{202F}name.txt");
let b = dir.path().join("file\u{00A0}name.txt");
tokio::fs::write(&a, b"a").await.unwrap();
crates/codegen/kigi-tools/src/util/git_detect.rs
-1
View File
@@ -62,7 +62,6 @@ impl PrRef {
/// Strip invocation prefixes that precede the actual binary in a statement:
/// `env` (with `-u NAME` args), `VAR=value` assignments, and an absolute /
/// relative path on the binary itself (`/opt/homebrew/bin/gh` → `gh`).
/// Covers common `env` / `VAR=value` / absolute-path wrappers around git/gh.
fn strip_invocation_prefixes(statement: &str) -> &str {
let mut rest = statement.trim_start();
loop {
crates/codegen/kigi-tools/src/util/hash.rs
-5
View File
@@ -87,7 +87,6 @@ mod tests {
#[test]
fn fnv1a_32_empty() {
// FNV-1a of empty input is the offset basis.
assert_eq!(fnv1a_32(b""), FNV_OFFSET);
}
@@ -112,7 +111,6 @@ mod tests {
#[test]
fn line_hash_whitespace_normalization_indentation() {
// Different indentation → same hash.
let a = line_hash(" let x = 1;");
let b = line_hash(" let x = 1;");
let c = line_hash("\tlet x = 1;");
@@ -131,7 +129,6 @@ mod tests {
#[test]
fn line_hash_whitespace_normalization_internal_collapse() {
// Multiple internal spaces collapse to one.
let a = line_hash("let x = 1;");
let b = line_hash("let x = 1;");
assert_eq!(a, b);
@@ -139,7 +136,6 @@ mod tests {
#[test]
fn line_hash_preserves_token_boundaries() {
// "return x" vs "returnx" must differ.
let a = line_hash("return x");
let b = line_hash("returnx");
assert_ne!(a, b);
@@ -147,7 +143,6 @@ mod tests {
#[test]
fn line_hash_empty_line() {
// Empty and whitespace-only lines should hash the same.
let a = line_hash("");
let b = line_hash(" ");
let c = line_hash("\t\t");
crates/codegen/kigi-tools/src/util/image_compress.rs
-5
View File
@@ -93,7 +93,6 @@ pub fn re_encode_under_limit(
let img: &DynamicImage = &scaled;
let (w, h) = (img.width(), img.height());
// --- PNG candidate ---------------------------------------------------
let png_candidate = {
let mut buf = Vec::new();
img.write_to(&mut std::io::Cursor::new(&mut buf), image::ImageFormat::Png)
@@ -102,7 +101,6 @@ pub fn re_encode_under_limit(
.map(|_| buf)
};
// --- JPEG candidate (best quality that fits) -------------------------
let jpeg_candidate = params.quality_steps.iter().find_map(|&quality| {
let mut buf = Vec::new();
let mut enc = JpegEncoder::new_with_quality(&mut buf, quality);
@@ -110,7 +108,6 @@ pub fn re_encode_under_limit(
(buf.len() <= params.max_bytes).then_some(buf)
});
// --- Pick the smaller candidate --------------------------------------
match (png_candidate, jpeg_candidate) {
(Some(png), Some(jpeg)) => {
if png.len() <= jpeg.len() {
@@ -193,8 +190,6 @@ mod tests {
fn does_not_upscale_images_smaller_than_the_side_cap() {
// 1280x960 is already under the test's 1568px side cap. Re-encoding
// must NOT enlarge it — output dimensions must never exceed the input.
// (Regression: the resize previously scaled small images up to
// `max_side_px`.)
let img = noise(1280, 960);
let (_bytes, w, h, _mime) =
re_encode_under_limit(&img, &params(5_000_000, 1568, u64::MAX)).unwrap();
crates/codegen/kigi-tools/src/util/image_validate.rs
+32 -22
View File
@@ -143,7 +143,8 @@ pub fn jpeg_reaches_eoi(bytes: &[u8]) -> bool {
match marker {
// Not a marker (stuffed/stray `FF00`): keep scanning.
0x00 => {}
0xD9 => return true, // EOI
// EOI
0xD9 => return true,
// Standalone markers without a length field.
0x01 | 0xD0..=0xD7 => {}
0xDA => {
@@ -504,7 +505,6 @@ mod tests {
));
}
/// Random non-image bytes → `Unsupported`.
#[test]
fn random_bytes_rejected_as_unsupported() {
let bytes = b"not an image at all, just plain text data";
@@ -527,7 +527,6 @@ mod tests {
);
}
/// Truncated PNG → `Truncated`.
#[test]
fn truncated_png_pinned_as_truncated() {
let mut bytes = png_bytes(64, 64);
@@ -642,7 +641,6 @@ mod tests {
assert!(!needs_endpoint_transcode(b"not an image"));
}
/// GIF/BMP/TIFF/ICO need client-side PNG conversion.
#[test]
fn needs_endpoint_transcode_true_for_gif_bmp_tiff_ico() {
assert!(needs_endpoint_transcode(&gif_bytes(4, 4)));
@@ -694,7 +692,6 @@ mod tests {
);
}
/// Already-large inputs are not upscaled further.
#[test]
fn transcode_to_endpoint_png_leaves_large_gif_dimensions() {
let png = transcode_to_endpoint_png(&gif_bytes(200, 150))
@@ -753,8 +750,6 @@ mod tests {
);
}
// ─── jpeg_reaches_eoi / png_structurally_valid structural walks ─────────
/// Noisy JPEG whose entropy stream is long enough to cut mid-scan.
fn noisy_jpeg(w: u32, h: u32) -> Vec<u8> {
use image::codecs::jpeg::JpegEncoder;
@@ -833,7 +828,8 @@ mod tests {
fn jpeg_reaches_eoi_rejects_degenerate_inputs() {
assert!(!jpeg_reaches_eoi(&[]));
assert!(!jpeg_reaches_eoi(&[0xFF]));
assert!(!jpeg_reaches_eoi(&[0xFF, 0xD8])); // bare SOI
// bare SOI
assert!(!jpeg_reaches_eoi(&[0xFF, 0xD8]));
assert!(!jpeg_reaches_eoi(b"not a jpeg"));
// Segment length running past the end of the buffer.
assert!(!jpeg_reaches_eoi(&[
@@ -850,13 +846,20 @@ mod tests {
fn jpeg_reaches_eoi_scans_through_restart_markers() {
#[rustfmt::skip]
let full: Vec<u8> = vec![
0xFF, 0xD8, // SOI
0xFF, 0xDD, 0x00, 0x04, 0x00, 0x02, // DRI, interval 2
0xFF, 0xDA, 0x00, 0x08, 1, 2, 3, 4, 5, 6, // SOS header
0x12, 0x34, 0xFF, 0x00, 0x56, // entropy + stuffed FF
0xFF, 0xD0, 0x78, 0x9A, // RST0, more entropy
0xFF, 0xD1, 0xBC, // RST1, more entropy
0xFF, 0xD9, // EOI
// SOI
0xFF, 0xD8,
// DRI, interval 2
0xFF, 0xDD, 0x00, 0x04, 0x00, 0x02,
// SOS header
0xFF, 0xDA, 0x00, 0x08, 1, 2, 3, 4, 5, 6,
// entropy + stuffed FF
0x12, 0x34, 0xFF, 0x00, 0x56,
// RST0, more entropy
0xFF, 0xD0, 0x78, 0x9A,
// RST1, more entropy
0xFF, 0xD1, 0xBC,
// EOI
0xFF, 0xD9,
];
assert!(jpeg_reaches_eoi(&full));
for cut in 3..full.len() {
@@ -873,15 +876,22 @@ mod tests {
fn jpeg_reaches_eoi_walks_multiple_scans() {
#[rustfmt::skip]
let full: Vec<u8> = vec![
0xFF, 0xD8, // SOI
0xFF, 0xDA, 0x00, 0x08, 1, 2, 3, 4, 5, 6, // SOS 1
0x11, 0x22, 0xFF, 0x00, 0x33, // entropy 1
0xFF, 0xDA, 0x00, 0x08, 1, 2, 3, 4, 5, 6, // SOS 2
0x44, 0x55, // entropy 2
0xFF, 0xD9, // EOI
// SOI
0xFF, 0xD8,
// SOS 1
0xFF, 0xDA, 0x00, 0x08, 1, 2, 3, 4, 5, 6,
// entropy 1
0x11, 0x22, 0xFF, 0x00, 0x33,
// SOS 2
0xFF, 0xDA, 0x00, 0x08, 1, 2, 3, 4, 5, 6,
// entropy 2
0x44, 0x55,
// EOI
0xFF, 0xD9,
];
assert!(jpeg_reaches_eoi(&full));
let cut = full.len() - 3; // inside entropy 2
// inside entropy 2
let cut = full.len() - 3;
assert!(!jpeg_reaches_eoi(&full[..cut]));
}
crates/codegen/kigi-tools/src/util/path_suggestions.rs
-10
View File
@@ -53,8 +53,6 @@ impl fmt::Display for PathNotFoundHint {
/// Build hints for a path-not-found error.
///
/// Returns [`PathNotFoundHint`].
///
/// `path` is the resolved (real) filesystem path that failed.
/// `display_cwd` is the model-facing working directory (for the CWD note).
#[tracing::instrument(name = "fs.path_not_found_hint", skip_all)]
@@ -220,8 +218,6 @@ mod tests {
// Display formatting). Broader integration fixtures live in
// tests/path_suggestions_production.rs.
// ── CWD note ──────────────────────────────────────────────────────
#[tokio::test]
async fn cwd_note_always_present() {
let tmp = TempDir::new().unwrap();
@@ -235,8 +231,6 @@ mod tests {
assert!(hint.similar.is_empty());
}
// ── "dropped repo folder" detection ───────────────────────────────
#[tokio::test]
async fn dropped_repo_folder_detected() {
let tmp = TempDir::new().unwrap();
@@ -294,8 +288,6 @@ mod tests {
assert!(hint.similar.is_empty());
}
// ── similar-name scan (internal invariants) ───────────────────────
#[tokio::test]
async fn similar_name_multi_match() {
let tmp = TempDir::new().unwrap();
@@ -346,8 +338,6 @@ mod tests {
);
}
// ── Display formatting ────────────────────────────────────────────
#[test]
fn display_with_suggestion() {
let hint = PathNotFoundHint {
crates/codegen/kigi-tools/src/util/query_tools.rs
-3
View File
@@ -13,13 +13,10 @@
/// detected; see [`kigi_config::shell::is_command_available`].
#[derive(Debug, Clone, Copy, Default)]
pub(crate) struct QueryTools {
/// `jq`, if present.
pub(crate) jq: Option<&'static str>,
/// Resolved python interpreter (`python3` preferred), if any.
pub(crate) python: Option<&'static str>,
/// `sed`, if present.
pub(crate) sed: Option<&'static str>,
/// `cut`, if present.
pub(crate) cut: Option<&'static str>,
}
crates/codegen/kigi-tools/src/util/remap.rs
-4
View File
@@ -55,7 +55,6 @@ pub fn remap_schema_properties(
let mut schema = schema.clone();
// Remap keys in "properties"
if let Some(serde_json::Value::Object(props)) = schema.get("properties").cloned() {
let mut new_props = serde_json::Map::new();
for (key, value) in props {
@@ -65,7 +64,6 @@ pub fn remap_schema_properties(
schema["properties"] = serde_json::Value::Object(new_props);
}
// Remap entries in "required" array
if let Some(serde_json::Value::Array(items)) = schema.get("required").cloned() {
let new_items: Vec<serde_json::Value> = items
.into_iter()
@@ -153,12 +151,10 @@ mod tests {
("new_string".to_string(), "replace_with".to_string()),
]);
let result = remap_schema_properties(&schema, &param_map);
// Properties remapped
assert!(result["properties"]["find"].is_object());
assert!(result["properties"]["replace_with"].is_object());
assert!(result["properties"]["file_path"].is_object());
assert!(result["properties"].get("old_string").is_none());
// Required array remapped
let required: Vec<String> = result["required"]
.as_array()
.unwrap()
crates/codegen/kigi-tools/src/util/serde_base64.rs
-4
View File
@@ -16,7 +16,6 @@ use base64::engine::general_purpose::STANDARD;
use serde::de::{self, Deserializer, SeqAccess, Visitor};
use serde::ser::Serializer;
/// Serialize a byte slice as a base64 string.
pub fn serialize<S>(bytes: &[u8], serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
@@ -24,8 +23,6 @@ where
serializer.serialize_str(&STANDARD.encode(bytes))
}
/// Deserialize `Vec<u8>` from either a base64 string (new form) or an integer
/// array (legacy form).
pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
where
D: Deserializer<'de>,
@@ -33,7 +30,6 @@ where
deserializer.deserialize_any(BytesVisitor)
}
/// Accepts both a base64 string and a legacy integer array.
struct BytesVisitor;
impl<'de> Visitor<'de> for BytesVisitor {
crates/codegen/kigi-tools/src/util/spawn.rs
+4 -4
View File
@@ -1,9 +1,9 @@
//! Cross-platform child-process lifecycle helpers for `tokio::process::Command`.
//!
//! All implementations now live in the lightweight [`kigi_tty_utils`] crate
//! so that every crate in the workspace can use them without pulling in the
//! heavyweight `kigi-tools` dependency. This module re-exports the public
//! API for backward compatibility.
//! The implementations live in the lightweight [`kigi_tty_utils`] crate so that
//! every crate in the workspace can use them without pulling in the heavyweight
//! `kigi-tools` dependency. This module re-exports the public API for callers
//! that reach it through `kigi-tools`.
pub use kigi_tty_utils::{
ProcessGroup, ProcessScope, detach_command, global_process_scope, new_process_group,
crates/codegen/kigi-tools/src/util/truncate.rs
+7 -34
View File
@@ -1,12 +1,9 @@
use std::borrow::Cow;
/// Default wrap width for soft-wrapping (used by bash, task_output).
pub const DEFAULT_SOFT_WRAP_WIDTH: usize = 2_000;
/// Default preview shown before a truncation footer.
pub const PREVIEW_SIZE: usize = 2_000;
/// Marker appended by `truncate_str_with_marker` when content is cut.
pub(crate) const TRUNCATION_MARKER: &str = "…";
/// Truncate a line to at most `max_chars` characters, respecting UTF-8 boundaries.
@@ -185,7 +182,7 @@ pub fn ceil_char_boundary(s: &str, index: usize) -> usize {
/// Estimate the number of tokens in a string using the bytes/4 heuristic.
/// Thin wrapper around [`kigi_token_estimation::estimate_tokens`] preserving
/// the historical `usize` return type used by tool-side callers
/// the `usize` return type used by tool-side callers
/// (`read_file`, `attach_file`, `inspect`, `compaction` file gates).
pub fn estimate_tokens(s: &str) -> usize {
kigi_token_estimation::estimate_tokens(s) as usize
@@ -282,13 +279,11 @@ pub fn truncate_middle(s: &str, max_chars: usize) -> String {
let front_count = remaining / 2;
let back_count = remaining - front_count;
// Front: first `front_count` chars.
let front_end = s
.char_indices()
.nth(front_count)
.map(|(i, _)| i)
.unwrap_or(s.len());
// Back: last `back_count` chars.
let back_start = if char_count <= back_count {
0
} else {
@@ -336,8 +331,6 @@ pub fn truncate_lines_to_char_budget(content: &str, budget: usize) -> (String, b
mod tests {
use super::*;
// ---- estimate_tokens ----
#[test]
fn estimate_tokens_empty() {
assert_eq!(estimate_tokens(""), 0);
@@ -359,8 +352,6 @@ mod tests {
assert_eq!(estimate_tokens(&"x".repeat(20_000)), 5_000);
}
// ---- truncate_line ----
#[test]
fn truncate_short_line_borrowed() {
let r = truncate_line("hello", 2_000);
@@ -393,12 +384,11 @@ mod tests {
#[test]
fn truncate_multibyte_char_count_under() {
let line = "é".repeat(1_999); // 2 bytes each, 1 char each
// 2 bytes each, 1 char each
let line = "é".repeat(1_999);
assert!(matches!(truncate_line(&line, 2_000), Cow::Borrowed(_)));
}
// ---- soft_wrap_line ----
#[test]
fn wrap_short_line_borrowed() {
assert!(matches!(soft_wrap_line("hello", 2_000), Cow::Borrowed(_)));
@@ -418,7 +408,7 @@ mod tests {
let line = "a".repeat(5_000);
let r = soft_wrap_line(&line, 2_000);
let lines: Vec<&str> = r.split('\n').collect();
assert_eq!(lines.len(), 3); // 2000 + 2000 + 1000
assert_eq!(lines.len(), 3);
assert_eq!(lines[0].len(), 2_000);
assert_eq!(lines[1].len(), 2_000);
assert_eq!(lines[2].len(), 1_000);
@@ -433,8 +423,6 @@ mod tests {
assert_eq!(lines[1].chars().count(), 1_000);
}
// ---- truncate_str ----
#[test]
fn truncate_str_returns_original_when_fits() {
assert_eq!(truncate_str("hello", 10), "hello");
@@ -470,8 +458,6 @@ mod tests {
assert_eq!(truncate_str("日本語", 0), "");
}
// ---- truncate_str_with_marker ----
#[test]
fn truncate_with_marker_exact_boundary_no_marker() {
// len == max_bytes: still fits, no marker.
@@ -501,8 +487,6 @@ mod tests {
assert_eq!(r.chars().count(), 3);
}
// ---- truncate_with_preview ----
#[test]
fn truncate_with_preview_short_output_unchanged() {
let (result, truncated) = truncate_with_preview("hello", 10, 5, None);
@@ -555,15 +539,13 @@ mod tests {
assert!(!result.contains("full content"));
}
// ---- soft_wrap_lines ----
#[test]
fn wrap_lines_mixed() {
let text = format!("short\n{}\nanother", "x".repeat(5_000));
let result = soft_wrap_lines(&text, 2_000);
let lines: Vec<&str> = result.split('\n').collect();
assert_eq!(lines[0], "short");
assert_eq!(lines[1].len(), 2_000); // first chunk of wrapped line
assert_eq!(lines[1].len(), 2_000);
assert_eq!(lines[4], "another");
// Total content preserved
let unwrapped: String = result.chars().filter(|c| *c != '\n').collect();
@@ -577,8 +559,6 @@ mod tests {
assert_eq!(soft_wrap_lines("hello", 2_000), "hello");
}
// ---- truncate_front_and_back ----
#[test]
fn front_and_back_short_string_not_truncated() {
let (result, truncated) = truncate_front_and_back("hello world", 100);
@@ -591,8 +571,8 @@ mod tests {
let s = "a".repeat(100);
let (result, truncated) = truncate_front_and_back(&s, 20);
assert!(truncated);
assert!(result.starts_with("aaaaaaaaaa")); // first 10
assert!(result.ends_with("aaaaaaaaaa")); // last 10
assert!(result.starts_with("aaaaaaaaaa"));
assert!(result.ends_with("aaaaaaaaaa"));
assert!(result.contains("... (output truncated) ..."));
}
@@ -604,8 +584,6 @@ mod tests {
assert!(!truncated);
}
// ---- truncate_lines_to_char_budget ----
#[test]
fn lines_budget_short_content_not_truncated() {
let (result, truncated) = truncate_lines_to_char_budget("line1\nline2\nline3", 100);
@@ -631,8 +609,6 @@ mod tests {
assert!(result.contains("character budget"));
}
// ---- floor_char_boundary / ceil_char_boundary ----
#[test]
fn floor_boundary_ascii() {
assert_eq!(floor_char_boundary("hello", 3), 3);
@@ -669,8 +645,6 @@ mod tests {
assert_eq!(ceil_char_boundary("hi", 100), 2);
}
// ---- truncate_middle ----
#[test]
fn truncate_middle_short_string_unchanged() {
assert_eq!(truncate_middle("hello", 10), "hello");
@@ -700,7 +674,6 @@ mod tests {
#[test]
fn truncate_middle_respects_budget() {
// Example: 50_000 chars at 12_000 limit.
let s = "a".repeat(50_000);
let result = truncate_middle(&s, 12_000);
assert_eq!(
crates/codegen/kigi-tools/src/util/unicode_confusables.rs
+47 -53
View File
@@ -34,14 +34,22 @@
/// (b) almost always produced by accidental rich-text auto-correction rather
/// than deliberate content authoring.
pub const CONFUSABLE_MAP: &[(char, &str)] = &[
('\u{201C}', "\""), // " left double quotation mark
('\u{201D}', "\""), // " right double quotation mark
('\u{2018}', "'"), // ' left single quotation mark
('\u{2019}', "'"), // ' right single quotation mark
('\u{2014}', "--"), // — em-dash
('\u{2013}', "-"), // – en-dash
('\u{2026}', "..."), // … horizontal ellipsis
('\u{00A0}', " "), // non-breaking space
// left double quotation mark
('\u{201C}', "\""),
// right double quotation mark
('\u{201D}', "\""),
// left single quotation mark
('\u{2018}', "'"),
// right single quotation mark
('\u{2019}', "'"),
// em-dash
('\u{2014}', "--"),
// en-dash
('\u{2013}', "-"),
// horizontal ellipsis
('\u{2026}', "..."),
// non-breaking space
('\u{00A0}', " "),
];
/// A single detected confusable character with its location metadata.
@@ -49,7 +57,6 @@ pub const CONFUSABLE_MAP: &[(char, &str)] = &[
pub struct ConfusableHit {
/// Byte offset of the confusable character in the source string.
pub byte_offset: usize,
/// The Unicode character that was detected.
pub unicode_char: char,
/// The ASCII replacement string from [`CONFUSABLE_MAP`].
pub ascii_replacement: &'static str,
@@ -215,8 +222,6 @@ pub fn build_offset_map(s: &str) -> (String, Vec<usize>) {
mod tests {
use super::*;
// ── CONFUSABLE_MAP coverage ─────────────────────────────────────────
#[test]
fn normalize_left_double_quote() {
assert_eq!(normalize_confusables("\u{201C}hello"), "\"hello");
@@ -257,8 +262,6 @@ mod tests {
assert_eq!(normalize_confusables("hello\u{00A0}world"), "hello world");
}
// ── Identity / passthrough ──────────────────────────────────────────
#[test]
fn normalize_pure_ascii_is_identity() {
let ascii = "The quick brown fox jumps over the lazy dog. 0123456789 !@#$%^&*()";
@@ -277,8 +280,6 @@ mod tests {
assert_eq!(normalize_confusables(s), s);
}
// ── has_confusables ─────────────────────────────────────────────────
#[test]
fn has_confusables_false_for_ascii() {
assert!(!has_confusables("plain ASCII text"));
@@ -304,8 +305,6 @@ mod tests {
assert!(has_confusables("a\u{2014}b"));
}
// ── detect_confusables ──────────────────────────────────────────────
#[test]
fn detect_returns_empty_for_ascii() {
assert!(detect_confusables("plain text").is_empty());
@@ -316,7 +315,7 @@ mod tests {
let hits = detect_confusables("say \u{201C}hi\u{201D}");
assert_eq!(hits.len(), 2);
assert_eq!(hits[0].byte_offset, 4); // "say " is 4 bytes
assert_eq!(hits[0].byte_offset, 4);
assert_eq!(hits[0].unicode_char, '\u{201C}');
assert_eq!(hits[0].ascii_replacement, "\"");
assert_eq!(hits[0].line_number, 1);
@@ -333,9 +332,9 @@ mod tests {
let s = "line one\nline\u{00A0}two\nline \u{201C}three\u{201D}\n";
let hits = detect_confusables(s);
assert_eq!(hits.len(), 3);
assert_eq!(hits[0].line_number, 2); // NBSP on line 2
assert_eq!(hits[1].line_number, 3); // left quote on line 3
assert_eq!(hits[2].line_number, 3); // right quote on line 3
assert_eq!(hits[0].line_number, 2);
assert_eq!(hits[1].line_number, 3);
assert_eq!(hits[2].line_number, 3);
}
#[test]
@@ -345,7 +344,8 @@ mod tests {
let hits = detect_confusables(s);
assert_eq!(hits.len(), 2);
assert_eq!(hits[0].byte_offset, 0);
assert_eq!(hits[1].byte_offset, 3); // em-dash is 3 bytes
// em-dash is 3 bytes
assert_eq!(hits[1].byte_offset, 3);
}
#[test]
@@ -360,8 +360,6 @@ mod tests {
assert_eq!(hits[1].unicode_char, '\u{2019}');
}
// ── build_offset_map ────────────────────────────────────────────────
#[test]
fn offset_map_pure_ascii() {
let s = "abc";
@@ -397,7 +395,7 @@ mod tests {
let (normalized, map) = build_offset_map(s);
assert_eq!(normalized, "\"hi\"");
assert_eq!(normalized.len(), 4);
assert_eq!(map.len(), 5); // 4 bytes + sentinel
assert_eq!(map.len(), 5);
// Byte 0 of normalized ('"') maps to byte 0 of original ('\u{201C}' start)
assert_eq!(map[0], 0);
@@ -419,13 +417,13 @@ mod tests {
let s = "a\u{2014}b";
let (normalized, map) = build_offset_map(s);
assert_eq!(normalized, "a--b");
assert_eq!(map.len(), 5); // 4 bytes + sentinel
assert_eq!(map.len(), 5);
assert_eq!(map[0], 0); // 'a' → 'a'
assert_eq!(map[1], 1); // first '-' → start of em-dash
assert_eq!(map[2], 1); // second '-' → start of em-dash
assert_eq!(map[3], 4); // 'b' → 'b'
assert_eq!(map[4], 5); // sentinel
assert_eq!(map[0], 0);
assert_eq!(map[1], 1);
assert_eq!(map[2], 1);
assert_eq!(map[3], 4);
assert_eq!(map[4], 5);
}
#[test]
@@ -436,13 +434,13 @@ mod tests {
let s = "\u{2026}";
let (normalized, map) = build_offset_map(s);
assert_eq!(normalized, "...");
assert_eq!(map.len(), 4); // 3 bytes + sentinel
assert_eq!(map.len(), 4);
// All three dots map to the start of the original ellipsis character.
assert_eq!(map[0], 0);
assert_eq!(map[1], 0);
assert_eq!(map[2], 0);
assert_eq!(map[3], 3); // sentinel
assert_eq!(map[3], 3);
}
#[test]
@@ -453,12 +451,12 @@ mod tests {
let s = "a\u{00A0}b";
let (normalized, map) = build_offset_map(s);
assert_eq!(normalized, "a b");
assert_eq!(map.len(), 4); // 3 bytes + sentinel
assert_eq!(map.len(), 4);
assert_eq!(map[0], 0); // 'a'
assert_eq!(map[1], 1); // ' ' maps to NBSP start
assert_eq!(map[2], 3); // 'b'
assert_eq!(map[3], 4); // sentinel
assert_eq!(map[0], 0);
assert_eq!(map[1], 1);
assert_eq!(map[2], 3);
assert_eq!(map[3], 4);
}
#[test]
@@ -467,16 +465,16 @@ mod tests {
// '🌍' is U+1F30D, 4 bytes in UTF-8.
let s = "a🌍b";
let (normalized, map) = build_offset_map(s);
assert_eq!(normalized, s); // no confusables → identical
assert_eq!(normalized, s);
// 'a'=1 byte, '🌍'=4 bytes, 'b'=1 byte → 6 bytes + sentinel
assert_eq!(map.len(), 7);
assert_eq!(map[0], 0); // 'a'
assert_eq!(map[1], 1); // '🌍' byte 0
assert_eq!(map[2], 2); // '🌍' byte 1
assert_eq!(map[3], 3); // '🌍' byte 2
assert_eq!(map[4], 4); // '🌍' byte 3
assert_eq!(map[5], 5); // 'b'
assert_eq!(map[6], 6); // sentinel
assert_eq!(map[0], 0);
assert_eq!(map[1], 1);
assert_eq!(map[2], 2);
assert_eq!(map[3], 3);
assert_eq!(map[4], 4);
assert_eq!(map[5], 5);
assert_eq!(map[6], 6);
}
#[test]
@@ -508,13 +506,11 @@ mod tests {
let s = "\u{2013}";
let (normalized, map) = build_offset_map(s);
assert_eq!(normalized, "-");
assert_eq!(map.len(), 2); // 1 byte + sentinel
assert_eq!(map[0], 0); // '-' maps to start of en-dash
assert_eq!(map[1], 3); // sentinel = original len
assert_eq!(map.len(), 2);
assert_eq!(map[0], 0);
assert_eq!(map[1], 3);
}
// ── Compound / integration scenarios ────────────────────────────────
#[test]
fn normalize_multiple_confusables_one_line() {
let s = "\u{201C}hello\u{201D}\u{2014}world\u{2026}";
@@ -540,8 +536,6 @@ mod tests {
assert_eq!(from_map, from_normalize);
}
// ── Consumer-contract tests ─────────────────────────────────────────
//
// These tests simulate the exact pattern that the normalized-fallback
// matcher will use: find a substring in the normalized text,
// remap the span back to the original via offset_map, and verify that