docs(comments): rewrite comments across all crates to the guidelines
Sweep every first-party crate source (1956 .rs files) to the project comment guidelines: delete redundant restatements, decorative banners, change narration, and end-of-line comments; keep and tighten the crucial ones (invariants, bug rationale, SAFETY blocks, ported-source attribution). No functional code changed. Every edit is proven comment-only against the prior tree by a comment-stripping lexer (string/char/raw-string aware) plus a separate doctest-fence check. Where removing a comment made rustfmt or clippy want to re-lay-out adjacent code, the minimal triggering comment is restored so code tokens stay byte-identical. Gates green: cargo fmt --all --check (0 diffs), cargo check and cargo clippy --workspace --all-targets (0 warnings). Adds scripts/check_codegen_comment_guidelines.py — the enforcement gate for these guidelines (flags banners, end-of-line comments, change narration, and commented-out code).
This commit is contained in:
@@ -45,7 +45,6 @@ fn main() {
|
||||
println!("git2 (index only): {} files in {:?}", files.len(), elapsed);
|
||||
}
|
||||
_ => {
|
||||
// Run all three methods multiple times for comparison
|
||||
println!("Benchmarking file listing for: {}", root_path.display());
|
||||
println!();
|
||||
|
||||
@@ -56,7 +55,6 @@ fn main() {
|
||||
let _ = collect_files_git2(root_path, ®istry);
|
||||
let _ = collect_files_git2_index_only(root_path, ®istry);
|
||||
|
||||
// CLI benchmark
|
||||
let mut cli_times = Vec::with_capacity(iterations);
|
||||
let mut cli_count = 0;
|
||||
for _ in 0..iterations {
|
||||
@@ -66,7 +64,6 @@ fn main() {
|
||||
cli_count = files.len();
|
||||
}
|
||||
|
||||
// git2 benchmark (with untracked)
|
||||
let mut git2_times = Vec::with_capacity(iterations);
|
||||
let mut git2_count = 0;
|
||||
for _ in 0..iterations {
|
||||
@@ -76,7 +73,6 @@ fn main() {
|
||||
git2_count = files.len();
|
||||
}
|
||||
|
||||
// git2 index-only benchmark
|
||||
let mut git2_index_times = Vec::with_capacity(iterations);
|
||||
let mut git2_index_count = 0;
|
||||
for _ in 0..iterations {
|
||||
@@ -86,7 +82,6 @@ fn main() {
|
||||
git2_index_count = files.len();
|
||||
}
|
||||
|
||||
// Print results
|
||||
let cli_avg = cli_times.iter().sum::<std::time::Duration>() / iterations as u32;
|
||||
let git2_avg = git2_times.iter().sum::<std::time::Duration>() / iterations as u32;
|
||||
let git2_index_avg =
|
||||
@@ -117,9 +112,7 @@ fn main() {
|
||||
}
|
||||
}
|
||||
|
||||
/// Collect files using git CLI (original approach)
|
||||
fn collect_files_cli(root_path: &Path, registry: &LanguageRegistry) -> Vec<std::path::PathBuf> {
|
||||
// Get tracked files
|
||||
let tracked_output = Command::new("git")
|
||||
.args(["ls-files"])
|
||||
.current_dir(root_path)
|
||||
@@ -130,7 +123,6 @@ fn collect_files_cli(root_path: &Path, registry: &LanguageRegistry) -> Vec<std::
|
||||
_ => return vec![],
|
||||
};
|
||||
|
||||
// Get untracked files
|
||||
let untracked_output = Command::new("git")
|
||||
.args(["ls-files", "--others", "--exclude-standard"])
|
||||
.current_dir(root_path)
|
||||
@@ -159,7 +151,6 @@ fn collect_files_cli(root_path: &Path, registry: &LanguageRegistry) -> Vec<std::
|
||||
files
|
||||
}
|
||||
|
||||
/// Collect files using git2 (new approach)
|
||||
fn collect_files_git2(root_path: &Path, registry: &LanguageRegistry) -> Vec<std::path::PathBuf> {
|
||||
let repo = match Repository::open(root_path) {
|
||||
Ok(r) => r,
|
||||
@@ -183,7 +174,6 @@ fn collect_files_git2(root_path: &Path, registry: &LanguageRegistry) -> Vec<std:
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Get untracked files
|
||||
let mut status_opts = StatusOptions::new();
|
||||
status_opts
|
||||
.include_untracked(true)
|
||||
@@ -204,7 +194,6 @@ fn collect_files_git2(root_path: &Path, registry: &LanguageRegistry) -> Vec<std:
|
||||
files
|
||||
}
|
||||
|
||||
/// Collect files using git2 index only (tracked files only, no untracked)
|
||||
fn collect_files_git2_index_only(
|
||||
root_path: &Path,
|
||||
registry: &LanguageRegistry,
|
||||
|
||||
@@ -21,7 +21,6 @@ fn main() {
|
||||
std::process::exit(1);
|
||||
};
|
||||
|
||||
// First, verify all queries compile
|
||||
println!("Verifying query compilation...");
|
||||
let registry = LanguageRegistry::new();
|
||||
for ext in &["ts", "tsx", "js", "jsx", "rs", "go", "py"] {
|
||||
|
||||
@@ -176,14 +176,12 @@ fn main() {
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the effective cache path - use custom if provided, otherwise default.
|
||||
fn effective_cache_path(repo_path: &Path, custom_cache: Option<&Path>) -> PathBuf {
|
||||
custom_cache
|
||||
.map(|p| p.to_path_buf())
|
||||
.unwrap_or_else(|| get_cache_path(repo_path))
|
||||
}
|
||||
|
||||
/// Load index from cache or build if necessary.
|
||||
fn load_or_build_index(repo_path: &Path, cache_path: &Path) -> ScopeGraphIndex {
|
||||
if let Ok(index) = load_index(cache_path) {
|
||||
println!("Loaded index from cache: {}", cache_path.display());
|
||||
@@ -204,7 +202,6 @@ fn load_or_build_index(repo_path: &Path, cache_path: &Path) -> ScopeGraphIndex {
|
||||
files, defs, refs, elapsed
|
||||
);
|
||||
|
||||
// Save to cache
|
||||
if let Err(e) = save_index(cache_path, &index) {
|
||||
println!("Warning: Failed to save cache: {}", e);
|
||||
} else {
|
||||
@@ -260,7 +257,6 @@ fn cmd_definition(
|
||||
let navigator = Navigator::new(index);
|
||||
|
||||
let result = match (file, row, col, symbol) {
|
||||
// Position-based lookup
|
||||
(Some(file_path), Some(r), Some(c), _) => {
|
||||
let abs_path = if file_path.is_absolute() {
|
||||
file_path
|
||||
@@ -276,7 +272,6 @@ fn cmd_definition(
|
||||
}
|
||||
}
|
||||
}
|
||||
// Symbol-based lookup
|
||||
(_, _, _, Some(sym)) => navigator.goto_definition_by_name(&sym, None),
|
||||
_ => {
|
||||
println!("Error: Must provide either --file, --row, --col OR --symbol");
|
||||
@@ -310,7 +305,6 @@ fn cmd_references(
|
||||
let navigator = Navigator::new(index);
|
||||
|
||||
let result = match (file, row, col, symbol) {
|
||||
// Position-based lookup
|
||||
(Some(file_path), Some(r), Some(c), _) => {
|
||||
let abs_path = if file_path.is_absolute() {
|
||||
file_path
|
||||
@@ -326,7 +320,6 @@ fn cmd_references(
|
||||
}
|
||||
}
|
||||
}
|
||||
// Symbol-based lookup
|
||||
(_, _, _, Some(sym)) => navigator.goto_references_by_name(&sym, None, include_definition),
|
||||
_ => {
|
||||
println!("Error: Must provide either --file, --row, --col OR --symbol");
|
||||
@@ -361,7 +354,6 @@ fn cmd_stats(path: &Path, custom_cache: Option<&Path>) {
|
||||
println!(" References: {}", refs);
|
||||
println!(" Aliases: {}", index.alias_count());
|
||||
|
||||
// Top symbols by reference count
|
||||
let ref_counts = index.top_referenced_symbols(10);
|
||||
|
||||
println!("\nTop 10 most referenced symbols:");
|
||||
|
||||
@@ -149,7 +149,6 @@ pub enum IndexCommand {
|
||||
BackgroundRefresh {
|
||||
/// Files that need reindexing (stale or new)
|
||||
stale_files: Vec<String>,
|
||||
/// Files that were deleted
|
||||
deleted_files: Vec<String>,
|
||||
},
|
||||
/// Get the number of indexed files (lightweight, no clone)
|
||||
@@ -330,7 +329,7 @@ impl IndexManagerHandle {
|
||||
self.command_tx.send(IndexCommand::Shutdown)
|
||||
}
|
||||
|
||||
// ========== Async Query APIs ==========
|
||||
// Async Query APIs
|
||||
|
||||
/// Go to definition at the given position (async).
|
||||
///
|
||||
@@ -417,7 +416,7 @@ impl IndexManagerHandle {
|
||||
Ok(rx.await.expect("IndexManager dropped before responding"))
|
||||
}
|
||||
|
||||
// ========== Blocking Query APIs ==========
|
||||
// Blocking Query APIs
|
||||
|
||||
/// Go to definition at the given position (blocking).
|
||||
pub fn goto_definition_blocking(
|
||||
@@ -518,7 +517,6 @@ impl IndexManagerConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the cache path.
|
||||
pub fn with_cache_path(mut self, path: PathBuf) -> Self {
|
||||
self.cache_path = Some(path);
|
||||
self
|
||||
@@ -1349,12 +1347,12 @@ fn background_index_refresh(
|
||||
if cached_meta.is_stale(path_ref) {
|
||||
// Check if file exists or is deleted
|
||||
if path_ref.exists() {
|
||||
Some((Some(path.clone()), None)) // Stale
|
||||
Some((Some(path.clone()), None))
|
||||
} else {
|
||||
Some((None, Some(path.clone()))) // Deleted
|
||||
Some((None, Some(path.clone())))
|
||||
}
|
||||
} else {
|
||||
None // Up to date
|
||||
None
|
||||
}
|
||||
})
|
||||
.fold(
|
||||
@@ -1391,8 +1389,10 @@ fn background_index_refresh(
|
||||
let registry = crate::languages::LanguageRegistry::new();
|
||||
|
||||
let new_files: Vec<String> = ignore::WalkBuilder::new(&root_path)
|
||||
.hidden(true) // Skip hidden files/dirs
|
||||
.git_ignore(true) // Respect .gitignore
|
||||
// Skip hidden files/dirs
|
||||
.hidden(true)
|
||||
// Respect .gitignore
|
||||
.git_ignore(true)
|
||||
.git_global(true)
|
||||
.git_exclude(true)
|
||||
.build()
|
||||
@@ -1530,7 +1530,7 @@ impl CoalescedEvents {
|
||||
|
||||
fn add(&mut self, event: FileEvent) {
|
||||
// Renames are special: they carry two paths. Process the "to" path
|
||||
// as Created (it needs indexing) and the "from" as Removed.
|
||||
// as `Created` (it needs indexing) and the "from" as `Removed`.
|
||||
if event.kind == FileEventKind::Renamed && event.paths.len() >= 2 {
|
||||
self.insert(event.paths[0].clone(), FileEventKind::Removed);
|
||||
self.insert(event.paths[1].clone(), FileEventKind::Created);
|
||||
@@ -1551,11 +1551,11 @@ impl CoalescedEvents {
|
||||
Entry::Occupied(mut e) => {
|
||||
let prev = *e.get();
|
||||
match (prev, kind) {
|
||||
// Created/Modified then Removed → cancel both
|
||||
// `Created`/`Modified` then `Removed` → cancel both
|
||||
(FileEventKind::Created | FileEventKind::Modified, FileEventKind::Removed) => {
|
||||
e.remove();
|
||||
}
|
||||
// Removed then Created/Modified → file replaced, treat as Created
|
||||
// `Removed` then `Created`/`Modified` → file replaced, treat as `Created`
|
||||
(FileEventKind::Removed, FileEventKind::Created | FileEventKind::Modified) => {
|
||||
e.insert(FileEventKind::Created);
|
||||
}
|
||||
@@ -1649,8 +1649,10 @@ fn is_identifier_like(node: &tree_sitter::Node<'_>) -> bool {
|
||||
|| kind == "field_identifier"
|
||||
|| kind == "shorthand_property_identifier"
|
||||
|| kind == "shorthand_property_identifier_pattern"
|
||||
|| kind == "attribute" // Python
|
||||
|| kind == "package_identifier" // Go
|
||||
// Python
|
||||
|| kind == "attribute"
|
||||
// Go
|
||||
|| kind == "package_identifier"
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1830,7 +1832,8 @@ mod tests {
|
||||
let dir = tempdir().unwrap();
|
||||
let file_path = dir.path().join("huge.rs");
|
||||
// Write a file larger than MAX_INDEXABLE_FILE_SIZE
|
||||
let content = "fn a() {}\n".repeat(600_000); // ~6MB
|
||||
// ~6MB
|
||||
let content = "fn a() {}\n".repeat(600_000);
|
||||
fs::write(&file_path, &content).unwrap();
|
||||
|
||||
let config = IndexManagerConfig::new(dir.path().to_path_buf())
|
||||
@@ -1891,7 +1894,8 @@ mod tests {
|
||||
fs::write(dir.path().join("binary.rs"), &binary).unwrap();
|
||||
|
||||
// Oversized file — should be skipped
|
||||
let big = "fn big() {}\n".repeat(500_000); // ~6MB
|
||||
// ~6MB
|
||||
let big = "fn big() {}\n".repeat(500_000);
|
||||
fs::write(dir.path().join("huge.rs"), &big).unwrap();
|
||||
|
||||
let index = IndexBuilder::new().build(dir.path()).unwrap();
|
||||
@@ -1930,12 +1934,13 @@ mod tests {
|
||||
|
||||
let stats = handle.get_stats().unwrap();
|
||||
assert_eq!(stats.files, 1);
|
||||
assert!(stats.definitions >= 2); // hello + world
|
||||
// hello + world
|
||||
assert!(stats.definitions >= 2);
|
||||
|
||||
handle.shutdown().unwrap();
|
||||
}
|
||||
|
||||
// ========== CoalescedEvents tests ==========
|
||||
// CoalescedEvents tests
|
||||
|
||||
#[test]
|
||||
fn test_coalesce_create_then_remove_cancels() {
|
||||
@@ -2003,7 +2008,7 @@ mod tests {
|
||||
let mut c = CoalescedEvents::new();
|
||||
c.add(FileEvent::renamed("/a.rs".into(), "/b.rs".into()));
|
||||
c.add(FileEvent::removed("/b.rs".into()));
|
||||
// /a.rs should still be Removed, /b.rs Created+Removed = cancelled
|
||||
// /a.rs should still be `Removed`, /b.rs `Created`+`Removed` = cancelled
|
||||
assert_eq!(c.events.len(), 1);
|
||||
assert_eq!(c.events[&PathBuf::from("/a.rs")], FileEventKind::Removed);
|
||||
}
|
||||
@@ -2013,7 +2018,7 @@ mod tests {
|
||||
let mut c = CoalescedEvents::new();
|
||||
c.add(FileEvent::renamed("/a.rs".into(), "/b.rs".into()));
|
||||
c.add(FileEvent::modified("/b.rs".into()));
|
||||
// /a.rs Removed, /b.rs Created+Modified → Modified (last writer wins)
|
||||
// /a.rs `Removed`, /b.rs `Created`+`Modified` → `Modified` (last writer wins)
|
||||
assert_eq!(c.events.len(), 2);
|
||||
assert_eq!(c.events[&PathBuf::from("/a.rs")], FileEventKind::Removed);
|
||||
assert_eq!(c.events[&PathBuf::from("/b.rs")], FileEventKind::Modified);
|
||||
|
||||
@@ -52,7 +52,6 @@ impl StringId {
|
||||
Self(id)
|
||||
}
|
||||
|
||||
/// Get the raw u32 value.
|
||||
#[inline]
|
||||
pub const fn as_u32(self) -> u32 {
|
||||
self.0
|
||||
@@ -209,7 +208,6 @@ impl StringInterner {
|
||||
self.offsets.is_empty()
|
||||
}
|
||||
|
||||
/// Total bytes used by the arena.
|
||||
#[inline]
|
||||
pub fn arena_bytes(&self) -> usize {
|
||||
self.arena.len()
|
||||
@@ -271,7 +269,7 @@ impl StringInterner {
|
||||
///
|
||||
/// After a bulk build the arena and offsets Vecs may hold up to 2× their
|
||||
/// actual content due to doubling growth. Calling this reclaims that
|
||||
/// wasted heap. The lookup table is intentionally left unshrunk because
|
||||
/// wasted heap. The lookup table is deliberately left unshrunk because
|
||||
/// it benefits from load-factor headroom.
|
||||
///
|
||||
/// This is an internal maintenance hook called by `ScopeGraphIndex::compact()`.
|
||||
@@ -313,7 +311,7 @@ mod tests {
|
||||
|
||||
let id1 = interner.intern("src");
|
||||
let id2 = interner.intern("lib");
|
||||
let id3 = interner.intern("src"); // duplicate
|
||||
let id3 = interner.intern("src");
|
||||
|
||||
assert_eq!(id1, id3);
|
||||
assert_ne!(id1, id2);
|
||||
@@ -348,7 +346,8 @@ mod tests {
|
||||
// Invalid UTF-8
|
||||
let invalid_utf8: &[u8] = &[0x80, 0x81, 0x82];
|
||||
let id2 = interner.intern_bytes(invalid_utf8);
|
||||
assert_eq!(interner.get(id2), None); // Not valid UTF-8
|
||||
// Not valid UTF-8
|
||||
assert_eq!(interner.get(id2), None);
|
||||
assert_eq!(interner.get_bytes(id2), Some(invalid_utf8));
|
||||
|
||||
// Duplicate bytes return same ID
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
//! JavaScript/JSX language configuration.
|
||||
|
||||
use crate::languages::types::TSLanguageConfig;
|
||||
|
||||
pub fn js_lang() -> TSLanguageConfig {
|
||||
|
||||
@@ -114,7 +114,7 @@ impl LanguageRegistry {
|
||||
/// Compute a hash of all tree-sitter queries across all languages.
|
||||
///
|
||||
/// This is used to detect when queries change, which should trigger
|
||||
/// a rebuild of the index even if file contents haven't changed.
|
||||
/// a rebuild of the index even if file contents are `unchanged`.
|
||||
///
|
||||
/// The hash is computed by:
|
||||
/// 1. Sorting languages by their primary ID for deterministic ordering
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
//! Python language configuration.
|
||||
|
||||
use crate::languages::types::TSLanguageConfig;
|
||||
|
||||
pub fn python_lang() -> TSLanguageConfig {
|
||||
@@ -12,7 +10,6 @@ pub fn python_lang() -> TSLanguageConfig {
|
||||
"variable".to_owned(),
|
||||
"module".to_owned(),
|
||||
]],
|
||||
// Python definitions query
|
||||
r#"
|
||||
; Class definitions
|
||||
(class_definition
|
||||
|
||||
@@ -19,7 +19,6 @@ pub fn ts_lang() -> TSLanguageConfig {
|
||||
"const".to_owned(),
|
||||
"let".to_owned(),
|
||||
]],
|
||||
// Comprehensive TypeScript query with full type coverage
|
||||
r#"
|
||||
;; === DEFINITIONS ===
|
||||
|
||||
|
||||
@@ -30,7 +30,6 @@ impl TSLanguageConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the language IDs.
|
||||
pub fn language_ids(&self) -> &[String] {
|
||||
&self.language_ids
|
||||
}
|
||||
@@ -43,17 +42,14 @@ impl TSLanguageConfig {
|
||||
.unwrap_or("unknown")
|
||||
}
|
||||
|
||||
/// Get the file extensions.
|
||||
pub fn file_extensions(&self) -> &[String] {
|
||||
&self.file_extensions
|
||||
}
|
||||
|
||||
/// Get the namespaces.
|
||||
pub fn namespaces(&self) -> &[Vec<String>] {
|
||||
&self.namespaces
|
||||
}
|
||||
|
||||
/// Get the file definition queries.
|
||||
pub fn file_definition_queries(&self) -> &str {
|
||||
&self.file_definition_queries
|
||||
}
|
||||
|
||||
@@ -235,7 +235,8 @@ impl IndexBuilder {
|
||||
.git_ignore(self.respect_gitignore)
|
||||
.git_global(self.respect_gitignore)
|
||||
.git_exclude(self.respect_gitignore)
|
||||
.threads(self.num_threads.min(12)) // Use parallel walking (capped at 12)
|
||||
// Use parallel walking (capped at 12)
|
||||
.threads(self.num_threads.min(12))
|
||||
.build_parallel();
|
||||
|
||||
walker.run(|| {
|
||||
@@ -309,7 +310,6 @@ impl IndexBuilder {
|
||||
//
|
||||
// New approach: for each batch of build_batch_size files:
|
||||
// 1. Parse in parallel (par_chunks preserves thread-local cache locality)
|
||||
// 2. Merge the batch into the index
|
||||
// 3. Drop the batch before starting the next one
|
||||
// Peak = O(build_batch_size) symbols + growing index simultaneously.
|
||||
for batch in file_paths.chunks(build_batch_size) {
|
||||
|
||||
@@ -1,25 +1,21 @@
|
||||
//! Index caching for fast loading.
|
||||
//!
|
||||
//! Uses a custom binary format with magic bytes "SGIX" for the new interned format.
|
||||
//! Automatically detects and skips legacy bincode format (returns error so caller can rebuild).
|
||||
//! The on-disk format is a custom binary layout tagged with the magic bytes
|
||||
//! "SGIX". Caches written by the earlier bincode format are detected and
|
||||
//! rejected rather than parsed, so the caller rebuilds from source.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use crate::scope_graph::ScopeGraphIndex;
|
||||
|
||||
/// Default cache file name.
|
||||
pub const CACHE_FILE_NAME: &str = ".goto_index.bin";
|
||||
|
||||
/// Error type for cache operations.
|
||||
#[derive(Debug)]
|
||||
pub enum CacheError {
|
||||
/// IO error.
|
||||
IoError(std::io::Error),
|
||||
/// Serialization error.
|
||||
SerializeError(String),
|
||||
/// Deserialization error.
|
||||
DeserializeError(String),
|
||||
/// Legacy format detected (caller should rebuild).
|
||||
/// A bincode-era cache was found; the caller is expected to rebuild.
|
||||
LegacyFormat,
|
||||
}
|
||||
|
||||
@@ -42,19 +38,14 @@ impl From<std::io::Error> for CacheError {
|
||||
}
|
||||
}
|
||||
|
||||
/// Result type for cache operations.
|
||||
pub type Result<T> = std::result::Result<T, CacheError>;
|
||||
|
||||
/// Get the default cache path for a repository.
|
||||
pub fn get_cache_path(root_path: &Path) -> std::path::PathBuf {
|
||||
root_path.join(CACHE_FILE_NAME)
|
||||
}
|
||||
|
||||
/// Load an index from cache.
|
||||
///
|
||||
/// Uses the new binary format with magic bytes "SGIX".
|
||||
/// Returns `CacheError::LegacyFormat` if the file uses the old bincode format,
|
||||
/// signaling to the caller that a rebuild is needed.
|
||||
/// Returns `CacheError::LegacyFormat` for a bincode-format cache, signaling to
|
||||
/// the caller that a rebuild is needed.
|
||||
pub fn load_index(cache_path: &Path) -> Result<ScopeGraphIndex> {
|
||||
if !cache_path.exists() {
|
||||
return Err(CacheError::IoError(std::io::Error::new(
|
||||
@@ -63,11 +54,10 @@ pub fn load_index(cache_path: &Path) -> Result<ScopeGraphIndex> {
|
||||
)));
|
||||
}
|
||||
|
||||
// Use ScopeGraphIndex::load which handles format detection
|
||||
match ScopeGraphIndex::load(cache_path) {
|
||||
Ok(Some(index)) => Ok(index),
|
||||
// `Ok(None)` is how the loader reports a legacy-format file.
|
||||
Ok(None) => {
|
||||
// None means legacy format was detected
|
||||
tracing::info!(
|
||||
cache_path = %cache_path.display(),
|
||||
"Legacy cache format detected, will rebuild"
|
||||
@@ -78,15 +68,12 @@ pub fn load_index(cache_path: &Path) -> Result<ScopeGraphIndex> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Save an index to cache using the new binary format.
|
||||
pub fn save_index(cache_path: &Path, index: &ScopeGraphIndex) -> Result<()> {
|
||||
index.save(cache_path).map_err(CacheError::IoError)
|
||||
}
|
||||
|
||||
/// Save an index to cache asynchronously (in a background thread).
|
||||
///
|
||||
/// Returns immediately and spawns a thread to do the actual saving.
|
||||
/// Useful for saving the index without blocking the main thread.
|
||||
/// Saves on a detached thread: the caller gets no join handle and no result,
|
||||
/// so a failed write is only visible in the logs.
|
||||
pub fn save_index_async(cache_path: std::path::PathBuf, index: ScopeGraphIndex) {
|
||||
std::thread::spawn(move || {
|
||||
if let Err(e) = save_index(&cache_path, &index) {
|
||||
@@ -95,12 +82,11 @@ pub fn save_index_async(cache_path: std::path::PathBuf, index: ScopeGraphIndex)
|
||||
});
|
||||
}
|
||||
|
||||
/// Check if a cache exists and return its metadata.
|
||||
pub fn cache_exists(cache_path: &Path) -> bool {
|
||||
cache_path.exists()
|
||||
}
|
||||
|
||||
/// Get cache file size in bytes.
|
||||
/// Size of the cache file in bytes, or `None` if it cannot be stat'd.
|
||||
pub fn cache_size(cache_path: &Path) -> Option<u64> {
|
||||
std::fs::metadata(cache_path).ok().map(|m| m.len())
|
||||
}
|
||||
|
||||
@@ -52,7 +52,8 @@ impl IndexOperation {
|
||||
/// Whether this operation requires exclusive access.
|
||||
pub fn is_exclusive(&self) -> bool {
|
||||
match self {
|
||||
Self::Load => false, // Shared/read access
|
||||
// Shared/read access
|
||||
Self::Load => false,
|
||||
Self::Save | Self::Build | Self::BackgroundRefresh => true,
|
||||
}
|
||||
}
|
||||
@@ -77,8 +78,10 @@ impl std::fmt::Display for IndexOperation {
|
||||
/// In-memory lock state for same-process deduplication.
|
||||
struct InMemoryLockState {
|
||||
operation: IndexOperation,
|
||||
readers: usize, // Count for shared locks
|
||||
exclusive: bool, // Whether an exclusive lock is held
|
||||
// Count for shared locks
|
||||
readers: usize,
|
||||
// Whether an exclusive lock is held
|
||||
exclusive: bool,
|
||||
}
|
||||
|
||||
/// Global registry of in-memory locks (same process).
|
||||
@@ -299,7 +302,6 @@ fn try_acquire_in_memory_lock(workspace: &Path, operation: IndexOperation) -> bo
|
||||
true
|
||||
}
|
||||
|
||||
/// Release an in-memory lock.
|
||||
fn release_in_memory_lock(workspace: &Path, operation: IndexOperation) {
|
||||
// Use entry API for atomic check-and-modify
|
||||
if let dashmap::mapref::entry::Entry::Occupied(mut entry) =
|
||||
@@ -439,7 +441,6 @@ mod tests {
|
||||
// Drop first lock
|
||||
drop(guard1);
|
||||
|
||||
// Now second should succeed
|
||||
let guard3 = try_lock(workspace, IndexOperation::Build);
|
||||
assert!(guard3.is_acquired());
|
||||
}
|
||||
@@ -490,7 +491,6 @@ mod tests {
|
||||
// Drop shared lock
|
||||
drop(guard1);
|
||||
|
||||
// Now exclusive should succeed
|
||||
let guard3 = try_lock(workspace, IndexOperation::Build);
|
||||
assert!(guard3.is_acquired());
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! Index management: building, caching, locking, and updating.
|
||||
//! Index management: building, caching, and workspace locking.
|
||||
|
||||
mod builder;
|
||||
pub mod cache;
|
||||
|
||||
@@ -238,7 +238,7 @@ impl Navigator {
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `file_path` - Path to the file
|
||||
/// * `row` - 1-indexed line number
|
||||
/// * `row` - 1-indexed line number
|
||||
/// * `col` - 1-indexed column number
|
||||
/// * `include_definition` - Whether to include the definition location in results
|
||||
///
|
||||
@@ -388,8 +388,8 @@ fn is_identifier_like(node: &tree_sitter::Node<'_>) -> bool {
|
||||
| "field_identifier"
|
||||
| "shorthand_property_identifier"
|
||||
| "shorthand_property_identifier_pattern"
|
||||
| "attribute" // Python
|
||||
| "package_identifier" // Go
|
||||
| "attribute"
|
||||
| "package_identifier"
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -2,21 +2,21 @@
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Describes the relation between two nodes in the ScopeGraph.
|
||||
/// Edge weight in the ScopeGraph. Every variant is directed source-to-target,
|
||||
/// in the order its name reads.
|
||||
#[derive(Serialize, Deserialize, PartialEq, Eq, Copy, Clone, Debug)]
|
||||
pub enum EdgeKind {
|
||||
/// The edge weight from a nested scope to its parent scope.
|
||||
/// Nested scope to its parent scope.
|
||||
ScopeToScope,
|
||||
|
||||
/// The edge weight from a definition to its definition scope.
|
||||
/// Definition to the scope that owns it, which for a hoisted def is the
|
||||
/// parent of the scope it was written in.
|
||||
DefToScope,
|
||||
|
||||
/// The edge weight from an import to its definition scope.
|
||||
/// Import to its defining scope.
|
||||
ImportToScope,
|
||||
|
||||
/// The edge weight from a reference to its definition.
|
||||
RefToDef,
|
||||
|
||||
/// The edge weight from a reference to its import.
|
||||
RefToImport,
|
||||
}
|
||||
|
||||
@@ -44,7 +44,7 @@ pub type ExtractedSymbols = (
|
||||
/// even if file contents haven't changed.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub enum QueryVersion {
|
||||
/// Legacy format - index was built before query versioning was added.
|
||||
/// Legacy format - index was built without query versioning.
|
||||
/// This triggers a rebuild since we don't know what queries were used.
|
||||
/// Default for backwards compatibility with old cached indexes.
|
||||
#[default]
|
||||
@@ -394,7 +394,6 @@ impl ScopeGraph {
|
||||
})
|
||||
}
|
||||
|
||||
/// Find all references to a given name
|
||||
pub fn find_references(&self, name: &str, src: &[u8]) -> Vec<Range> {
|
||||
self.graph
|
||||
.node_indices()
|
||||
@@ -703,9 +702,7 @@ impl ScopeGraphIndex {
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// String interning helpers
|
||||
// ========================================================================
|
||||
|
||||
/// Intern a string and return its ID.
|
||||
#[inline]
|
||||
@@ -725,9 +722,7 @@ impl ScopeGraphIndex {
|
||||
self.interner.get_id(s)
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// File metadata operations
|
||||
// ========================================================================
|
||||
|
||||
/// Update file metadata (size and mtime) for staleness tracking.
|
||||
pub fn update_file_meta(&mut self, path: &Path) {
|
||||
@@ -751,9 +746,7 @@ impl ScopeGraphIndex {
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Alias operations
|
||||
// ========================================================================
|
||||
|
||||
/// Register an alias relationship: alias_name is an alias for original_name
|
||||
pub fn add_alias(&mut self, alias_name: &str, original_name: &str) {
|
||||
@@ -771,9 +764,7 @@ impl ScopeGraphIndex {
|
||||
self.add_alias(&alias_name, &original_name);
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Symbol insertion (for builder/manager use)
|
||||
// ========================================================================
|
||||
|
||||
/// Add a definition occurrence for a symbol.
|
||||
pub fn add_definition(&mut self, symbol: &str, path: &str, line: usize) {
|
||||
@@ -847,9 +838,7 @@ impl ScopeGraphIndex {
|
||||
.filter_map(|(&id, meta)| self.get_str(id).map(|path| (path, meta)))
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// File operations
|
||||
// ========================================================================
|
||||
|
||||
/// Add a file's scope graph to the index
|
||||
pub fn add_file(&mut self, file_path: PathBuf, graph: ScopeGraph, src: &[u8]) {
|
||||
@@ -996,9 +985,7 @@ impl ScopeGraphIndex {
|
||||
self.file_meta.len()
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Query operations
|
||||
// ========================================================================
|
||||
|
||||
/// Find where a symbol is defined (includes resolving aliases)
|
||||
pub fn find_definitions(&self, symbol: &str) -> Vec<(&str, usize)> {
|
||||
@@ -1284,9 +1271,7 @@ impl ScopeGraphIndex {
|
||||
.collect()
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Statistics and metadata
|
||||
// ========================================================================
|
||||
|
||||
/// Get statistics: (files_count, total_definitions, total_references).
|
||||
///
|
||||
@@ -1301,7 +1286,6 @@ impl ScopeGraphIndex {
|
||||
)
|
||||
}
|
||||
|
||||
/// Get alias count
|
||||
pub fn alias_count(&self) -> usize {
|
||||
self.aliases.len()
|
||||
}
|
||||
@@ -1356,9 +1340,7 @@ impl ScopeGraphIndex {
|
||||
self.interner.shrink_to_fit();
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Binary serialization (custom format with magic bytes)
|
||||
// ========================================================================
|
||||
|
||||
/// Save the index to a file in binary format.
|
||||
pub fn save(&self, path: &Path) -> io::Result<()> {
|
||||
@@ -1617,7 +1599,8 @@ impl ScopeGraphIndex {
|
||||
|
||||
Ok(Self {
|
||||
interner,
|
||||
graphs: HashMap::new(), // Not serialized
|
||||
// Not serialized
|
||||
graphs: HashMap::new(),
|
||||
definitions,
|
||||
references,
|
||||
aliases,
|
||||
@@ -1716,7 +1699,8 @@ mod tests {
|
||||
index.compact();
|
||||
let (f1, d1, r1) = index.stats();
|
||||
|
||||
index.compact(); // second call must be a no-op
|
||||
// second call must be a no-op
|
||||
index.compact();
|
||||
let (f2, d2, r2) = index.stats();
|
||||
|
||||
assert_eq!(f1, f2);
|
||||
|
||||
@@ -16,17 +16,12 @@ pub use nodes::{LocalDef, LocalImport, LocalScope, NodeKind, Reference, Symbol,
|
||||
|
||||
use crate::languages::TSLanguageConfig;
|
||||
|
||||
/// Result of building a scope graph, including alias pairs.
|
||||
pub struct ScopeGraphResult {
|
||||
/// The scope graph for the file.
|
||||
pub graph: ScopeGraph,
|
||||
/// Alias pairs: (alias_name, original_name).
|
||||
/// Each pair is `(alias_name, original_name)`.
|
||||
pub aliases: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
/// Build a ScopeGraph from tree-sitter query and source.
|
||||
///
|
||||
/// This is a convenience wrapper around `scope_graph_from_definitions_query`.
|
||||
pub fn build_scope_graph(
|
||||
query: &tree_sitter::Query,
|
||||
root_node: tree_sitter::Node<'_>,
|
||||
|
||||
@@ -89,7 +89,6 @@ impl LocalDef {
|
||||
&src[self.range.start_byte()..self.range.end_byte()]
|
||||
}
|
||||
|
||||
/// Get the scope range.
|
||||
pub fn scope_range(&self) -> &Range {
|
||||
&self.scope.range
|
||||
}
|
||||
|
||||
@@ -25,7 +25,6 @@ pub enum FileEvent {
|
||||
|
||||
/// A file was renamed/moved.
|
||||
Renamed {
|
||||
/// Original path.
|
||||
from: PathBuf,
|
||||
/// New path.
|
||||
to: PathBuf,
|
||||
@@ -49,7 +48,8 @@ impl FileEvent {
|
||||
FileEvent::Created { .. } => true,
|
||||
FileEvent::Modified { .. } => true,
|
||||
FileEvent::Deleted { .. } => false,
|
||||
FileEvent::Renamed { .. } => false, // Only path update needed
|
||||
// Only path update needed
|
||||
FileEvent::Renamed { .. } => false,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -49,7 +49,6 @@ impl Location {
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the file path.
|
||||
pub fn file_path(&self) -> &PathBuf {
|
||||
&self.file_path
|
||||
}
|
||||
|
||||
@@ -113,7 +113,8 @@ impl FileMeta {
|
||||
let current = Self::from_metadata(&meta);
|
||||
*self != current
|
||||
}
|
||||
Err(_) => true, // File deleted or inaccessible
|
||||
// File deleted or inaccessible
|
||||
Err(_) => true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,7 +70,6 @@ impl Position {
|
||||
self.character
|
||||
}
|
||||
|
||||
/// Get the byte offset.
|
||||
pub fn byte_offset(&self) -> usize {
|
||||
self.byte_offset
|
||||
}
|
||||
@@ -80,7 +79,6 @@ impl Position {
|
||||
self.byte_offset
|
||||
}
|
||||
|
||||
/// Set the byte offset.
|
||||
pub fn set_byte_offset(&mut self, byte_offset: usize) {
|
||||
self.byte_offset = byte_offset;
|
||||
}
|
||||
@@ -120,7 +118,6 @@ impl Position {
|
||||
}
|
||||
}
|
||||
|
||||
/// Move to the next line.
|
||||
pub fn move_to_next_line(mut self) -> Self {
|
||||
self.line += 1;
|
||||
self.character = 0;
|
||||
@@ -188,12 +185,10 @@ impl Range {
|
||||
Self::for_tree_node(node)
|
||||
}
|
||||
|
||||
/// Get the start position.
|
||||
pub fn start_position(&self) -> Position {
|
||||
self.start_position
|
||||
}
|
||||
|
||||
/// Get the end position.
|
||||
pub fn end_position(&self) -> Position {
|
||||
self.end_position
|
||||
}
|
||||
@@ -208,12 +203,10 @@ impl Range {
|
||||
&self.end_position
|
||||
}
|
||||
|
||||
/// Set the start position.
|
||||
pub fn set_start_position(&mut self, position: Position) {
|
||||
self.start_position = position;
|
||||
}
|
||||
|
||||
/// Set the end position.
|
||||
pub fn set_end_position(&mut self, position: Position) {
|
||||
self.end_position = position;
|
||||
}
|
||||
|
||||
@@ -1,29 +1,20 @@
|
||||
//! Isolated RSS test for incremental reindexing.
|
||||
//!
|
||||
//! This test lives in its own integration-test file (and therefore its own
|
||||
//! Bazel `rust_test` target / process) so that its whole-process RSS samples
|
||||
//! are not polluted by the other allocation-heavy tests in
|
||||
//! `memory_integration.rs` (e.g. `test_fresh_build_rss`,
|
||||
//! `test_build_batch_peak_rss_is_bounded`, `test_compact_reduces_rss_vs_uncompacted`).
|
||||
//! `libtest` runs a test binary's tests concurrently across `num_cpus` threads,
|
||||
//! but VmRSS is measured per-*process*. Sharing a binary with the other
|
||||
//! allocation-heavy tests in `memory_integration.rs` made this test observe
|
||||
//! their allocator churn, intermittently pushing the measured incremental
|
||||
//! growth delta over the 20 MB budget on aarch64 fastbuild CI (~31 MB).
|
||||
//!
|
||||
//! Background: `libtest` runs tests in a single binary concurrently across
|
||||
//! `num_cpus` threads, and VmRSS is measured per-*process*. When this test
|
||||
//! ran inside `memory_integration.rs` it observed allocator churn from the
|
||||
//! other tests on the same process, intermittently pushing the measured
|
||||
//! "incremental growth" delta over the 20 MB budget on aarch64 fastbuild CI
|
||||
//! (`run_1_of_2` and `run_2_of_2` both failed at ~31 MB).
|
||||
//!
|
||||
//! Keep this file to a single test. If you need to add another RSS-sensitive
|
||||
//! test, give it its own file too rather than reintroducing the
|
||||
//! noisy-neighbor problem.
|
||||
//! Hence its own integration-test file, and therefore its own Bazel
|
||||
//! `rust_test` target and process. Keep this file to a single test; any other
|
||||
//! RSS-sensitive test needs a file of its own rather than a noisy neighbor.
|
||||
|
||||
use kigi_codebase_graph::{FileEvent, IndexManager, IndexManagerConfig};
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use tempfile::tempdir;
|
||||
|
||||
/// Read current process RSS in bytes. Supports Linux and macOS.
|
||||
/// Returns `None` on unsupported platforms.
|
||||
fn rss_bytes() -> Option<usize> {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
@@ -65,7 +56,6 @@ fn fmt_rss(rss: Option<f64>) -> String {
|
||||
rss.map_or("N/A".to_string(), |v| format!("{:.1}MB", v))
|
||||
}
|
||||
|
||||
/// Create N Rust source files in `dir`, each with `defs_per_file` function defs.
|
||||
fn create_rust_files(dir: &Path, count: usize, defs_per_file: usize) {
|
||||
for i in 0..count {
|
||||
let mut content = String::new();
|
||||
@@ -122,7 +112,6 @@ fn test_bulk_incremental_indexing_memory() {
|
||||
);
|
||||
println!("RSS after incremental: {}", fmt_rss(rss_after_incremental));
|
||||
|
||||
// Incremental reindexing should not grow memory significantly.
|
||||
if let (Some(after_inc), Some(after_build)) = (rss_after_incremental, rss_after_build) {
|
||||
let growth = after_inc - after_build;
|
||||
assert!(
|
||||
|
||||
@@ -80,9 +80,7 @@ fn create_binary_files(dir: &Path, count: usize, size: usize) {
|
||||
}
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Tests
|
||||
// =========================================================================
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
@@ -221,11 +219,14 @@ fn test_builder_skips_binary_and_oversized_in_bulk() {
|
||||
let root = dir.path();
|
||||
|
||||
// Mix of valid, binary, and oversized files
|
||||
create_rust_files(root, 100, 5); // 100 valid files
|
||||
create_binary_files(root, 50, 10_000); // 50 binary files
|
||||
// 100 valid files
|
||||
create_rust_files(root, 100, 5);
|
||||
// 50 binary files
|
||||
create_binary_files(root, 50, 10_000);
|
||||
|
||||
// One oversized file
|
||||
let big = "fn x() {}\n".repeat(600_000); // ~6MB
|
||||
// ~6MB
|
||||
let big = "fn x() {}\n".repeat(600_000);
|
||||
fs::write(root.join("oversized.rs"), &big).unwrap();
|
||||
drop(big);
|
||||
|
||||
@@ -234,7 +235,8 @@ fn test_builder_skips_binary_and_oversized_in_bulk() {
|
||||
|
||||
// Only the 100 valid files should be indexed
|
||||
assert_eq!(files, 100);
|
||||
assert!(defs >= 500); // 100 files × 5 defs
|
||||
// 100 files × 5 defs
|
||||
assert!(defs >= 500);
|
||||
}
|
||||
|
||||
/// Measure RSS growth from a single `get_snapshot()` call on a representative index.
|
||||
@@ -248,7 +250,8 @@ fn test_builder_skips_binary_and_oversized_in_bulk() {
|
||||
fn test_single_snapshot_rss() {
|
||||
let dir = tempdir().unwrap();
|
||||
let root = dir.path();
|
||||
create_rust_files(root, 500, 10); // 500 files, 5 000 defs
|
||||
// 500 files, 5 000 defs
|
||||
create_rust_files(root, 500, 10);
|
||||
|
||||
let config = IndexManagerConfig::new(root.to_path_buf())
|
||||
.without_cache_load()
|
||||
@@ -353,7 +356,8 @@ fn test_repeated_snapshots_rss_bounded() {
|
||||
fn test_fresh_build_rss() {
|
||||
let dir = tempdir().unwrap();
|
||||
let root = dir.path();
|
||||
create_rust_files(root, 500, 10); // 500 files, 5 000 defs
|
||||
// 500 files, 5 000 defs
|
||||
create_rust_files(root, 500, 10);
|
||||
|
||||
let rss_before = rss_mb();
|
||||
|
||||
@@ -451,7 +455,8 @@ fn test_cache_load_rss() {
|
||||
fn test_build_batch_size_produces_correct_index() {
|
||||
let dir = tempdir().unwrap();
|
||||
let root = dir.path();
|
||||
create_rust_files(root, 200, 5); // 200 files, 1 000 defs
|
||||
// 200 files, 1 000 defs
|
||||
create_rust_files(root, 200, 5);
|
||||
|
||||
// Build with a very small batch size (10 files per merge batch)
|
||||
let batched = IndexBuilder::new()
|
||||
@@ -587,9 +592,7 @@ fn test_build_batch_peak_rss_is_bounded() {
|
||||
assert_eq!(b_refs, u_refs, "reference count must match");
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Structural compaction tests
|
||||
// =============================================================================
|
||||
|
||||
/// Verify that an index survives a save/load round-trip after compact().
|
||||
///
|
||||
@@ -601,7 +604,8 @@ fn test_build_batch_peak_rss_is_bounded() {
|
||||
fn test_compact_then_save_load_roundtrip() {
|
||||
let dir = tempdir().unwrap();
|
||||
let root = dir.path();
|
||||
create_rust_files(root, 50, 4); // 50 files, 200 defs
|
||||
// 50 files, 200 defs
|
||||
create_rust_files(root, 50, 4);
|
||||
|
||||
// build() calls compact() internally via build_fast()
|
||||
let original = IndexBuilder::new().build(root).unwrap();
|
||||
|
||||
Reference in New Issue
Block a user