docs(comments): rewrite comments across all crates to the guidelines

Sweep every first-party crate source (1956 .rs files) to the project comment
guidelines: delete redundant restatements, decorative banners, change
narration, and end-of-line comments; keep and tighten the crucial ones
(invariants, bug rationale, SAFETY blocks, ported-source attribution).

No functional code changed. Every edit is proven comment-only against the
prior tree by a comment-stripping lexer (string/char/raw-string aware) plus a
separate doctest-fence check. Where removing a comment made rustfmt or clippy
want to re-lay-out adjacent code, the minimal triggering comment is restored so
code tokens stay byte-identical.

Gates green: cargo fmt --all --check (0 diffs), cargo check and cargo clippy
--workspace --all-targets (0 warnings).

Adds scripts/check_codegen_comment_guidelines.py — the enforcement gate for
these guidelines (flags banners, end-of-line comments, change narration, and
commented-out code).
This commit is contained in:
2026-07-23 16:55:39 -04:00
parent ff0fb56c67
commit a02b555e66
1458 changed files with 10729 additions and 21750 deletions
+26 -57
View File
@@ -8,12 +8,10 @@ use kigi_markdown::{
MarkdownStyle, StreamingMarkdownRenderer, Syntect, render_markdown_ratatui_full,
};
/// Default style for benchmarking.
fn default_style() -> MarkdownStyle {
MarkdownStyle::default()
}
/// Create syntect highlighter for benchmarking.
fn create_syntect() -> Syntect {
Syntect::new(include_bytes!("../assets/tokyo-night.tmTheme"))
}
@@ -94,7 +92,6 @@ This is *some <a href="https://google.com">inline html</a> block*.
});
}
/// Generate a markdown document with multiple blocks for streaming simulation.
fn generate_streaming_content(num_blocks: usize) -> String {
let mut content = String::new();
for i in 0..num_blocks {
@@ -129,7 +126,7 @@ fn generate_streaming_content(num_blocks: usize) -> String {
content
}
/// Benchmark streaming with full re-render on each token (O(N²) baseline).
/// O(N²) baseline: a full re-render on every token.
fn bench_streaming_full_rerender(c: &mut Criterion) {
let mut group = c.benchmark_group("streaming");
let syntect = create_syntect();
@@ -164,11 +161,9 @@ fn bench_streaming_full_rerender(c: &mut Criterion) {
group.finish();
}
/// Generate a hyperlink-heavy markdown document.
///
/// Each block contains 4 inline links and 1 autolink so the renderer's
/// link-translation path is exercised on most rendered lines. Designed
/// to surface O(lines * link_targets) costs in `translate_link_targets`.
/// Each block carries 4 inline links and 1 autolink, so most rendered lines hit
/// the link-translation path — surfacing O(lines * link_targets) costs in
/// `translate_link_targets`.
fn generate_hyperlink_content(num_blocks: usize) -> String {
let mut content = String::new();
for i in 0..num_blocks {
@@ -189,11 +184,8 @@ fn generate_hyperlink_content(num_blocks: usize) -> String {
content
}
/// Benchmark a single full render of a hyperlink-heavy document.
///
/// Surfaces the cost of the parse-time `link_targets` collection plus the
/// post-render translation step. Pair with `bench_render_markdown` to see
/// the link-translation overhead in isolation.
/// Parse-time `link_targets` collection plus the post-render translation step.
/// Pair with `bench_render_markdown` to isolate the link-translation overhead.
fn bench_render_markdown_hyperlinks(c: &mut Criterion) {
let syntect = create_syntect();
let mut group = c.benchmark_group("render_markdown_hyperlinks");
@@ -219,10 +211,8 @@ fn bench_render_markdown_hyperlinks(c: &mut Criterion) {
group.finish();
}
/// Benchmark incremental streaming of a hyperlink-heavy document.
///
/// Exercises `rerender_tail` repeatedly, which calls the link-translation
/// path on the unfrozen tail every push.
/// `rerender_tail` runs the link-translation path over the unfrozen tail on
/// every push.
fn bench_streaming_hyperlinks_incremental(c: &mut Criterion) {
let mut group = c.benchmark_group("streaming_hyperlinks");
let syntect = create_syntect();
@@ -251,7 +241,7 @@ fn bench_streaming_hyperlinks_incremental(c: &mut Criterion) {
group.finish();
}
/// Benchmark streaming with incremental renderer (O(N) target).
/// O(N) target against the `bench_streaming_full_rerender` baseline.
fn bench_streaming_incremental(c: &mut Criterion) {
let mut group = c.benchmark_group("streaming");
let syntect = create_syntect();
@@ -280,12 +270,10 @@ fn bench_streaming_incremental(c: &mut Criterion) {
group.finish();
}
/// Generate a math-heavy markdown document.
///
/// Each block exercises all four delimiter forms (`$...$`, `$$...$$`,
/// `\(...\)`, `\[...\]`) plus the expensive converter paths: scripts,
/// fractions, roots, symbol lookups, alphabets, and multi-row environments
/// (aligned / pmatrix / cases) that go through the MathBox 2D layout.
/// Each block covers all four delimiter forms (`$...$`, `$$...$$`, `\(...\)`,
/// `\[...\]`) plus the expensive converter paths: scripts, fractions, roots,
/// symbol lookups, alphabets, and the multi-row environments (aligned / pmatrix
/// / cases) that go through the MathBox 2D layout.
fn generate_math_content(num_blocks: usize) -> String {
let mut content = String::new();
for i in 0..num_blocks {
@@ -311,10 +299,8 @@ fn generate_math_content(num_blocks: usize) -> String {
content
}
/// Benchmark a single full render of a math-heavy document.
///
/// Surfaces the cost of the LaTeX → Unicode converter plus the parse-time
/// `\(...\)` / `\[...\]` source scans and block replacements.
/// The LaTeX → Unicode converter plus the parse-time `\(...\)` / `\[...\]`
/// source scans and block replacements.
fn bench_render_markdown_math(c: &mut Criterion) {
let syntect = create_syntect();
let mut group = c.benchmark_group("render_markdown_math");
@@ -340,11 +326,8 @@ fn bench_render_markdown_math(c: &mut Criterion) {
group.finish();
}
/// Benchmark incremental streaming of a math-heavy document.
///
/// Exercises the streaming hot path the `MAX_MATH_SOURCE_LEN` guard
/// protects: every push re-renders the unfrozen tail, re-running the math
/// scans and conversions on it.
/// The hot path the `MAX_MATH_SOURCE_LEN` guard protects: every push re-renders
/// the unfrozen tail, re-running the math scans and conversions over it.
fn bench_streaming_math_incremental(c: &mut Criterion) {
let mut group = c.benchmark_group("streaming_math");
let syntect = create_syntect();
@@ -373,7 +356,7 @@ fn bench_streaming_math_incremental(c: &mut Criterion) {
group.finish();
}
/// Generate a document with plain URLs in prose (no markdown link syntax).
/// Bare URLs in prose, with no markdown link syntax to pick them up.
fn generate_plain_url_content(num_blocks: usize) -> String {
let mut content = String::new();
for i in 0..num_blocks {
@@ -384,12 +367,8 @@ fn generate_plain_url_content(num_blocks: usize) -> String {
content
}
/// Benchmark streaming + finish() of a plain-URL-heavy document.
///
/// Exercises the `detect_plain_urls` scan, which after the multi-line-URL
/// fix runs inside both `rerender_tail` (every `push_and_render`) and
/// `finish()`. Bench numbers from this point forward are not comparable
/// to historical runs that measured the prior `finish()`-only path.
/// The `detect_plain_urls` scan runs inside both `rerender_tail` (on every
/// `push_and_render`) and `finish()`, so this measures both.
fn bench_streaming_plain_urls_incremental(c: &mut Criterion) {
let mut group = c.benchmark_group("streaming_plain_urls");
let syntect = create_syntect();
@@ -416,11 +395,9 @@ fn bench_streaming_plain_urls_incremental(c: &mut Criterion) {
group.finish();
}
/// Generate a realistic nested YAML document of roughly `num_lines` lines.
///
/// Produces nested keys, lists, and scalars (the kind of config an LLM streams
/// into a single fenced block) so the syntect highlighter does real work per
/// line rather than trivial whitespace.
/// Nested keys, lists, and scalars — the kind of config an LLM streams into a
/// single fenced block — so the syntect highlighter does real work per line
/// rather than trivial whitespace.
fn generate_yaml_lines(num_lines: usize) -> Vec<String> {
let mut lines = Vec::with_capacity(num_lines);
let mut i = 0usize;
@@ -450,13 +427,11 @@ fn generate_yaml_lines(num_lines: usize) -> Vec<String> {
lines
}
/// Benchmark streaming a SINGLE open ```yaml fenced block line-by-line WITHOUT
/// ever closing the fence.
///
/// A single yaml fence that is opened and never closed, streamed line by line.
/// This reproduces the UI-freeze pathology: while the fence is open the block
/// never checkpoints, so every `push_and_render` re-highlights the whole tail.
/// With the incremental open-code cache, per-line cost should stay roughly flat
/// in block size instead of growing linearly (overall O(N) instead of O(N²)).
/// The open-code cache should keep per-line cost roughly flat in block size
/// instead of growing linearly (overall O(N) instead of O(N²)).
fn bench_streaming_open_yaml_incremental(c: &mut Criterion) {
let mut group = c.benchmark_group("streaming_open_yaml");
let syntect = create_syntect();
@@ -464,16 +439,12 @@ fn bench_streaming_open_yaml_incremental(c: &mut Criterion) {
for num_lines in [100, 500, 1081] {
let lines = generate_yaml_lines(num_lines);
// Only the cache-on path ships; the parameter is just the line count
// (no A/B baseline here — the no-cache baseline was measured ad hoc and
// is not committed).
group.bench_with_input(
BenchmarkId::from_parameter(num_lines),
&lines,
|b, lines| {
b.iter(|| {
let mut renderer = StreamingMarkdownRenderer::new(default_style(), true);
// Open the fence but never close it.
renderer.push_and_render("```yaml\n", Some(&syntect));
let mut total_lines = 0;
for line in lines.iter() {
@@ -497,7 +468,6 @@ fn bench_streaming_open_yaml_incremental(c: &mut Criterion) {
fn generate_fence_in_list_content(trailing_words: usize) -> String {
let mut s = String::new();
s.push_str("Here is where you're stuck:\n\n");
// Two list items embedding closed scala fences (the pathological shape).
for i in 0..2 {
s.push_str(&format!(
"- **[File{i}.scala:{}](https://example.com/f{i})** (domain)\n \
@@ -513,7 +483,6 @@ fn generate_fence_in_list_content(trailing_words: usize) -> String {
100 + i,
));
}
// Continued streaming within the same (never-closing) list context.
for w in 0..trailing_words {
if w % 12 == 0 {
s.push_str("\n- item: ");
@@ -31,7 +31,7 @@ use kigi_markdown::{
};
use kigi_ratatui_textarea::{TextArea, TextAreaState};
// ── Tokyo Night Storm palette (matches kigi-tui) ──────────────────────
// Tokyo Night Storm palette (matches kigi-tui)
#[path = "playground_common.rs"]
mod playground_common;
@@ -39,7 +39,7 @@ use playground_common::{get_syntect, md_style};
const MD_STYLE: MarkdownStyle = md_style(anstyle::Style::new());
// ── Compute minimum render width ─────────────────────────────────────────────
// Compute minimum render width
/// Minimum width = 4k + 1, where k = number of table columns.
/// Columns = max(`|` count per line) - 1 (the outer pipes are borders).
@@ -58,7 +58,7 @@ fn min_render_width(source: &str) -> usize {
}
}
// ── Render helpers ───────────────────────────────────────────────────────────
// Render helpers
/// One-shot full render at a given width.
fn render_full(source: &str, width: usize) -> Vec<Line<'static>> {
@@ -84,7 +84,7 @@ fn render_streaming(source: &str, width: usize) -> Vec<Line<'static>> {
renderer.view().lines.to_vec()
}
// ── App state ────────────────────────────────────────────────────────────────
// App state
const DEFAULT_MARKDOWN: &str = "\
| A | B | C |
@@ -145,7 +145,7 @@ impl App {
}
}
// ── Main ─────────────────────────────────────────────────────────────────────
// Main
fn main() -> io::Result<()> {
// Terminal setup
@@ -164,14 +164,14 @@ fn main() -> io::Result<()> {
let ev = event::read()?;
match &ev {
// ── Global: Ctrl-Q quits from anywhere ──
// Global: Ctrl-Q quits from anywhere
Event::Key(KeyEvent {
code: KeyCode::Char('q'),
modifiers: KeyModifiers::CONTROL,
..
}) => break,
// ── Focused: defocus on Esc or Tab ──
// Focused: defocus on Esc or Tab
Event::Key(KeyEvent {
code: KeyCode::Esc | KeyCode::Tab,
..
@@ -179,7 +179,7 @@ fn main() -> io::Result<()> {
app.textarea_focused = false;
}
// ── Unfocused: quit on q, Ctrl-C, Ctrl-D ──
// Unfocused: quit on q, Ctrl-C, Ctrl-D
Event::Key(KeyEvent {
code: KeyCode::Char('q'),
..
@@ -190,7 +190,7 @@ fn main() -> io::Result<()> {
..
}) if !app.textarea_focused => break,
// ── Unfocused: Space or Enter to re-focus ──
// Unfocused: Space or Enter to re-focus
Event::Key(KeyEvent {
code: KeyCode::Char(' ') | KeyCode::Enter,
..
@@ -198,7 +198,7 @@ fn main() -> io::Result<()> {
app.textarea_focused = true;
}
// ── Unfocused: width controls ──
// Unfocused: width controls
Event::Key(KeyEvent {
code: KeyCode::Char('h') | KeyCode::Left,
..
@@ -224,13 +224,13 @@ fn main() -> io::Result<()> {
app.adjust_width(5);
}
// ── Focused: forward keys to textarea, live re-render ──
// Focused: forward keys to textarea, live re-render
Event::Key(key) if app.textarea_focused => {
app.textarea.input(*key);
app.rerender();
}
// ── Focused: forward mouse to textarea ──
// Focused: forward mouse to textarea
Event::Mouse(mouse) if app.textarea_focused => {
app.textarea
.handle_mouse(*mouse, app.textarea_area, app.textarea_state);
@@ -248,7 +248,7 @@ fn main() -> io::Result<()> {
Ok(())
}
// ── Drawing ──────────────────────────────────────────────────────────────────
// Drawing
fn draw(f: &mut ratatui::Frame, app: &mut App) {
let size = f.area();
@@ -259,7 +259,8 @@ fn draw(f: &mut ratatui::Frame, app: &mut App) {
// to the textarea.
let render_w = app.render_width as u16;
let full_height = wrapped_line_count(&app.full_lines, render_w).max(1) + 2; // +2 for border
// +2 for border
let full_height = wrapped_line_count(&app.full_lines, render_w).max(1) + 2;
let stream_height = wrapped_line_count(&app.streaming_lines, render_w).max(1) + 2;
// Detect mismatches between full and streaming
@@ -281,7 +282,7 @@ fn draw(f: &mut ratatui::Frame, app: &mut App) {
])
.split(size);
// ── Header ──
// Header
let focus_indicator = if app.textarea_focused {
Span::styled(
" EDITING ",
@@ -339,7 +340,7 @@ fn draw(f: &mut ratatui::Frame, app: &mut App) {
]);
f.render_widget(header, chunks[0]);
// ── Textarea ──
// Textarea
let border_color = if app.textarea_focused {
Color::Green
} else {
@@ -367,11 +368,11 @@ fn draw(f: &mut ratatui::Frame, app: &mut App) {
f.set_cursor_position((cx, cy));
}
// ── Full render panel ──
// Full render panel
let full_title = format!(" full: {} ", app.render_width);
render_panel(f, chunks[2], &full_title, &app.full_lines, render_w, false);
// ── Streaming render panel ──
// Streaming render panel
let stream_title = format!(" stream: {} ", app.render_width);
render_panel(
f,
@@ -403,7 +404,8 @@ fn wrapped_line_count(lines: &[Line<'_>], width: u16) -> u16 {
if display_w == 0 {
1u16
} else {
display_w.div_ceil(w) as u16 // ceil division
// ceil division
display_w.div_ceil(w) as u16
}
})
.sum()
@@ -11,19 +11,16 @@ fuzz_target!(|data: &[u8]| {
return;
};
// Full render: pretty / non-pretty
for pretty in [true, false] {
let _ = render_markdown_ratatui_full(s, STYLE, pretty, None);
}
// Streaming with rotating chunk sizes: pretty / non-pretty
for pretty in [true, false] {
let mut r = StreamingMarkdownRenderer::new(STYLE, pretty);
let mut pos = 0;
let mut ci = 0;
while pos < s.len() {
let mut end = (pos + CHUNK_SIZES[ci]).min(s.len());
// snap to char boundary
while end < s.len() && !s.is_char_boundary(end) {
end += 1;
}
+10 -39
View File
@@ -1,7 +1,4 @@
//! Reusable buffers and internal data types for markdown parsing and rendering.
//!
//! This module contains all the intermediate data structures used by
//! MarkdownHighlighter during parsing and rendering.
use std::ops::Range;
@@ -9,7 +6,6 @@ use anstyle::Style as AnsiStyle;
use ratatui::text::{Line, Span};
use syntect::highlighting::Style as SyntectStyle;
/// A range of text with optional styling.
#[derive(Debug, Clone)]
pub struct Highlight {
pub style: Option<AnsiStyle>,
@@ -18,12 +14,11 @@ pub struct Highlight {
/// Syntax-highlighted code block replacement.
///
/// Stores the raw highlighted spans per line (intermediate representation).
/// This allows rendering to either ANSI strings or ratatui Lines on demand.
/// Spans are kept in their intermediate form so the block can be rendered to
/// either ANSI strings or ratatui Lines on demand.
#[derive(Debug, Clone)]
pub struct Replace {
/// Raw highlighted spans per line: Vec<(style, text)>.
/// Each inner Vec represents one line of the code block.
/// One inner Vec per line of the code block.
pub highlighted: Vec<Vec<(SyntectStyle, String)>>,
/// Source byte range this replaces.
pub range: Range<usize>,
@@ -37,7 +32,6 @@ pub struct Replace {
pub struct LinkTarget {
/// Source byte range of the *link text* (not the full `[text](url)` span).
pub source_range: Range<usize>,
/// Destination URL.
pub url: String,
/// Monotonically increasing identifier assigned during parsing.
pub id: u32,
@@ -118,12 +112,10 @@ impl StyledCell {
Self { spans: Vec::new() }
}
/// Get plain text content (for width calculation).
pub fn plain_text(&self) -> String {
self.spans.iter().map(|s| s.text.as_str()).collect()
}
/// Clear the cell content.
pub fn clear(&mut self) {
self.spans.clear();
}
@@ -134,15 +126,10 @@ impl StyledCell {
pub struct TableState {
/// Column alignments from the table header.
pub alignments: Vec<pulldown_cmark::Alignment>,
/// Header row cells.
pub header: Vec<StyledCell>,
/// Body rows (each row is a Vec of styled cells).
pub rows: Vec<Vec<StyledCell>>,
/// Current row being built.
pub current_row: Vec<StyledCell>,
/// Current cell content being accumulated.
pub current_cell: StyledCell,
/// Current style state for the cell.
pub cell_bold: bool,
pub cell_italic: bool,
pub cell_code: bool,
@@ -151,7 +138,6 @@ pub struct TableState {
/// is set produce link-tagged `CellSpan`s so the table renderer can
/// apply link styling and emit `HyperlinkTarget`s.
pub cell_link: Option<(String, u32)>,
/// Whether we're in the header section.
pub in_header: bool,
/// Source byte range of the entire table.
pub range: Range<usize>,
@@ -194,11 +180,9 @@ impl TableState {
/// `HyperlinkTarget`.
#[derive(Debug, Clone)]
pub struct TableHyperlink {
/// Index within `TableReplace::styled_lines`.
pub line_offset: usize,
/// Column range (display cells) on that line.
/// Column range in display cells, not bytes.
pub column_range: Range<usize>,
/// Destination URL.
pub url: String,
/// Stable identifier shared with the paragraph link path.
pub id: u32,
@@ -245,7 +229,6 @@ pub struct MermaidReplace {
pub range: Range<usize>,
}
/// Calculate the display width of a string (accounting for Unicode).
pub fn unicode_display_width(s: &str) -> usize {
use unicode_width::UnicodeWidthStr;
s.width()
@@ -287,8 +270,10 @@ pub enum RenderEventKind {
Mermaid = 3,
}
/// Render event: marks where a highlight/replace/table starts or ends.
/// Derives Ord for sorting by (pos, kind, index, is_end).
/// Marks where a highlight/replace/table starts or ends.
///
/// Field order is load-bearing: the derived `Ord` sorts the event queue by
/// (pos, kind, index, is_end).
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct RenderEvent {
pub pos: usize,
@@ -301,22 +286,8 @@ pub struct RenderEvent {
///
/// All vectors are cleared (keeping capacity) between renders, eliminating
/// allocation overhead in the streaming hot path.
///
/// # Buffer Categories
///
/// **Parse output buffers** - populated during `run()`, read-only during `render()`:
/// - `highlights`: Style ranges for inline formatting
/// - `replaces`: Syntax-highlighted code blocks
/// - `transforms`: Character substitutions (e.g., bullets)
/// - `untagged_code_ranges`: Code blocks without language tags
/// - `table_replaces`: Formatted table replacements
///
/// **Render scratch buffers** - temporary storage during `render()`:
/// - `render_events`: Sorted event queue for the render loop
/// - `current_spans`: Building current line's spans
/// - `active_highlights`: Stack of active highlight indices
pub struct MarkdownBuffers {
// Parse output buffers (written by run(), read by render())
// Parse output buffers: written by run(), read-only during render().
pub highlights: Vec<Highlight>,
pub replaces: Vec<Replace>,
pub transforms: Vec<Transform>,
@@ -327,7 +298,7 @@ pub struct MarkdownBuffers {
/// Closed fenced code blocks, in document order (see [`CodeBlockMeta`]).
pub code_blocks: Vec<CodeBlockMeta>,
// Render scratch buffers (used only during render())
// Render scratch buffers: used only during render().
pub render_events: Vec<RenderEvent>,
pub current_spans: Vec<Span<'static>>,
pub active_highlights: Vec<usize>,
@@ -1,8 +1,8 @@
//! Checkpoint types for incremental markdown rendering.
//!
//! This module defines types for identifying stable boundaries in markdown text
//! where rendered output can be "frozen" and cached. Content before a checkpoint
//! will not change regardless of what text is appended after it.
//! A checkpoint marks a stable boundary in markdown text where rendered output
//! can be "frozen" and cached: content before it will not change regardless of
//! what text is appended after it.
//!
//! # Design
//!
+9 -5
View File
@@ -165,7 +165,7 @@ pub fn set_color_level(level: ColorLevel) -> Result<(), ColorLevel> {
/// Convert an `anstyle::Color` to the appropriate level based on terminal support.
///
/// This will downgrade colors as needed:
/// - TrueColor terminals: pass through unchanged
/// - TrueColor terminals: pass through `unchanged`
/// - 256-color terminals: RGB colors are converted to closest ANSI 256 color
/// - Basic terminals: colors are converted to closest ANSI 16 color
/// - No color: returns None
@@ -234,22 +234,26 @@ mod tests {
// Medium gray
let result = rgb_to_ansi256(RgbColor(128, 128, 128));
assert!(result.index() >= 232); // Should be in grayscale range
// Should be in grayscale range
assert!(result.index() >= 232);
}
#[test]
fn test_rgb_to_ansi256_colors() {
// Pure red
let result = rgb_to_ansi256(RgbColor(255, 0, 0));
assert_eq!(result.index(), 196); // Bright red in the cube
// Bright red in the cube
assert_eq!(result.index(), 196);
// Pure green
let result = rgb_to_ansi256(RgbColor(0, 255, 0));
assert_eq!(result.index(), 46); // Bright green in the cube
// Bright green in the cube
assert_eq!(result.index(), 46);
// Pure blue
let result = rgb_to_ansi256(RgbColor(0, 0, 255));
assert_eq!(result.index(), 21); // Bright blue in the cube
// Bright blue in the cube
assert_eq!(result.index(), 21);
}
#[test]
@@ -60,7 +60,7 @@ pub(crate) struct ChunkLinkRange {
/// transform's replacement string. Both endpoints (start/end) clamp the
/// same direction, so a link whose endpoint straddles a transform produces
/// a column range that excludes the straddling bytes. This is intentional
/// rather than precise — a future transform that intentionally rewrites
/// rather than precise — a future transform that deliberately rewrites
/// link text should add a typed mapping instead of relying on this clamp.
pub(crate) fn source_to_chunk_offset(
src_pos: usize,
@@ -231,7 +231,7 @@ mod hyperlink_tests {
/// one whose `column_range` slices to `expected_slice` in the
/// rendered output. Since `render_markdown_ratatui_full` now also
/// emits a url_scan target for the pretty-mode `(url)` suffix, tests
/// that previously checked `hyperlinks.len() == 1` must explicitly
/// that earlier checked `hyperlinks.len() == 1` must explicitly
/// pick the parser-produced entry.
fn parser_link_text<'a>(
out: &'a crate::output::MarkdownRenderOutput,
@@ -448,7 +448,7 @@ mod hyperlink_tests {
let view = renderer.view();
// Compare on `(url, line_index, column_range)` — ids are
// intentionally independent between the two code paths (full
// Deliberately independent between the two code paths (full
// re-render restarts id counters; streaming preserves continuity).
let extract = |hs: &[HyperlinkTarget]| -> Vec<(String, usize, std::ops::Range<usize>)> {
let mut v: Vec<_> = hs
@@ -623,7 +623,7 @@ mod hyperlink_tests {
}
/// Paragraph links must keep the `link_text` foreground color even when
/// the `text` style sets its own foreground. Previously the parser
/// the `text` style sets its own foreground. earlier the parser
/// pushed `ms.text` as a highlight after the link_text highlight whenever
/// no `Heading`/`Emphasis`/`Strong`/`Strikethrough` ancestor was present
/// — and `merge_styles` lets the later fg color win, so `ms.text`'s color
@@ -205,7 +205,7 @@ fn script_atom_is_wordlike(atom: &str, rendered: &str) -> bool {
fn render_command(cursor: &mut Cursor<'_>, out: &mut MathBox, depth: usize, mode: Mode) {
let name = cursor.read_command_name();
match name {
// ── Structure ────────────────────────────────────────────────────
// Structure
"" => out.push('\\'),
"\\" => out.push('\n'),
"begin" => render_environment(cursor, out, depth, mode),
@@ -232,7 +232,7 @@ fn render_command(cursor: &mut Cursor<'_>, out: &mut MathBox, depth: usize, mode
}
}
// ── Fractions / binomials / roots ────────────────────────────────
// Fractions / binomials / roots
"frac" | "dfrac" | "tfrac" | "cfrac" => {
let num = take_brace_arg(cursor).map(|a| render_atom(a, depth, mode));
let den = take_brace_arg(cursor).map(|a| render_atom(a, depth, mode));
@@ -261,7 +261,8 @@ fn render_command(cursor: &mut Cursor<'_>, out: &mut MathBox, depth: usize, mode
cursor.bump();
}
let idx = &cursor.src[start..cursor.pos];
cursor.bump(); // consume `]`
// consume `]`
cursor.bump();
Some(render_atom(idx, depth, mode))
} else {
None
@@ -290,7 +291,7 @@ fn render_command(cursor: &mut Cursor<'_>, out: &mut MathBox, depth: usize, mode
}
}
// ── Boxes (frame dropped; content preserved) ─────────────────────
// Boxes (frame dropped; content preserved)
"boxed" => {
if let Some(arg) = take_brace_arg(cursor) {
out.push_str(&render_atom(arg, depth, mode));
@@ -302,7 +303,7 @@ fn render_command(cursor: &mut Cursor<'_>, out: &mut MathBox, depth: usize, mode
}
}
// ── Text / alphabets ─────────────────────────────────────────────
// Text / alphabets
"text" | "textrm" | "textit" | "textbf" | "textsf" | "texttt" | "textnormal" | "mbox"
| "hbox" => {
if let Some(arg) = take_brace_arg(cursor) {
@@ -321,7 +322,7 @@ fn render_command(cursor: &mut Cursor<'_>, out: &mut MathBox, depth: usize, mode
render_mapped_alphabet(cursor, out, depth, mode, map_mathbf)
}
// ── Accents (combining marks) ────────────────────────────────────
// Accents (combining marks)
"hat" | "widehat" => render_accent(cursor, out, depth, mode, '\u{0302}'),
"bar" | "overline" => render_accent(cursor, out, depth, mode, '\u{0304}'),
"tilde" | "widetilde" => render_accent(cursor, out, depth, mode, '\u{0303}'),
@@ -335,7 +336,7 @@ fn render_command(cursor: &mut Cursor<'_>, out: &mut MathBox, depth: usize, mode
"mathring" => render_accent(cursor, out, depth, mode, '\u{030A}'),
"underline" => render_accent(cursor, out, depth, mode, '\u{0332}'),
// ── Negation ─────────────────────────────────────────────────────
// Negation
"not" => {
if let Some(atom) = cursor.read_atom() {
let rendered = render_atom(atom, depth, mode);
@@ -359,7 +360,7 @@ fn render_command(cursor: &mut Cursor<'_>, out: &mut MathBox, depth: usize, mode
}
}
// ── Decorations rendered as base + script ────────────────────────
// Decorations rendered as base + script
"overset" | "stackrel" => {
let over = take_brace_arg(cursor).map(|a| render_atom(a, depth, mode));
let base = take_brace_arg(cursor).map(|a| render_atom(a, depth, mode));
@@ -385,7 +386,7 @@ fn render_command(cursor: &mut Cursor<'_>, out: &mut MathBox, depth: usize, mode
}
}
// ── Modular arithmetic ───────────────────────────────────────────
// Modular arithmetic
"pmod" => {
if let Some(arg) = take_brace_arg(cursor) {
if !out.at_line_start() && !out.ends_with_space() {
@@ -401,7 +402,7 @@ fn render_command(cursor: &mut Cursor<'_>, out: &mut MathBox, depth: usize, mode
out.push_str("mod ");
}
// ── Spacing ──────────────────────────────────────────────────────
// Spacing
"," | ";" | ":" | ">" | " " | "space" | "thinspace" | "medspace" | "thickspace"
| "enspace" => {
if !out.at_line_start() && !out.ends_with_space() {
@@ -412,7 +413,7 @@ fn render_command(cursor: &mut Cursor<'_>, out: &mut MathBox, depth: usize, mode
"qquad" => out.push_str(" "),
"!" | "negthinspace" | "negmedspace" | "negthickspace" => {}
// ── No-ops (sizing/styling/structure hints) ──────────────────────
// No-ops (sizing/styling/structure hints)
"limits" | "nolimits" | "displaystyle" | "textstyle" | "scriptstyle"
| "scriptscriptstyle" | "big" | "Big" | "bigg" | "Bigg" | "bigl" | "Bigl" | "biggl"
| "Biggl" | "bigr" | "Bigr" | "biggr" | "Biggr" | "bigm" | "Bigm" | "biggm" | "Biggm"
@@ -425,7 +426,7 @@ fn render_command(cursor: &mut Cursor<'_>, out: &mut MathBox, depth: usize, mode
}
}
// ── Symbol table ─────────────────────────────────────────────────
// Symbol table
_ => {
if let Some(sym) = symbol(name) {
out.push_str(sym);
@@ -1,6 +1,5 @@
//! Byte cursor over TeX source.
/// Byte cursor over the TeX source.
pub(super) struct Cursor<'a> {
pub(super) src: &'a str,
pub(super) pos: usize,
@@ -24,7 +23,7 @@ impl<'a> Cursor<'a> {
/// Consume `\command` (alphabetic name) or `\<single char>`; the leading
/// backslash must already be consumed. Returns the command name.
///
/// Unlike TeX we do NOT consume trailing whitespace: the caller's
/// Unlike TeX, trailing whitespace is NOT consumed: the caller's
/// whitespace collapsing keeps `\to 0` rendering as `→ 0`.
pub(super) fn read_command_name(&mut self) -> &'a str {
let start = self.pos;
@@ -43,7 +42,6 @@ impl<'a> Cursor<'a> {
}
}
/// Skip whitespace (TeX collapses it; meaning comes from commands).
pub(super) fn skip_ws(&mut self) {
while matches!(self.peek(), Some(c) if c.is_whitespace()) {
self.bump();
@@ -59,7 +57,7 @@ impl<'a> Cursor<'a> {
while let Some(ch) = self.bump() {
match ch {
'\\' => {
// Skip escaped char so `\{`/`\}` don't affect depth.
// Skip the escaped char so `\{`/`\}` don't affect depth.
self.bump();
}
'{' => depth += 1,
@@ -19,8 +19,8 @@ pub(super) fn render_environment(
};
let env_name = env_name.trim().trim_end_matches('*');
// Capture body source until the matching `\end{name}`, tracking nesting
// of same-named environments. Scans raw source from the cursor.
// Scan raw source for the matching `\end{name}`, tracking nesting of
// same-named environments.
let body_start = cursor.pos;
let mut body_end = cursor.src.len();
let mut resume = cursor.src.len();
@@ -38,8 +38,8 @@ pub(super) fn render_environment(
} else if command_at(after_bs, "end") {
"end".len()
} else {
// Not begin/end: skip the backslash and the char after it (so
// `\\` and `\{` never confuse the scan).
// Skip the backslash and the char after it so that `\\` and `\{`
// never confuse the scan.
let skip = after_bs.chars().next().map_or(0, char::len_utf8);
search = bs_pos + 1 + skip.max(1);
continue;
@@ -66,7 +66,7 @@ pub(super) fn render_environment(
cursor.pos = resume;
let mut body = &cursor.src[body_start..body_end.min(cursor.src.len())];
// Optional column spec for array environments: `\begin{array}{ll}`.
// Discard the optional column spec: `\begin{array}{ll}`.
if env_name == "array" || env_name == "alignat" {
let mut probe = Cursor::new(body);
probe.skip_ws();
@@ -92,9 +92,9 @@ fn command_at(rest: &str, word: &str) -> bool {
/// Split an environment body into rows (`\\`) and cells (`&`) at brace and
/// environment depth 0, render each cell, then lay the rows out according to
/// the environment. Returns one string per visual row; the caller attaches
/// them as a box. In `flat` mode, matrix/cases environments render as a
/// single row with `; ` between matrix rows.
/// the environment. Returns one string per visual row. In `flat` mode,
/// matrix/cases environments collapse to a single row with `; ` between
/// matrix rows.
fn env_rows_to_strings(
body: &str,
env_name: &str,
@@ -130,7 +130,7 @@ fn env_rows_to_strings(
env_depth = env_depth.saturating_sub(1);
}
// Skip the backslash plus the char after it so escaped
// delimiters (`\&`, `\{`, `\}`) never affect depth/splits.
// delimiters (`\&`, `\{`, `\}`) never affect depth or splits.
let skip = rest.chars().next().map_or(0, char::len_utf8);
i += 1 + skip.max(1);
continue;
@@ -148,7 +148,6 @@ fn env_rows_to_strings(
row.push(body[cell_start.min(bytes.len())..].to_string());
rows.push(row);
// Render each cell, drop fully-empty rows.
let mut rendered_rows: Vec<Vec<String>> = rows
.into_iter()
.map(|cells| {
@@ -177,15 +176,14 @@ fn env_rows_to_strings(
let n_rows = rendered_rows.len();
if is_matrix {
// Flat (inline) mode: one row, single delimiter pair, rows joined
// with `; ` — `(1 2; 3 4)`.
if flat {
let inner = rendered_rows
.iter()
.map(|cells| cells.join(" "))
.collect::<Vec<_>>()
.join("; ");
// Single-row delimiter pair; plain `matrix` has none (' ').
// Ask for the single-row form; plain `matrix` has no delimiter
// and reports ' '.
let (l, r) = matrix_delims(env_name, 0, 1);
let mut s = String::new();
if l != ' ' {
@@ -197,7 +195,6 @@ fn env_rows_to_strings(
}
return vec![s];
}
// Pad columns to equal width so rows align.
let n_cols = rendered_rows.iter().map(Vec::len).max().unwrap_or(0);
let mut widths = vec![0usize; n_cols];
for cells in &rendered_rows {
@@ -243,9 +240,7 @@ fn env_rows_to_strings(
.collect()
} else {
// aligned/align/gather/split/equation/…: `&` is an invisible
// alignment marker; rejoin cells with a single space. One string per
// row; the caller's box attachment (or flat `; ` join) handles the
// rest.
// alignment marker, so cells rejoin with a single space.
rendered_rows
.iter()
.map(|cells| {
@@ -255,7 +250,7 @@ fn env_rows_to_strings(
.cloned()
.collect::<Vec<_>>()
.join(" ");
// Collapse any double spaces introduced around markers.
// Empty alignment cells leave runs of spaces behind.
while s.contains(" ") {
s = s.replace(" ", " ");
}
@@ -124,7 +124,7 @@ impl MathBox {
self.lines.push(String::new());
}
}
// Place the box rows, left-padded to the attach column.
// Place the box rows, left-`padded` to the attach column.
for (i, row) in rows.iter().enumerate() {
let target = self.anchor - box_anchor + i;
let line = &mut self.lines[target];
@@ -79,7 +79,6 @@ pub(crate) fn latex_to_unicode_display(src: &str) -> Option<Vec<String>> {
Some(lines)
}
/// Run the converter and return the output lines.
fn convert(src: &str, flat: bool) -> Vec<String> {
let mut cursor = Cursor::new(src);
let mut out = MathBox::new(flat);
@@ -1,4 +1,10 @@
//! Character and symbol mapping tables.
//!
//! In the `map_math*` alphabets the explicit letter arms come first on
//! purpose: those letters were encoded as Letterlike Symbols before the
//! contiguous Mathematical Alphanumeric blocks existed, so their block slots
//! are unassigned and the arithmetic arms below would yield a reserved
//! codepoint.
pub(super) fn to_superscript(c: char) -> Option<char> {
Some(match c {
@@ -29,10 +29,10 @@ fn subscripts_map_to_unicode() {
#[test]
fn script_fallback_uses_parens() {
// φ has no superscript form fall back to ^(...)
// Greek letters have no superscript forms; a multi-char run falls back to
// ^(...) while a lone char keeps the bare marker.
assert_eq!(inline("x^{\\alpha\\beta}"), "x^(αβ)");
assert_eq!(inline("x^\\alpha"), "x^α");
// Single unmappable subscript char.
assert_eq!(inline("a_q"), "a_q");
}
@@ -24,7 +24,7 @@
//! located and the ASCII whitespace immediately inside the delimiters is
//! trimmed, so the emitted `$…$` has no space right after the opening `$` or
//! before the closing `$`. pulldown-cmark's dollar-math flanking rule rejects
//! `$ … $` (whitespace next to a delimiter) and would otherwise leave a padded
//! `$ … $` (whitespace next to a delimiter) and would otherwise leave a `padded`
//! span as raw `$ … $` text. Interior newlines join to spaces (TeX treats them
//! as spaces) so a span wrapped across source lines cannot be re-parsed as
//! block structure.
@@ -199,7 +199,8 @@ impl LatexDelimiterNormalizer {
b'`' => {
let run = count_run(bytes, i, b'`');
if i + run == n && !final_flush {
break; // run may extend; hold it back
// run may extend; hold it back
break;
}
out.push_str(&buf[i..i + run]);
i += run;
@@ -277,7 +278,8 @@ impl LatexDelimiterNormalizer {
b'$' => {
let run = count_run(bytes, i, b'$');
if run == 1 && i + 1 == n && !final_flush {
break; // may become `$$`; hold it back
// may become `$$`; hold it back
break;
}
if run >= 2 {
// A display opener is exactly two `$`; any further
@@ -340,7 +342,8 @@ impl LatexDelimiterNormalizer {
let r = count_run(bytes, i, b'`');
if i + r == n && !final_flush {
out.push_str(&buf[start..i]);
return (out, i); // hold back the trailing run
// hold back the trailing run
return (out, i);
}
if r == run {
i += r;
@@ -350,13 +353,15 @@ impl LatexDelimiterNormalizer {
handled = true;
break;
}
i += r; // non-matching run is literal content
// non-matching run is literal content
i += r;
}
_ => i += 1,
}
}
if !handled {
out.push_str(&buf[start..i]); // EOF inside code
// EOF inside code
out.push_str(&buf[start..i]);
}
}
State::Fenced { ch, len } => {
@@ -379,7 +384,8 @@ impl LatexDelimiterNormalizer {
i += 1;
}
if i < n {
i += 1; // include the newline
// include the newline
i += 1;
self.at_line_start = true;
} else {
self.at_line_start = false;
@@ -430,13 +436,15 @@ fn scan_fence_open(bytes: &[u8], i: usize, final_flush: bool) -> FenceScan {
j += 1;
}
if spaces >= 4 {
return FenceScan::No; // indented; not treated as a fence opener
// indented; not treated as a fence opener
return FenceScan::No;
}
if j == n {
return if final_flush {
FenceScan::No
} else {
FenceScan::NeedMore // ≤3 spaces then EOF: a fence may still start
// ≤3 spaces then EOF: a fence may still start
FenceScan::NeedMore
};
}
let ch = bytes[j];
@@ -445,10 +453,12 @@ fn scan_fence_open(bytes: &[u8], i: usize, final_flush: bool) -> FenceScan {
}
let run = count_run(bytes, j, ch);
if j + run == n && !final_flush {
return FenceScan::NeedMore; // run may extend
// run may extend
return FenceScan::NeedMore;
}
if run < 3 {
return FenceScan::No; // inline code / stray tildes, not a fence
// inline code / stray tildes, not a fence
return FenceScan::No;
}
FenceScan::Match {
ch,
@@ -482,7 +492,8 @@ fn scan_fence_close(bytes: &[u8], i: usize, ch: u8, len: usize, final_flush: boo
}
let run = count_run(bytes, j, ch);
if j + run == n && !final_flush {
return FenceScan::NeedMore; // run may still grow to >= len
// run may still grow to >= len
return FenceScan::NeedMore;
}
if run < len {
return FenceScan::No;
@@ -510,7 +521,8 @@ fn scan_fence_close(bytes: &[u8], i: usize, ch: u8, len: usize, final_flush: boo
end: j + run,
}
} else {
FenceScan::No // non-whitespace after the run → info string → content
// non-whitespace after the run → info string → content
FenceScan::No
}
}
@@ -617,9 +629,11 @@ fn find_inline_close(bytes: &[u8], open: usize, final_flush: bool) -> InlineClos
}
if bytes[k] == b'\\' {
match bytes.get(k + 1) {
None => break, // trailing `\`: need the next byte to classify
// trailing `\`: need the next byte to classify
None => break,
Some(b')') => return InlineClose::Found { close: k },
Some(_) => k += 2, // `\\` pair or `\x` escape: skip both bytes
// `\\` pair or `\x` escape: skip both bytes
Some(_) => k += 2,
}
} else {
k += 1;
@@ -678,7 +692,8 @@ fn find_display_close(buf: &str, content_start: usize, final_flush: bool) -> Dis
}
match bytes[k] {
b'\\' => match bytes.get(k + 1) {
None => break, // trailing `\`: need the next byte to classify
// trailing `\`: need the next byte to classify
None => break,
Some(b']') => {
return DisplayClose::Found {
close: k,
@@ -709,9 +724,11 @@ fn find_display_close(buf: &str, content_start: usize, final_flush: bool) -> Dis
if could_extend && !final_flush {
return DisplayClose::NeedMore;
}
k += 2; // `\e…` of something else: span content
// `\e…` of something else: span content
k += 2;
}
Some(_) => k += 2, // `\\` pair or `\x` escape: span content
// `\\` pair or `\x` escape: span content
Some(_) => k += 2,
},
b'$' => {
let run = count_run(bytes, k, b'$');
@@ -722,7 +739,8 @@ fn find_display_close(buf: &str, content_start: usize, final_flush: bool) -> Dis
};
}
if k + run == n && !final_flush {
return DisplayClose::NeedMore; // lone `$` at EOB may extend
// lone `$` at EOB may extend
return DisplayClose::NeedMore;
}
k += run;
}
@@ -734,7 +752,8 @@ fn find_display_close(buf: &str, content_start: usize, final_flush: bool) -> Dis
j += 1;
}
if j == n {
break; // need the next line's first byte to decide
// need the next line's first byte to decide
break;
}
if matches!(bytes[j], b'\n' | b'>') {
return DisplayClose::Unmatched;
@@ -757,7 +776,7 @@ fn find_display_close(buf: &str, content_start: usize, final_flush: bool) -> Dis
/// through byte-for-byte, keeping the pass idempotent). Multi-line interiors
/// have each line trimmed and joined with a single space so CommonMark block
/// parsing (setext underlines, list items, headings) cannot split the span;
/// TeX treats the newlines as spaces, so rendering is unchanged.
/// TeX treats the newlines as spaces, so rendering is `unchanged`.
fn emit_display_span(out: &mut String, interior: &str) {
out.push_str("$$");
push_joined_lines(out, interior);
@@ -825,7 +844,7 @@ mod tests {
normalize_latex_delimiters(s)
}
// ── Basic conversions ────────────────────────────────────────────────
// Basic conversions
#[test]
fn inline_paren_converts() {
@@ -833,7 +852,7 @@ mod tests {
assert_eq!(norm("a \\(x\\) b"), "a $x$ b");
}
// ── Inline `\( … \)` boundary-whitespace trimming (the regression) ────
// Inline `\( … \)` boundary-whitespace trimming (the regression)
#[test]
fn normalize_inline_paren_trims_boundary_ws() {
@@ -858,7 +877,7 @@ mod tests {
fn normalize_inline_paren_trim_leaves_escapes_and_dollars_alone() {
// Escaped `\\(`/`\\)` is a literal backslash + paren, not a math span.
assert_eq!(norm("\\\\( x \\\\)"), "\\\\( x \\\\)");
// Only the backslash forms are ours: a space-padded bare `$ x $` is NOT
// Only the backslash forms are ours: a space-`padded` bare `$ x $` is NOT
// trimmed (currency untouched-ness is covered by `currency_not_misconverted`).
assert_eq!(norm("$ x $"), "$ x $");
}
@@ -880,7 +899,7 @@ mod tests {
assert_eq!(norm("a\n\\[x\\]\nb"), "a\n$$x$$\nb");
}
// ── Multi-line display spans join onto one line ──────────────────────
// Multi-line display spans join onto one line
#[test]
fn multiline_display_with_setext_hazard_joins() {
@@ -971,7 +990,7 @@ mod tests {
assert_eq!(norm("$$\nprice \\$5\n=\nz\n$$"), "$$price \\$5 = z$$");
}
// ── Inline `\(…\)` spans join interior newlines ──────────────────────
// Inline `\(…\)` spans join interior newlines
#[test]
fn multiline_inline_paren_joins() {
@@ -1016,7 +1035,7 @@ mod tests {
}
}
// ── Escapes & currency ───────────────────────────────────────────────
// Escapes & currency
#[test]
fn escaped_backslash_paren_stays_literal() {
@@ -1037,7 +1056,7 @@ mod tests {
assert_eq!(norm("\\(a\\) costs $5"), "$a$ costs $5");
}
// ── Code is left verbatim ────────────────────────────────────────────
// Code is left verbatim
#[test]
fn inline_code_latex_untouched() {
@@ -1072,7 +1091,7 @@ mod tests {
);
}
// ── Math inside tables (the bug) ─────────────────────────────────────
// Math inside tables (the bug)
#[test]
fn table_cell_backslash_math_converts() {
@@ -1081,7 +1100,7 @@ mod tests {
assert_eq!(norm(input), expected);
}
// ── Streaming equivalence (the key invariant) ────────────────────────
// Streaming equivalence (the key invariant)
const RICH_DOC: &str = concat!(
"Inline \\(a+b\\), dollar $c+d$, display \\[e=mc^2\\].\n\n",
@@ -1146,13 +1165,13 @@ mod tests {
" ",
"\\\\(escaped\\\\)",
"`unterminated \\(x\\)\nafter \\(y\\)",
// Padded inline spans exercise the look-ahead + trim hold-back.
// `Padded` inline spans exercise the look-ahead + trim hold-back.
"\\( x \\)",
"a \\( x+y \\) b",
"\\( \\alpha + \\beta \\)",
"\\( \\{x\\} \\)",
"\\( \\) empty",
// Unclosed padded open: held back until finish() flushes a lone `$`.
// Unclosed `padded` open: held back until finish() flushes a lone `$`.
"unclosed padded \\( x + y",
// Display spans exercise the close-scan hold-back and its aborts.
"$$\nx\n=\ny\n$$",
@@ -1172,7 +1191,7 @@ mod tests {
}
}
// ── finish() flushes held-back partials literally ────────────────────
// finish() flushes held-back partials literally
#[test]
fn finish_flushes_partial_backslash() {
+1 -1
View File
@@ -129,7 +129,7 @@ pub fn render_markdown_ratatui_with_buffers_width(
/// still-open fenced code block: only the streaming tail re-render passes
/// `Some(cache)`; `finish()` and non-streaming callers pass `None`. Everything
/// other than that one open block (closed code blocks, HTML, math, tables,
/// inline) always goes through the unchanged batch highlighter, so output is
/// inline) always goes through the `unchanged` batch highlighter, so output is
/// byte-for-byte identical to the cache-less path. See [`open_code_highlighter`].
#[allow(clippy::too_many_arguments)]
pub(crate) fn render_markdown_ratatui_with_link_id(
@@ -366,7 +366,7 @@ mod tests {
);
}
// ── highlight_closed (closed-fence memo) ─────────────────────────
// highlight_closed (closed-fence memo)
#[test]
fn closed_memo_matches_batch_and_is_idempotent() {
+12 -7
View File
@@ -262,8 +262,10 @@ pub(crate) fn cell_word_separator<'a>(
{
let mut in_whitespace = false;
let mut after_break_char = false;
let mut prev_is_digit = false; // was the *previous* char a digit?
let mut digit_before_break = false; // was the char before the break char a digit?
// was the *previous* char a digit?
let mut prev_is_digit = false;
// was the char before the break char a digit?
let mut digit_before_break = false;
let mut last_break_ch: char = '\0';
let mut break_char_start: usize = 0;
for (idx, ch) in line.char_indices() {
@@ -740,7 +742,7 @@ impl<'a, 'b, 'syn, 'oc> MarkdownParser<'a, 'b, 'syn, 'oc> {
Event::Html(_) => {
// Render HTML block content as regular text (not code).
// pulldown-cmark treats XML-like tags (e.g. <example>) as HTML
// blocks, which previously got code-block styling via Replace.
// blocks, which would otherwise get code-block styling via Replace.
self.push_highlight(Some(self.ms.text), &range);
}
Event::InlineHtml(html) => {
@@ -1115,7 +1117,7 @@ impl<'a, 'b, 'syn, 'oc> MarkdownParser<'a, 'b, 'syn, 'oc> {
}
}
// We intentionally use allow_outside=true here (instead of the previous
// Use allow_outside=true here (instead of the alternate
// pointer-based allow_outside=false) and then do an rfind on the prefix
// before the (last) dest_url occurrence. This is required because dest_url
// may be a CowStr::Owned (after percent-decoding or HTML entity expansion)
@@ -1224,7 +1226,8 @@ impl<'a, 'b, 'syn, 'oc> MarkdownParser<'a, 'b, 'syn, 'oc> {
}
None
}
TagEnd::Strikethrough => None, // No highlight pushed
// No highlight pushed
TagEnd::Strikethrough => None,
TagEnd::CodeBlock => {
// pulldown synthesizes a block end at end-of-input even for an
// unterminated fence, so the end event alone does not prove
@@ -1591,7 +1594,8 @@ impl<'a, 'b, 'syn, 'oc> MarkdownParser<'a, 'b, 'syn, 'oc> {
// = 1 + sum(col_width) + num_cols * 2 * padding + (num_cols - 1) + 1
// = num_cols * (2 * padding + 1) + sum(col_width) + 2 - 1
if let Some(max_width) = self.max_table_width {
let overhead = num_cols * (2 * padding + 1) + 1; // borders + padding
// borders + padding
let overhead = num_cols * (2 * padding + 1) + 1;
let content_budget = max_width.saturating_sub(overhead);
let total_content: usize = col_widths.iter().sum();
@@ -1740,7 +1744,8 @@ impl<'a, 'b, 'syn, 'oc> MarkdownParser<'a, 'b, 'syn, 'oc> {
// Body rows
for (i, row) in state.rows.iter().enumerate() {
let row_offset = separator_offset + 1 + i; // offset 2, 3, ...
// offset 2, 3, ...
let row_offset = separator_offset + 1 + i;
let (row_plains, row_styleds, row_links) = self.format_styled_content_lines(
row,
+9 -5
View File
@@ -1546,7 +1546,8 @@ mod tests {
true,
&mut buffers,
None,
Some(30), // narrow enough to force wrapping in column B
// narrow enough to force wrapping in column B
Some(30),
);
// Find the lines that contain "abc" — they should have a styled span
@@ -1587,7 +1588,8 @@ mod tests {
true,
&mut buffers,
None,
Some(20), // narrow enough to force wrapping around the em-dash
// narrow enough to force wrapping around the em-dash
Some(20),
);
let text = lines_to_text(&output.lines);
let all_text: String = text.join("");
@@ -1628,7 +1630,8 @@ mod tests {
let md = "| A | B |\n|---|---|\n| x | y |\n| w | z |\n\n";
let table_start_line = 0usize;
let table_source_lines = 4usize; // header + separator + 2 rows
// header + separator + 2 rows
let table_source_lines = 4usize;
let (output, _) = render_markdown_ratatui_full(md, test_style::STYLE, true, None);
@@ -1694,7 +1697,8 @@ mod tests {
Some(30),
);
let table_source_lines = 3; // header + separator + 1 row
// header + separator + 1 row
let table_source_lines = 3;
for (i, &src_line) in output.line_source_map.iter().enumerate() {
assert!(
src_line < table_source_lines,
@@ -2428,7 +2432,7 @@ mod math_tests {
#[test]
fn paren_inline_math_in_table_cell_renders_unicode() {
// `\(…\)` inside a table cell must convert. Previously the
// `\(…\)` inside a table cell must convert. Historically the
// backslash-form scanner was disabled inside tables, leaving raw TeX.
// Normalization rewrites `\(…\)` → `$…$` before parsing, so the existing
// in-cell `$` path converts it.
+2 -33
View File
@@ -105,37 +105,6 @@ impl SourceMap {
}
}
// ## Restoring Byte-Level Source Maps (if ever needed)
// Ratatui path tracks line-level mapping only (`line_source_map`) for
// copy/selection. Byte-level `SourceMap` is unused here (~6% faster).
//
// The ratatui rendering path currently only tracks line-level source mapping
// (`line_source_map`), which is sufficient for copy/selection operations.
// Byte-level `SourceMap` was removed for simplicity and ~6% speedup.
//
// To restore byte-level source maps:
//
// 1. Add field to MarkdownRenderOutput and MarkdownRenderView:
// ```
// pub source_map: SourceMap,
// ```
//
// 2. In render_ratatui(), add tracking variables:
// ```
// let mut source_map = SourceMap::new();
// let mut rendered_offset: usize = 0;
// ```
//
// 3. For each text segment emitted, record the mapping:
// ```
// source_map.add(rendered_offset, source_start..source_end);
// rendered_offset += emitted_text.len();
// ```
//
// 4. In streaming.rs, update FrozenState to track:
// ```
// source_map_len: usize,
// rendered_bytes: usize,
// ```
//
// 5. Use SourceMap::extend_with_offsets() to merge tail source maps.
//
// See git history for the removed implementation.
@@ -1899,7 +1899,8 @@ The frozen lines are **never re-rendered**, making streaming O(N) instead of O(N
let first_splits: Vec<usize> = if first_half.len() > 1 {
vec![first_half.len() / 2]
} else {
vec![first_half.len()] // No split, use whole thing
// No split, use whole thing
vec![first_half.len()]
};
// Split second half (if possible)
@@ -1927,7 +1928,8 @@ The frozen lines are **never re-rendered**, making streaming O(N) instead of O(N
.collect();
if chunks.len() < 2 {
continue; // Need at least 2 chunks
// Need at least 2 chunks
continue;
}
tested += 1;
@@ -2482,9 +2484,7 @@ The frozen lines are **never re-rendered**, making streaming O(N) instead of O(N
assert_streaming_matches_full_both(text);
}
// ----------------------------------------------------------------------
// Syntect-enabled streaming equivalence (incremental open-code highlighter)
// ----------------------------------------------------------------------
/// Build a nested YAML body of at least `num_lines` lines (no fences).
fn yaml_body(num_lines: usize) -> String {
@@ -2866,7 +2866,8 @@ The frozen lines are **never re-rendered**, making streaming O(N) instead of O(N
#[test]
fn clone_preserves_held_back_pending() {
let mut r = StreamingMarkdownRenderer::new(test_style::STYLE, true);
r.push_and_render("ab \\(\\alpha\\) cd \\", None); // trailing `\` held back
// trailing `\` held back
r.push_and_render("ab \\(\\alpha\\) cd \\", None);
let mut cloned = r.clone();
r.push_and_render("(\\beta\\) ef\n\n", None);
cloned.push_and_render("(\\beta\\) ef\n\n", None);
+14 -17
View File
@@ -1,13 +1,9 @@
//! Markdown styling types.
//!
//! This module provides the `MarkdownStyle` struct which defines colors and
//! effects for all markdown elements.
use anstyle::{Effects, Style};
use crate::colors::adapt_style;
/// Table border characters for rendering tables in pretty mode.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub struct TableBorders {
chars: [char; 11],
@@ -42,7 +38,8 @@ impl TableBorders {
Self { chars }
}
// Short names (used in table formatting)
// Every character is reachable under two names: a terse one for dense
// table-formatting expressions and a spelled-out one for everything else.
pub const fn h(&self) -> char {
self.chars[Self::H]
}
@@ -77,7 +74,6 @@ impl TableBorders {
self.chars[Self::X]
}
// Long names (for readability)
pub const fn horizontal(&self) -> char {
self.chars[Self::H]
}
@@ -121,9 +117,8 @@ impl Default for TableBorders {
/// Style configuration for markdown rendering.
///
/// Each field controls the styling for a specific markdown element.
/// The `_inner` variants are applied to the content, while `_outer` variants
/// are applied to the syntax markers (which are hidden in pretty mode).
/// `_inner` variants style an element's content; `_outer` variants style its
/// syntax markers, which pretty mode hides.
#[derive(Copy, Clone, Default, Debug, PartialEq, Eq, Hash)]
pub struct MarkdownStyle {
pub heading_inner: [Style; 6],
@@ -161,9 +156,8 @@ pub struct MarkdownStyle {
}
impl MarkdownStyle {
/// Adapt all styles for the terminal's color capabilities.
///
/// This downgrades RGB colors to 256-color or 16-color as needed.
/// Downgrade every style's RGB colors to 256-color or 16-color to match
/// the terminal's capabilities.
pub fn adapt(self) -> Self {
Self {
heading_inner: [
@@ -210,8 +204,8 @@ impl MarkdownStyle {
}
}
/// Check if ALL active styles have HIDDEN effect.
/// Used in pretty mode to determine if text should be skipped.
/// Pretty mode skips a span when every style covering it is HIDDEN. An empty
/// style set is not hidden.
pub(crate) fn all_hidden(styles: impl IntoIterator<Item = Option<Style>>) -> bool {
let mut has_any = false;
let mut all_are_hidden = true;
@@ -230,11 +224,15 @@ pub(crate) fn all_hidden(styles: impl IntoIterator<Item = Option<Style>>) -> boo
}
/// Merge multiple styles into one for rendering.
/// Strips HIDDEN from final output - it's a semantic marker, not a visual style.
///
/// HIDDEN is a semantic marker rather than a visual effect, so it never
/// reaches the output: a style that raised it contributes nothing at all, and
/// a trailing HIDDEN is dropped from the result.
pub(crate) fn merge_styles(styles: impl IntoIterator<Item = Option<Style>>) -> Style {
let mut out = Style::new();
let mut prev = Style::new();
for style in styles {
// Rewind past a style that raised HIDDEN before folding in the next one.
if out.get_effects().contains(Effects::HIDDEN) {
out = prev;
} else {
@@ -265,13 +263,12 @@ pub(crate) fn merge_styles(styles: impl IntoIterator<Item = Option<Style>>) -> S
out.effects(out.get_effects().remove(Effects::HIDDEN))
}
// Simple default style for testing (no colors, just effects)
#[cfg(any(test, fuzzing))]
pub mod test_style {
use super::MarkdownStyle;
use anstyle::Style;
/// A minimal style for testing with no colors.
/// Effects only, no colors, so assertions compare stable escape sequences.
pub const STYLE: MarkdownStyle = MarkdownStyle {
heading_inner: [Style::new().bold(); 6],
heading_outer: [Style::new().dimmed().hidden(); 6],
+8 -20
View File
@@ -1,7 +1,4 @@
//! Syntax highlighting support using syntect.
//!
//! This module provides the `Syntect` struct which holds the syntax definitions
//! and theme for code block highlighting.
use std::io::Cursor;
use std::path::Path;
@@ -14,20 +11,16 @@ use syntect::{
/// Syntax highlighting configuration.
///
/// Holds the theme and syntax definitions for code highlighting.
/// Create one instance and pass it to the markdown renderer.
/// Loading the syntax set is expensive; create one instance and pass it to
/// the markdown renderer.
pub struct Syntect {
/// The color theme for syntax highlighting.
pub theme: SyntectTheme,
/// The syntax definitions (supports 250+ languages via two-face).
pub syntax_set: SyntaxSet,
}
impl Syntect {
/// Create a new Syntect instance from theme bytes.
///
/// The theme bytes should be a TextMate `.tmTheme` file.
/// Uses two-face's extended syntax set with 250+ languages.
/// `theme_bytes` must be a TextMate `.tmTheme` file. The syntax set is
/// two-face's extended one, covering the 250+ languages bat ships.
///
/// # Example
///
@@ -37,12 +30,11 @@ impl Syntect {
pub fn new(theme_bytes: &[u8]) -> Self {
let mut cursor = Cursor::new(theme_bytes);
let theme = ThemeSet::load_from_reader(&mut cursor).expect("Failed to load theme");
// Use two-face's extended syntax set which includes 250+ languages from bat
let syntax_set = two_face::syntax::extra_newlines();
Self { theme, syntax_set }
}
/// Find a syntax definition by file path extension.
/// Resolves on the extension alone; the rest of the file name is ignored.
pub fn find_syntax_by_file_path(&self, file_path: &Path) -> Option<&SyntaxReference> {
let ext = file_path.extension()?.to_str()?;
self.syntax_set.find_syntax_by_extension(ext)
@@ -53,7 +45,6 @@ impl Syntect {
self.syntax_set.find_syntax_by_token(token)
}
/// Create a highlighter for the given file path.
pub fn highlight_lines_by_file_path(&self, file_path: &Path) -> Option<HighlightLines<'_>> {
Some(HighlightLines::new(
self.find_syntax_by_file_path(file_path)?,
@@ -61,7 +52,6 @@ impl Syntect {
))
}
/// Create a highlighter for the given language token.
pub fn highlight_lines_for_token(&self, token: &str) -> Option<HighlightLines<'_>> {
Some(HighlightLines::new(
self.find_syntax_by_token(token)?,
@@ -78,7 +68,7 @@ impl Syntect {
/// If the string matches the citation form but no syntax is found for the
/// path, this falls back to [`Syntect::find_syntax_by_token`] with the full
/// `fence_info` string, so plain ` ```lang` blocks keep working and odd
/// citations degrade like the pre-citation code path.
/// citations degrade to a plain token lookup.
pub fn highlight_lines_for_fence_info(&self, fence_info: &str) -> Option<HighlightLines<'_>> {
Some(HighlightLines::new(
self.find_syntax_for_fence_info(fence_info)?,
@@ -159,10 +149,8 @@ pub(crate) fn syntax_highlight_raw(
Some(lines)
}
/// Get a shared Syntect instance for tests.
///
/// This loads the tokyo-night theme bundled with the crate.
/// Uses a static OnceLock for efficiency in test runs.
/// Shared Syntect instance for tests, using the crate's bundled tokyo-night
/// theme.
#[cfg(any(test, fuzzing))]
#[allow(dead_code)]
pub fn test_syntect() -> &'static Syntect {