From c35fdac2e35c300a5e25b36aa6163f1bc9dbaad4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Wed, 14 Oct 2026 11:34:50 -0400 Subject: [PATCH] documents: read presses as the parser decoded them Round 3 of R4.5 found five ways the reverse reading drifted from markdown-rs: kept indents and marks were eaten as joins, autolink text was decoded, table code kept its escaped pipes, decoded white space could not join or drop, and C0/C1 numeric references kept their controls. Matching now tries an equal character first, a drawn space against any line end next, and drops white space decoded or not; autolinks read as written, table cell code reads \| as |, and numeric references follow markdown-rs. The reviewer's probes are regressions. Co-Authored-By: Claude Opus 5.5 (1M context) --- AGENTS.md | 2 +- src/documents/document/tests/mod.rs | 1 + src/documents/document/tests/point.rs | 17 ++-- src/documents/document/tests/presses.rs | 122 ++++++++++++++++++++++++ src/documents/document/tests/reading.rs | 58 +++++++++++ src/documents/tree/mod.rs | 12 ++- src/documents/tree/sources.rs | 43 ++++----- 7 files changed, 223 insertions(+), 32 deletions(-) create mode 100644 src/documents/document/tests/presses.rs diff --git a/AGENTS.md b/AGENTS.md index 564de49..79379ed 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,7 @@ Ely GPUI Component. A component library for GPUI, in light and dark. - Web views: `wry` 0.57 (Apache-2.0 OR MIT, default features off, macOS only) lays a child WKWebView over the window. - Terminal: `alacritty_terminal` 0.26 (Apache-2.0, default features off, native only) for the grid, its parser and the pseudo-terminal; `futures` carries its events. `vte` 0.15 (Apache-2.0 OR MIT), the parser it re-exports, reads `AnsiText`'s codes alone, so colored output builds for wasm32. - Diffs: `similar` 3.2 (Apache-2.0) for line and word diffs and three-way merges; its `unicode` feature splits words at punctuation. -- Markdown: `markdown` 1.0.0 (markdown-rs, MIT) reads GitHub's flavor and math into a syntax tree with byte positions for `MarkdownRenderer`, `MarkdownEditor`, the highlights, block ranges and outline. `MarkdownRenderer` draws display math through `typography::Latex`, a ```` ```mermaid ```` block through `documents::Diagram`, and pictures from the owner's folder (`base`) or `data:` addresses (`base64` 0.22, MIT OR Apache-2.0); web pictures stay unloaded. Inline math shows as TeX: gpui text carries no inline pictures. Inline it reads as Typora's defaults: `:shortcode:` turns to its emoji as GitHub reads it (the `emojis` crate's names, GitHub's today, so `:cricket:` is the insect; read after escapes and references, so `\:smile:` turns too; inside an autolink or `` it stays text; within one DOM text node, which filtered tags do not split, the first `:name:`-shaped run decides for all: blank text or a space before it turns every shortcode, else none, so `x:smile: :smile:` stays), inline HTML's `u`, `b`/`strong`, `i`/`em`, `s`/`del`/`strike`, `code`/`kbd`, `mark`, `br` and `a href` draw as what they mean GFM's filtered tags (`script`, `style`, `title`, `textarea`, `iframe`, `xmp`, `noembed`, `noframes`, `plaintext`) show as text, and other tags, hyphenated names too, vanish, leaving their text; a tag's name runs to white space or `/`, as HTML5 reads it; attribute values split on ASCII whitespace alone and decode as HTML decodes them; each Markdown container is the element GitHub writes (`em`, `strong`, `del`, `a`), on one stack with the tags and repaired as HTML5 repairs: an end tag closes the innermost element of its name and all above it, of which only formatting elements (`a`, `b`, `code`, `em`, `i`, `s`, `strike`, `strong`, `u`) reopen, a new `a` ends the open one and an `a` without `href` links nothing; other tags are not tracked, and `
` breaks as `
` does; and a footnote reference is its number in superscript, numbered by first reference, as its definition is; one `~` still strikes. At the root, blocks read as GitHub shows them: a quote whose first line is plain text reading `[!NOTE]`, `[!TIP]`, `[!IMPORTANT]`, `[!WARNING]` or `[!CAUTION]` in any case, after escapes and references, spaces and tabs around it, with more after it, is an alert under its title (Important in the palette's violet, `syntax.keyword`, read at 4.5:1); a paragraph that is `[toc]` alone, as Typora reads it, lists every heading by depth, and a press on one reveals it as a link press does (`MarkdownDocument`'s `reveal`), the heading found where `CodeEditor` anchors carried it through each edit since the read, gone if an edit or an undo took it, and a secondary press acts on press, since a block writing back moves the list; front matter shows in a box without its fences; an HTML block shows its text with the inline tags above, white space collapsed and owed as a browser lays it, block elements breaking lines and closing what opened inside them, HTML5's start tags ending an open `p`, `fieldset` and `menu` too, and an `li` start an `li`, a `dd` or `dt` start either, as HTML5 ends them, comments dropped, the filtered tags shown as text, and an unclosed tag dropping the rest; a rule is a 2px `border_strong` line; bullets go •, ◦, ▪ by list depth, ordered lists counted. Exports read a tree as the drawing does through `MarkdownReader` (`block` gives its `MarkdownBlock`s, `inline` its text, `note` and `label` its footnotes, `headings` the `[toc]` list), one fresh stack per paragraph, heading or cell. `MarkdownDocument` holds a whole file in one `CodeEditor` (text and undo, colored by `CodeEditor::highlighter` with `markdown_highlights`): visual draws the file's top-level blocks in a gpui `list`, a pressed block opening as its source in a `TextInput` whose typing is the document's text at once and whose write-back is one undo step; source is the editor; preview draws without presses. A written-back block, a mode switch or a new text is read at once up to 128 KiB, longer ones on the background executor; typing in source or split reads 300 ms after the last key, off the main thread (markdown-rs grows with the square of a paragraph's lines: 40,000 lines take 0.9 s). Texts past `LARGEST_READ` (2 MiB) are never read: no blocks, no outline. A new read keeps the list's blocks alike at both ends. One caret serves every mode: the editor's selection, which a written-back block moves; `set_caret` and `set_top_line` set caret and view apart; a top line asked (numbered, so a late frame callback finishes only its own ask; new text places it again, clamped to the text as it then is) waits for the read, lands on its block, then a frame later at its share of the block's height; `reveal` is such an ask with the caret at its line; a switch between source and the list keeps the top line, one between visual and preview keeps the list's scroll as it is. A block opens only when the read is of the text as it is now. A pressed block keeps its identity across the write-back of the one before it, and opens with the caret on the pressed character: each drawn piece keeps its written span and how it decodes (Markdown's escapes and references for text, HTML's references for an HTML block's runs, none for code spans, cut as CommonMark cuts their end spaces), and is matched from its end, so quote marks and indents at a line's start are never taken for text and an alert's text reads past its marker; text before an emoji lands at its piece's start; code, front matter, a link definition and an HTML block's runs read back line for line, LF, CRLF and CR alike, each drawn line ending its written one, a tab cut to spaces landing on the tab; a block's drawn positions count from its own start, so an edit before it keeps it alike. A secondary press opens nothing, an empty document's open on focus included: it writes back an open block, refuses the press if that changed the pressed block's kind, text or drawing (a definition's new address too), moves the caret there and emits `DocumentEvent::Pointed`, whose `DocumentTarget` names the block's kind, a link's or picture's address, and whether it lies in a table or code. Blocks are Tab stops that Enter or Space opens; preview takes no presses, undo or redo. `CodeEditor` keeps at most 64 MiB of undo and redo text, dropping the oldest undo and then the farthest redo after every record, composition, undo and redo. `BlockData::from_markdown` and its export still read with `pulldown-cmark` 0.13 (MIT, default features off); it adds GitHub's alerts, a callout's form, and reads each field's inline Markdown from the source by offset, as written. +- Markdown: `markdown` 1.0.0 (markdown-rs, MIT) reads GitHub's flavor and math into a syntax tree with byte positions for `MarkdownRenderer`, `MarkdownEditor`, the highlights, block ranges and outline. `MarkdownRenderer` draws display math through `typography::Latex`, a ```` ```mermaid ```` block through `documents::Diagram`, and pictures from the owner's folder (`base`) or `data:` addresses (`base64` 0.22, MIT OR Apache-2.0); web pictures stay unloaded. Inline math shows as TeX: gpui text carries no inline pictures. Inline it reads as Typora's defaults: `:shortcode:` turns to its emoji as GitHub reads it (the `emojis` crate's names, GitHub's today, so `:cricket:` is the insect; read after escapes and references, so `\:smile:` turns too; inside an autolink or `` it stays text; within one DOM text node, which filtered tags do not split, the first `:name:`-shaped run decides for all: blank text or a space before it turns every shortcode, else none, so `x:smile: :smile:` stays), inline HTML's `u`, `b`/`strong`, `i`/`em`, `s`/`del`/`strike`, `code`/`kbd`, `mark`, `br` and `a href` draw as what they mean GFM's filtered tags (`script`, `style`, `title`, `textarea`, `iframe`, `xmp`, `noembed`, `noframes`, `plaintext`) show as text, and other tags, hyphenated names too, vanish, leaving their text; a tag's name runs to white space or `/`, as HTML5 reads it; attribute values split on ASCII whitespace alone and decode as HTML decodes them; each Markdown container is the element GitHub writes (`em`, `strong`, `del`, `a`), on one stack with the tags and repaired as HTML5 repairs: an end tag closes the innermost element of its name and all above it, of which only formatting elements (`a`, `b`, `code`, `em`, `i`, `s`, `strike`, `strong`, `u`) reopen, a new `a` ends the open one and an `a` without `href` links nothing; other tags are not tracked, and `
` breaks as `
` does; and a footnote reference is its number in superscript, numbered by first reference, as its definition is; one `~` still strikes. At the root, blocks read as GitHub shows them: a quote whose first line is plain text reading `[!NOTE]`, `[!TIP]`, `[!IMPORTANT]`, `[!WARNING]` or `[!CAUTION]` in any case, after escapes and references, spaces and tabs around it, with more after it, is an alert under its title (Important in the palette's violet, `syntax.keyword`, read at 4.5:1); a paragraph that is `[toc]` alone, as Typora reads it, lists every heading by depth, and a press on one reveals it as a link press does (`MarkdownDocument`'s `reveal`), the heading found where `CodeEditor` anchors carried it through each edit since the read, gone if an edit or an undo took it, and a secondary press acts on press, since a block writing back moves the list; front matter shows in a box without its fences; an HTML block shows its text with the inline tags above, white space collapsed and owed as a browser lays it, block elements breaking lines and closing what opened inside them, HTML5's start tags ending an open `p`, `fieldset` and `menu` too, and an `li` start an `li`, a `dd` or `dt` start either, as HTML5 ends them, comments dropped, the filtered tags shown as text, and an unclosed tag dropping the rest; a rule is a 2px `border_strong` line; bullets go •, ◦, ▪ by list depth, ordered lists counted. Exports read a tree as the drawing does through `MarkdownReader` (`block` gives its `MarkdownBlock`s, `inline` its text, `note` and `label` its footnotes, `headings` the `[toc]` list), one fresh stack per paragraph, heading or cell. `MarkdownDocument` holds a whole file in one `CodeEditor` (text and undo, colored by `CodeEditor::highlighter` with `markdown_highlights`): visual draws the file's top-level blocks in a gpui `list`, a pressed block opening as its source in a `TextInput` whose typing is the document's text at once and whose write-back is one undo step; source is the editor; preview draws without presses. A written-back block, a mode switch or a new text is read at once up to 128 KiB, longer ones on the background executor; typing in source or split reads 300 ms after the last key, off the main thread (markdown-rs grows with the square of a paragraph's lines: 40,000 lines take 0.9 s). Texts past `LARGEST_READ` (2 MiB) are never read: no blocks, no outline. A new read keeps the list's blocks alike at both ends. One caret serves every mode: the editor's selection, which a written-back block moves; `set_caret` and `set_top_line` set caret and view apart; a top line asked (numbered, so a late frame callback finishes only its own ask; new text places it again, clamped to the text as it then is) waits for the read, lands on its block, then a frame later at its share of the block's height; `reveal` is such an ask with the caret at its line; a switch between source and the list keeps the top line, one between visual and preview keeps the list's scroll as it is. A block opens only when the read is of the text as it is now. A pressed block keeps its identity across the write-back of the one before it, and opens with the caret on the pressed character: each drawn piece keeps its written span and how it decodes (Markdown's escapes and references for text, as markdown-rs decodes them, autolink text as written, HTML's references for an HTML block's runs, none for code spans but a table cell's `\|`, cut as CommonMark cuts their end spaces), and is matched from its end, an equal character first, a drawn space next against a line end, decoded or not, so quote marks and indents at a line's start are never taken for text and an alert's text reads past its marker; text before an emoji lands at its piece's start; code, front matter, a link definition and an HTML block's runs read back line for line, LF, CRLF and CR alike, each drawn line ending its written one, a tab cut to spaces landing on the tab; a block's drawn positions count from its own start, so an edit before it keeps it alike. A secondary press opens nothing, an empty document's open on focus included: it writes back an open block, refuses the press if that changed the pressed block's kind, text or drawing (a definition's new address too), moves the caret there and emits `DocumentEvent::Pointed`, whose `DocumentTarget` names the block's kind, a link's or picture's address, and whether it lies in a table or code. Blocks are Tab stops that Enter or Space opens; preview takes no presses, undo or redo. `CodeEditor` keeps at most 64 MiB of undo and redo text, dropping the oldest undo and then the farthest redo after every record, composition, undo and redo. `BlockData::from_markdown` and its export still read with `pulldown-cmark` 0.13 (MIT, default features off); it adds GitHub's alerts, a callout's form, and reads each field's inline Markdown from the source by offset, as written. - Pictures: `image` 0.25 (MIT OR Apache-2.0), the crate gpui decodes with, default features off; it turns decoded frames and rims map tiles. - Math and diagrams: RaTeX 0.1.14 (`ratex-parser`, `ratex-layout`, `ratex-svg` with `embed-fonts`, `ratex-types`; MIT, KaTeX's fonts OFL) typesets `Latex` to SVG with outlined glyphs; `mermaid-rs-renderer` 0.3.1 (MIT, default features off) draws `Diagram`. Both go through gpui's `SvgRenderer` at the window's scale, loaded as gpui assets off the main thread; `usvg` 0.46, gpui's own, reads a diagram's size. A RaTeX or Mermaid panic on parsed input becomes the element's error. Each element holds its picture through `primitives::held`: a new source, or the element leaving the window, removes the old asset from gpui's cache and drops its image from the atlas after the frame. - Maps: Natural Earth's 110m countries (public domain). `assets/maps/countries.geojson`, trimmed by `scripts/countries.py` to an ISO code, a name and positions to 0.01°, backs `maps::WorldMap`; the gallery's tiles are cut from the same file, z0 to z3, by `scripts/tiles.py` with Pillow. diff --git a/src/documents/document/tests/mod.rs b/src/documents/document/tests/mod.rs index 43871d0..a0fc09f 100644 --- a/src/documents/document/tests/mod.rs +++ b/src/documents/document/tests/mod.rs @@ -371,6 +371,7 @@ mod links; mod look; mod place; mod point; +mod presses; mod reading; mod toc; mod tools; diff --git a/src/documents/document/tests/point.rs b/src/documents/document/tests/point.rs index 99f1bd4..dd209c8 100644 --- a/src/documents/document/tests/point.rs +++ b/src/documents/document/tests/point.rs @@ -11,7 +11,7 @@ use crate::{ }; /// Presses block `ix` on its first line's character `k`. -fn press_on(ix: usize, k: usize, button: MouseButton, cx: &mut VisualTestContext) { +pub(super) fn press_on(ix: usize, k: usize, button: MouseButton, cx: &mut VisualTestContext) { let bounds = cx .debug_bounds(Box::leak(format!("markdown-block-{ix}").into_boxed_str())) .expect("the block is drawn"); @@ -29,7 +29,7 @@ fn press_on(ix: usize, k: usize, button: MouseButton, cx: &mut VisualTestContext cx.run_until_parked(); } -fn heard( +pub(super) fn heard( document: &Entity, cx: &mut VisualTestContext, ) -> Rc>> { @@ -50,7 +50,10 @@ fn caret(document: &Entity, cx: &mut VisualTestContext) -> usi document.read_with(cx, |document, cx| document.editor.read(cx).primary().head) } -fn opened_caret(document: &Entity, cx: &mut VisualTestContext) -> usize { +pub(super) fn opened_caret( + document: &Entity, + cx: &mut VisualTestContext, +) -> usize { document.read_with(cx, |document, cx| { document .open @@ -62,7 +65,7 @@ fn opened_caret(document: &Entity, cx: &mut VisualTestContext) }) } -fn target(at: usize, block: TargetBlock) -> DocumentTarget { +pub(super) fn target(at: usize, block: TargetBlock) -> DocumentTarget { DocumentTarget { at, block: Some(block), @@ -165,21 +168,21 @@ fn a_secondary_press_past_the_end_goes_to_the_end(cx: &mut TestAppContext) { } /// Base and small text sizes, in pixels. -fn sizes(cx: &mut VisualTestContext) -> (Pixels, Pixels) { +pub(super) fn sizes(cx: &mut VisualTestContext) -> (Pixels, Pixels) { cx.update(|window, cx| { let size = |size| cx.theme().text_size(size).to_pixels(window.rem_size()); (size(TextSize::Base), size(TextSize::Sm)) }) } -fn press_at(x: Pixels, y: Pixels, button: MouseButton, cx: &mut VisualTestContext) { +pub(super) fn press_at(x: Pixels, y: Pixels, button: MouseButton, cx: &mut VisualTestContext) { cx.simulate_mouse_down(point(x, y), button, Modifiers::none()); cx.simulate_mouse_up(point(x, y), button, Modifiers::none()); cx.run_until_parked(); } /// Presses block 0's code glyph `k` on `line`. -fn press_lines(k: usize, line: usize, button: MouseButton, cx: &mut VisualTestContext) { +pub(super) fn press_lines(k: usize, line: usize, button: MouseButton, cx: &mut VisualTestContext) { let bounds = cx.debug_bounds("markdown-block-0").expect("drawn"); let (body, small) = sizes(cx); let x = bounds.left() + px(25.0) + small * 0.6 * (k as f32 + 0.25); diff --git a/src/documents/document/tests/presses.rs b/src/documents/document/tests/presses.rs new file mode 100644 index 0000000..a3a383e --- /dev/null +++ b/src/documents/document/tests/presses.rs @@ -0,0 +1,122 @@ +//! Presses on text the parser rewrote (R4.5). + +use gpui::{MouseButton, TestAppContext, px}; + +use super::{ + open, + point::{heard, opened_caret, press_at, press_lines, press_on, sizes, target}, +}; +use crate::documents::{MarkdownMode, TargetBlock}; + +#[gpui::test] +fn inline_code_kept_indent_press(cx: &mut TestAppContext) { + let source = "a `fox\n y` z"; + let (document, cx) = open(source, MarkdownMode::Visual, cx); + let events = heard(&document, cx); + press_on(0, 3, MouseButton::Right, cx); + assert_eq!(*events.borrow(), [target(4, TargetBlock::Paragraph)]); + press_on(0, 3, MouseButton::Left, cx); + assert_eq!(opened_caret(&document, cx), 4); +} + +#[gpui::test] +fn quoted_html_kept_mark_press(cx: &mut TestAppContext) { + let source = ">
\n> > alpha beta
"; + let (document, cx) = open(source, MarkdownMode::Visual, cx); + let events = heard(&document, cx); + let bounds = cx.debug_bounds("markdown-block-0").expect("drawn"); + let (body, _) = sizes(cx); + let x = bounds.left() + px(31.0) + body * 0.6 * 0.25; + press_at(x, bounds.top() + px(10.0), MouseButton::Right, cx); + assert_eq!(*events.borrow(), [target(10, TargetBlock::Quote)]); +} + +#[gpui::test] +fn decoded_line_end_press(cx: &mut TestAppContext) { + let source = "ab y"; + let (document, cx) = open(source, MarkdownMode::Visual, cx); + let events = heard(&document, cx); + press_on(0, 1, MouseButton::Right, cx); + assert_eq!(*events.borrow(), [target(1, TargetBlock::Paragraph)]); +} + +#[gpui::test] +fn numeric_control_press(cx: &mut TestAppContext) { + let source = "ab € y"; + let (document, cx) = open(source, MarkdownMode::Visual, cx); + let events = heard(&document, cx); + press_on(0, 1, MouseButton::Right, cx); + assert_eq!(*events.borrow(), [target(1, TargetBlock::Paragraph)]); +} + +#[gpui::test] +fn html_decoded_space_press(cx: &mut TestAppContext) { + let source = "
ab
"; + let (document, cx) = open(source, MarkdownMode::Visual, cx); + let events = heard(&document, cx); + press_on(0, 1, MouseButton::Right, cx); + assert_eq!(*events.borrow(), [target(6, TargetBlock::Html)]); +} + +#[gpui::test] +fn inline_code_kept_indent_left_press(cx: &mut TestAppContext) { + let source = "a `fox\n y` z"; + let (document, cx) = open(source, MarkdownMode::Visual, cx); + press_on(0, 3, MouseButton::Left, cx); + assert_eq!(opened_caret(&document, cx), 4); +} + +#[gpui::test] +fn autolink_literal_reference_press(cx: &mut TestAppContext) { + let source = ""; + let (document, cx) = open(source, MarkdownMode::Visual, cx); + let events = heard(&document, cx); + press_on(0, 9, MouseButton::Right, cx); + assert_eq!(events.borrow().len(), 1); + assert_eq!(events.borrow()[0].at, source.find('x').unwrap()); +} + +#[gpui::test] +fn table_escaped_pipe_press(cx: &mut TestAppContext) { + let source = "| `ab\\|cd` |\n| --- |\n| cell |\n"; + let (document, cx) = open(source, MarkdownMode::Visual, cx); + let events = heard(&document, cx); + let bounds = cx.debug_bounds("markdown-block-0").expect("drawn"); + let (body, _) = sizes(cx); + let x = bounds.left() + px(26.0) + body * 0.6 * 1.25; + press_at(x, bounds.top() + px(17.0), MouseButton::Right, cx); + assert_eq!(events.borrow().len(), 1); + assert!(events.borrow()[0].in_table); + assert_eq!(events.borrow()[0].at, source.find('b').unwrap()); +} + +#[gpui::test] +fn prior_crlf_code_press(cx: &mut TestAppContext) { + let source = "```\r\nalpha\r\nbeta\r\n```\r\n"; + let (document, cx) = open(source, MarkdownMode::Visual, cx); + let events = heard(&document, cx); + press_lines(2, 0, MouseButton::Right, cx); + assert_eq!(events.borrow()[0].at, source.find("pha").unwrap()); + press_lines(2, 0, MouseButton::Left, cx); + assert_eq!(opened_caret(&document, cx), source.find("pha").unwrap()); +} + +#[gpui::test] +fn prior_crlf_front_matter_press(cx: &mut TestAppContext) { + let source = "---\r\ntitle: value\r\nauthor: person\r\n---\r\n"; + let (document, cx) = open(source, MarkdownMode::Visual, cx); + let events = heard(&document, cx); + press_lines(7, 0, MouseButton::Right, cx); + assert_eq!(events.borrow()[0].at, source.find("value").unwrap()); + press_lines(7, 0, MouseButton::Left, cx); + assert_eq!(opened_caret(&document, cx), source.find("value").unwrap()); +} + +#[gpui::test] +fn prior_crlf_definition_press(cx: &mut TestAppContext) { + let source = "[ref]: https://ely.dev\r\n \"title\"\r\n"; + let (document, cx) = open(source, MarkdownMode::Visual, cx); + let events = heard(&document, cx); + press_on(0, 10, MouseButton::Right, cx); + assert_eq!(*events.borrow(), [target(10, TargetBlock::Definition)]); +} diff --git a/src/documents/document/tests/reading.rs b/src/documents/document/tests/reading.rs index 3914627..8a6baa7 100644 --- a/src/documents/document/tests/reading.rs +++ b/src/documents/document/tests/reading.rs @@ -121,3 +121,61 @@ fn a_decoded_line_end_reads_without_a_written_one() { let alert = parsed("> [!NOTE] alpha beta"); assert_eq!(alert.blocks[0].kind, TargetBlock::Alert); } + +#[test] +fn kept_leading_marks_still_map() { + let source = "a `fox\n y` z"; + assert_eq!(read_back(source, "ox"), source.find("ox")); +} + +#[test] +fn decoded_line_end_maps_nearby_text() { + for (source, shown) in [ + ("ab y", "b"), + ("ab y", "b"), + ("ab y", "b"), + ("
ab y
", "b"), + ("
ab y
", "b"), + ] { + assert_eq!(read_back(source, shown), source.find(shown), "{source:?}"); + } +} + +#[test] +fn numeric_references_match_the_parser() { + for source in ["ab € y", "ab  y", "ab ﷐ y"] { + assert_eq!(read_back(source, "b"), source.find('b'), "{source:?}"); + } +} + +#[test] +fn html_trailing_encoded_space_maps_previous_word() { + for source in [ + "
ab
", + "
ab word
", + "
ab word
", + ] { + assert_eq!(read_back(source, "b"), source.find('b'), "{source:?}"); + } +} + +#[test] +fn code_pipe_in_a_table_maps_previous_text() { + let source = "| `ab\\|cd` |\n| --- |\n| cell |\n"; + let parsed = parsed(source); + let nodes = parsed.blocks[0].nodes.as_ref().unwrap(); + let Node::Table { head, .. } = &nodes[0] else { + panic!("table") + }; + let Node::Paragraph(inline) = &head[0][0] else { + panic!("text") + }; + assert_eq!(inline.text, "ab|cd"); + assert_eq!(inline.source_at(source, 1), source.find('b')); +} + +#[test] +fn autolink_does_not_decode_literal_reference() { + let source = ""; + assert_eq!(read_back(source, "xample"), source.find("xample")); +} diff --git a/src/documents/tree/mod.rs b/src/documents/tree/mod.rs index fcecc35..4286ba6 100644 --- a/src/documents/tree/mod.rs +++ b/src/documents/tree/mod.rs @@ -120,6 +120,8 @@ struct Run { named: HashMap, /// Inside an autolink: its text stays as written. verbatim: bool, + /// In a table cell, whose code reads `\|` as `|`. + table: bool, /// The current DOM text node's shortcodes. shortcodes: emoji::Shortcodes, /// White space owed, and the open elements before it. @@ -306,7 +308,10 @@ impl<'a> Reader<'a> { { return out.push(Node::Math(math.value.clone())); } - let mut run = Run::default(); + let mut run = Run { + table: matches!(node, mdast::Node::TableCell(_)), + ..Run::default() + }; for child in children { self.inline_into(child, &mut run, &mut Images::Split(out)); } @@ -344,6 +349,11 @@ impl<'a> Reader<'a> { run.shortcodes.split(); } if let Some((written, escapes)) = sources::written(node, self.source) { + let escapes = match (escapes, node) { + (Escapes::Markdown, _) if run.verbatim => Escapes::Plain, + (_, mdast::Node::InlineCode(_)) if run.table => Escapes::Pipes, + _ => escapes, + }; let at = run.text.text.len(); run.text.sources.push((at, written, escapes)); } diff --git a/src/documents/tree/sources.rs b/src/documents/tree/sources.rs index e9b4c58..4df17b5 100644 --- a/src/documents/tree/sources.rs +++ b/src/documents/tree/sources.rs @@ -12,6 +12,8 @@ pub enum Escapes { Markdown, /// References, as HTML reads them. Html, + /// Escaped pipes, as a table cell's code reads them. + Pipes, } /// A leaf's written span and how it reads. @@ -208,7 +210,7 @@ struct Read { impl Read { fn ends_line(&self) -> bool { - self.raw && matches!(self.ch, '\n' | '\r') + matches!(self.ch, '\n' | '\r') } } @@ -226,6 +228,9 @@ fn read(written: &str, escapes: Escapes) -> Vec { (Escapes::Html, '&') => { html::reference(rest, false).map(|(value, used)| (value, used, at)) } + (Escapes::Pipes, '\\') => { + (rest[1..].starts_with('|')).then(|| ("|".to_string(), 2, at + 1)) + } _ => None, }; if let Some((value, used, place)) = decoded { @@ -260,7 +265,10 @@ fn reference(text: &str) -> Option<(String, usize)> { let fits = (1..=most).contains(&digits.len()) && digits.chars().all(|ch| ch.is_digit(radix)); let code = u32::from_str_radix(digits, radix).ok().filter(|_| fits)?; - let ch = char::from_u32(code).filter(|ch| *ch != '\0'); + // Controls become U+FFFD, as markdown-rs reads them. + let ch = char::from_u32(code).filter(|ch| { + !matches!(ch, '\0'..='\u{08}' | '\u{0B}' | '\u{0E}'..='\u{1F}' | '\u{7F}'..='\u{9F}') + }); Some(ch.unwrap_or(char::REPLACEMENT_CHARACTER).to_string()) }; let value = match name.strip_prefix('#') { @@ -279,34 +287,23 @@ fn align(drawn: &str, written: &str, escapes: Escapes, at: usize) -> Option 0; - if crlf && source[j - 1].ends_line() && source[j - 1].ch == '\r' { - j -= 1; - } - if d == '\n' && shown.peek().is_some_and(|(_, ch)| *ch == '\r') { - let (cr, _) = shown.next().expect("peeked"); - if cr == at { - return Some(source[j].at); - } - } - break source[j].at; - } if d == s.ch { j -= 1; break s.at; } + // A join: a line's end drawn as white space. + if d.is_whitespace() && s.ends_line() { + j -= 1; + if s.ch == '\n' && j > 0 && source[j - 1].ch == '\r' && source[j - 1].raw == s.raw { + j -= 1; + } + break source[j].at; + } // White space, line ends and marks the text dropped. - if s.raw && (matches!(s.ch, ' ' | '\t') || s.ends_line() || s.lead) { + if s.lead || matches!(s.ch, ' ' | '\t') || s.ends_line() { j -= 1; continue; }