documents: read presses as the parser decoded them

Round 3 of R4.5 found five ways the reverse reading drifted from
markdown-rs: kept indents and marks were eaten as joins, autolink text
was decoded, table code kept its escaped pipes, decoded white space
could not join or drop, and C0/C1 numeric references kept their
controls. Matching now tries an equal character first, a drawn space
against any line end next, and drops white space decoded or not;
autolinks read as written, table cell code reads \| as |, and numeric
references follow markdown-rs. The reviewer's probes are regressions.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

src/documents/document/tests/mod.rs

@@ -371,6 +371,7 @@ mod links;
 mod look;
 mod place;
 mod point;
+mod presses;
 mod reading;
 mod toc;
 mod tools;

src/documents/document/tests/point.rs

@@ -11,7 +11,7 @@ use crate::{
 };
 
 /// Presses block `ix` on its first line's character `k`.
-fn press_on(ix: usize, k: usize, button: MouseButton, cx: &mut VisualTestContext) {
+pub(super) fn press_on(ix: usize, k: usize, button: MouseButton, cx: &mut VisualTestContext) {
     let bounds = cx
         .debug_bounds(Box::leak(format!("markdown-block-{ix}").into_boxed_str()))
         .expect("the block is drawn");
@@ -29,7 +29,7 @@ fn press_on(ix: usize, k: usize, button: MouseButton, cx: &mut VisualTestContext
     cx.run_until_parked();
 }
 
-fn heard(
+pub(super) fn heard(
     document: &Entity<MarkdownDocument>,
     cx: &mut VisualTestContext,
 ) -> Rc<RefCell<Vec<DocumentTarget>>> {
@@ -50,7 +50,10 @@ fn caret(document: &Entity<MarkdownDocument>, cx: &mut VisualTestContext) -> usi
     document.read_with(cx, |document, cx| document.editor.read(cx).primary().head)
 }
 
-fn opened_caret(document: &Entity<MarkdownDocument>, cx: &mut VisualTestContext) -> usize {
+pub(super) fn opened_caret(
+    document: &Entity<MarkdownDocument>,
+    cx: &mut VisualTestContext,
+) -> usize {
     document.read_with(cx, |document, cx| {
         document
             .open
@@ -62,7 +65,7 @@ fn opened_caret(document: &Entity<MarkdownDocument>, cx: &mut VisualTestContext)
     })
 }
 
-fn target(at: usize, block: TargetBlock) -> DocumentTarget {
+pub(super) fn target(at: usize, block: TargetBlock) -> DocumentTarget {
     DocumentTarget {
         at,
         block: Some(block),
@@ -165,21 +168,21 @@ fn a_secondary_press_past_the_end_goes_to_the_end(cx: &mut TestAppContext) {
 }
 
 /// Base and small text sizes, in pixels.
-fn sizes(cx: &mut VisualTestContext) -> (Pixels, Pixels) {
+pub(super) fn sizes(cx: &mut VisualTestContext) -> (Pixels, Pixels) {
     cx.update(|window, cx| {
         let size = |size| cx.theme().text_size(size).to_pixels(window.rem_size());
         (size(TextSize::Base), size(TextSize::Sm))
     })
 }
 
-fn press_at(x: Pixels, y: Pixels, button: MouseButton, cx: &mut VisualTestContext) {
+pub(super) fn press_at(x: Pixels, y: Pixels, button: MouseButton, cx: &mut VisualTestContext) {
     cx.simulate_mouse_down(point(x, y), button, Modifiers::none());
     cx.simulate_mouse_up(point(x, y), button, Modifiers::none());
     cx.run_until_parked();
 }
 
 /// Presses block 0's code glyph `k` on `line`.
-fn press_lines(k: usize, line: usize, button: MouseButton, cx: &mut VisualTestContext) {
+pub(super) fn press_lines(k: usize, line: usize, button: MouseButton, cx: &mut VisualTestContext) {
     let bounds = cx.debug_bounds("markdown-block-0").expect("drawn");
     let (body, small) = sizes(cx);
     let x = bounds.left() + px(25.0) + small * 0.6 * (k as f32 + 0.25);

src/documents/document/tests/presses.rs

@@ -0,0 +1,122 @@
+//! Presses on text the parser rewrote (R4.5).
+
+use gpui::{MouseButton, TestAppContext, px};
+
+use super::{
+    open,
+    point::{heard, opened_caret, press_at, press_lines, press_on, sizes, target},
+};
+use crate::documents::{MarkdownMode, TargetBlock};
+
+#[gpui::test]
+fn inline_code_kept_indent_press(cx: &mut TestAppContext) {
+    let source = "a `fox\n  y` z";
+    let (document, cx) = open(source, MarkdownMode::Visual, cx);
+    let events = heard(&document, cx);
+    press_on(0, 3, MouseButton::Right, cx);
+    assert_eq!(*events.borrow(), [target(4, TargetBlock::Paragraph)]);
+    press_on(0, 3, MouseButton::Left, cx);
+    assert_eq!(opened_caret(&document, cx), 4);
+}
+
+#[gpui::test]
+fn quoted_html_kept_mark_press(cx: &mut TestAppContext) {
+    let source = "> <div>\n> > alpha beta</div>";
+    let (document, cx) = open(source, MarkdownMode::Visual, cx);
+    let events = heard(&document, cx);
+    let bounds = cx.debug_bounds("markdown-block-0").expect("drawn");
+    let (body, _) = sizes(cx);
+    let x = bounds.left() + px(31.0) + body * 0.6 * 0.25;
+    press_at(x, bounds.top() + px(10.0), MouseButton::Right, cx);
+    assert_eq!(*events.borrow(), [target(10, TargetBlock::Quote)]);
+}
+
+#[gpui::test]
+fn decoded_line_end_press(cx: &mut TestAppContext) {
+    let source = "ab &#10; y";
+    let (document, cx) = open(source, MarkdownMode::Visual, cx);
+    let events = heard(&document, cx);
+    press_on(0, 1, MouseButton::Right, cx);
+    assert_eq!(*events.borrow(), [target(1, TargetBlock::Paragraph)]);
+}
+
+#[gpui::test]
+fn numeric_control_press(cx: &mut TestAppContext) {
+    let source = "ab &#128; y";
+    let (document, cx) = open(source, MarkdownMode::Visual, cx);
+    let events = heard(&document, cx);
+    press_on(0, 1, MouseButton::Right, cx);
+    assert_eq!(*events.borrow(), [target(1, TargetBlock::Paragraph)]);
+}
+
+#[gpui::test]
+fn html_decoded_space_press(cx: &mut TestAppContext) {
+    let source = "<div>ab&#32;</div>";
+    let (document, cx) = open(source, MarkdownMode::Visual, cx);
+    let events = heard(&document, cx);
+    press_on(0, 1, MouseButton::Right, cx);
+    assert_eq!(*events.borrow(), [target(6, TargetBlock::Html)]);
+}
+
+#[gpui::test]
+fn inline_code_kept_indent_left_press(cx: &mut TestAppContext) {
+    let source = "a `fox\n  y` z";
+    let (document, cx) = open(source, MarkdownMode::Visual, cx);
+    press_on(0, 3, MouseButton::Left, cx);
+    assert_eq!(opened_caret(&document, cx), 4);
+}
+
+#[gpui::test]
+fn autolink_literal_reference_press(cx: &mut TestAppContext) {
+    let source = "<https://example.com/?a=&amp;b=2>";
+    let (document, cx) = open(source, MarkdownMode::Visual, cx);
+    let events = heard(&document, cx);
+    press_on(0, 9, MouseButton::Right, cx);
+    assert_eq!(events.borrow().len(), 1);
+    assert_eq!(events.borrow()[0].at, source.find('x').unwrap());
+}
+
+#[gpui::test]
+fn table_escaped_pipe_press(cx: &mut TestAppContext) {
+    let source = "| `ab\\|cd` |\n| --- |\n| cell |\n";
+    let (document, cx) = open(source, MarkdownMode::Visual, cx);
+    let events = heard(&document, cx);
+    let bounds = cx.debug_bounds("markdown-block-0").expect("drawn");
+    let (body, _) = sizes(cx);
+    let x = bounds.left() + px(26.0) + body * 0.6 * 1.25;
+    press_at(x, bounds.top() + px(17.0), MouseButton::Right, cx);
+    assert_eq!(events.borrow().len(), 1);
+    assert!(events.borrow()[0].in_table);
+    assert_eq!(events.borrow()[0].at, source.find('b').unwrap());
+}
+
+#[gpui::test]
+fn prior_crlf_code_press(cx: &mut TestAppContext) {
+    let source = "```\r\nalpha\r\nbeta\r\n```\r\n";
+    let (document, cx) = open(source, MarkdownMode::Visual, cx);
+    let events = heard(&document, cx);
+    press_lines(2, 0, MouseButton::Right, cx);
+    assert_eq!(events.borrow()[0].at, source.find("pha").unwrap());
+    press_lines(2, 0, MouseButton::Left, cx);
+    assert_eq!(opened_caret(&document, cx), source.find("pha").unwrap());
+}
+
+#[gpui::test]
+fn prior_crlf_front_matter_press(cx: &mut TestAppContext) {
+    let source = "---\r\ntitle: value\r\nauthor: person\r\n---\r\n";
+    let (document, cx) = open(source, MarkdownMode::Visual, cx);
+    let events = heard(&document, cx);
+    press_lines(7, 0, MouseButton::Right, cx);
+    assert_eq!(events.borrow()[0].at, source.find("value").unwrap());
+    press_lines(7, 0, MouseButton::Left, cx);
+    assert_eq!(opened_caret(&document, cx), source.find("value").unwrap());
+}
+
+#[gpui::test]
+fn prior_crlf_definition_press(cx: &mut TestAppContext) {
+    let source = "[ref]: https://ely.dev\r\n  \"title\"\r\n";
+    let (document, cx) = open(source, MarkdownMode::Visual, cx);
+    let events = heard(&document, cx);
+    press_on(0, 10, MouseButton::Right, cx);
+    assert_eq!(*events.borrow(), [target(10, TargetBlock::Definition)]);
+}

src/documents/document/tests/reading.rs

@@ -121,3 +121,61 @@ fn a_decoded_line_end_reads_without_a_written_one() {
     let alert = parsed("> [!NOTE]
alpha beta");
     assert_eq!(alert.blocks[0].kind, TargetBlock::Alert);
 }
+
+#[test]
+fn kept_leading_marks_still_map() {
+    let source = "a `fox\n  y` z";
+    assert_eq!(read_back(source, "ox"), source.find("ox"));
+}
+
+#[test]
+fn decoded_line_end_maps_nearby_text() {
+    for (source, shown) in [
+        ("ab 
 y", "b"),
+        ("ab 
 y", "b"),
+        ("ab 
 y", "b"),
+        ("<div>ab &#10; y</div>", "b"),
+        ("<div>ab &#9; y</div>", "b"),
+    ] {
+        assert_eq!(read_back(source, shown), source.find(shown), "{source:?}");
+    }
+}
+
+#[test]
+fn numeric_references_match_the_parser() {
+    for source in ["ab &#128; y", "ab &#1; y", "ab &#xFDD0; y"] {
+        assert_eq!(read_back(source, "b"), source.find('b'), "{source:?}");
+    }
+}
+
+#[test]
+fn html_trailing_encoded_space_maps_previous_word() {
+    for source in [
+        "<div>ab&#32;</div>",
+        "<div>ab&#32;<b>word</b></div>",
+        "<div>ab &#32; word</div>",
+    ] {
+        assert_eq!(read_back(source, "b"), source.find('b'), "{source:?}");
+    }
+}
+
+#[test]
+fn code_pipe_in_a_table_maps_previous_text() {
+    let source = "| `ab\\|cd` |\n| --- |\n| cell |\n";
+    let parsed = parsed(source);
+    let nodes = parsed.blocks[0].nodes.as_ref().unwrap();
+    let Node::Table { head, .. } = &nodes[0] else {
+        panic!("table")
+    };
+    let Node::Paragraph(inline) = &head[0][0] else {
+        panic!("text")
+    };
+    assert_eq!(inline.text, "ab|cd");
+    assert_eq!(inline.source_at(source, 1), source.find('b'));
+}
+
+#[test]
+fn autolink_does_not_decode_literal_reference() {
+    let source = "<https://example.com/?a=&amp;b=2>";
+    assert_eq!(read_back(source, "xample"), source.find("xample"));
+}

src/documents/tree/mod.rs

@@ -120,6 +120,8 @@ struct Run {
     named: HashMap<String, usize>,
     /// Inside an autolink: its text stays as written.
     verbatim: bool,
+    /// In a table cell, whose code reads `\|` as `|`.
+    table: bool,
     /// The current DOM text node's shortcodes.
     shortcodes: emoji::Shortcodes,
     /// White space owed, and the open elements before it.
@@ -306,7 +308,10 @@ impl<'a> Reader<'a> {
         {
             return out.push(Node::Math(math.value.clone()));
         }
-        let mut run = Run::default();
+        let mut run = Run {
+            table: matches!(node, mdast::Node::TableCell(_)),
+            ..Run::default()
+        };
         for child in children {
             self.inline_into(child, &mut run, &mut Images::Split(out));
         }
@@ -344,6 +349,11 @@ impl<'a> Reader<'a> {
             run.shortcodes.split();
         }
         if let Some((written, escapes)) = sources::written(node, self.source) {
+            let escapes = match (escapes, node) {
+                (Escapes::Markdown, _) if run.verbatim => Escapes::Plain,
+                (_, mdast::Node::InlineCode(_)) if run.table => Escapes::Pipes,
+                _ => escapes,
+            };
             let at = run.text.text.len();
             run.text.sources.push((at, written, escapes));
         }

src/documents/tree/sources.rs

@@ -12,6 +12,8 @@ pub enum Escapes {
     Markdown,
     /// References, as HTML reads them.
     Html,
+    /// Escaped pipes, as a table cell's code reads them.
+    Pipes,
 }
 
 /// A leaf's written span and how it reads.
@@ -208,7 +210,7 @@ struct Read {
 
 impl Read {
     fn ends_line(&self) -> bool {
-        self.raw && matches!(self.ch, '\n' | '\r')
+        matches!(self.ch, '\n' | '\r')
     }
 }
 
@@ -226,6 +228,9 @@ fn read(written: &str, escapes: Escapes) -> Vec<Read> {
             (Escapes::Html, '&') => {
                 html::reference(rest, false).map(|(value, used)| (value, used, at))
             }
+            (Escapes::Pipes, '\\') => {
+                (rest[1..].starts_with('|')).then(|| ("|".to_string(), 2, at + 1))
+            }
             _ => None,
         };
         if let Some((value, used, place)) = decoded {
@@ -260,7 +265,10 @@ fn reference(text: &str) -> Option<(String, usize)> {
         let fits =
             (1..=most).contains(&digits.len()) && digits.chars().all(|ch| ch.is_digit(radix));
         let code = u32::from_str_radix(digits, radix).ok().filter(|_| fits)?;
-        let ch = char::from_u32(code).filter(|ch| *ch != '\0');
+        // Controls become U+FFFD, as markdown-rs reads them.
+        let ch = char::from_u32(code).filter(|ch| {
+            !matches!(ch, '\0'..='\u{08}' | '\u{0B}' | '\u{0E}'..='\u{1F}' | '\u{7F}'..='\u{9F}')
+        });
         Some(ch.unwrap_or(char::REPLACEMENT_CHARACTER).to_string())
     };
     let value = match name.strip_prefix('#') {
@@ -279,34 +287,23 @@ fn align(drawn: &str, written: &str, escapes: Escapes, at: usize) -> Option<usiz
     }
     let source = read(written, escapes);
     let mut j = source.len();
-    let mut shown = drawn.char_indices().rev().peekable();
-    while let Some((byte, d)) = shown.next() {
+    for (byte, d) in drawn.char_indices().rev() {
         let found = loop {
             let s = source[..j].last()?;
-            // A join: a line's end and the marks after it.
-            if d.is_whitespace() && (s.ends_line() || s.lead) {
-                while source[..j].last().is_some_and(|s| s.lead) {
-                    j -= 1;
-                }
-                j -= 1;
-                let crlf = source[j].ch == '\n' && j > 0;
-                if crlf && source[j - 1].ends_line() && source[j - 1].ch == '\r' {
-                    j -= 1;
-                }
-                if d == '\n' && shown.peek().is_some_and(|(_, ch)| *ch == '\r') {
-                    let (cr, _) = shown.next().expect("peeked");
-                    if cr == at {
-                        return Some(source[j].at);
-                    }
-                }
-                break source[j].at;
-            }
             if d == s.ch {
                 j -= 1;
                 break s.at;
             }
+            // A join: a line's end drawn as white space.
+            if d.is_whitespace() && s.ends_line() {
+                j -= 1;
+                if s.ch == '\n' && j > 0 && source[j - 1].ch == '\r' && source[j - 1].raw == s.raw {
+                    j -= 1;
+                }
+                break source[j].at;
+            }
             // White space, line ends and marks the text dropped.
-            if s.raw && (matches!(s.ch, ' ' | '\t') || s.ends_line() || s.lead) {
+            if s.lead || matches!(s.ch, ' ' | '\t') || s.ends_line() {
                 j -= 1;
                 continue;
             }