documents: Markdown containers are the elements GitHub writes, repaired with tags as HTML5 repairs; shortcodes read from decoded text, kept in autolinks and <code>

A raw end tag ends the Markdown element of its name; an end tag pops every
element above its target and only formatting elements reopen, so mark, kbd
and del stop where the browser stops them; a new anchor ends the open one.
Emoji read after escapes and references, as GitHub's Markdown API shows.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

src/documents/tree/html.rs

@@ -1,4 +1,4 @@
-use markdown::{decode_named, decode_numeric};
+use markdown::decode_named;
 
 use super::Style;
 
@@ -33,6 +33,14 @@ pub(crate) fn inline_style(name: &str) -> Option<Style> {
     }
 }
 
+/// HTML5's formatting elements reopen past a crossed close.
+pub(crate) fn reopens(name: &str) -> bool {
+    matches!(
+        name,
+        "a" | "b" | "code" | "em" | "i" | "s" | "strike" | "strong" | "u"
+    )
+}
+
 /// `html` as one tag; comments and the rest are none.
 pub(crate) fn tag(html: &str) -> Option<Tag> {
     let inner = html.trim().strip_prefix('<')?.strip_suffix('>')?;
@@ -184,56 +192,28 @@ fn numeric(code: Option<u32>) -> char {
     .unwrap_or(char::REPLACEMENT_CHARACTER)
 }
 
-/// The reference `text` starts with, as CommonMark reads one.
-fn reference(text: &str) -> Option<(String, usize)> {
-    let body = text.strip_prefix('&')?;
-    let end = body
-        .bytes()
-        .take(LONGEST + 1)
-        .position(|byte| byte == b';')?;
-    let name = &body[..end];
-    let value = match name.strip_prefix('#') {
-        Some(number) => {
-            let (digits, radix, most) = match number.strip_prefix(['x', 'X']) {
-                Some(hex) => (hex, 16, 6),
-                None => (number, 10, 7),
-            };
-            let valid =
-                (1..=most).contains(&digits.len()) && digits.chars().all(|ch| ch.is_digit(radix));
-            valid.then(|| decode_numeric(digits, radix))?
-        }
-        None if !name.is_empty() && name.chars().all(|ch| ch.is_ascii_alphanumeric()) => {
-            decode_named(name, true)?
-        }
-        None => return None,
-    };
-    Some((value, end + 2))
-}
-
-/// `value` with each `:shortcode:` written plainly in `source` as emoji.
-pub(crate) fn emoji(source: &str, value: &str) -> String {
-    let mut codes = literal_codes(source).into_iter().peekable();
-    let mut out = String::with_capacity(value.len());
-    let (mut colons, mut rest) = (0, value);
+/// `text` with each known `:shortcode:` as its emoji.
+pub(crate) fn emoji(text: &str) -> String {
+    let mut out = String::with_capacity(text.len());
+    let mut rest = text;
     while let Some(at) = rest.find(':') {
         out.push_str(&rest[..at]);
-        rest = &rest[at..];
-        match codes.peek() {
-            Some((ordinal, code, glyph))
-                if *ordinal == colons
-                    && rest[1..].starts_with(code)
-                    && rest[1 + code.len()..].starts_with(':') =>
-            {
-                out.push_str(glyph);
-                rest = &rest[code.len() + 2..];
-                colons += 2;
-                codes.next();
+        let after = &rest[at + 1..];
+        let len = after
+            .find(|ch: char| !shortcode_char(ch))
+            .unwrap_or(after.len());
+        let found = after[len..]
+            .starts_with(':')
+            .then(|| emojis::get_by_shortcode(&after[..len]))
+            .flatten();
+        match found {
+            Some(found) => {
+                out.push_str(found.as_str());
+                rest = &after[len + 1..];
             }
-            _ => {
+            None => {
                 out.push(':');
-                rest = &rest[1..];
-                colons += 1;
-                while codes.next_if(|(ordinal, _, _)| *ordinal < colons).is_some() {}
+                rest = after;
             }
         }
     }
@@ -241,45 +221,6 @@ pub(crate) fn emoji(source: &str, value: &str) -> String {
     out
 }
 
-/// Plain shortcodes in `source`, with colons counted before each.
-fn literal_codes(source: &str) -> Vec<(usize, &str, &'static str)> {
-    let (mut out, mut colons, mut at) = (Vec::new(), 0, 0);
-    while let Some(ch) = source[at..].chars().next() {
-        let (rest, after) = (&source[at..], &source[at + ch.len_utf8()..]);
-        if ch == '\\'
-            && let Some(next) = after.chars().next().filter(char::is_ascii_punctuation)
-        {
-            colons += usize::from(next == ':');
-            at += 1 + next.len_utf8();
-            continue;
-        }
-        if ch == '&'
-            && let Some((text, len)) = reference(rest)
-        {
-            colons += text.matches(':').count();
-            at += len;
-            continue;
-        }
-        if ch == ':' {
-            let len = after
-                .find(|ch: char| !shortcode_char(ch))
-                .unwrap_or(after.len());
-            let found = Some(&after[..len])
-                .filter(|code| !code.is_empty() && after[len..].starts_with(':'))
-                .and_then(|code| emojis::get_by_shortcode(code).map(|found| (code, found)));
-            if let Some((code, found)) = found {
-                out.push((colons, code, found.as_str()));
-                colons += 2;
-                at += code.len() + 2;
-                continue;
-            }
-            colons += 1;
-        }
-        at += ch.len_utf8();
-    }
-    out
-}
-
 fn shortcode_char(ch: char) -> bool {
     ch.is_ascii_alphanumeric() || matches!(ch, '_' | '+' | '-')
 }
@@ -359,26 +300,17 @@ mod tests {
         let (amps, colons) = ("&".repeat(1 << 18), ":zq".repeat(1 << 16));
         let started = web_time::Instant::now();
         assert_eq!(decoded(&amps).len(), amps.len());
-        assert_eq!(emoji(&amps, &amps).len(), amps.len());
-        assert_eq!(emoji(&colons, &colons).len(), colons.len());
+        assert_eq!(emoji(&colons).len(), colons.len());
         let spent = started.elapsed();
         assert!(spent.as_secs() < 2, "256 KiB took {spent:?}");
     }
 
     #[test]
-    fn shortcodes_written_plainly_become_emoji() {
-        let same = |text: &str| emoji(text, text);
-        assert_eq!(same("hi :smile: :+1:"), "hi 😄 👍");
-        assert_eq!(same(":nope: at 10:30 and ::"), ":nope: at 10:30 and ::");
-        assert_eq!(same("a:smile:b"), "a😄b");
-        assert_eq!(same("é :smile: 中\\é"), "é 😄 中\\é");
-    }
-
-    #[test]
-    fn escaped_or_referenced_colons_stay() {
-        assert_eq!(emoji("\\:smile: :smile:", ":smile: :smile:"), ":smile: 😄");
-        assert_eq!(emoji("&#58;smile: :tada:", ":smile: :tada:"), ":smile: 🎉");
-        assert_eq!(emoji(":sm&#105;le:", ":smile:"), ":smile:");
+    fn known_shortcodes_become_emoji() {
+        assert_eq!(emoji("hi :smile: :+1:"), "hi 😄 👍");
+        assert_eq!(emoji(":nope: at 10:30 and ::"), ":nope: at 10:30 and ::");
+        assert_eq!(emoji("a:smile:b :x:smile:"), "a😄b ❌smile:");
+        assert_eq!(emoji("é :smile: 中"), "é 😄 中");
     }
 
     #[test]

src/documents/tree/mod.rs

@@ -72,12 +72,12 @@ pub(crate) struct Reader<'a> {
     notes: HashMap<String, usize>,
 }
 
-/// An open style; `tag` names an HTML one.
+/// An open element, as HTML names it.
 struct Opened {
+    name: String,
     start: usize,
-    style: Style,
+    style: Option<Style>,
     url: Option<String>,
-    tag: Option<String>,
 }
 
 /// Inline text being built, and the containers open around it.
@@ -106,7 +106,7 @@ impl Run {
         self.open.iter().rev().find_map(|opened| opened.url.clone())
     }
 
-    /// Opens or closes a style for an inline HTML tag.
+    /// Opens or closes an element for an inline HTML tag.
     fn html(&mut self, written: &str) {
         let Some(tag) = html::tag(written) else {
             return;
@@ -117,41 +117,61 @@ impl Run {
         let Some(style) = html::inline_style(&tag.name) else {
             return;
         };
-        // A close meets its open anywhere, as a browser repairs.
         if tag.closing {
-            let found = self
-                .open
-                .iter()
-                .rposition(|opened| opened.tag.as_ref() == Some(&tag.name));
-            if let Some(at) = found {
-                let (opened, end) = (self.open.remove(at), self.text.text.len());
-                close(&mut self.text, &opened, end);
-            }
-            return;
+            return self.end(&tag.name);
         }
         let url = match style {
-            Style::Link => match tag.attribute("href") {
-                Some(href) => Some(href.to_string()),
-                None => return,
-            },
+            Style::Link => tag.attribute("href").map(str::to_string),
             _ => None,
         };
+        let style = (style != Style::Link || url.is_some()).then_some(style);
+        self.start(&tag.name, style, url);
+    }
+
+    /// Opens `name`; a new `a` ends the open one.
+    fn start(&mut self, name: &str, style: Option<Style>, url: Option<String>) {
+        if name == "a" {
+            self.end("a");
+        }
         self.open.push(Opened {
+            name: name.to_string(),
             start: self.text.text.len(),
             style,
             url,
-            tag: Some(tag.name),
         });
     }
+
+    /// Ends the innermost `name`, as HTML5 repairs nesting.
+    fn end(&mut self, name: &str) {
+        let Some(at) = self.open.iter().rposition(|opened| opened.name == name) else {
+            return;
+        };
+        let end = self.text.text.len();
+        let above = self.open.split_off(at + 1);
+        let ended = self.open.pop().expect("found at `at`");
+        close(&mut self.text, &ended, end);
+        for opened in above {
+            close(&mut self.text, &opened, end);
+            if html::reopens(&opened.name) {
+                self.open.push(Opened {
+                    start: end,
+                    ..opened
+                });
+            }
+        }
+    }
 }
 
 /// Records `opened` as styled up to `end`.
 fn close(text: &mut Inline, opened: &Opened, end: usize) {
-    if opened.start < end {
-        text.styles.push((opened.start..end, opened.style));
-        if let Some(url) = &opened.url {
-            text.links.push((opened.start..end, url.clone()));
-        }
+    if opened.start >= end {
+        return;
+    }
+    if let Some(style) = opened.style {
+        text.styles.push((opened.start..end, style));
+    }
+    if let Some(url) = &opened.url {
+        text.links.push((opened.start..end, url.clone()));
     }
 }
 
@@ -303,11 +323,12 @@ impl<'a> Reader<'a> {
 
     fn inline_into(&self, node: &mdast::Node, run: &mut Run, images: &mut Images) {
         let text = &mut run.text;
-        let (children, style, url) = match node {
+        let (children, (name, style), url) = match node {
             mdast::Node::Text(words) => {
-                let shown = match run.verbatim {
+                let code = run.open.iter().any(|opened| opened.name == "code");
+                let shown = match run.verbatim || code {
                     true => words.value.clone(),
-                    false => html::emoji(&self.source[span(node)], &words.value),
+                    false => html::emoji(&words.value),
                 };
                 let soft = shown.replace("\r\n", " ").replace(['\r', '\n'], " ");
                 return run.text.text.push_str(&soft);
@@ -338,23 +359,19 @@ impl<'a> Reader<'a> {
                 };
             }
             mdast::Node::Html(written) => return run.html(&written.value),
-            mdast::Node::Strong(strong) => (&strong.children, Style::Strong, None),
-            mdast::Node::Emphasis(emphasis) => (&emphasis.children, Style::Emphasis, None),
-            mdast::Node::Delete(delete) => (&delete.children, Style::Strike, None),
-            mdast::Node::Link(link) => (&link.children, Style::Link, Some(link.url.clone())),
+            mdast::Node::Strong(strong) => (&strong.children, ("strong", Style::Strong), None),
+            mdast::Node::Emphasis(emphasis) => (&emphasis.children, ("em", Style::Emphasis), None),
+            mdast::Node::Delete(delete) => (&delete.children, ("del", Style::Strike), None),
+            mdast::Node::Link(link) => (&link.children, ("a", Style::Link), Some(link.url.clone())),
             mdast::Node::LinkReference(link) => (
                 &link.children,
-                Style::Link,
+                ("a", Style::Link),
                 Some(self.url(&link.identifier)),
             ),
             other => unreachable!("{other:?} is no inline outside MDX"),
         };
-        run.open.push(Opened {
-            start: run.text.text.len(),
-            style,
-            url,
-            tag: None,
-        });
+        // A container is the element GitHub writes.
+        run.start(name, Some(style), url);
         let autolink =
             matches!(node, mdast::Node::Link(_)) && !self.source[span(node)].starts_with('[');
         let outer = run.verbatim;
@@ -363,14 +380,7 @@ impl<'a> Reader<'a> {
             self.inline_into(child, run, images);
         }
         run.verbatim = outer;
-        // Tags opened inside stay open past it.
-        let at = run
-            .open
-            .iter()
-            .rposition(|opened| opened.tag.is_none())
-            .expect("the container opened above");
-        let (opened, end) = (run.open.remove(at), run.text.text.len());
-        close(&mut run.text, &opened, end);
+        run.end(name);
     }
 }
 

src/documents/tree/tests.rs

@@ -225,21 +225,78 @@ fn inline_html_shows_as_typora_shows_it() {
     assert_eq!(text.links, [(13..14, "https://ely.dev".to_string())]);
 }
 
+/// Where `style` holds, touching stretches joined.
+fn marked(inline: &Inline, style: Style) -> Vec<(usize, usize)> {
+    let mut ranges: Vec<Range<usize>> = inline
+        .styles
+        .iter()
+        .filter(|(_, held)| *held == style)
+        .map(|(range, _)| range.clone())
+        .collect();
+    ranges.sort_by_key(|range| range.start);
+    let mut out: Vec<Range<usize>> = Vec::new();
+    for range in ranges {
+        match out.last_mut() {
+            Some(last) if last.end >= range.start => last.end = last.end.max(range.end),
+            _ => out.push(range),
+        }
+    }
+    out.into_iter()
+        .map(|range| (range.start, range.end))
+        .collect()
+}
+
 #[test]
 fn tags_and_emphasis_cross_as_a_browser_repairs_them() {
     let opened_inside = line("**a <u>b** c</u> d");
     assert_eq!(opened_inside.text, "a b c d");
-    let mut styles = opened_inside.styles.clone();
-    styles.sort_by_key(|(range, _)| (range.start, range.end));
-    assert_eq!(styles, [(0..3, Style::Strong), (2..5, Style::Underline)]);
+    assert_eq!(marked(&opened_inside, Style::Strong), [(0, 3)]);
+    assert_eq!(marked(&opened_inside, Style::Underline), [(2, 5)]);
     let closed_inside = line("a <u>*text</u> outside* end");
     assert_eq!(closed_inside.text, "a text outside end");
-    let mut styles = closed_inside.styles.clone();
-    styles.sort_by_key(|(range, _)| (range.start, range.end));
-    assert_eq!(styles, [(2..6, Style::Underline), (2..14, Style::Emphasis)]);
+    assert_eq!(marked(&closed_inside, Style::Underline), [(2, 6)]);
+    assert_eq!(marked(&closed_inside, Style::Emphasis), [(2, 14)]);
     assert!(line("a </u> b").styles.is_empty(), "a stray close");
 }
 
+#[test]
+fn a_raw_close_ends_the_markdown_element() {
+    let em = line("a *inside</em> outside* end");
+    assert_eq!(em.text, "a inside outside end");
+    assert_eq!(marked(&em, Style::Emphasis), [(2, 8)]);
+    let strong = line("a **inside</strong> outside** end");
+    assert_eq!(marked(&strong, Style::Strong), [(2, 8)]);
+}
+
+#[test]
+fn only_formatting_elements_reopen_past_a_close() {
+    for (tag, style) in [
+        ("mark", Style::Mark),
+        ("kbd", Style::Code),
+        ("del", Style::Strike),
+    ] {
+        let text = line(&format!("a **bold <{tag}>x** outside</{tag}> end"));
+        assert_eq!(text.text, "a bold x outside end", "{tag}");
+        assert_eq!(marked(&text, style), [(7, 8)], "{tag}");
+    }
+    let reopened = line("<b>1<i>2</b>3</i>4");
+    assert_eq!(marked(&reopened, Style::Strong), [(0, 2)]);
+    assert_eq!(marked(&reopened, Style::Emphasis), [(1, 3)]);
+}
+
+#[test]
+fn a_new_anchor_ends_the_open_one() {
+    let text = line("[one <a href='/inner'>two</a> three](/outer)");
+    assert_eq!(text.text, "one two three");
+    assert_eq!(
+        text.links,
+        [(0..4, "/outer".to_string()), (4..7, "/inner".to_string())]
+    );
+    assert_eq!(marked(&text, Style::Link), [(0, 7)]);
+    let named = line("[x <a name=n>y</a> z](/u)");
+    assert_eq!(named.links, [(0..2, "/u".to_string())]);
+}
+
 #[test]
 fn a_closing_br_breaks_as_html_reads_it() {
     assert_eq!(line("before<br></br>after").text, "before\n\nafter");
@@ -279,9 +336,11 @@ fn one_tilde_still_strikes() {
 }
 
 #[test]
-fn escapes_references_and_autolinks_keep_their_shortcodes() {
-    assert_eq!(line(r"Show \:smile: now.").text, "Show :smile: now.");
-    assert_eq!(line("Show &#58;smile: now.").text, "Show :smile: now.");
+fn shortcodes_read_as_github_reads_them() {
+    let decoded = line(r"Show \:smile: and &#58;smile: and :sm&#105;le:.");
+    assert_eq!(decoded.text, "Show 😄 and 😄 and 😄.");
+    let tags = line("a <code>:smile:</code> <kbd>:smile:</kbd> <tt>:smile:</tt> :smile:");
+    assert_eq!(tags.text, "a :smile: 😄 😄 😄");
     assert_eq!(
         line("<https://x.test/:smile:>").text,
         "https://x.test/:smile:"