documents: shortcodes read per DOM text node as GitHub reads them; HTML5's p closers end a p; a TOC press follows editor anchors
CI / platforms (windows-latest) (push) Failing after 9s
CI / site (push) Successful in 22s
CI / crate (push) Failing after 10m54s
CI / platforms (ubuntu-24.04) (push) Canceled after 0s

- The first `:name:`-shaped run in a text node decides for the whole node:
  blank text or a space before it turns every shortcode, else none. Filtered
  tags are text and do not split the node. Probed on GitHub's markdown API.
- `fieldset`, `menu` and the rest of HTML5's list end an open `p`.
- `CodeEditor` carries owner anchors through each edit; the document anchors
  every heading it reads, and a TOC press goes where its anchor went.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
ZacharyZhang-NYandClaude Opus 5.5 committed 2026-10-14 00:48:26 -04:00
1 parent a744316274
commit 8e5dad8177
12 files changed
+396 -138

No files matched your search

+1 -31
View File
@@ -12,27 +12,8 @@ use crate::{
forms::{InputEvent, TextInput},
};
/// The one edit from `old` to `new`, by shared ends.
pub(super) fn between(old: &str, new: &str) -> Option<(Range<usize>, usize)> {
let head: usize = old
.chars()
.zip(new.chars())
.take_while(|(a, b)| a == b)
.map(|(ch, _)| ch.len_utf8())
.sum();
let room = old.len().min(new.len()) - head;
let mut tail = 0;
for (a, b) in old.chars().rev().zip(new.chars().rev()) {
if a != b || tail + a.len_utf8() > room {
break;
}
tail += a.len_utf8();
}
(old != new).then(|| (head..old.len() - tail, new.len() - tail - head))
}
/// Where byte `at` lies after `edit`, if it survives.
pub(super) fn moved(at: usize, edit: &Option<(Range<usize>, usize)>) -> Option<usize> {
fn moved(at: usize, edit: &Option<(Range<usize>, usize)>) -> Option<usize> {
match edit {
None => Some(at),
Some((replaced, _)) if at < replaced.start => Some(at),
@@ -200,15 +181,4 @@ mod tests {
assert_eq!(super::moved(3, &edit), None, "inside is gone");
assert_eq!(super::moved(7, &None), Some(7));
}
#[test]
fn two_texts_part_at_their_shared_ends() {
assert_eq!(super::between("abc", "abc"), None);
assert_eq!(super::between("before\nB", "## B\nB"), Some((0..6, 4)));
assert_eq!(super::between("ab", "aXb"), Some((1..1, 1)));
assert_eq!(super::between("aa", "aaa"), Some((2..2, 1)));
assert_eq!(super::between("aXYb", "ab"), Some((1..3, 0)));
assert_eq!(super::between("é😄", "é"), Some((2..6, 0)));
assert_eq!(super::between("x😄", "y😄"), Some((0..1, 1)));
}
}
+3
View File
@@ -316,6 +316,9 @@ impl MarkdownDocument {
let old = self.parsed.blocks.len();
self.list
.splice(head..old - tail, read.blocks.len() - head - tail);
let headings: Vec<usize> = read.headings.iter().map(|heading| heading.at).collect();
self.editor
.update(cx, |editor, _| editor.set_anchors(headings));
self.parsed = Arc::new(read);
cx.emit(DocumentEvent::Parsed);
cx.notify();
+19 -17
View File
@@ -1,11 +1,6 @@
use std::sync::Arc;
use gpui::{App, Context, Focusable as _, ListOffset, ScrollStrategy, Window, px};
use super::{
MarkdownDocument, Placement,
edit::{between, moved},
};
use super::{MarkdownDocument, Placement};
use crate::documents::{MarkdownMode, modes::joint};
/// Line and column, from zero, of byte `at` in `text`.
@@ -194,24 +189,31 @@ impl MarkdownDocument {
.saturating_sub(1)
}
/// Reveals the pressed heading, moved by the edit since.
/// Reveals the pressed heading where the edits since carried it.
pub(super) fn reveal_at(&mut self, at: usize, window: &mut Window, cx: &mut Context<Self>) {
if !self.parsed.headings.iter().any(|heading| heading.at == at) {
let headings = &self.parsed.headings;
let Some(index) = headings.iter().position(|heading| heading.at == at) else {
return log::warn!("markdown document: a heading read away");
}
// The heading list drew from this read.
let drawn = Arc::clone(&self.parsed);
if self.commit(cx).is_some() || !self.read_now(cx) {
};
self.commit(cx);
let anchors = self.editor.read(cx).anchors();
let Some(found) = *anchors.get(index).expect("an anchor per heading read") else {
return log::warn!("markdown document: the pressed heading changed as one wrote");
};
if !self.read_now(cx) {
self.reparse(true, cx);
}
if !self.read_now(cx) {
return log::warn!("markdown document: a heading pressed while the text is read again");
}
let edit = between(&drawn.text, self.editor.read(cx).text());
let headings = &self.parsed.headings;
let Some(found) = moved(at, &edit).filter(|at| headings.iter().any(|h| h.at == *at)) else {
return log::warn!("markdown document: the pressed heading changed as one wrote");
};
if !self
.parsed
.headings
.iter()
.any(|heading| heading.at == found)
{
return log::warn!("markdown document: the pressed heading reads as no heading now");
}
let line = self.editor.read(cx).buffer.line_of(found);
self.reveal(line, window, cx);
}
+56
View File
@@ -132,3 +132,59 @@ fn a_press_while_the_text_waits_to_be_read_finds_the_heading(cx: &mut TestAppCon
(26, 0)
);
}
/// Makes `edits` while the text waits, caret at the top.
fn edit_waiting<'a>(
text: &str,
edits: Vec<(std::ops::Range<usize>, String)>,
cx: &'a mut TestAppContext,
) -> (Entity<MarkdownDocument>, &'a mut VisualTestContext) {
let (document, cx) = open(text, MarkdownMode::Split, cx);
document.update(cx, |document, cx| {
document
.editor
.update(cx, |editor, cx| editor.edit(edits, cx));
document.set_caret(0, 0, cx);
});
frames(cx);
assert!(!document.read_with(cx, |document, cx| document.read_now(cx)));
(document, cx)
}
fn caret(document: &Entity<MarkdownDocument>, cx: &mut VisualTestContext) -> (usize, usize) {
document.read_with(cx, |document, cx| document.caret(cx))
}
#[gpui::test]
fn a_twin_written_right_at_the_heading_goes_before_it(cx: &mut TestAppContext) {
let edits = vec![(7..7, "## B\n\n".to_string())];
let (document, cx) = edit_waiting("[toc]\n\n## B\n\ntext\n", edits, cx);
press_entry(0, Modifiers::none(), cx);
assert_eq!(caret(&document, cx), (4, 0));
}
#[gpui::test]
fn adjacent_twins_keep_who_was_deleted(cx: &mut TestAppContext) {
let twins = "[toc]\n\n## B\n\n## B\n\ntext\n";
let (document, cx) = edit_waiting(twins, vec![(7..13, String::new())], cx);
press_entry(0, Modifiers::none(), cx);
assert_eq!(
caret(&document, cx),
(0, 0),
"the deleted one moves nothing"
);
press_entry(1, Modifiers::none(), cx);
assert_eq!(caret(&document, cx), (2, 0), "the kept one moved up");
}
#[gpui::test]
fn a_heading_between_two_edits_stays_found(cx: &mut TestAppContext) {
let text = "before\n\n[toc]\n\n# A\n\nafter\n";
let edits = vec![
(0..0, "start\n".to_string()),
(text.len()..text.len(), "end\n".to_string()),
];
let (document, cx) = edit_waiting(text, edits, cx);
press_entry(0, Modifiers::none(), cx);
assert_eq!(caret(&document, cx), (5, 0));
}
+55 -22
View File
@@ -99,6 +99,7 @@ impl Reader<'_> {
rest = &rest[at..];
if let Some(comment) = rest.strip_prefix("<!--") {
rest = comment.split_once("-->").map_or("", |(_, after)| after);
run.shortcodes.split();
continue;
}
let opens =
@@ -116,10 +117,7 @@ impl Reader<'_> {
let (written, after) = rest.split_at(end);
rest = after;
match html::tag(written) {
Some(tag) if html::filtered(&tag.name) => {
run.filter(&tag);
run.spaced(&html::text(written));
}
Some(tag) if html::filtered(&tag.name) => run.words(written),
// A block closes what opened inside it.
Some(tag) if html::breaks(&tag.name) => {
run.line();
@@ -127,12 +125,18 @@ impl Reader<'_> {
true => run.end(&tag.name),
false => {
run.implied(&tag.name);
if tag.name != "hr" {
run.start(&tag.name, None, None);
match tag.name == "hr" {
true => run.shortcodes.split(),
false => run.start(&tag.name, None, None),
}
}
}
}
// It ends a `p` but lays no block.
Some(tag) if !tag.closing && html::closes_p(&tag.name) => {
run.implied(&tag.name);
run.html(written);
}
// A line's end drops its space.
Some(tag) if tag.closing || tag.name == "br" => {
run.owed = run.owed.filter(|_| tag.name != "br");
@@ -142,7 +146,7 @@ impl Reader<'_> {
run.pay();
run.html(written);
}
None => {}
None => run.shortcodes.split(),
}
}
run.words(rest);
@@ -153,11 +157,8 @@ impl Reader<'_> {
impl Run {
/// HTML text: references decoded, white space collapsed.
fn words(&mut self, written: &str) {
let decoded = html::text(written);
match self.reads_emoji() {
true => self.spaced(&html::emoji(&decoded)),
false => self.spaced(&decoded),
}
let shown = self.shown(&html::text(written));
self.spaced(&shown);
}
/// Text with its white space collapsed.
@@ -215,8 +216,8 @@ impl Run {
if let Some(item) = found {
self.end(&item);
}
// A table start closes it, so it is in scope.
if name != "tr" && self.count("p") > 0 {
if html::closes_p(name) && self.count("p") > 0 {
self.line();
self.end("p");
}
}
@@ -417,18 +418,50 @@ mod tests {
}
#[test]
fn filtered_tags_keep_shortcodes_and_names_end_as_html5_ends_them() {
let block = html("<script title=\":smile:\">:smile:</script>").expect("text");
assert_eq!(block.text, "<script title=\":smile:\">:smile:</script>");
assert_eq!(html("<div>:smile:</div>").expect("text").text, "😄");
let attribute = html("a <script title=\":smile:\">x</script> c").expect("text");
assert_eq!(attribute.text, "a <script title=\":smile:\">x</script> c");
let body = html("a <script>:smile:</script> :smile:").expect("text");
assert_eq!(body.text, "a <script>:smile:</script> 😄");
fn filtered_tags_are_text_and_names_end_as_html5_ends_them() {
// GitHub's answers, `.work/R4.3-fix3/github-*.log`.
for (source, shown) in [
(
"<div><script>:smile:</script> :smile:</div>",
"<script>:smile:</script> :smile:",
),
(
"<div><script><b>:smile:</b></script> :smile:</div>",
"<script>😄</script> 😄",
),
(
"<div><script> :smile:</script> :smile:</div>",
"<script> 😄</script> 😄",
),
(
"<div><script title=\" :smile:\"> x:smile:</div>",
"<script title=\" 😄\"> x😄",
),
(
"<div><iframe title=\":smile:\"> :smile:</div>",
"<iframe title=\":smile:\"> :smile:",
),
("<div>\n:smile: x:smile:\n</div>", "😄 x😄"),
("<div>\t:smile: x:smile:</div>", "😄 x😄"),
("<div>a\n :smile: x:smile:</div>", "a 😄 x😄"),
("<div>y\n:smile: x:smile:</div>", "y :smile: x:smile:"),
("<div>:smile:</div>", "😄"),
] {
assert_eq!(html(source).expect("text").text, shown, "{source:?}");
}
let named = "<div>a<script:custom>x</script:custom>b<script!>y</script!>c</div>";
assert_eq!(html(named).expect("text").text, "axbyc");
}
#[test]
fn every_html5_p_closer_ends_a_p() {
for name in ["fieldset", "menu"] {
let line = html(&format!("<div><p><mark>x<{name}>y</{name}>z</div>")).expect("text");
let mark = [(0..1, Style::Mark)];
assert_eq!((line.text.as_str(), &line.styles[..]), ("x\nyz", &mark[..]));
}
}
#[test]
fn hostile_html_reads_in_linear_time() {
let started = web_time::Instant::now();
+103
View File
@@ -0,0 +1,103 @@
/// `text` with each known `:shortcode:` as its emoji.
fn emoji(text: &str) -> String {
let mut out = String::with_capacity(text.len());
let mut rest = text;
while let Some(at) = rest.find(':') {
out.push_str(&rest[..at]);
let after = &rest[at + 1..];
let len = after
.find(|ch: char| !shortcode_char(ch))
.unwrap_or(after.len());
let found = after[len..]
.starts_with(':')
.then(|| emojis::get_by_shortcode(&after[..len]))
.flatten();
match found {
Some(found) => {
out.push_str(found.as_str());
rest = &after[len + 1..];
}
None => {
out.push(':');
rest = after;
}
}
}
out.push_str(rest);
out
}
fn shortcode_char(ch: char) -> bool {
ch.is_ascii_alphanumeric() || matches!(ch, '_' | '+' | '-')
}
/// Where the first `:name:` starts.
fn candidate(text: &str) -> Option<usize> {
let mut from = 0;
while let Some(at) = text[from..].find(':').map(|at| from + at) {
let after = &text[at + 1..];
let len = after
.find(|ch: char| !shortcode_char(ch))
.unwrap_or(after.len());
if len > 0 && after[len..].starts_with(':') {
return Some(at);
}
from = at + 1 + len;
}
None
}
/// One DOM text node's shortcodes, as GitHub reads them.
#[derive(Default)]
pub(crate) struct Shortcodes {
/// Settled by the node's first `:name:`.
reads: Option<bool>,
/// Text before it held more than white space.
inked: bool,
last: Option<char>,
}
impl Shortcodes {
/// A new DOM text node starts.
pub(crate) fn split(&mut self) {
*self = Self::default();
}
/// Blank or spaced text before the first reads them all.
pub(crate) fn read(&mut self, text: &str) -> String {
if self.reads.is_none() {
let at = candidate(text);
let lead = &text[..at.unwrap_or(text.len())];
self.inked |= !lead.chars().all(|ch| ch.is_ascii_whitespace());
self.last = lead.chars().next_back().or(self.last);
self.reads = at.map(|_| !self.inked || self.last == Some(' '));
}
match self.reads {
Some(true) => emoji(text),
_ => text.to_string(),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn known_shortcodes_become_emoji() {
assert_eq!(emoji("hi :smile: :+1:"), "hi 😄 👍");
assert_eq!(emoji(":nope: at 10:30 and ::"), ":nope: at 10:30 and ::");
assert_eq!(emoji("a:smile:b :x:smile:"), "a😄b ❌smile:");
assert_eq!(emoji("é :smile: 中"), "é 😄 中");
}
#[test]
fn long_runs_scan_in_linear_time() {
let colons = ":zq".repeat(1 << 16);
let started = web_time::Instant::now();
assert_eq!(emoji(&colons).len(), colons.len());
assert_eq!(Shortcodes::default().read(&colons).len(), colons.len());
let spent = started.elapsed();
assert!(spent.as_secs() < 2, "192 KiB took {spent:?}");
}
}
+47 -43
View File
@@ -96,6 +96,52 @@ pub(crate) fn breaks(name: &str) -> bool {
)
}
/// Start tags that end an open `p`, as HTML5 lists them.
pub(crate) fn closes_p(name: &str) -> bool {
matches!(
name,
"address"
| "article"
| "aside"
| "blockquote"
| "center"
| "details"
| "dialog"
| "dir"
| "div"
| "dl"
| "dd"
| "dt"
| "fieldset"
| "figcaption"
| "figure"
| "footer"
| "form"
| "h1"
| "h2"
| "h3"
| "h4"
| "h5"
| "h6"
| "header"
| "hgroup"
| "hr"
| "li"
| "listing"
| "main"
| "menu"
| "nav"
| "ol"
| "p"
| "pre"
| "search"
| "section"
| "summary"
| "table"
| "ul"
)
}
/// Bytes through the `>` ending the tag at `html`'s start.
pub(crate) fn tag_end(html: &str) -> Option<usize> {
let (mut quote, mut last) = (None, '<');
@@ -280,39 +326,6 @@ fn numeric(code: Option<u32>) -> char {
.unwrap_or(char::REPLACEMENT_CHARACTER)
}
/// `text` with each known `:shortcode:` as its emoji.
pub(crate) fn emoji(text: &str) -> String {
let mut out = String::with_capacity(text.len());
let mut rest = text;
while let Some(at) = rest.find(':') {
out.push_str(&rest[..at]);
let after = &rest[at + 1..];
let len = after
.find(|ch: char| !shortcode_char(ch))
.unwrap_or(after.len());
let found = after[len..]
.starts_with(':')
.then(|| emojis::get_by_shortcode(&after[..len]))
.flatten();
match found {
Some(found) => {
out.push_str(found.as_str());
rest = &after[len + 1..];
}
None => {
out.push(':');
rest = after;
}
}
}
out.push_str(rest);
out
}
fn shortcode_char(ch: char) -> bool {
ch.is_ascii_alphanumeric() || matches!(ch, '_' | '+' | '-')
}
/// `n` in superscript digits.
pub(crate) fn superscript(n: usize) -> String {
const DIGITS: [char; 10] = ['⁰', '¹', '²', '³', '⁴', '⁵', '⁶', '⁷', '⁸', '⁹'];
@@ -385,23 +398,14 @@ mod tests {
#[test]
fn long_runs_scan_in_linear_time() {
let (amps, colons) = ("&".repeat(1 << 18), ":zq".repeat(1 << 16));
let amps = "&".repeat(1 << 18);
let started = web_time::Instant::now();
assert_eq!(decoded(&amps, true).len(), amps.len());
assert_eq!(text(&amps).len(), amps.len());
assert_eq!(emoji(&colons).len(), colons.len());
let spent = started.elapsed();
assert!(spent.as_secs() < 2, "256 KiB took {spent:?}");
}
#[test]
fn known_shortcodes_become_emoji() {
assert_eq!(emoji("hi :smile: :+1:"), "hi 😄 👍");
assert_eq!(emoji(":nope: at 10:30 and ::"), ":nope: at 10:30 and ::");
assert_eq!(emoji("a:smile:b :x:smile:"), "a😄b ❌smile:");
assert_eq!(emoji("é :smile: 中"), "é 😄 中");
}
#[test]
fn text_takes_the_longest_legacy_name() {
assert_eq!(
+22 -22
View File
@@ -5,6 +5,7 @@ use markdown::mdast::{self, AlignKind};
use super::markdown::{span, tree, walk};
mod blocks;
mod emoji;
mod html;
pub use blocks::{AlertKind, DocumentHeading};
@@ -102,8 +103,8 @@ struct Run {
named: HashMap<String, usize>,
/// Inside an autolink: its text stays as written.
verbatim: bool,
/// Filtered tags open: their text keeps shortcodes.
filtered: usize,
/// The current DOM text node's shortcodes.
shortcodes: emoji::Shortcodes,
/// White space owed, and the open elements before it.
owed: Option<(char, usize)>,
}
@@ -124,17 +125,18 @@ impl Run {
self.named.get(name).copied().unwrap_or(0)
}
/// Outside autolinks, `<code>` and filtered tags.
fn reads_emoji(&self) -> bool {
!self.verbatim && self.count("code") == 0 && self.filtered == 0
/// Text with its shortcodes read, outside autolinks and `<code>`.
fn shown(&mut self, written: &str) -> String {
match !self.verbatim && self.count("code") == 0 {
true => self.shortcodes.read(written),
false => written.to_string(),
}
}
/// Text as shown: shortcodes read, lines joined.
fn push_text(&mut self, written: &str) {
match self.reads_emoji() {
true => self.push_plain(&html::emoji(written)),
false => self.push_plain(written),
}
let shown = self.shown(written);
self.push_plain(&shown);
}
fn push_plain(&mut self, shown: &str) {
@@ -142,14 +144,6 @@ impl Run {
self.text.text.push_str(&soft);
}
/// Counts a filtered tag opened or closed.
fn filter(&mut self, tag: &html::Tag) {
self.filtered = match tag.closing {
true => self.filtered.saturating_sub(1),
false => self.filtered + 1,
};
}
fn push(&mut self, opened: Opened) {
*self.named.entry(opened.name.clone()).or_default() += 1;
self.open.push(opened);
@@ -163,15 +157,15 @@ impl Run {
/// Opens or closes an element for an inline HTML tag.
fn html(&mut self, written: &str) {
let Some(tag) = html::tag(written) else {
return;
return self.shortcodes.split();
};
if html::filtered(&tag.name) {
return self.push_text(&html::text(written));
}
self.shortcodes.split();
if tag.name == "br" {
return self.text.text.push('\n');
}
if html::filtered(&tag.name) {
self.filter(&tag);
return self.push_plain(&html::text(written));
}
let Some(style) = html::inline_style(&tag.name) else {
return;
};
@@ -191,6 +185,7 @@ impl Run {
if name == "a" {
self.end("a");
}
self.shortcodes.split();
self.push(Opened {
name: name.to_string(),
start: self.text.text.len(),
@@ -204,6 +199,7 @@ impl Run {
if self.count(name) == 0 {
return;
}
self.shortcodes.split();
let at = self
.open
.iter()
@@ -402,6 +398,10 @@ impl<'a> Reader<'a> {
}
fn inline_into(&self, node: &mdast::Node, run: &mut Run, images: &mut Images) {
// Any element ends the DOM text node.
if !matches!(node, mdast::Node::Text(_) | mdast::Node::Html(_)) {
run.shortcodes.split();
}
let text = &mut run.text;
let (children, (name, style), url) = match node {
mdast::Node::Text(words) => return run.push_text(&words.value),
+37
View File
@@ -362,3 +362,40 @@ fn unknown_tags_show_text_and_only_anchors_link() {
let urls: Vec<&str> = query.links.iter().map(|(_, url)| url.as_str()).collect();
assert_eq!(urls, ["https://x.test/?q=1&p=2", "/path/"]);
}
#[test]
fn the_first_shortcode_decides_its_text_node() {
// GitHub's answers, `.work/R4.3-fix3/github-*.log`.
for (source, shown) in [
("a>:smile: :smile:", "a>:smile: :smile:"),
("a> :smile: :smile:", "a> 😄 😄"),
("x:smile: :smile:", "x:smile: :smile:"),
("a;:smile: :smile:", "a;:smile: :smile:"),
("a :smile: b>:smile:", "a 😄 b>😄"),
("a :zzzq: x:smile: :smile:", "a :zzzq: x😄 😄"),
("ax:zzzq: :smile:", "ax:zzzq: :smile:"),
("a :x-y: z:smile:", "a :x-y: z😄"),
("a :x.y: z:smile:", "a :x.y: z:smile:"),
("a.:x.y: :smile:", "a.:x.y: 😄"),
("x\t:smile: y:smile:", "x\t:smile: y:smile:"),
("x\u{a0}:smile: y:smile:", "x\u{a0}:smile: y:smile:"),
("x\n:smile: y:smile:", "x :smile: y:smile:"),
("*x*:smile: :smile:", "x😄 😄"),
("*x*\n:smile: y:smile:", "x 😄 y😄"),
("x:smile: `c` :smile:", "x:smile: c 😄"),
("x:smile:<!-- c --> :smile:", "x:smile: 😄"),
("x:smile:<span>s</span> :smile:", "x:smile:s 😄"),
("x&amp;:smile: :smile:", "x&:smile: :smile:"),
("x:smile: \n:smile:", "x:smile:\n😄"),
(
"a <script>x</style>:smile:</script> :smile:",
"a <script>x</style>:smile:</script> :smile:",
),
(
"a <script><b>:smile:</b></script> :smile:",
"a <script>😄</script> 😄",
),
] {
assert_eq!(line(source).text, shown, "{source:?}");
}
}