BlockData::to_markdown and from_markdown: a document of blocks as GitHub Markdown and back, each field's inline Markdown kept as written (#7)
This commit is contained in:
ZacharyZhang-NY committed
1 parent
5490869f23
commit
f5eb82ee1b
8 files changed
+1006
-2
No files matched your search
@@ -0,0 +1,199 @@
|
||||
use std::ops::Range;
|
||||
|
||||
use pulldown_cmark::{Event, Parser};
|
||||
|
||||
use super::{BlockData, BlockKind, Media};
|
||||
|
||||
impl BlockData {
|
||||
/// The document as Markdown, GitHub's flavor: a callout is a `[!NOTE]` quote, a toggle a `<details>`, a diagram a `mermaid` fence, math a `math` fence. Fields already hold Markdown and pass through as written, so a blank line in prose starts a new block; a heading, caption, toggle title or table cell keeps one line, and a caption's lone bracket comes back escaped. HTML in a field is read as HTML. What Markdown has no form for takes its nearest: a table's merges, media's width and alignment, columns as paragraphs, a synced block as its text, video and embeds as links.
|
||||
pub fn to_markdown(blocks: &[BlockData]) -> String {
|
||||
let mut out = String::new();
|
||||
let mut previous: Option<&BlockKind> = None;
|
||||
let mut number = 0;
|
||||
for block in blocks {
|
||||
number = match block.kind {
|
||||
BlockKind::Numbered => number + 1,
|
||||
_ => 0,
|
||||
};
|
||||
let Some(text) = write(block, number) else {
|
||||
continue;
|
||||
};
|
||||
if let Some(previous) = previous {
|
||||
out.push_str(if same_list(previous, &block.kind) {
|
||||
"\n"
|
||||
} else {
|
||||
"\n\n"
|
||||
});
|
||||
}
|
||||
out.push_str(&text);
|
||||
previous = Some(&block.kind);
|
||||
}
|
||||
out.push('\n');
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether two kinds follow on in one Markdown list.
|
||||
fn same_list(previous: &BlockKind, next: &BlockKind) -> bool {
|
||||
let bulleted = |kind: &BlockKind| matches!(kind, BlockKind::Bullet | BlockKind::Todo(_));
|
||||
(bulleted(previous) && bulleted(next))
|
||||
|| (*previous == BlockKind::Numbered && *next == BlockKind::Numbered)
|
||||
}
|
||||
|
||||
/// One block as Markdown; nothing for an empty paragraph, which Markdown cannot hold.
|
||||
fn write(block: &BlockData, number: usize) -> Option<String> {
|
||||
let texts = &block.texts;
|
||||
let text = || texts[0].as_str();
|
||||
let written = match &block.kind {
|
||||
BlockKind::Paragraph | BlockKind::Synced(_) if text().trim().is_empty() => return None,
|
||||
BlockKind::Paragraph | BlockKind::Synced(_) => text().to_string(),
|
||||
BlockKind::Heading(level) => {
|
||||
let level = usize::from((*level).clamp(1, 6));
|
||||
format!("{} {}", "#".repeat(level), text().replace('\n', " "))
|
||||
}
|
||||
BlockKind::Bullet => item("- ", text()),
|
||||
BlockKind::Numbered => item(&format!("{number}. "), text()),
|
||||
BlockKind::Todo(done) => item(if *done { "- [x] " } else { "- [ ] " }, text()),
|
||||
BlockKind::Quote => quoted(text()),
|
||||
BlockKind::Callout => format!("> [!NOTE]\n{}", quoted(text())),
|
||||
BlockKind::Toggle(open) => {
|
||||
let open = if *open { " open" } else { "" };
|
||||
let title = texts[0].replace('\n', " ");
|
||||
let head = format!("<details{open}>\n<summary>{title}</summary>");
|
||||
match texts[1].trim() {
|
||||
"" => format!("{head}\n\n</details>"),
|
||||
body => format!("{head}\n\n{body}\n\n</details>"),
|
||||
}
|
||||
}
|
||||
BlockKind::Code => fenced(text(), ""),
|
||||
BlockKind::Diagram => fenced(text(), "mermaid"),
|
||||
BlockKind::Math => fenced(text(), "math"),
|
||||
BlockKind::Divider => "---".to_string(),
|
||||
BlockKind::Table { columns, .. } => table(texts, *columns),
|
||||
BlockKind::Image(media) => {
|
||||
let caption = brackets(&text().replace('\n', " "));
|
||||
format!("", address(media))
|
||||
}
|
||||
BlockKind::Video(media) | BlockKind::Embed(media) => {
|
||||
let words = if text().is_empty() {
|
||||
&media.source
|
||||
} else {
|
||||
text()
|
||||
};
|
||||
format!("[{words}]({})", address(media))
|
||||
}
|
||||
BlockKind::Columns(_) => {
|
||||
let filled: Vec<&str> = texts
|
||||
.iter()
|
||||
.map(|text| text.trim())
|
||||
.filter(|text| !text.is_empty())
|
||||
.collect();
|
||||
if filled.is_empty() {
|
||||
return None;
|
||||
}
|
||||
filled.join("\n\n")
|
||||
}
|
||||
};
|
||||
Some(written)
|
||||
}
|
||||
|
||||
/// A list item: the marker, then each further line under its text.
|
||||
fn item(marker: &str, text: &str) -> String {
|
||||
let indent = " ".repeat(marker.len());
|
||||
let mut lines = text.lines();
|
||||
let mut out = format!("{marker}{}", lines.next().unwrap_or_default());
|
||||
for line in lines {
|
||||
out.push('\n');
|
||||
out.push_str(&indent);
|
||||
out.push_str(line);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn quoted(text: &str) -> String {
|
||||
let lines: Vec<String> = text
|
||||
.split('\n')
|
||||
.map(|line| match line {
|
||||
"" => ">".to_string(),
|
||||
line => format!("> {line}"),
|
||||
})
|
||||
.collect();
|
||||
lines.join("\n")
|
||||
}
|
||||
|
||||
/// Code between fences longer than any run of backticks inside it.
|
||||
fn fenced(text: &str, language: &str) -> String {
|
||||
let longest = text
|
||||
.split(|ch| ch != '`')
|
||||
.map(str::len)
|
||||
.max()
|
||||
.unwrap_or_default();
|
||||
let fence = "`".repeat(longest.max(2) + 1);
|
||||
format!("{fence}{language}\n{text}\n{fence}")
|
||||
}
|
||||
|
||||
fn table(texts: &[String], columns: usize) -> String {
|
||||
let row = |cells: &[String]| {
|
||||
let cells: Vec<String> = cells
|
||||
.iter()
|
||||
.map(|cell| pipes(cell).replace('\n', " "))
|
||||
.collect();
|
||||
format!("| {} |", cells.join(" | "))
|
||||
};
|
||||
let mut rows = texts.chunks(columns);
|
||||
let head = rows.next().expect("a table holds its header row");
|
||||
let mut out = vec![row(head), format!("|{}", " --- |".repeat(columns))];
|
||||
out.extend(rows.map(row));
|
||||
out.join("\n")
|
||||
}
|
||||
|
||||
/// A link's address, in angle brackets when it holds a space or a bracket.
|
||||
fn address(media: &Media) -> String {
|
||||
match media.source.contains([' ', '(', ')', '<', '>']) {
|
||||
true => format!("<{}>", media.source.replace('<', "\\<").replace('>', "\\>")),
|
||||
false => media.source.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A caption that cannot end its image early: a bracket left open or shut alone outside code, or a last backslash, is escaped.
|
||||
fn brackets(text: &str) -> String {
|
||||
let code: Vec<Range<usize>> = Parser::new(text)
|
||||
.into_offset_iter()
|
||||
.filter_map(|(event, range)| matches!(event, Event::Code(_)).then_some(range))
|
||||
.collect();
|
||||
let (mut open, mut lone, mut slashes) = (Vec::new(), Vec::new(), 0);
|
||||
for (at, ch) in text.char_indices() {
|
||||
let free = slashes % 2 == 0 && !code.iter().any(|span| span.contains(&at));
|
||||
match ch {
|
||||
'[' if free => open.push(at),
|
||||
']' if free && open.pop().is_none() => lone.push(at),
|
||||
_ => {}
|
||||
}
|
||||
slashes = if ch == '\\' { slashes + 1 } else { 0 };
|
||||
}
|
||||
lone.extend(open);
|
||||
let mut out = String::with_capacity(text.len() + lone.len() + 1);
|
||||
for (at, ch) in text.char_indices() {
|
||||
if lone.contains(&at) {
|
||||
out.push('\\');
|
||||
}
|
||||
out.push(ch);
|
||||
}
|
||||
if slashes % 2 == 1 {
|
||||
out.push('\\');
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// A cell's pipes escaped where they are not already, so none splits its row.
|
||||
fn pipes(cell: &str) -> String {
|
||||
let (mut out, mut slashes) = (String::with_capacity(cell.len()), 0);
|
||||
for ch in cell.chars() {
|
||||
if ch == '|' && slashes % 2 == 0 {
|
||||
out.push('\\');
|
||||
}
|
||||
slashes = if ch == '\\' { slashes + 1 } else { 0 };
|
||||
out.push(ch);
|
||||
}
|
||||
out
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
use std::ops::Range;
|
||||
|
||||
use pulldown_cmark::{Event, Tag};
|
||||
|
||||
use super::{Reader, Toggle};
|
||||
use crate::documents::blocks::{BlockData, BlockKind};
|
||||
|
||||
/// An HTML tag: where it starts and ends, its name, whether it closes, and its attributes' names.
|
||||
struct Mark {
|
||||
at: usize,
|
||||
end: usize,
|
||||
closes: bool,
|
||||
name: String,
|
||||
attributes: Vec<String>,
|
||||
}
|
||||
|
||||
/// The tags in `html`, past comments; a quoted value is read as a value.
|
||||
fn marks(html: &str) -> Vec<Mark> {
|
||||
let bytes = html.as_bytes();
|
||||
let (mut marks, mut at) = (Vec::new(), 0);
|
||||
while let Some(found) = html[at..].find('<').map(|found| at + found) {
|
||||
if html[found..].starts_with("<!--") {
|
||||
at = html[found..]
|
||||
.find("-->")
|
||||
.map_or(html.len(), |end| found + end + 3);
|
||||
continue;
|
||||
}
|
||||
let closes = bytes.get(found + 1) == Some(&b'/');
|
||||
let name_at = found + 1 + usize::from(closes);
|
||||
let name_end = html[name_at..]
|
||||
.find(|ch: char| !(ch.is_ascii_alphanumeric() || ch == '-'))
|
||||
.map_or(html.len(), |end| name_at + end);
|
||||
if !bytes.get(name_at).is_some_and(u8::is_ascii_alphabetic) {
|
||||
at = found + 1;
|
||||
continue;
|
||||
}
|
||||
let (end, attributes) = attributes(html, name_end);
|
||||
marks.push(Mark {
|
||||
at: found,
|
||||
end,
|
||||
closes,
|
||||
name: html[name_at..name_end].to_ascii_lowercase(),
|
||||
attributes,
|
||||
});
|
||||
at = end;
|
||||
}
|
||||
marks
|
||||
}
|
||||
|
||||
/// Reads a tag's attributes from `from` to its `>`: where the tag ends, and the names.
|
||||
fn attributes(html: &str, from: usize) -> (usize, Vec<String>) {
|
||||
let bytes = html.as_bytes();
|
||||
let (mut names, mut word, mut value, mut quote) = (Vec::new(), None, false, None);
|
||||
for (at, byte) in bytes.iter().copied().enumerate().skip(from) {
|
||||
if let Some(open) = quote {
|
||||
if byte == open {
|
||||
(quote, value) = (None, false);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let ends =
|
||||
byte.is_ascii_whitespace() || byte == b'>' || (!value && matches!(byte, b'=' | b'/'));
|
||||
if !ends {
|
||||
match byte {
|
||||
b'"' | b'\'' => quote = Some(byte),
|
||||
_ => word = word.or(Some(at)),
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if let Some(start) = word.take() {
|
||||
if !value {
|
||||
names.push(html[start..at].to_ascii_lowercase());
|
||||
}
|
||||
value = false;
|
||||
}
|
||||
match byte {
|
||||
b'=' => value = true,
|
||||
b'>' => return (at + 1, names),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if let Some(start) = word.filter(|_| !value) {
|
||||
names.push(html[start..].to_ascii_lowercase());
|
||||
}
|
||||
(html.len(), names)
|
||||
}
|
||||
|
||||
fn named<'a>(marks: &'a [Mark], name: &str, closes: bool, from: usize) -> Option<&'a Mark> {
|
||||
marks
|
||||
.iter()
|
||||
.find(|mark| mark.name == name && mark.closes == closes && mark.at >= from)
|
||||
}
|
||||
|
||||
impl Reader<'_> {
|
||||
pub(super) fn html_block(&mut self, range: Range<usize>) {
|
||||
let source = self.source;
|
||||
let block = &source[range.clone()];
|
||||
let marks = marks(block);
|
||||
let first = marks
|
||||
.first()
|
||||
.filter(|mark| block[..mark.at].trim().is_empty() && mark.name == "details");
|
||||
let Some(first) = first.filter(|mark| !mark.closes) else {
|
||||
self.html = true;
|
||||
return self.put(BlockKind::Paragraph, block.trim().to_string());
|
||||
};
|
||||
let summary = named(&marks, "summary", false, first.end);
|
||||
let shut = summary.and_then(|open| named(&marks, "summary", true, open.end));
|
||||
let (title, head) = match (summary, shut) {
|
||||
(Some(open), Some(shut)) => (block[open.end..shut.at].trim(), shut.end),
|
||||
_ => ("", first.end),
|
||||
};
|
||||
let head = range.start + head;
|
||||
self.toggle = Some(Toggle {
|
||||
open: first.attributes.iter().any(|name| name == "open"),
|
||||
title: title.to_string(),
|
||||
body: head,
|
||||
depth: 0,
|
||||
});
|
||||
self.scan_toggle(head, range.end);
|
||||
}
|
||||
|
||||
/// Skips a toggle's body, Markdown kept as written, to the `</details>` that closes it.
|
||||
pub(super) fn in_toggle(&mut self, event: Event, range: Range<usize>) {
|
||||
if let Event::Start(Tag::HtmlBlock) = event {
|
||||
self.scan_toggle(range.start, range.end);
|
||||
}
|
||||
}
|
||||
|
||||
/// Follows `<details>` tags in `from..to`; the close that matches the toggle's open ends it.
|
||||
fn scan_toggle(&mut self, from: usize, to: usize) {
|
||||
let source = self.source;
|
||||
let Some(mut depth) = self.toggle.as_ref().map(|toggle| toggle.depth) else {
|
||||
return;
|
||||
};
|
||||
let mut closed = None;
|
||||
let marks = marks(&source[from..to]);
|
||||
for mark in marks.into_iter().filter(|mark| mark.name == "details") {
|
||||
match (mark.closes, depth) {
|
||||
(false, _) => depth += 1,
|
||||
(true, 0) => {
|
||||
closed = Some(mark);
|
||||
break;
|
||||
}
|
||||
(true, _) => depth -= 1,
|
||||
}
|
||||
}
|
||||
let Some(close) = closed else {
|
||||
if let Some(toggle) = &mut self.toggle {
|
||||
toggle.depth = depth;
|
||||
}
|
||||
return;
|
||||
};
|
||||
self.end_toggle(from + close.at);
|
||||
self.html = true;
|
||||
let rest = source[from + close.end..to].trim();
|
||||
if !rest.is_empty() {
|
||||
self.put(BlockKind::Paragraph, rest.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
fn end_toggle(&mut self, end: usize) {
|
||||
let Some(toggle) = self.toggle.take() else {
|
||||
return;
|
||||
};
|
||||
let body = self.source[toggle.body..end].trim().to_string();
|
||||
self.bodies.push(toggle.body..end);
|
||||
if let Some((_, _, parts)) = &mut self.quote {
|
||||
parts.push(body);
|
||||
return;
|
||||
}
|
||||
self.emit_item();
|
||||
let kind = BlockKind::Toggle(toggle.open);
|
||||
self.out.push(BlockData::new(kind, [toggle.title, body]));
|
||||
}
|
||||
|
||||
/// A toggle the source never closes keeps the rest as its body.
|
||||
pub(super) fn unclosed(&mut self) {
|
||||
if let Some(toggle) = &self.toggle {
|
||||
log::error!(
|
||||
"block editor: the <details> at byte {} never closes; its body runs to the end",
|
||||
toggle.body
|
||||
);
|
||||
self.end_toggle(self.source.len());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,378 @@
|
||||
use std::ops::Range;
|
||||
|
||||
use pulldown_cmark::{CodeBlockKind, Event, Options, Parser, Tag, TagEnd};
|
||||
|
||||
use super::{BlockData, BlockKind, Media};
|
||||
|
||||
mod details;
|
||||
|
||||
impl BlockData {
|
||||
/// Blocks from Markdown, as `to_markdown` writes it: inline Markdown stays as written in each field. A nested list comes out flat, and a quote or callout holds its paragraphs as text.
|
||||
pub fn from_markdown(source: &str) -> Vec<BlockData> {
|
||||
let options = Options::ENABLE_TABLES
|
||||
| Options::ENABLE_STRIKETHROUGH
|
||||
| Options::ENABLE_TASKLISTS
|
||||
| Options::ENABLE_MATH
|
||||
| Options::ENABLE_GFM;
|
||||
let mut reader = Reader {
|
||||
source,
|
||||
..Reader::default()
|
||||
};
|
||||
let mut events = Parser::new_ext(source, options).into_offset_iter();
|
||||
for (event, range) in events.by_ref() {
|
||||
reader.read(event, range);
|
||||
}
|
||||
reader.unclosed();
|
||||
let mut spans: Vec<Range<usize>> = events
|
||||
.reference_definitions()
|
||||
.iter()
|
||||
.map(|(_, definition)| definition.span.clone())
|
||||
.filter(|span| !reader.bodies.iter().any(|body| body.contains(&span.start)))
|
||||
.collect();
|
||||
spans.sort_by_key(|span| span.start);
|
||||
let definitions: Vec<&str> = spans.into_iter().map(|span| source[span].trim()).collect();
|
||||
if !definitions.is_empty() {
|
||||
reader.put(
|
||||
BlockKind::Paragraph,
|
||||
definitions.join(
|
||||
"
|
||||
",
|
||||
),
|
||||
);
|
||||
}
|
||||
log::debug!(
|
||||
"block editor: {} blocks read from markdown",
|
||||
reader.out.len()
|
||||
);
|
||||
reader.out
|
||||
}
|
||||
}
|
||||
|
||||
/// What a paragraph holds alone, which makes it a block of its own.
|
||||
#[derive(Default)]
|
||||
enum Sole {
|
||||
#[default]
|
||||
Nothing,
|
||||
Math(String),
|
||||
Image(String, Option<Range<usize>>),
|
||||
Mixed,
|
||||
}
|
||||
|
||||
/// Inline text being read: its stretch of the source, and how deep in inline tags.
|
||||
#[derive(Default)]
|
||||
struct Leaf {
|
||||
range: Option<Range<usize>>,
|
||||
depth: usize,
|
||||
sole: Sole,
|
||||
}
|
||||
|
||||
struct Item {
|
||||
kind: BlockKind,
|
||||
parts: Vec<String>,
|
||||
emitted: bool,
|
||||
}
|
||||
|
||||
struct Toggle {
|
||||
open: bool,
|
||||
title: String,
|
||||
body: usize,
|
||||
depth: usize,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Reader<'a> {
|
||||
source: &'a str,
|
||||
out: Vec<BlockData>,
|
||||
leaf: Option<Leaf>,
|
||||
/// The outermost quote: callout or not, depth, and its paragraphs.
|
||||
quote: Option<(bool, usize, Vec<String>)>,
|
||||
lists: Vec<bool>,
|
||||
items: Vec<Item>,
|
||||
code: Option<(BlockKind, String)>,
|
||||
/// Columns, cells so far, and the row being read.
|
||||
table: Option<(usize, Vec<String>, Vec<String>)>,
|
||||
toggle: Option<Toggle>,
|
||||
/// Toggle bodies, kept as written.
|
||||
bodies: Vec<Range<usize>>,
|
||||
html: bool,
|
||||
}
|
||||
|
||||
/// Inline lines as one field: each further line loses its indent and quote marks.
|
||||
fn clean(slice: &str, quotes: usize) -> String {
|
||||
let mut lines = slice.split('\n');
|
||||
let mut out = lines.next().unwrap_or_default().to_string();
|
||||
for line in lines {
|
||||
let mut line = line.trim_start();
|
||||
for _ in 0..quotes {
|
||||
line = line.strip_prefix('>').unwrap_or(line).trim_start();
|
||||
}
|
||||
out.push('\n');
|
||||
out.push_str(line);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
impl Reader<'_> {
|
||||
fn read(&mut self, event: Event, range: Range<usize>) {
|
||||
if self.toggle.is_some() {
|
||||
return self.in_toggle(event, range);
|
||||
}
|
||||
if self.html {
|
||||
self.html = !matches!(event, Event::End(TagEnd::HtmlBlock));
|
||||
return;
|
||||
}
|
||||
if let Some((_, code)) = &mut self.code {
|
||||
match event {
|
||||
Event::Text(text) => code.push_str(&text),
|
||||
Event::End(TagEnd::CodeBlock) => self.end_code(),
|
||||
_ => {}
|
||||
}
|
||||
return;
|
||||
}
|
||||
match event {
|
||||
Event::Start(Tag::HtmlBlock) => self.html_block(range),
|
||||
Event::Start(Tag::Paragraph | Tag::Heading { .. } | Tag::TableCell) => {
|
||||
self.flush_leaf();
|
||||
self.leaf = Some(Leaf::default());
|
||||
}
|
||||
Event::End(TagEnd::Paragraph) => self.end_paragraph(),
|
||||
Event::End(TagEnd::Heading(level)) => {
|
||||
let text = self.leaf_text();
|
||||
self.put(BlockKind::Heading(level as u8), text);
|
||||
}
|
||||
Event::Start(Tag::BlockQuote(kind)) => match &mut self.quote {
|
||||
Some((_, depth, _)) => *depth += 1,
|
||||
None => {
|
||||
self.flush_leaf();
|
||||
self.quote = Some((kind.is_some(), 1, Vec::new()));
|
||||
}
|
||||
},
|
||||
Event::End(TagEnd::BlockQuote(_)) => self.end_quote(),
|
||||
Event::Start(Tag::CodeBlock(kind)) => {
|
||||
self.flush_leaf();
|
||||
let fence = match kind {
|
||||
CodeBlockKind::Fenced(language) => language.trim().to_string(),
|
||||
CodeBlockKind::Indented => String::new(),
|
||||
};
|
||||
let kind = match fence.as_str() {
|
||||
"mermaid" => BlockKind::Diagram,
|
||||
"math" => BlockKind::Math,
|
||||
_ => BlockKind::Code,
|
||||
};
|
||||
self.code = Some((kind, String::new()));
|
||||
}
|
||||
Event::Start(Tag::List(start)) => {
|
||||
self.flush_leaf();
|
||||
self.emit_item();
|
||||
self.lists.push(start.is_some());
|
||||
}
|
||||
Event::End(TagEnd::List(_)) => {
|
||||
self.lists.pop();
|
||||
}
|
||||
Event::Start(Tag::Item) => {
|
||||
let numbered = self.lists.last().copied().unwrap_or_default();
|
||||
let kind = match numbered {
|
||||
true => BlockKind::Numbered,
|
||||
false => BlockKind::Bullet,
|
||||
};
|
||||
self.items.push(Item {
|
||||
kind,
|
||||
parts: Vec::new(),
|
||||
emitted: false,
|
||||
});
|
||||
self.leaf = Some(Leaf::default());
|
||||
}
|
||||
Event::TaskListMarker(done) => {
|
||||
if let Some(item) = self.items.last_mut() {
|
||||
item.kind = BlockKind::Todo(done);
|
||||
}
|
||||
}
|
||||
Event::End(TagEnd::Item) => {
|
||||
self.flush_leaf();
|
||||
self.emit_item();
|
||||
self.items.pop();
|
||||
}
|
||||
Event::Rule => self.put(BlockKind::Divider, String::new()),
|
||||
Event::Start(Tag::Table(aligns)) => {
|
||||
self.table = Some((aligns.len(), Vec::new(), Vec::new()));
|
||||
}
|
||||
Event::End(TagEnd::TableCell) => {
|
||||
let text = self.leaf_text().replace("\\|", "|");
|
||||
if let Some((_, _, row)) = &mut self.table {
|
||||
row.push(text);
|
||||
}
|
||||
}
|
||||
Event::End(TagEnd::TableHead | TagEnd::TableRow) => {
|
||||
if let Some((columns, cells, row)) = &mut self.table {
|
||||
row.resize(*columns, String::new());
|
||||
cells.append(row);
|
||||
}
|
||||
}
|
||||
Event::End(TagEnd::Table) => self.end_table(),
|
||||
event => self.inline(event, range),
|
||||
}
|
||||
}
|
||||
|
||||
fn inline(&mut self, event: Event, range: Range<usize>) {
|
||||
let Some(leaf) = &mut self.leaf else {
|
||||
return;
|
||||
};
|
||||
let (start, end) = leaf.range.as_ref().map_or((range.start, range.end), |had| {
|
||||
(had.start.min(range.start), had.end.max(range.end))
|
||||
});
|
||||
leaf.range = Some(start..end);
|
||||
let top = leaf.depth == 0;
|
||||
if let (false, Sole::Image(_, alt)) =
|
||||
(top || matches!(event, Event::End(_)), &mut leaf.sole)
|
||||
{
|
||||
let had = alt.get_or_insert(range.clone());
|
||||
*had = had.start.min(range.start)..had.end.max(range.end);
|
||||
}
|
||||
match event {
|
||||
Event::Start(Tag::Image { dest_url, .. }) if top => {
|
||||
leaf.sole = match leaf.sole {
|
||||
Sole::Nothing => Sole::Image(dest_url.to_string(), None),
|
||||
_ => Sole::Mixed,
|
||||
};
|
||||
leaf.depth += 1;
|
||||
}
|
||||
Event::Start(_) => {
|
||||
if top {
|
||||
leaf.sole = Sole::Mixed;
|
||||
}
|
||||
leaf.depth += 1;
|
||||
}
|
||||
Event::End(_) => leaf.depth = leaf.depth.saturating_sub(1),
|
||||
Event::DisplayMath(math) if top => {
|
||||
leaf.sole = match leaf.sole {
|
||||
Sole::Nothing => Sole::Math(math.to_string()),
|
||||
_ => Sole::Mixed,
|
||||
};
|
||||
}
|
||||
_ if top => leaf.sole = Sole::Mixed,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn quotes(&self) -> usize {
|
||||
self.quote.as_ref().map_or(0, |(_, depth, _)| *depth)
|
||||
}
|
||||
|
||||
fn leaf_text(&mut self) -> String {
|
||||
let (source, quotes) = (self.source, self.quotes());
|
||||
self.leaf
|
||||
.take()
|
||||
.and_then(|leaf| leaf.range)
|
||||
.map(|range| clean(&source[range], quotes))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Text an item read before a block inside it began.
|
||||
fn flush_leaf(&mut self) {
|
||||
if self.leaf.as_ref().is_some_and(|leaf| leaf.range.is_some()) {
|
||||
let text = self.leaf_text();
|
||||
self.put(BlockKind::Paragraph, text);
|
||||
}
|
||||
self.leaf = None;
|
||||
}
|
||||
|
||||
/// Text where it belongs: in the open quote, the open item, or a block of its own.
|
||||
fn put(&mut self, kind: BlockKind, text: String) {
|
||||
if let Some((_, _, parts)) = &mut self.quote {
|
||||
parts.push(text);
|
||||
return;
|
||||
}
|
||||
if kind == BlockKind::Paragraph
|
||||
&& let Some(item) = self.items.last_mut()
|
||||
&& !item.emitted
|
||||
{
|
||||
item.parts.push(text);
|
||||
return;
|
||||
}
|
||||
self.emit_item();
|
||||
let data = match kind {
|
||||
BlockKind::Divider => BlockData::new(kind, Vec::<String>::new()),
|
||||
kind => BlockData::new(kind, [text]),
|
||||
};
|
||||
self.out.push(data);
|
||||
}
|
||||
|
||||
fn emit_item(&mut self) {
|
||||
let quoted = self.quote.is_some();
|
||||
let Some(item) = self.items.last_mut().filter(|item| !item.emitted) else {
|
||||
return;
|
||||
};
|
||||
item.emitted = true;
|
||||
if quoted {
|
||||
return;
|
||||
}
|
||||
let data = BlockData::new(item.kind.clone(), [item.parts.join("\n\n")]);
|
||||
self.out.push(data);
|
||||
}
|
||||
|
||||
fn end_paragraph(&mut self) {
|
||||
let alone = self.quote.is_none() && self.items.is_empty();
|
||||
let sole = self
|
||||
.leaf
|
||||
.as_mut()
|
||||
.map(|leaf| std::mem::take(&mut leaf.sole));
|
||||
match sole {
|
||||
Some(Sole::Math(math)) if alone => {
|
||||
self.leaf = None;
|
||||
self.put(BlockKind::Math, math.trim().to_string());
|
||||
}
|
||||
Some(Sole::Image(source, alt)) if alone => {
|
||||
self.leaf = None;
|
||||
let source_text = self.source;
|
||||
let caption = alt.map_or(String::new(), |alt| source_text[alt].to_string());
|
||||
self.put(BlockKind::Image(Media::new(source)), caption);
|
||||
}
|
||||
_ => {
|
||||
let text = self.leaf_text();
|
||||
self.put(BlockKind::Paragraph, text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn end_quote(&mut self) {
|
||||
let Some((callout, depth, parts)) = &mut self.quote else {
|
||||
return;
|
||||
};
|
||||
*depth -= 1;
|
||||
if *depth > 0 {
|
||||
return;
|
||||
}
|
||||
let kind = match callout {
|
||||
true => BlockKind::Callout,
|
||||
false => BlockKind::Quote,
|
||||
};
|
||||
let text = parts.join("\n\n");
|
||||
self.quote = None;
|
||||
self.put(kind, text);
|
||||
}
|
||||
|
||||
fn end_code(&mut self) {
|
||||
let Some((kind, mut code)) = self.code.take() else {
|
||||
return;
|
||||
};
|
||||
if code.ends_with('\n') {
|
||||
code.pop();
|
||||
}
|
||||
self.put(kind, code);
|
||||
}
|
||||
|
||||
fn end_table(&mut self) {
|
||||
let Some((columns, mut cells, _)) = self.table.take() else {
|
||||
return;
|
||||
};
|
||||
if cells.len() < columns * 2 {
|
||||
cells.resize(columns * 2, String::new());
|
||||
}
|
||||
let kind = BlockKind::Table {
|
||||
columns,
|
||||
merged: Vec::new(),
|
||||
};
|
||||
self.emit_item();
|
||||
self.out.push(BlockData::new(kind, cells));
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,7 @@
|
||||
mod content;
|
||||
mod edit;
|
||||
mod export;
|
||||
mod import;
|
||||
mod keys;
|
||||
mod kind;
|
||||
mod media;
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
use super::{BlockData, BlockKind};
|
||||
use crate::documents::Media;
|
||||
|
||||
fn block(kind: BlockKind, texts: &[&str]) -> BlockData {
|
||||
BlockData::new(kind, texts.iter().copied())
|
||||
}
|
||||
|
||||
fn table(columns: usize, cells: &[&str]) -> BlockData {
|
||||
let kind = BlockKind::Table {
|
||||
columns,
|
||||
merged: Vec::new(),
|
||||
};
|
||||
block(kind, cells)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_kind_markdown_holds_comes_back_as_it_went() {
|
||||
let document = vec![
|
||||
block(BlockKind::Heading(1), &["Project *notes*"]),
|
||||
block(
|
||||
BlockKind::Paragraph,
|
||||
&["Blend **toward white**,\nthen `clamp`."],
|
||||
),
|
||||
block(BlockKind::Bullet, &["Tokens"]),
|
||||
block(BlockKind::Todo(true), &["Light and dark"]),
|
||||
block(BlockKind::Todo(false), &["Motion"]),
|
||||
block(BlockKind::Numbered, &["First"]),
|
||||
block(BlockKind::Numbered, &["Second"]),
|
||||
block(BlockKind::Quote, &["One line\nand the next"]),
|
||||
block(BlockKind::Callout, &["Drag blocks by their handle."]),
|
||||
block(
|
||||
BlockKind::Toggle(true),
|
||||
&["Why gamma?", "It keeps **midtones**."],
|
||||
),
|
||||
block(BlockKind::Toggle(false), &["Closed", ""]),
|
||||
block(BlockKind::Code, &["let fence = \"```\";\nlift(color)"]),
|
||||
block(BlockKind::Diagram, &["graph LR; a --> b"]),
|
||||
block(BlockKind::Math, &["c' = c + (1 - c)\\,t"]),
|
||||
block(BlockKind::Divider, &[]),
|
||||
table(2, &["Token", "Light", "a|b", "0.50"]),
|
||||
block(BlockKind::Image(Media::new("atrium.jpg")), &["An atrium"]),
|
||||
block(BlockKind::Heading(4), &["Deep"]),
|
||||
];
|
||||
let markdown = BlockData::to_markdown(&document);
|
||||
assert_eq!(BlockData::from_markdown(&markdown), document, "{markdown}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lists_stay_tight_and_count_up() {
|
||||
let document = vec![
|
||||
block(BlockKind::Bullet, &["a"]),
|
||||
block(BlockKind::Todo(false), &["b"]),
|
||||
block(BlockKind::Numbered, &["c"]),
|
||||
block(BlockKind::Numbered, &["d\ne"]),
|
||||
block(BlockKind::Paragraph, &[""]),
|
||||
block(BlockKind::Callout, &["Note"]),
|
||||
];
|
||||
assert_eq!(
|
||||
BlockData::to_markdown(&document),
|
||||
"- a\n- [ ] b\n\n1. c\n2. d\n e\n\n> [!NOTE]\n> Note\n"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn markdown_written_by_hand_reads_as_blocks() {
|
||||
let source = "#### Deep\n\n- one\n - inner\n- two\n\n> quoted\nlazy\n\n| a | b |\n|---|---|\n| 1 |\n\nInline $x$ math\n\n<div>raw</div>\n";
|
||||
assert_eq!(
|
||||
BlockData::from_markdown(source),
|
||||
vec![
|
||||
block(BlockKind::Heading(4), &["Deep"]),
|
||||
block(BlockKind::Bullet, &["one"]),
|
||||
block(BlockKind::Bullet, &["inner"]),
|
||||
block(BlockKind::Bullet, &["two"]),
|
||||
block(BlockKind::Quote, &["quoted\nlazy"]),
|
||||
table(2, &["a", "b", "1", ""]),
|
||||
block(BlockKind::Paragraph, &["Inline $x$ math"]),
|
||||
block(BlockKind::Paragraph, &["<div>raw</div>"]),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_markdown_cannot_hold_takes_its_nearest_form() {
|
||||
let document = vec![
|
||||
block(BlockKind::Columns(2), &["Left", "Right"]),
|
||||
block(BlockKind::Video(Media::new("clip.mp4")), &[""]),
|
||||
block(BlockKind::Synced("shared".into()), &["Kept"]),
|
||||
];
|
||||
assert_eq!(
|
||||
BlockData::to_markdown(&document),
|
||||
"Left\n\nRight\n\n[clip.mp4](clip.mp4)\n\nKept\n"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_caption_keeps_its_marks_and_a_toggle_its_nested_details() {
|
||||
let document = vec![
|
||||
block(
|
||||
BlockKind::Image(Media::new("a.png")),
|
||||
&["**bold** and *slant*"],
|
||||
),
|
||||
block(
|
||||
BlockKind::Toggle(false),
|
||||
&[
|
||||
"Outer",
|
||||
"<details><summary>Inner</summary>x</details>\n\nAfter",
|
||||
],
|
||||
),
|
||||
block(BlockKind::Paragraph, &["Last"]),
|
||||
];
|
||||
let markdown = BlockData::to_markdown(&document);
|
||||
assert_eq!(BlockData::from_markdown(&markdown), document, "{markdown}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reference_definitions_stay_with_the_document() {
|
||||
let source = "See [the site][id].\n\n[id]: https://ely.dev \"Ely\"\n";
|
||||
let blocks = BlockData::from_markdown(source);
|
||||
assert_eq!(
|
||||
blocks,
|
||||
vec![
|
||||
block(BlockKind::Paragraph, &["See [the site][id]."]),
|
||||
block(BlockKind::Paragraph, &["[id]: https://ely.dev \"Ely\""]),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
BlockData::from_markdown(&BlockData::to_markdown(&blocks)),
|
||||
blocks
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_toggle_left_open_keeps_the_rest_as_its_body() {
|
||||
assert_eq!(
|
||||
BlockData::from_markdown("<details>\n<summary>Open</summary>\n\nBody\n\nMore\n"),
|
||||
vec![block(BlockKind::Toggle(false), &["Open", "Body\n\nMore"])]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn brackets_and_comments_keep_their_blocks() {
|
||||
let document = vec![
|
||||
block(
|
||||
BlockKind::Image(Media::new("a)b c<d>.png")),
|
||||
&[r"`[x` \[y\] [z]"],
|
||||
),
|
||||
block(
|
||||
BlockKind::Toggle(true),
|
||||
&["Closes", "<!-- </details> -->\n\nkept"],
|
||||
),
|
||||
block(
|
||||
BlockKind::Toggle(true),
|
||||
&["Opens", "<!-- <details> -->\n\nkept"],
|
||||
),
|
||||
block(
|
||||
BlockKind::Toggle(false),
|
||||
&["Quoted", r#"<div title="</details>">kept</div>"#],
|
||||
),
|
||||
block(BlockKind::Paragraph, &["After"]),
|
||||
];
|
||||
let markdown = BlockData::to_markdown(&document);
|
||||
assert_eq!(BlockData::from_markdown(&markdown), document, "{markdown}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bracket_left_alone_comes_back_escaped_in_its_image() {
|
||||
let image = || BlockKind::Image(Media::new("a.png"));
|
||||
let document = vec![block(image(), &[r"a ] b [c\"])];
|
||||
assert_eq!(
|
||||
BlockData::from_markdown(&BlockData::to_markdown(&document)),
|
||||
vec![block(image(), &[r"a \] b \[c\\"])]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn math_with_a_blank_line_and_a_piped_cell_come_back() {
|
||||
let document = vec![
|
||||
block(BlockKind::Math, &["a = b\n\nc = d $$"]),
|
||||
table(2, &["Code", "Note", "`a|b`", "x | y"]),
|
||||
block(BlockKind::Paragraph, &["$$\nx\n$$"]),
|
||||
];
|
||||
let markdown = BlockData::to_markdown(&document);
|
||||
let mut expected = document.clone();
|
||||
expected[2] = block(BlockKind::Math, &["x"]);
|
||||
assert_eq!(BlockData::from_markdown(&markdown), expected, "{markdown}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_line_fields_keep_one_line() {
|
||||
let document = vec![
|
||||
block(BlockKind::Heading(2), &["Two\nlines"]),
|
||||
block(BlockKind::Toggle(true), &["Title\nrest", "Body"]),
|
||||
block(BlockKind::Image(Media::new("a.png")), &["Cap\ntion"]),
|
||||
];
|
||||
assert_eq!(
|
||||
BlockData::from_markdown(&BlockData::to_markdown(&document)),
|
||||
vec![
|
||||
block(BlockKind::Heading(2), &["Two lines"]),
|
||||
block(BlockKind::Toggle(true), &["Title rest", "Body"]),
|
||||
block(BlockKind::Image(Media::new("a.png")), &["Cap tion"]),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn toggle_tags_read_as_html_reads_them() {
|
||||
let document = vec![
|
||||
block(
|
||||
BlockKind::Toggle(true),
|
||||
&[
|
||||
r#"a <span title="</summary>">b</span>"#,
|
||||
"<details-extra>\n\nkept",
|
||||
],
|
||||
),
|
||||
block(BlockKind::Paragraph, &["After"]),
|
||||
];
|
||||
let markdown = BlockData::to_markdown(&document);
|
||||
assert_eq!(BlockData::from_markdown(&markdown), document, "{markdown}");
|
||||
let source = "<DETAILS class=\"x\" OPEN>\n<summary>T</summary>\n\nB\n\n</details>\n";
|
||||
assert_eq!(
|
||||
BlockData::from_markdown(source),
|
||||
vec![block(BlockKind::Toggle(true), &["T", "B"])]
|
||||
);
|
||||
let bare = "<details title=\"open\">\n\nBody\n\n</details>\n";
|
||||
assert_eq!(
|
||||
BlockData::from_markdown(bare),
|
||||
vec![block(BlockKind::Toggle(false), &["", "Body"])]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unquoted_value_runs_to_a_space() {
|
||||
let source = "<details data-url=https://ely.dev/open>\n\nBody\n\n</details>\n";
|
||||
assert_eq!(
|
||||
BlockData::from_markdown(source),
|
||||
vec![block(BlockKind::Toggle(false), &["", "Body"])]
|
||||
);
|
||||
}
|
||||
@@ -3,6 +3,7 @@ use gpui::{AppContext as _, Entity, Focusable, TestAppContext, VisualTestContext
|
||||
use super::{BlockData, BlockEditor, BlockKind};
|
||||
use crate::{forms, theme::Theme};
|
||||
|
||||
mod markdown;
|
||||
mod scroll;
|
||||
mod stale;
|
||||
|
||||
|
||||
Reference in new issue
Block a user