Markdown: display math through RaTeX, Mermaid through mermaid-rs-renderer, pictures from the owner's folder

Latex typesets with RaTeX to outlined SVG drawn by gpui's SvgRenderer at the
window's scale, loaded as an asset off the main thread. Diagram draws a
mermaid block in Mermaid's theme for the mode. MarkdownRenderer::base names
the folder pictures load from; data addresses decode, web ones stay unloaded.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

src/documents/diagram.rs

@@ -0,0 +1,175 @@
+use std::{
+    panic::{AssertUnwindSafe, catch_unwind},
+    sync::Arc,
+};
+
+use gpui::{
+    App, Asset, Hsla, IntoElement, ParentElement, Pixels, RenderImage, RenderOnce, SharedString,
+    Size, Styled, Window, div, img, px, size,
+};
+use mermaid_rs_renderer::{LayoutConfig, RenderOptions, Theme, render_strict};
+
+use crate::theme::{ActiveTheme, Radius, TextSize};
+
+/// A Mermaid diagram; a failure shows why and the source.
+#[derive(IntoElement)]
+pub struct Diagram {
+    source: SharedString,
+}
+
+impl Diagram {
+    pub fn new(source: impl Into<SharedString>) -> Self {
+        Self {
+            source: source.into(),
+        }
+    }
+}
+
+/// What a drawn diagram depends on.
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+struct Chart {
+    source: SharedString,
+    dark: bool,
+    /// Background and text, as `#rrggbb`.
+    ground: SharedString,
+    ink: SharedString,
+    /// Window scale, in hundredths.
+    scale: u32,
+}
+
+type Drawn = Result<(Arc<RenderImage>, Size<Pixels>), SharedString>;
+
+enum Drawer {}
+
+impl Asset for Drawer {
+    type Source = Chart;
+    type Output = Drawn;
+
+    fn load(chart: Chart, cx: &mut App) -> impl Future<Output = Drawn> + Send + 'static {
+        let renderer = cx.svg_renderer();
+        async move {
+            let svg = svg(&chart)?;
+            let tree = usvg::Tree::from_data(svg.as_bytes(), &usvg::Options::default())
+                .map_err(|error| SharedString::from(format!("{error}")))?;
+            let shape = size(px(tree.size().width()), px(tree.size().height()));
+            let picture = renderer
+                .render_single_frame(svg.as_bytes(), chart.scale as f32 / 100.0)
+                .map_err(|error| SharedString::from(format!("{error}")))?;
+            Ok((picture, shape))
+        }
+    }
+}
+
+/// The diagram as SVG in Mermaid's theme for the mode.
+fn svg(chart: &Chart) -> Result<String, SharedString> {
+    let mut theme = match chart.dark {
+        true => Theme::dark(),
+        false => Theme::modern(),
+    };
+    theme.background = chart.ground.to_string();
+    theme.edge_label_background = chart.ground.to_string();
+    theme.text_color = chart.ink.to_string();
+    let options = RenderOptions {
+        theme,
+        layout: LayoutConfig::default(),
+    };
+    // The renderer may panic on input it parsed.
+    let drawn = catch_unwind(AssertUnwindSafe(|| render_strict(&chart.source, options)));
+    match drawn {
+        Ok(Ok(svg)) => Ok(svg),
+        Ok(Err(error)) => Err(error.to_string().into()),
+        Err(_) => {
+            log::error!("diagram: the renderer failed on a parsed diagram");
+            Err("the Mermaid renderer could not draw this diagram".into())
+        }
+    }
+}
+
+/// `#rrggbb` of a theme color.
+fn hex(color: Hsla) -> SharedString {
+    let rgb = color.to_rgb();
+    let channel = |value: f32| (value.clamp(0.0, 1.0) * 255.0).round() as u8;
+    format!(
+        "#{:02x}{:02x}{:02x}",
+        channel(rgb.r),
+        channel(rgb.g),
+        channel(rgb.b)
+    )
+    .into()
+}
+
+impl RenderOnce for Diagram {
+    fn render(self, window: &mut Window, cx: &mut App) -> impl IntoElement {
+        let theme = cx.theme();
+        let colors = theme.colors.clone();
+        let chart = Chart {
+            source: self.source.clone(),
+            dark: theme.is_dark(),
+            ground: hex(colors.surface),
+            ink: hex(colors.fg),
+            scale: (window.scale_factor() * 100.0).round() as u32,
+        };
+        let (radius, small, mono) = (
+            theme.radius(Radius::Md),
+            theme.text_size(TextSize::Sm),
+            theme.mono_family.clone(),
+        );
+        match window.use_asset::<Drawer>(&chart, cx) {
+            Some(Ok((picture, shape))) => div()
+                .flex()
+                .justify_center()
+                .child(img(picture).w(shape.width).h(shape.height).max_w_full())
+                .into_any_element(),
+            Some(Err(why)) => div()
+                .flex()
+                .flex_col()
+                .gap_1()
+                .p_3()
+                .rounded(radius)
+                .bg(colors.sunken)
+                .text_size(small)
+                .child(div().text_color(colors.danger).child(why))
+                .child(
+                    div()
+                        .font_family(mono)
+                        .text_color(colors.fg_muted)
+                        .child(self.source),
+                )
+                .into_any_element(),
+            None => div().h_12().into_any_element(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn chart(source: &str, dark: bool) -> Chart {
+        Chart {
+            source: source.to_string().into(),
+            dark,
+            ground: "#2c2c2c".into(),
+            ink: "#eeeeee".into(),
+            scale: 100,
+        }
+    }
+
+    #[test]
+    fn a_flowchart_draws_with_the_ground_and_ink() {
+        let svg = svg(&chart("flowchart LR\n  A --> B", true)).expect("draws");
+        assert!(svg.contains("<svg"), "{svg}");
+        assert!(svg.to_lowercase().contains("#2c2c2c"), "the ground: {svg}");
+    }
+
+    #[test]
+    fn a_broken_diagram_says_why() {
+        let why = svg(&chart("flowchart LR\n  A -->", false)).expect_err("an error");
+        assert!(!why.is_empty());
+    }
+
+    #[test]
+    fn colors_write_as_hex() {
+        assert_eq!(hex(gpui::rgb(0x2c2c2c).into()), "#2c2c2c");
+    }
+}

src/documents/mod.rs

@@ -1,5 +1,6 @@
 mod blocks;
 mod counts;
+mod diagram;
 mod epub;
 mod find;
 mod format;
@@ -9,6 +10,7 @@ mod link;
 mod markdown;
 mod modes;
 mod pages;
+mod picture;
 mod print;
 mod reading;
 mod render;
@@ -28,6 +30,7 @@ use gpui::App;
 pub(crate) use blocks::source;
 pub use blocks::{Align, BlockData, BlockEditor, BlockEvent, BlockKind, Media};
 pub use counts::{ReadingTime, WordCount};
+pub use diagram::Diagram;
 pub use epub::{Chapter, EpubReader};
 pub use format::{Format, format};
 pub(crate) use history::Change;

src/documents/picture.rs

@@ -0,0 +1,272 @@
+use std::{
+    path::{Path, PathBuf},
+    sync::Arc,
+};
+
+use base64::Engine as _;
+use gpui::{
+    AnyElement, App, Asset, Hsla, Image, ImageAssetLoader, ImageFormat, InteractiveElement,
+    IntoElement, ParentElement, RenderImage, RenderOnce, Resource, SMOOTH_SVG_SCALE_FACTOR,
+    SharedString, Styled, Window, div, img, px,
+};
+
+use crate::{
+    i18n,
+    primitives::{Icon, IconName},
+    theme::{ActiveTheme, IconSize, Radius},
+};
+
+/// Where a picture's bytes come from.
+#[derive(Clone, Debug, PartialEq)]
+pub(crate) enum Origin {
+    File(PathBuf),
+    /// Bytes inline in the address, and their type.
+    Data(ImageFormat, Arc<[u8]>),
+    Web,
+    /// A relative path with no folder to read it in.
+    Loose,
+    /// An address that names no picture, and why.
+    Bad(SharedString),
+}
+
+/// `%XX` decoded; a broken escape stays as written.
+fn decoded(text: &str) -> Vec<u8> {
+    let bytes = text.as_bytes();
+    let (mut out, mut at) = (Vec::with_capacity(bytes.len()), 0);
+    while at < bytes.len() {
+        let pair = bytes.get(at + 1..at + 3);
+        let hex =
+            pair.and_then(|pair| u8::from_str_radix(std::str::from_utf8(pair).ok()?, 16).ok());
+        match (bytes[at], hex) {
+            (b'%', Some(byte)) => (out.push(byte), at += 3),
+            (byte, _) => (out.push(byte), at += 1),
+        };
+    }
+    out
+}
+
+/// A path written in a link, `%XX` decoded.
+fn path(written: &str) -> Result<PathBuf, SharedString> {
+    String::from_utf8(decoded(written))
+        .map(PathBuf::from)
+        .map_err(|_| "the path is not UTF-8".into())
+}
+
+/// Where `url` points, read from the folder `base`.
+pub(crate) fn origin(url: &str, base: Option<&Path>) -> Origin {
+    let url = url.trim();
+    let lower = url.to_ascii_lowercase();
+    if lower.starts_with("http://") || lower.starts_with("https://") || url.starts_with("//") {
+        return Origin::Web;
+    }
+    if lower.starts_with("data:") {
+        return inline(url);
+    }
+    if let Some(rest) = url.strip_prefix("file://") {
+        let rest = rest.strip_prefix("localhost").unwrap_or(rest);
+        // Windows writes file:///C:/…
+        let rest = match rest.as_bytes() {
+            [b'/', _, b':', ..] if cfg!(windows) => &rest[1..],
+            _ => rest,
+        };
+        return path(rest).map_or_else(Origin::Bad, Origin::File);
+    }
+    let scheme = url.split_once(':').is_some_and(|(scheme, _)| {
+        scheme.len() > 1
+            && scheme
+                .chars()
+                .all(|ch| ch.is_ascii_alphanumeric() || "+-.".contains(ch))
+    });
+    if scheme {
+        return Origin::Bad(format!("{url} is no picture address").into());
+    }
+    let written = url.split(['?', '#']).next().unwrap_or_default();
+    match path(written) {
+        Ok(path) if path.is_absolute() => Origin::File(path),
+        Ok(path) => base.map_or(Origin::Loose, |base| Origin::File(base.join(path))),
+        Err(why) => Origin::Bad(why),
+    }
+}
+
+/// A `data:` address: base64 bytes of an image type.
+fn inline(url: &str) -> Origin {
+    let Some((head, payload)) = url[5..].split_once(',') else {
+        return Origin::Bad("the data address has no comma".into());
+    };
+    let Some(mime) = head.strip_suffix(";base64") else {
+        return Origin::Bad("only base64 data addresses draw".into());
+    };
+    let Some(format) = ImageFormat::from_mime_type(&mime.to_ascii_lowercase()) else {
+        return Origin::Bad(format!("{mime} is no image type").into());
+    };
+    match base64::engine::general_purpose::STANDARD.decode(payload.trim()) {
+        Ok(bytes) => Origin::Data(format, bytes.into()),
+        Err(error) => Origin::Bad(format!("{error}").into()),
+    }
+}
+
+/// Inline bytes decoded as gpui decodes files.
+enum Unpacker {}
+
+impl Asset for Unpacker {
+    type Source = (ImageFormat, Arc<[u8]>);
+    type Output = Result<Arc<RenderImage>, SharedString>;
+
+    fn load(
+        (format, bytes): Self::Source,
+        cx: &mut App,
+    ) -> impl Future<Output = Self::Output> + Send + 'static {
+        let renderer = cx.svg_renderer();
+        async move {
+            Image::from_bytes(format, bytes.to_vec())
+                .to_image_data(renderer)
+                .map_err(|error| format!("{error:#}").into())
+        }
+    }
+}
+
+/// The picture at `url`, natural size up to the column.
+#[derive(IntoElement)]
+pub(crate) struct Picture {
+    pub alt: String,
+    pub url: String,
+    /// The folder relative addresses start from.
+    pub base: Option<Arc<Path>>,
+}
+
+impl RenderOnce for Picture {
+    fn render(self, window: &mut Window, cx: &mut App) -> impl IntoElement {
+        picture(&self.alt, &self.url, self.base.as_deref(), window, cx)
+    }
+}
+
+fn picture(
+    alt: &str,
+    url: &str,
+    base: Option<&Path>,
+    window: &mut Window,
+    cx: &mut App,
+) -> AnyElement {
+    let colors = cx.theme().colors.clone();
+    let (loaded, svg) = match origin(url, base) {
+        Origin::File(path) => {
+            let svg = path
+                .extension()
+                .is_some_and(|ext| ext.eq_ignore_ascii_case("svg"));
+            let resource = Resource::Path(path.into());
+            let loaded = window
+                .use_asset::<ImageAssetLoader>(&resource, cx)
+                .map(|loaded| loaded.map_err(|error| SharedString::from(error.to_string())));
+            (loaded, svg)
+        }
+        Origin::Data(format, bytes) => {
+            let svg = format == ImageFormat::Svg;
+            (window.use_asset::<Unpacker>(&(format, bytes), cx), svg)
+        }
+        Origin::Web => {
+            let said = i18n::text(cx, "markdown.picture.web", &[]);
+            return note(alt, said, colors.fg_subtle, cx);
+        }
+        Origin::Loose => {
+            let said = i18n::text(cx, "markdown.picture.folder", &[]);
+            return note(alt, said, colors.fg_subtle, cx);
+        }
+        Origin::Bad(why) => return note(alt, why, colors.danger, cx),
+    };
+    match loaded {
+        Some(Ok(image)) => {
+            let natural = image.size(0);
+            // The picture's own height follows its ratio.
+            let smooth = if svg { SMOOTH_SVG_SCALE_FACTOR } else { 1.0 };
+            let width = i32::from(natural.width) as f32 / smooth;
+            div()
+                .debug_selector(|| "markdown-picture".into())
+                .w_full()
+                .max_w(px(width))
+                .child(img(image).size_full())
+                .into_any_element()
+        }
+        Some(Err(why)) => {
+            log::warn!("markdown: a picture did not load: {why}");
+            note(alt, why, colors.danger, cx)
+        }
+        None => note(alt, url.to_string().into(), colors.fg_subtle, cx),
+    }
+}
+
+/// A picture not drawn: its alt text and why.
+fn note(alt: &str, said: SharedString, tone: Hsla, cx: &App) -> AnyElement {
+    let theme = cx.theme();
+    let colors = &theme.colors;
+    div()
+        .flex()
+        .items_center()
+        .gap_2()
+        .p_3()
+        .rounded(theme.radius(Radius::Md))
+        .border_1()
+        .border_color(colors.border)
+        .text_color(colors.fg_muted)
+        .child(
+            Icon::new(IconName::Image)
+                .size(IconSize::Sm)
+                .color(colors.fg_subtle),
+        )
+        .child(div().flex_1().child(alt.to_string()))
+        .child(div().text_color(tone).child(said))
+        .into_any_element()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn addresses_resolve_against_the_folder() {
+        let base = Path::new("/notes");
+        assert_eq!(
+            origin("assets/a%20b.png?x#y", Some(base)),
+            Origin::File(base.join("assets/a b.png"))
+        );
+        assert_eq!(origin("assets/a.png", None), Origin::Loose);
+        assert_eq!(origin("https://e.x/a.png", Some(base)), Origin::Web);
+        assert_eq!(origin("//e.x/a.png", Some(base)), Origin::Web);
+        assert!(matches!(origin("mailto:a@b", Some(base)), Origin::Bad(_)));
+        let absolute = std::env::current_dir().expect("a dir").join("a.png");
+        assert_eq!(
+            origin(&absolute.display().to_string(), Some(base)),
+            Origin::File(absolute)
+        );
+    }
+
+    #[test]
+    fn file_addresses_decode() {
+        assert_eq!(
+            origin("file:///tmp/a%23b.png", None),
+            Origin::File(PathBuf::from("/tmp/a#b.png"))
+        );
+        if cfg!(windows) {
+            assert_eq!(
+                origin("file:///C:/notes/a.png", None),
+                Origin::File(PathBuf::from("C:/notes/a.png"))
+            );
+        }
+    }
+
+    #[test]
+    fn data_addresses_carry_their_bytes() {
+        assert_eq!(
+            origin("data:image/png;base64,cG5n", None),
+            Origin::Data(ImageFormat::Png, Arc::from(&b"png"[..]))
+        );
+        assert!(matches!(origin("data:image/png,raw", None), Origin::Bad(_)));
+        assert!(matches!(
+            origin("data:text/plain;base64,eA==", None),
+            Origin::Bad(_)
+        ));
+        assert!(matches!(
+            origin("data:image/png;base64,***", None),
+            Origin::Bad(_)
+        ));
+    }
+}

src/documents/render.rs

@@ -1,4 +1,4 @@
-use std::rc::Rc;
+use std::{path::Path, rc::Rc, sync::Arc};
 
 use gpui::{
     AnyElement, App, ElementId, Entity, FontStyle, FontWeight, HighlightStyle, InteractiveText,
@@ -7,7 +7,11 @@ use gpui::{
 };
 use markdown::mdast::AlignKind;
 
-use super::tree::{Inline, Node, Style, parse};
+use super::{
+    diagram::Diagram,
+    picture::Picture,
+    tree::{Inline, Node, Style, parse},
+};
 use crate::{
     editor::code_colors,
     navigation::{Anchor, Sections},
@@ -28,6 +32,7 @@ pub struct MarkdownRenderer {
     size: TextSize,
     sections: Option<Entity<Sections>>,
     code: Option<DrawCode>,
+    base: Option<Arc<Path>>,
 }
 
 impl MarkdownRenderer {
@@ -39,9 +44,16 @@ impl MarkdownRenderer {
             size: TextSize::Base,
             sections: None,
             code: None,
+            base: None,
         }
     }
 
+    /// The folder relative picture addresses start from.
+    pub fn base(mut self, folder: impl Into<Arc<Path>>) -> Self {
+        self.base = Some(folder.into());
+        self
+    }
+
     /// The body's size; headings keep theirs.
     pub fn text_size(mut self, size: TextSize) -> Self {
         self.size = size;
@@ -78,6 +90,7 @@ struct Draw<'a> {
     sections: &'a Option<Entity<Sections>>,
     headings: std::cell::Cell<usize>,
     code: &'a Option<DrawCode>,
+    base: &'a Option<Arc<Path>>,
 }
 
 impl Draw<'_> {
@@ -200,6 +213,9 @@ impl Draw<'_> {
                 .children(self.nodes(inner, cx))
                 .into_any_element(),
             Node::Code { language, text } => {
+                if language.as_deref() == Some("mermaid") {
+                    return Diagram::new(text.clone()).into_any_element();
+                }
                 if let Some(draw) = self.code {
                     return draw(
                         self.key("code"),
@@ -336,7 +352,7 @@ impl Draw<'_> {
                 .justify_center()
                 .py_2()
                 .child(match Latex::parse(tex) {
-                    Ok(math) => math.into_any_element(),
+                    Ok(math) => math.display().into_any_element(),
                     Err(error) => div()
                         .text_color(colors.danger)
                         .child(error.to_string())
@@ -344,23 +360,11 @@ impl Draw<'_> {
                 })
                 .into_any_element(),
             Node::Image { alt, url, link } => {
-                let picture = div()
-                    .id(self.key("image"))
-                    .flex()
-                    .items_center()
-                    .gap_2()
-                    .p_3()
-                    .rounded(theme.radius(Radius::Md))
-                    .border_1()
-                    .border_color(colors.border)
-                    .text_color(colors.fg_muted)
-                    .child(
-                        Icon::new(IconName::Image)
-                            .size(IconSize::Sm)
-                            .color(colors.fg_subtle),
-                    )
-                    .child(div().flex_1().child(alt.clone()))
-                    .child(div().text_color(colors.fg_subtle).child(url.clone()));
+                let picture = div().id(self.key("image")).child(Picture {
+                    alt: alt.clone(),
+                    url: url.clone(),
+                    base: self.base.clone(),
+                });
                 let Some(link) = link.clone() else {
                     return picture.into_any_element();
                 };
@@ -443,6 +447,7 @@ impl RenderOnce for MarkdownRenderer {
             sections: &self.sections,
             headings: std::cell::Cell::new(0),
             code: &self.code,
+            base: &self.base,
         };
         div()
             .flex()

src/documents/tests.rs

@@ -301,3 +301,60 @@ fn a_selected_version_past_the_history_selects_none(cx: &mut TestAppContext) {
     });
     settle(cx);
 }
+
+/// Markdown drawn `width` points wide, its pictures read in `base`.
+struct Page {
+    source: &'static str,
+    base: std::path::PathBuf,
+    width: f32,
+}
+
+impl Render for Page {
+    fn render(&mut self, _: &mut Window, _: &mut Context<Self>) -> impl IntoElement {
+        div()
+            .w(px(self.width))
+            .child(super::MarkdownRenderer::new("page", self.source).base(self.base.as_path()))
+    }
+}
+
+/// A 200×100 picture drawn in a `width` column.
+fn drawn_picture(width: f32, cx: &mut TestAppContext) -> gpui::Size<Pixels> {
+    let folder = std::env::temp_dir().join(format!("ely-picture-{}-{width}", std::process::id()));
+    std::fs::create_dir_all(folder.join("assets")).expect("a folder");
+    let mut png = Vec::new();
+    image::RgbaImage::new(200, 100)
+        .write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png)
+        .expect("a png");
+    std::fs::write(folder.join("assets/wide shot.png"), png).expect("the picture");
+    cx.update(Theme::init);
+    let (_, cx) = cx.add_window_view(|_, _| Page {
+        source: "Before
+
+![shot](assets/wide%20shot.png)
+",
+        base: folder.clone(),
+        width,
+    });
+    let mut drawn = None;
+    // The decode runs on the background executor.
+    for _ in 0..50 {
+        settle(cx);
+        drawn = cx.debug_bounds("markdown-picture");
+        if drawn.is_some() {
+            break;
+        }
+    }
+    std::fs::remove_dir_all(&folder).expect("cleaned");
+    drawn.expect("the picture drew").size
+}
+
+#[gpui::test]
+fn a_picture_beside_the_file_draws_at_its_ratio_within_the_column(cx: &mut TestAppContext) {
+    let shrunk = drawn_picture(120.0, cx);
+    assert_eq!(shrunk, size(px(120.0), px(60.0)), "shrunk at 2:1");
+}
+
+#[gpui::test]
+fn a_picture_narrower_than_the_column_keeps_its_size(cx: &mut TestAppContext) {
+    assert_eq!(drawn_picture(400.0, cx), size(px(200.0), px(100.0)));
+}