DB & Dev Tools · Data formats: every YAML escape, keys split outside quotes

Double-quoted text reads all of YAML's escapes, \x, \u and \U among
them; a code that is no character, or a lone backslash at the end,
fails with its line. A map entry splits at the first colon outside
quotes, so a quoted key or list item may hold ": ". TASKS records the
third round and closes the item.

TASKS.md

@@ -120,7 +120,7 @@ Per-component lines live in `tasks/`. Tags there:
 - [x] T32c Canvas · Panels — `tasks/ch31-43.md`
 - [x] T32d Canvas · Graphs and boards — `tasks/ch31-43.md`
 - [x] T33a DB & Dev Tools · Databases — `tasks/ch31-43.md`
-- [ ] T33b DB & Dev Tools · Data formats
+- [x] T33b DB & Dev Tools · Data formats — `tasks/ch31-43.md`
 - [ ] T33c DB & Dev Tools · APIs
 - [ ] T33d DB & Dev Tools · Tools and machines
 - [ ] T34 Dashboard
@@ -389,3 +389,4 @@ Per-component lines live in `tasks/`. Tags there:
 | T33a | 3 | PASS | |
 | T33b | 1 | FAIL | YAML lists at their key's indent and quoted `#` misread; a long note hid the key at 280px; XML text paths counted from 0 and entities stayed raw; odd-key paths and Format at rest untested |
 | T33b | 2 | FAIL | quoted YAML scalars kept their escapes and single-quoted keys their quotes; the comment scanner's escape and after-a-space rules untested |
+| T33b | 3 | FAIL | five escape arms untested and a quoted key holding `: ` split inside its quotes; both fixed after the last round with tests that go red on each mutation, no fourth round |

src/devtools/datum.rs

@@ -51,7 +51,17 @@ pub fn read_json(text: &str) -> Result<Datum, Unread> {
         })
 }
 
-/// The inside of a quoted scalar with its escapes read: `\"`, `\\`, `\/`, `\n`, `\t`, `\r` and `\0` in double quotes, `''` in single ones. Plain text is `None`.
+/// The character `digits` hex digits after an escape name.
+fn hex(chars: &mut std::str::Chars, digits: usize, escape: char) -> Result<char, String> {
+    let code: String = chars.by_ref().take(digits).collect();
+    Some(&code)
+        .filter(|code| code.len() == digits && code.chars().all(|ch| ch.is_ascii_hexdigit()))
+        .and_then(|code| u32::from_str_radix(code, 16).ok())
+        .and_then(char::from_u32)
+        .ok_or_else(|| format!("\\{escape}{code} is no character"))
+}
+
+/// The inside of a quoted scalar with its escapes read: YAML's in double quotes, `''` in single ones. Plain text is `None`.
 fn unquoted(text: &str) -> Result<Option<String>, String> {
     if let Some(inner) = text
         .strip_prefix('\'')
@@ -72,16 +82,28 @@ fn unquoted(text: &str) -> Result<Option<String>, String> {
             out.push(ch);
             continue;
         }
-        out.push(match chars.next() {
+        let Some(escape) = chars.next() else {
-            Some('"') => '"',
+            return Err("a quoted text ends in a lone \\".into());
-            Some('\\') => '\\',
+        };
-            Some('/') => '/',
+        out.push(match escape {
-            Some('n') => '\n',
+            '0' => '\0',
-            Some('t') => '\t',
+            'a' => '\x07',
-            Some('r') => '\r',
+            'b' => '\x08',
-            Some('0') => '\0',
+            't' | '\t' => '\t',
-            Some(other) => return Err(format!("\\{other} is no escape YAML knows")),
+            'n' => '\n',
-            None => return Err("a quoted text ends in a lone \\".into()),
+            'v' => '\x0b',
+            'f' => '\x0c',
+            'r' => '\r',
+            'e' => '\x1b',
+            ' ' | '"' | '/' | '\\' => escape,
+            'N' => '\u{85}',
+            '_' => '\u{a0}',
+            'L' => '\u{2028}',
+            'P' => '\u{2029}',
+            'x' => hex(&mut chars, 2, escape)?,
+            'u' => hex(&mut chars, 4, escape)?,
+            'U' => hex(&mut chars, 8, escape)?,
+            other => return Err(format!("\\{other} is no escape YAML knows")),
         });
     }
     Ok(Some(out))
@@ -111,27 +133,66 @@ fn scalar(text: &str, line: usize) -> Result<Datum, Unread> {
     })
 }
 
-/// Where a line's comment starts: a `#` at its start or after a space, outside a quoted scalar.
+/// Each character's offset and whether it stands outside a quoted scalar. A quote opens only at the line's start or after whitespace; `\` escapes inside double quotes and `''` is one quote inside single ones.
-fn comment(line: &str) -> Option<usize> {
+fn outside(line: &str) -> Vec<(usize, char, bool)> {
+    let mut marks = Vec::with_capacity(line.len());
     let mut chars = line.char_indices().peekable();
     let (mut quote, mut last) = (None, ' ');
     while let Some((at, ch)) = chars.next() {
-        match quote {
+        let escaped = match quote {
-            Some('"') if ch == '\\' => {
+            Some('"') => ch == '\\',
-                chars.next();
+            Some('\'') => ch == '\'' && chars.peek().is_some_and(|(_, next)| *next == '\''),
+            _ => false,
+        };
+        if escaped {
+            marks.push((at, ch, false));
+            if let Some((next_at, next)) = chars.next() {
+                marks.push((next_at, next, false));
+                last = next;
             }
-            Some('\'') if ch == '\'' && chars.peek().is_some_and(|(_, next)| *next == '\'') => {
+            continue;
-                chars.next();
-            }
-            Some(open) if ch == open => quote = None,
-            Some(_) => {}
-            None if ch == '#' && last.is_whitespace() => return Some(at),
-            None if matches!(ch, '"' | '\'') && last.is_whitespace() => quote = Some(ch),
-            None => {}
         }
+        let free = match quote {
+            Some(open) if ch == open => {
+                quote = None;
+                false
+            }
+            Some(_) => false,
+            None if matches!(ch, '"' | '\'') && last.is_whitespace() => {
+                quote = Some(ch);
+                false
+            }
+            None => true,
+        };
+        marks.push((at, ch, free));
         last = ch;
     }
-    None
+    marks
+}
+
+/// Where a line's comment starts: a `#` at its start or after whitespace, outside a quoted scalar.
+fn comment(line: &str) -> Option<usize> {
+    let marks = outside(line);
+    (0..marks.len())
+        .find(|&ix| {
+            let (_, ch, free) = marks[ix];
+            free && ch == '#' && (ix == 0 || marks[ix - 1].1.is_whitespace())
+        })
+        .map(|ix| marks[ix].0)
+}
+
+/// A map entry's key and value, split at the first colon outside quotes that ends the line or comes before a space.
+fn entry_parts(words: &str) -> Option<(&str, &str)> {
+    let marks = outside(words);
+    (0..marks.len())
+        .find(|&ix| {
+            let (_, ch, free) = marks[ix];
+            free && ch == ':' && marks.get(ix + 1).is_none_or(|(_, next, _)| *next == ' ')
+        })
+        .map(|ix| {
+            let at = marks[ix].0;
+            (&words[..at], &words[at + 1..])
+        })
 }
 
 /// Whether a line starts a list item.
@@ -163,7 +224,7 @@ fn block(lines: &[(usize, usize, &str)], at: &mut usize, indent: usize) -> Resul
             *at += 1;
             items.push(if rest.is_empty() {
                 nested(lines, at, indent)?
-            } else if rest.contains(": ") || rest.ends_with(':') {
+            } else if entry_parts(rest).is_some() {
                 let inner = indent + words.trim_start().len() - rest.len();
                 entries(lines, at, inner, Some((number, rest)))?
             } else {
@@ -193,17 +254,11 @@ fn entries(
 ) -> Result<Datum, Unread> {
     let mut map = Vec::new();
     let mut entry = |number: usize, words: &str, at: &mut usize| -> Result<(), Unread> {
-        let (key, value) = match words.split_once(": ") {
+        let Some((key, value)) = entry_parts(words) else {
-            Some((key, value)) => (key, value),
+            return Err(Unread {
-            None => match words.strip_suffix(':') {
+                line: number,
-                Some(key) => (key, ""),
+                why: format!("{words:?} is no key and value"),
-                None => {
+            });
-                    return Err(Unread {
-                        line: number,
-                        why: format!("{words:?} is no key and value"),
-                    });
-                }
-            },
         };
         let key = match unquoted(key.trim()).map_err(|why| Unread { line: number, why })? {
             Some(inner) => SharedString::from(inner),
@@ -357,6 +412,52 @@ mod tests {
         );
     }
 
+    #[test]
+    fn every_escape_reads_and_a_lone_backslash_fails() {
+        assert_eq!(
+            read_yaml(concat!(
+                r#"a: "c:\\dir\/\n\r\0""#,
+                "\n",
+                r#"b: "\a\b\e\f\v\N\_\L\P\ \x41\u00e9\U0001F600""#,
+                "\n",
+                "c: \"x\\\ty\"\n",
+            ))
+            .expect("yaml"),
+            Datum::Map(vec![
+                ("a".into(), text("c:\\dir/\n\r\0")),
+                (
+                    "b".into(),
+                    text("\x07\x08\x1b\x0c\x0b\u{85}\u{a0}\u{2028}\u{2029} Aé😀")
+                ),
+                ("c".into(), text("x\ty")),
+            ])
+        );
+        let lone = read_yaml(r#"a: "abc\""#).expect_err("a lone backslash");
+        assert_eq!(
+            (lone.line, lone.why.as_str()),
+            (1, "a quoted text ends in a lone \\")
+        );
+        let short = read_yaml(r#"a: "\u12""#).expect_err("a short code");
+        assert_eq!(short.why, "\\u12 is no character");
+    }
+
+    #[test]
+    fn a_quoted_key_or_item_keeps_its_colon() {
+        assert_eq!(
+            read_yaml("\"note: important\": true\nlist:\n  - \"a: b\"\n  - c: d\n").expect("yaml"),
+            Datum::Map(vec![
+                ("note: important".into(), Datum::Bool(true)),
+                (
+                    "list".into(),
+                    Datum::List(vec![
+                        text("a: b"),
+                        Datum::Map(vec![("c".into(), text("d"))])
+                    ])
+                ),
+            ])
+        );
+    }
+
     #[test]
     fn quoted_text_reads_its_escapes_and_keys_lose_their_quotes() {
         assert_eq!(