oxedyne/fe2o3/fe2o3_text/tests/djot_write.rs
13.4 KiB, 29 runs
created by r1870400018:72233, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! The Djot writer's invariant, `parse(&render(&d)) == d`, over generated trees and the escaping |
| 2 | //! cases one by one. |
| 3 | |
| 4 | use oxedyne_fe2o3_text::doc::{ |
| 5 | Align, |
| 6 | Attrs, |
| 7 | Block, |
| 8 | Cell, |
| 9 | Doc, |
| 10 | Inline, |
| 11 | Row, |
| 12 | Term, |
| 13 | djot, |
| 14 | }; |
| 15 | |
| 16 | use oxedyne_fe2o3_core::prelude::*; |
| 17 | |
| 18 | /// A small deterministic generator, so a failure names its seed and reproduces. |
| 19 | struct Gen(u64); |
| 20 | |
| 21 | impl Gen { |
| 22 | fn next(&mut self) -> u64 { |
| 23 | self.0 ^= self.0 << 13; |
| 24 | self.0 ^= self.0 >> 7; |
| 25 | self.0 ^= self.0 << 17; |
| 26 | self.0 |
| 27 | } |
| 28 | fn below(&mut self, n: usize) -> usize { |
| 29 | (self.next() % n as u64) as usize |
| 30 | } |
| 31 | fn pick<'a>(&mut self, xs: &[&'a str]) -> &'a str { |
| 32 | xs[self.below(xs.len())] |
| 33 | } |
| 34 | fn chance(&mut self, n: usize) -> bool { |
| 35 | self.below(n) == 0 |
| 36 | } |
| 37 | } |
| 38 | |
| 39 | // Every Djot metacharacter, a leading block opener of each kind, and plain words. |
| 40 | const WORDS: &[&str] = &[ |
| 41 | "word", "café", "naïve", "x", "*", "_", "[", "]", "{", "}", "\\", "<", ">", "#", "-", "+", |
| 42 | "1.", "2)", "12.", "|", "^", "~", "!", ":::", ":", "`", "``", "a_b", "*star*", "{_", "_}", |
| 43 | "(", ")", "&", "$", "%", "\"", "'", "=", "---", "***", ">x", "#x", "^ cap", "!", |
| 44 | ]; |
| 45 | const NAMES: &[&str] = &["note", "warn", "a1", "x-y", "big_one", "c"]; |
| 46 | const VALUES: &[&str] = &["v", "two words", "1/2", "a}b", "", "x-y.z", "{q}"]; |
| 47 | const DESTS: &[&str] = &[ |
| 48 | "https://example.com/a", "page.html", "a b.png", "", "x(1).png", "q\\r", "w[1]", "t`t", "p|q", |
| 49 | "<x>", "a)b", |
| 50 | ]; |
| 51 | const CODES: &[&str] = &["x", "a|b", "`", "a``b", " lead", "trail ", " both ", " ", "*_[{"]; |
| 52 | const LINES: &[&str] = &["fn main() {}", "", " indented", "```", "~~~ x", "a | b", "\t tab", "> q"]; |
| 53 | |
| 54 | /// One text run, never empty, and trimmed where `trim` says the run stands at an edge. |
| 55 | fn text(g: &mut Gen) -> String { |
| 56 | let n = 1 + g.below(3); |
| 57 | let mut s = String::new(); |
| 58 | for i in 0..n { |
| 59 | if i > 0 && !g.chance(3) { |
| 60 | s.push(' '); |
| 61 | } |
| 62 | s.push_str(g.pick(WORDS)); |
| 63 | } |
| 64 | s |
| 65 | } |
| 66 | |
| 67 | fn attrs(g: &mut Gen) -> Attrs { |
| 68 | let mut a = Attrs::default(); |
| 69 | if g.chance(2) { |
| 70 | a.id = Some(g.pick(NAMES).to_string()); |
| 71 | } |
| 72 | for _ in 0..g.below(3) { |
| 73 | a.classes.push(g.pick(NAMES).to_string()); |
| 74 | } |
| 75 | for _ in 0..g.below(3) { |
| 76 | a.pairs.push((g.pick(NAMES).to_string(), g.pick(VALUES).to_string())); |
| 77 | } |
| 78 | a |
| 79 | } |
| 80 | |
| 81 | /// A run of inlines: `brk` allows hard breaks, which a cell and a caption cannot hold. |
| 82 | fn inlines(g: &mut Gen, depth: usize, brk: bool) -> Vec<Inline> { |
| 83 | let n = 1 + g.below(4); |
| 84 | let mut out: Vec<Inline> = Vec::new(); |
| 85 | for _ in 0..n { |
| 86 | let k = if depth == 0 { 0 } else { g.below(11) }; |
| 87 | let item = match k { |
| 88 | 1 => Inline::Emph { strong: g.chance(2), content: inlines(g, depth - 1, false) }, |
| 89 | 2 => Inline::Link { to: g.pick(DESTS).to_string(), content: inlines(g, depth - 1, false) }, |
| 90 | 3 => Inline::Image { |
| 91 | src: g.pick(DESTS).to_string(), |
| 92 | alt: if g.chance(4) { String::new() } else { text(g) }, |
| 93 | attrs: if g.chance(2) { Attrs::default() } else { attrs(g) }, |
| 94 | }, |
| 95 | 4 => Inline::Code(g.pick(CODES).to_string()), |
| 96 | 5 => Inline::Span { attrs: attrs(g), content: inlines(g, depth - 1, false) }, |
| 97 | 6 if brk => Inline::Break, |
| 98 | 7 => Inline::Sup(inlines(g, depth - 1, false)), |
| 99 | 8 => Inline::Sub(inlines(g, depth - 1, false)), |
| 100 | 9 => Inline::Emph { strong: true, content: inlines(g, depth - 1, false) }, |
| 101 | _ => Inline::Text(text(g)), |
| 102 | }; |
| 103 | // Two code spans side by side are one run of backticks in Djot, so no reader makes them. |
| 104 | let item = match (out.last(), item) { |
| 105 | (Some(Inline::Code(_)), Inline::Code(_)) => Inline::Text("x".into()), |
| 106 | (_, item) => item, |
| 107 | }; |
| 108 | // Text merges with text beside it, as the reader would merge it. |
| 109 | match (out.last_mut(), &item) { |
| 110 | (Some(Inline::Text(a)), Inline::Text(b)) => a.push_str(b), |
| 111 | _ => out.push(item), |
| 112 | } |
| 113 | } |
| 114 | out |
| 115 | } |
| 116 | |
| 117 | /// A run fit to stand as a whole block: no whitespace at either end, and no hard break last, nor a |
| 118 | /// break followed by whitespace. |
| 119 | fn edge(mut v: Vec<Inline>) -> Vec<Inline> { |
| 120 | let mut out: Vec<Inline> = Vec::new(); |
| 121 | for (i, item) in v.drain(..).enumerate() { |
| 122 | match item { |
| 123 | Inline::Text(t) => { |
| 124 | let after_break = matches!(out.last(), Some(Inline::Break)); |
| 125 | let t = if i == 0 || after_break { t.trim_start().to_string() } else { t }; |
| 126 | if !t.is_empty() { |
| 127 | out.push(Inline::Text(t)); |
| 128 | } |
| 129 | } |
| 130 | other => out.push(other), |
| 131 | } |
| 132 | } |
| 133 | while matches!(out.last(), Some(Inline::Break)) { |
| 134 | out.pop(); |
| 135 | } |
| 136 | if let Some(Inline::Text(t)) = out.last_mut() { |
| 137 | let n = t.trim_end().len(); |
| 138 | t.truncate(n); |
| 139 | if t.is_empty() { |
| 140 | out.pop(); |
| 141 | } |
| 142 | } |
| 143 | if out.is_empty() { |
| 144 | out.push(Inline::Text("w".to_string())); |
| 145 | } |
| 146 | out |
| 147 | } |
| 148 | |
| 149 | /// A row of cells, each a header where `header` says, or now and then the other way. |
| 150 | fn cells(g: &mut Gen, n: usize, depth: usize, header: bool) -> Row { |
| 151 | Row((0..n).map(|_| { |
| 152 | let content = match g.chance(5) { |
| 153 | true => Vec::new(), |
| 154 | false => edge(inlines(g, depth, false)), |
| 155 | }; |
| 156 | Cell { content, header: header != g.chance(6) } |
| 157 | }).collect()) |
| 158 | } |
| 159 | |
| 160 | fn blocks(g: &mut Gen, depth: usize) -> Vec<Block> { |
| 161 | (0..1 + g.below(3)).map(|_| block(g, depth)).collect() |
| 162 | } |
| 163 | |
| 164 | fn block(g: &mut Gen, depth: usize) -> Block { |
| 165 | let k = if depth == 0 { g.below(4) } else { g.below(11) }; |
| 166 | let d = depth.saturating_sub(1); |
| 167 | match k { |
| 168 | 0 => Block::Para(edge(inlines(g, depth.min(3), true))), |
| 169 | 1 => Block::Heading { level: 1 + g.below(6) as u8, content: edge(inlines(g, depth.min(3), true)) }, |
| 170 | 2 => Block::Rule, |
| 171 | 3 => { |
| 172 | let mut text = String::new(); |
| 173 | for _ in 0..g.below(4) { |
| 174 | text.push_str(g.pick(LINES)); |
| 175 | text.push('\n'); |
| 176 | } |
| 177 | Block::Code { lang: if g.chance(2) { Some("rust".to_string()) } else { None }, text } |
| 178 | } |
| 179 | 4 => Block::List { |
| 180 | ordered: g.chance(2), |
| 181 | items: (0..1 + g.below(3)).map(|_| match g.chance(6) { |
| 182 | true => Vec::new(), |
| 183 | false => blocks(g, d), |
| 184 | }).collect(), |
| 185 | }, |
| 186 | 5 => Block::Quote(blocks(g, d)), |
| 187 | 6 => { |
| 188 | let n = 1 + g.below(3); |
| 189 | let cols = (0..n).map(|_| [Align::None, Align::Start, Align::Centre, Align::End][g.below(4)]).collect(); |
| 190 | let head = if g.chance(3) { None } else { Some(cells(g, n, d.min(2), true)) }; |
| 191 | let mut rows: Vec<Row> = (0..g.below(4)).map(|_| cells(g, n, d.min(2), false)).collect(); |
| 192 | // The page case: the first cell of every body row heads its row. |
| 193 | if g.chance(4) { |
| 194 | for r in rows.iter_mut() { |
| 195 | for (i, c) in r.0.iter_mut().enumerate() { |
| 196 | c.header = i == 0; |
| 197 | } |
| 198 | } |
| 199 | } |
| 200 | Block::Table { |
| 201 | head, |
| 202 | rows, |
| 203 | cols, |
| 204 | caption: if g.chance(2) { Vec::new() } else { edge(inlines(g, d.min(2), false)) }, |
| 205 | } |
| 206 | } |
| 207 | 9 => Block::Figure { |
| 208 | attrs: if g.chance(2) { Attrs::default() } else { attrs(g) }, |
| 209 | content: if g.chance(6) { Vec::new() } else { blocks(g, d) }, |
| 210 | caption: if g.chance(3) { Vec::new() } else { edge(inlines(g, d.min(2), true)) }, |
| 211 | }, |
| 212 | 10 => Block::Terms { |
| 213 | attrs: if g.chance(2) { Attrs::default() } else { attrs(g) }, |
| 214 | items: (0..1 + g.below(3)).map(|_| Term { |
| 215 | term: if g.chance(6) { Vec::new() } else { edge(inlines(g, d.min(2), true)) }, |
| 216 | defs: (0..g.below(3)).map(|_| match g.chance(5) { |
| 217 | true => Vec::new(), |
| 218 | false => blocks(g, d), |
| 219 | }).collect(), |
| 220 | }).collect(), |
| 221 | }, |
| 222 | _ => Block::Div { |
| 223 | attrs: if g.chance(4) { Attrs::default() } else { attrs(g) }, |
| 224 | content: if g.chance(6) { Vec::new() } else { blocks(g, d) }, |
| 225 | }, |
| 226 | } |
| 227 | } |
| 228 | |
| 229 | /// Lists side by side merge unless the second takes the other marker, and a list with a sibling |
| 230 | /// list of the other kind is its own; both are what the writer must keep apart. |
| 231 | fn round(d: &Doc) -> Outcome<()> { |
| 232 | let text = djot::render(d); |
| 233 | let back = res!(djot::parse(&text)); |
| 234 | if back != *d { |
| 235 | return Err(err!("Round trip changed the tree.\n--- djot ---\n{}\n--- want ---\n{:?}\n--- got ---\n{:?}", |
| 236 | text, d, back; Invalid, Output)); |
| 237 | } |
| 238 | Ok(()) |
| 239 | } |
| 240 | |
| 241 | #[test] |
| 242 | fn test_round_trip_over_generated_trees() -> Outcome<()> { |
| 243 | let mut seen = std::collections::BTreeSet::new(); |
| 244 | for seed in 1..=5000u64 { |
| 245 | let mut g = Gen(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1); |
| 246 | let doc = Doc { blocks: blocks(&mut g, 4) }; |
| 247 | let dbg = fmt!("{:?}", doc); |
| 248 | for name in ["Heading", "Para", "List", "Code", "Quote", "Table", "Rule", "Div", "Text", "Emph", |
| 249 | "Link", "Image", "Span", "Break", "Sup", "Sub", "caption: [", "head: None", "Centre", |
| 250 | "Figure", "Terms", "defs: []", "term: []", "header: true", "header: false"] |
| 251 | { |
| 252 | if dbg.contains(name) { |
| 253 | seen.insert(name); |
| 254 | } |
| 255 | } |
| 256 | if round(&doc).is_err() { |
| 257 | let small = shrink(doc.blocks.clone()); |
| 258 | let e = round(&Doc { blocks: small }).err().map(|e| fmt!("{}", e)).unwrap_or_default(); |
| 259 | return Err(err!("Seed {}: {}", seed, e; Invalid, Output)); |
| 260 | } |
| 261 | } |
| 262 | assert_eq!(seen.len(), 25, "variants seen: {:?}", seen); |
| 263 | Ok(()) |
| 264 | } |
| 265 | |
| 266 | /// The smallest run of blocks within a failing one that still fails, for a readable report. |
| 267 | fn shrink(bs: Vec<Block>) -> Vec<Block> { |
| 268 | let fails = |v: &Vec<Block>| round(&Doc { blocks: v.clone() }).is_err(); |
| 269 | // One block alone, or any pair of neighbours, that still fails. |
| 270 | for i in 0..bs.len() { |
| 271 | for j in i + 1..=bs.len().min(i + 2) { |
| 272 | let v = bs[i..j].to_vec(); |
| 273 | if v.len() < bs.len() && fails(&v) { |
| 274 | return shrink(v); |
| 275 | } |
| 276 | } |
| 277 | } |
| 278 | // Then the children of each block. |
| 279 | for b in &bs { |
| 280 | let kids: Vec<Vec<Block>> = match b { |
| 281 | Block::Div { content, .. } => vec![content.clone()], |
| 282 | Block::Quote(c) => vec![c.clone()], |
| 283 | Block::List { items, .. } => items.clone(), |
| 284 | Block::Figure { content, .. } => vec![content.clone()], |
| 285 | Block::Terms { items, .. } => items.iter().flat_map(|t| t.defs.clone()).collect(), |
| 286 | _ => Vec::new(), |
| 287 | }; |
| 288 | for k in kids { |
| 289 | if fails(&k) { |
| 290 | return shrink(k); |
| 291 | } |
| 292 | } |
| 293 | } |
| 294 | bs |
| 295 | } |
| 296 | |
| 297 | fn para(v: Vec<Inline>) -> Doc { |
| 298 | Doc { blocks: vec![Block::Para(v)] } |
| 299 | } |
| 300 | |
| 301 | fn t(s: &str) -> Inline { |
| 302 | Inline::Text(s.to_string()) |
| 303 | } |
| 304 | |
| 305 | #[test] |
| 306 | fn test_escaping_case_by_case() -> Outcome<()> { |
| 307 | // A paragraph that starts like an ordered list, and its parenthesis form. |
| 308 | res!(round(¶(vec![t("1. not a list")]))); |
| 309 | res!(round(¶(vec![t("1) nor this")]))); |
| 310 | // Each mark alone. |
| 311 | for s in ["a literal _ here", "a * star", "an open [ bracket", "a { brace", "a } brace", |
| 312 | "x^2 and H~2", "<https://not.a/link>", "trailing!", "a ` tick"] |
| 313 | { |
| 314 | res!(round(¶(vec![t(s)]))); |
| 315 | } |
| 316 | // A backslash at the end of a line, before a break and at the very end. |
| 317 | res!(round(¶(vec![t("ends \\"), Inline::Break, t("next")]))); |
| 318 | res!(round(¶(vec![t("ends \\")]))); |
| 319 | // Every block opener at the start of a line, and again after a break. |
| 320 | for s in ["# no heading", "> no quote", "- no item", "+ no item", "::: no div", "| no table", |
| 321 | "^ no caption", "*** no rule", "``` no fence", "~~~ no fence", "{.no attrs}", "[no]: ref"] |
| 322 | { |
| 323 | res!(round(¶(vec![t(s)]))); |
| 324 | res!(round(¶(vec![t("lead"), Inline::Break, t(s)]))); |
| 325 | } |
| 326 | // A `!` before a link is not an image. |
| 327 | res!(round(¶(vec![t("bang!"), Inline::Link { to: "x".into(), content: vec![t("y")] }]))); |
| 328 | Ok(()) |
| 329 | } |
| 330 | |
| 331 | #[test] |
| 332 | fn test_reader_learns_sup_sub_and_captions() -> Outcome<()> { |
| 333 | let d = res!(djot::parse("H~2~O and x^2^.\n")); |
| 334 | assert_eq!(d.blocks, vec![Block::Para(vec![ |
| 335 | t("H"), Inline::Sub(vec![t("2")]), t("O and x"), Inline::Sup(vec![t("2")]), t("."), |
| 336 | ])]); |
| 337 | let d = res!(djot::parse("| a |\n|---|\n| b |\n\n^ The caption.\n")); |
| 338 | match &d.blocks[0] { |
| 339 | Block::Table { caption, .. } => assert_eq!(caption, &vec![t("The caption.")]), |
| 340 | other => return Err(err!("Not a table: {:?}", other; Invalid, Output)), |
| 341 | } |
| 342 | // Forced emphasis around a space. |
| 343 | let d = res!(djot::parse("{_ spaced _}\n")); |
| 344 | assert_eq!(d.blocks, vec![Block::Para(vec![Inline::Emph { strong: false, content: vec![t(" spaced ")] }])]); |
| 345 | Ok(()) |
| 346 | } |
| 347 | |
| 348 | #[test] |
| 349 | fn test_reader_learns_the_page_constructs() -> Outcome<()> { |
| 350 | // Two terms, the second with two definitions kept apart, and attrs above the list. |
| 351 | let d = res!(djot::parse("{.terms}\n: Bark\n\n A tree's skin.\n\n ::\n\n A dog's call.\n\n: Jaw\n")); |
| 352 | assert_eq!(d.blocks, vec![Block::Terms { |
| 353 | attrs: Attrs { id: None, classes: vec!["terms".into()], pairs: Vec::new() }, |
| 354 | items: vec![ |
| 355 | Term { term: vec![t("Bark")], defs: vec![ |
| 356 | vec![Block::Para(vec![t("A tree's skin.")])], |
| 357 | vec![Block::Para(vec![t("A dog's call.")])], |
| 358 | ] }, |
| 359 | Term { term: vec![t("Jaw")], defs: Vec::new() }, |
| 360 | ], |
| 361 | }]); |
| 362 | // A figure and its caption; a division classed `figure` stays a division. |
| 363 | let d = res!(djot::parse("::: figure {#f}\n\n\n^ A caption.\n:::\n\n::: {.figure}\nP\n:::\n")); |
| 364 | assert_eq!(d.blocks[0], Block::Figure { |
| 365 | attrs: Attrs { id: Some("f".into()), classes: Vec::new(), pairs: Vec::new() }, |
| 366 | content: vec![Block::Para(vec![Inline::Image { src: "a.png".into(), alt: "x".into(), attrs: Attrs::default() }])], |
| 367 | caption: vec![t("A caption.")], |
| 368 | }); |
| 369 | assert!(matches!(&d.blocks[1], Block::Div { .. }), "got {:?}", d.blocks[1]); |
| 370 | // Row headers in the short form and the long, and a heads value that fits nothing stays a pair. |
| 371 | let d = res!(djot::parse("{heads=\"rows\"}\n| --- | --- |\n| a | 1 |\n")); |
| 372 | match &d.blocks[0] { |
| 373 | Block::Table { rows, .. } => assert_eq!((rows[0].0[0].header, rows[0].0[1].header), (true, false)), |
| 374 | other => return Err(err!("Not a table: {:?}", other; Invalid, Output)), |
| 375 | } |
| 376 | let d = res!(djot::parse("{heads=\"01 10\"}\n| h | i |\n|---|---|\n| a | 1 |\n")); |
| 377 | match &d.blocks[0] { |
| 378 | Block::Table { head: Some(h), rows, .. } => { |
| 379 | assert_eq!((h.0[0].header, h.0[1].header), (false, true)); |
| 380 | assert_eq!((rows[0].0[0].header, rows[0].0[1].header), (true, false)); |
| 381 | } |
| 382 | other => return Err(err!("Not a headed table: {:?}", other; Invalid, Output)), |
| 383 | } |
| 384 | let d = res!(djot::parse("{heads=\"x\"}\n| a |\n|---|\n")); |
| 385 | assert!(matches!(&d.blocks[0], Block::Div { .. }), "got {:?}", d.blocks[0]); |
| 386 | // A line of prose that opens with a colon does not break a paragraph. |
| 387 | let d = res!(djot::parse("Note\n: not a term\n")); |
| 388 | assert_eq!(d.blocks, vec![Block::Para(vec![t("Note : not a term")])]); |
| 389 | Ok(()) |
| 390 | } |