oxedyne/fe2o3/fe2o3_austenite/tests/eval_model_sweep.rs
5.4 KiB, 1 run
created by r1870400018:66080, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! Every model element set once, against the `typst` 0.15.1 oracle (U6b-S2): no element may be set otherwise |
| 2 | //! than Typst sets it without a warning saying so. |
| 3 | //! |
| 4 | //! Each fixture under `tests/fixtures/eval/sweep/` is a minimal document that sets one element of Typst's |
| 5 | //! model (a heading, a list, a figure, a table, a quote, a reference, a footnote, an outline...) with its default |
| 6 | //! show, and, where the element has settable parts, a set rule on them. Austenite lays each out through the |
| 7 | //! whole pipeline and the page count, every line's text, baseline and start are compared with Typst's. A fixture |
| 8 | //! passes when the two agree, or when they differ and Austenite said so with a warning of kind `unsupported`; |
| 9 | //! one that differs in silence fails the suite. So a construct can never again be set wrongly without a warning: |
| 10 | //! when a layout is made right its warning goes, and the fixture is then held to the comparison alone. |
| 11 | //! |
| 12 | //! The suite also fails when a model element has no fixture, or fewer fixtures than `MIN_FIXTURES` were found. |
| 13 | |
| 14 | #![allow(dead_code)] |
| 15 | |
| 16 | #[path = "eval_oracle/mod.rs"] |
| 17 | mod harness; |
| 18 | |
| 19 | use harness::austenite::{ |
| 20 | pages, |
| 21 | plain, |
| 22 | Keep, |
| 23 | }; |
| 24 | use harness::corpus::{ |
| 25 | self, |
| 26 | Filter, |
| 27 | }; |
| 28 | use harness::layout::compare; |
| 29 | use harness::oracle::Oracle; |
| 30 | |
| 31 | use oxedyne_fe2o3_austenite::diag::{ |
| 32 | DiagnosticKind, |
| 33 | Severity, |
| 34 | }; |
| 35 | use oxedyne_fe2o3_austenite::eval::content::{ |
| 36 | ElemKind, |
| 37 | Family, |
| 38 | }; |
| 39 | use oxedyne_fe2o3_austenite::eval::fixpoint; |
| 40 | use oxedyne_fe2o3_austenite::eval::{ |
| 41 | eval_source, |
| 42 | Engine, |
| 43 | World, |
| 44 | }; |
| 45 | |
| 46 | use oxedyne_fe2o3_core::prelude::*; |
| 47 | |
| 48 | const MIN_FIXTURES: usize = 19; |
| 49 | |
| 50 | /// The model elements a fixture of their own sets: its name is the fixture's. The others are held by one of |
| 51 | /// these (an item by its list, a caption by its figure) or are what realisation makes. |
| 52 | const ELEMENTS: &[&str] = &[ |
| 53 | "heading", "par", "list", "enum", "terms", "figure", "caption", "table", "quote", "quote_inline", "raw", |
| 54 | "link", "ref", "cite", "footnote", "outline", "title", "bibliography", "strong", "emph", |
| 55 | ]; |
| 56 | |
| 57 | /// How one fixture came out. |
| 58 | #[derive(Debug)] |
| 59 | enum Verdict { |
| 60 | Match, |
| 61 | Warned(String), // differs, with an `unsupported` warning: the head of its message |
| 62 | Silent(String), // differs with no such warning: the fault |
| 63 | } |
| 64 | |
| 65 | #[test] |
| 66 | fn every_model_element_matches_typst_or_warns_unsupported() -> Outcome<()> { |
| 67 | let oracle = match res!(Oracle::find()) { |
| 68 | Some(o) => o, |
| 69 | None => return Ok(()), |
| 70 | }; |
| 71 | let only = std::env::var("EVAL_SWEEP_FIXTURE").ok(); |
| 72 | let (mut matched, mut warned, mut failures) = (0usize, 0usize, Vec::new()); |
| 73 | let mut found = Vec::new(); |
| 74 | for fx in res!(corpus::discover(&Filter::only("sweep"))) { |
| 75 | found.push(fx.name.clone()); |
| 76 | if let Some(o) = &only { |
| 77 | if !fx.name.contains(o.as_str()) { |
| 78 | continue; |
| 79 | } |
| 80 | } |
| 81 | let want = match res!(oracle.layout(&fx)) { |
| 82 | Ok(w) => w, |
| 83 | Err(e) => { |
| 84 | failures.push(fmt!("{}: typst: {}", fx.id(), e)); |
| 85 | continue; |
| 86 | }, |
| 87 | }; |
| 88 | let mut world = World::new(fx.root.clone()); |
| 89 | let id = match world.load(&fx.path) { |
| 90 | Ok(id) => id, |
| 91 | Err(e) => { |
| 92 | failures.push(fmt!("{}: load: {}", fx.id(), plain(&e))); |
| 93 | continue; |
| 94 | }, |
| 95 | }; |
| 96 | let mut engine = Engine::new(world); |
| 97 | let module = match eval_source(&mut engine, id) { |
| 98 | Ok(m) => m, |
| 99 | Err(e) => { |
| 100 | failures.push(fmt!("{}: eval: {}", fx.id(), plain(&e))); |
| 101 | continue; |
| 102 | }, |
| 103 | }; |
| 104 | let mut kept = Keep::default(); |
| 105 | if let Err(e) = fixpoint::run(&mut engine, &module, &mut kept) { |
| 106 | failures.push(fmt!("{}: layout: {}", fx.id(), plain(&e))); |
| 107 | continue; |
| 108 | } |
| 109 | let got = pages(&kept.pages); |
| 110 | let mut diffs = Vec::new(); |
| 111 | compare(&want, &got, &mut diffs); |
| 112 | let warning = engine.diags.iter() |
| 113 | .find(|d| d.severity == Severity::Warning && d.kind == DiagnosticKind::Unsupported) |
| 114 | .map(|d| d.head().to_string()); |
| 115 | let verdict = match (diffs.is_empty(), warning) { |
| 116 | (true, _) => Verdict::Match, |
| 117 | (false, Some(w)) => Verdict::Warned(w), |
| 118 | (false, None) => Verdict::Silent(diffs.iter().take(3).cloned().collect::<Vec<_>>().join(" | ")), |
| 119 | }; |
| 120 | match &verdict { |
| 121 | Verdict::Match => { |
| 122 | matched += 1; |
| 123 | println!("match {}", fx.id()); |
| 124 | }, |
| 125 | Verdict::Warned(w) => { |
| 126 | warned += 1; |
| 127 | println!("warned {} ({})", fx.id(), w); |
| 128 | }, |
| 129 | Verdict::Silent(d) => { |
| 130 | failures.push(fmt!("{} differs from typst and carries no `unsupported` warning: {}", fx.id(), d)); |
| 131 | }, |
| 132 | } |
| 133 | } |
| 134 | if only.is_none() { |
| 135 | for e in ELEMENTS { |
| 136 | if !found.iter().any(|f| f == e) { |
| 137 | failures.push(fmt!("the model element `{}` has no fixture under sweep/", e)); |
| 138 | } |
| 139 | } |
| 140 | // Every model element of the registry is one of those above or part of one. |
| 141 | for k in ElemKind::ALL.iter().filter(|k| k.family() == Family::Model) { |
| 142 | let head = k.path().split('.').next().unwrap_or(""); |
| 143 | if !ELEMENTS.contains(&head) && !PARTS.contains(&head) { |
| 144 | failures.push(fmt!("the model element `{}` is in neither ELEMENTS nor PARTS", k.path())); |
| 145 | } |
| 146 | } |
| 147 | } |
| 148 | println!("[sweep] {} fixture(s) match typst, {} warn `unsupported`, {} failure(s)", matched, warned, failures.len()); |
| 149 | for f in &failures { |
| 150 | println!("FAIL {}", f); |
| 151 | } |
| 152 | assert!(failures.is_empty(), "{} failure(s):\n{}", failures.len(), failures.join("\n")); |
| 153 | if only.is_none() { |
| 154 | assert!(matched + warned >= MIN_FIXTURES, "only {} sweep fixture(s) compared; {} are expected", |
| 155 | matched + warned, MIN_FIXTURES); |
| 156 | } |
| 157 | Ok(()) |
| 158 | } |
| 159 | |
| 160 | /// Model elements that are parts of the ones above, or what the model itself builds from them. |
| 161 | const PARTS: &[&str] = &["document", "parbreak", "divider", "bibliography_entry", "outline_entry", "footnote_entry", "cite-group"]; |