oxedyne/fe2o3/fe2o3_austenite/tests/eval_oracle.rs
24.2 KiB, 7 runs
created by r1870400018:63610, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! The evaluator's structure oracle (U12): Austenite against the host's `typst` 0.15.x on a synthetic |
| 2 | //! fixture corpus, at four levels -- values, positions, realised structure, layout. The driver is |
| 3 | //! `tests/eval_oracle/mod.rs`; this file states what is asserted. |
| 4 | //! |
| 5 | //! Two kinds of test live here. The harness's own tests prove each reader and comparator against the |
| 6 | //! oracle on fixtures whose answer is fixed by construction, and prove each comparator goes red on a |
| 7 | //! difference -- a harness that cannot fail measures nothing. The conformance test runs every |
| 8 | //! fixture and prints the scoreboard; it fails on any Austenite panic or unbounded run, on any |
| 9 | //! fixture Typst rejects (or accepts, when it is marked `oracle: rejects`), and, in the areas `EVAL_ORACLE_STRICT` names, on any shortfall. |
| 10 | //! |
| 11 | //! cargo test -p oxedyne_fe2o3_austenite --test eval_oracle -- --nocapture |
| 12 | //! EVAL_ORACLE_AREA=general EVAL_ORACLE_STRICT=general cargo test ... --test eval_oracle |
| 13 | |
| 14 | // `#[path]` because this test crate's root is itself `eval_oracle.rs`; a plain `mod eval_oracle;` |
| 15 | // would find the module at two paths. |
| 16 | #[path = "eval_oracle/mod.rs"] |
| 17 | mod harness; |
| 18 | |
| 19 | use harness::austenite::{ |
| 20 | self, |
| 21 | RunEnd, |
| 22 | }; |
| 23 | use harness::corpus::{ |
| 24 | self, |
| 25 | Expect, |
| 26 | Filter, |
| 27 | }; |
| 28 | use harness::json::{ |
| 29 | self, |
| 30 | J, |
| 31 | }; |
| 32 | use harness::layout::{ |
| 33 | self, |
| 34 | APage, |
| 35 | ARun, |
| 36 | }; |
| 37 | use harness::oracle::Oracle; |
| 38 | use harness::structure::{ |
| 39 | self, |
| 40 | Sk, |
| 41 | }; |
| 42 | use harness::{ |
| 43 | Board, |
| 44 | PosRow, |
| 45 | }; |
| 46 | |
| 47 | use oxedyne_fe2o3_austenite::eval::content::{ |
| 48 | Content, |
| 49 | Elem, |
| 50 | ElemKind, |
| 51 | }; |
| 52 | use oxedyne_fe2o3_austenite::eval::fixpoint::Laid; |
| 53 | use oxedyne_fe2o3_austenite::eval::intro::Builder; |
| 54 | use oxedyne_fe2o3_austenite::eval::locate::Location; |
| 55 | use oxedyne_fe2o3_austenite::eval::value::{ |
| 56 | Dict, |
| 57 | Label, |
| 58 | Value, |
| 59 | }; |
| 60 | use oxedyne_fe2o3_austenite::font::ShapedText; |
| 61 | use oxedyne_fe2o3_austenite::fonts; |
| 62 | use oxedyne_fe2o3_austenite::ir::Sp; |
| 63 | use oxedyne_fe2o3_austenite::ledger::Position; |
| 64 | use oxedyne_fe2o3_austenite::page::{ |
| 65 | Frame, |
| 66 | Page, |
| 67 | PageGeometry, |
| 68 | Placed, |
| 69 | PlacedKind, |
| 70 | }; |
| 71 | use oxedyne_fe2o3_austenite::syntax::{ |
| 72 | FileId, |
| 73 | Source, |
| 74 | Span, |
| 75 | }; |
| 76 | |
| 77 | use oxedyne_fe2o3_core::prelude::*; |
| 78 | use oxedyne_fe2o3_font::face::Role; |
| 79 | use oxedyne_fe2o3_font::shape::Dir; |
| 80 | |
| 81 | use std::path::PathBuf; |
| 82 | use std::process::Command; |
| 83 | use std::sync::Arc; |
| 84 | |
| 85 | /// The oracle, or `None` after saying loudly that it was skipped on request. |
| 86 | fn oracle() -> Outcome<Option<Oracle>> { |
| 87 | let o = res!(Oracle::find()); |
| 88 | if o.is_none() { |
| 89 | println!("[eval-oracle] SKIPPED: EVAL_ORACLE_SKIP=1 and no typst 0.15.x -- nothing was compared"); |
| 90 | } |
| 91 | Ok(o) |
| 92 | } |
| 93 | |
| 94 | // An account whose own font folder holds a face Typst does not carry: Libertinus Mono, copied beside a fake |
| 95 | // home. Typst finds it unless the run hides the account's fonts, and every run of the harness must. |
| 96 | #[test] |
| 97 | fn typst_runs_hide_the_accounts_own_fonts_and_go_red_when_they_do_not() -> Outcome<()> { |
| 98 | let o = match res!(oracle()) { |
| 99 | Some(o) => o, |
| 100 | None => return Ok(()), |
| 101 | }; |
| 102 | let home = o.work.join("account-home"); |
| 103 | let fonts = home.join(".local/share/fonts"); |
| 104 | res!(std::fs::create_dir_all(&fonts)); |
| 105 | let face = "LibertinusMono-Regular.otf"; |
| 106 | res!(std::fs::copy(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("fonts").join(face), fonts.join(face))); |
| 107 | let typ = o.work.join("account-home.typ"); |
| 108 | res!(std::fs::write(&typ, "#set text(font: \"Libertinus Mono\")\nMono\n")); |
| 109 | let listed = |mut c: Command| -> Outcome<String> { |
| 110 | let out = res!(c.arg("fonts").env("HOME", &home).output()); |
| 111 | Ok(String::from_utf8_lossy(&out.stdout).to_string()) |
| 112 | }; |
| 113 | let embedded = |mut c: Command, name: &str| -> Outcome<Vec<String>> { |
| 114 | let pdf = home.join(name); |
| 115 | let status = res!(c.env("HOME", &home).status()); |
| 116 | assert!(status.success(), "typst did not compile {}", typ.display()); |
| 117 | harness::pdf::font_names(&pdf) |
| 118 | }; |
| 119 | // The premise: with nothing hidden, the face is there to be found. |
| 120 | let mut open = Command::new(&o.bin); |
| 121 | open.env_remove("XDG_DATA_HOME"); |
| 122 | assert!(res!(listed(open)).lines().any(|l| l == "Libertinus Mono"), "the fake home's face was not found by an open run"); |
| 123 | assert!(!res!(listed(harness::typst_command::typst_command(&o.bin, None, &o.work))).lines().any(|l| l == "Libertinus Mono"), |
| 124 | "the oracle's own run saw the account's face"); |
| 125 | // The PDF builder the comparators use. |
| 126 | let mut open = Command::new(harness::pdf::TYPST); |
| 127 | open.env_remove("XDG_DATA_HOME").arg("compile").arg(&typ).arg(home.join("open.pdf")); |
| 128 | assert!(res!(embedded(open, "open.pdf")).iter().any(|n| n.starts_with("LibertinusMono")), "an open run did not embed the face"); |
| 129 | let hidden = res!(harness::pdf::typst_pdf_command(&typ, &home.join("hidden.pdf"), &[])); |
| 130 | assert!(!res!(embedded(hidden, "hidden.pdf")).iter().any(|n| n.starts_with("LibertinusMono")), |
| 131 | "the PDF builder's run embedded the account's face"); |
| 132 | Ok(()) |
| 133 | } |
| 134 | |
| 135 | fn harness_fixture(name: &str) -> Outcome<corpus::Fixture> { |
| 136 | let all = res!(corpus::discover(&Filter::only("harness"))); |
| 137 | match all.into_iter().find(|f| f.name == name) { |
| 138 | Some(f) => Ok(f), |
| 139 | None => Err(err!("The harness fixture {} is missing from tests/fixtures/eval/harness", name; Missing)), |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | #[test] |
| 144 | fn oracle_is_present_and_pinned() -> Outcome<()> { |
| 145 | if let Some(o) = res!(oracle()) { |
| 146 | assert!(o.version.starts_with(harness::oracle::PINNED), "oracle version {}", o.version); |
| 147 | println!("[eval-oracle] {} (cache {})", o.version, o.work.display()); |
| 148 | } |
| 149 | Ok(()) |
| 150 | } |
| 151 | |
| 152 | #[test] |
| 153 | fn fixture_corpus_is_synthetic_and_accepted_by_typst() -> Outcome<()> { |
| 154 | let o = match res!(oracle()) { |
| 155 | Some(o) => o, |
| 156 | None => return Ok(()), |
| 157 | }; |
| 158 | let all = res!(corpus::discover(&Filter::from_env())); |
| 159 | assert!(!all.is_empty(), "no fixtures under {}", corpus::fixtures_dir().display()); |
| 160 | let mut bad = Vec::new(); |
| 161 | for fx in &all { |
| 162 | if let Some(f) = corpus::provenance_fault(fx) { |
| 163 | bad.push(f); |
| 164 | continue; |
| 165 | } |
| 166 | match (res!(o.compiles(fx)), fx.expect) { |
| 167 | (Err(e), Expect::Accepts) => bad.push(fmt!("{}: typst rejects it: {}", fx.id(), e)), |
| 168 | (Ok(()), Expect::Rejects) => bad.push(fmt!("{}: marked `oracle: rejects`, but typst accepts it", fx.id())), |
| 169 | _ => (), |
| 170 | } |
| 171 | } |
| 172 | assert!(bad.is_empty(), "corpus faults:\n{}", bad.join("\n")); |
| 173 | Ok(()) |
| 174 | } |
| 175 | |
| 176 | // The values of `harness/values.typ`, built by hand from Austenite's own value types. |
| 177 | fn hand_values() -> Vec<Value> { |
| 178 | let mut inner = Dict::new(); |
| 179 | inner.insert("c", Value::str("x")); |
| 180 | let mut d = Dict::new(); |
| 181 | d.insert("b", Value::Int(1)); |
| 182 | d.insert("a", Value::dict(inner)); |
| 183 | vec![ |
| 184 | Value::Int(1), |
| 185 | Value::Int(-7), |
| 186 | Value::Float(1.5), |
| 187 | Value::Float(1.0), |
| 188 | Value::str("quoted \"text\", tab\tand ünïcode →"), |
| 189 | Value::Bool(true), |
| 190 | Value::None, |
| 191 | Value::array(vec![Value::Int(1), Value::array(vec![Value::Int(2), Value::Int(3)]), Value::array(vec![])]), |
| 192 | Value::dict(d), |
| 193 | Value::Float(f64::INFINITY), |
| 194 | Value::Int(9_007_199_254_740_993), |
| 195 | ] |
| 196 | } |
| 197 | |
| 198 | #[test] |
| 199 | fn value_serialisation_agrees_with_the_oracle_and_goes_red_on_a_difference() -> Outcome<()> { |
| 200 | let o = match res!(oracle()) { |
| 201 | Some(o) => o, |
| 202 | None => return Ok(()), |
| 203 | }; |
| 204 | let fx = res!(harness_fixture("values")); |
| 205 | let want = match res!(o.probes(&fx)) { |
| 206 | Ok(v) => J::Arr(v), |
| 207 | Err(e) => return Err(err!("the oracle could not read harness/values: {}", e; Invalid)), |
| 208 | }; |
| 209 | let vals = hand_values(); |
| 210 | let got = J::Arr(vals.iter().map(austenite::to_json).collect()); |
| 211 | let mut d = Vec::new(); |
| 212 | json::diff("probes", &want, &got, &mut d); |
| 213 | assert!(d.is_empty(), "hand-built values differ from typst:\n{}", d.join("\n")); |
| 214 | |
| 215 | // Each perturbation is a distinction Typst makes and the comparison must keep. |
| 216 | let perturb: [(&str, fn(&mut Vec<Value>)); 7] = [ |
| 217 | ("int for float", |v| v[3] = Value::Int(1)), |
| 218 | ("float for int", |v| v[0] = Value::Float(1.0)), |
| 219 | ("changed string", |v| v[4] = Value::str("quoted")), |
| 220 | ("dictionary order", |v| { |
| 221 | let mut d = Dict::new(); |
| 222 | d.insert("a", Value::Int(0)); |
| 223 | d.insert("b", Value::Int(1)); |
| 224 | v[8] = Value::dict(d); |
| 225 | }), |
| 226 | ("missing item", |v| { v.pop(); }), |
| 227 | ("none for false", |v| v[5] = Value::None), |
| 228 | ("near integer", |v| v[10] = Value::Int(9_007_199_254_740_992)), |
| 229 | ]; |
| 230 | for (what, f) in perturb { |
| 231 | let mut vals = hand_values(); |
| 232 | f(&mut vals); |
| 233 | let got = J::Arr(vals.iter().map(austenite::to_json).collect()); |
| 234 | let mut d = Vec::new(); |
| 235 | json::diff("probes", &want, &got, &mut d); |
| 236 | assert!(!d.is_empty(), "the comparison missed a {}", what); |
| 237 | } |
| 238 | Ok(()) |
| 239 | } |
| 240 | |
| 241 | #[test] |
| 242 | fn content_comparison_ignores_field_order_but_not_text() -> Outcome<()> { |
| 243 | // Content in the shape the oracle writes it (see `markup_content.typ`), and variations of it. |
| 244 | let want = res!(json::parse(r#"{"func":"strong","body":{"func":"text","text":"b"},"label":"<x>"}"#)); |
| 245 | let same = res!(json::parse(r#"{"label":"<x>","body":{"text":"b","func":"text"},"func":"strong"}"#)); |
| 246 | let other = res!(json::parse(r#"{"func":"strong","body":{"func":"text","text":"c"},"label":"<x>"}"#)); |
| 247 | let unlabelled = res!(json::parse(r#"{"func":"strong","body":{"func":"text","text":"b"}}"#)); |
| 248 | let mut d = Vec::new(); |
| 249 | json::diff("v", &want, &same, &mut d); |
| 250 | assert!(d.is_empty(), "field order is not significant in content: {:?}", d); |
| 251 | json::diff("v", &want, &other, &mut d); |
| 252 | assert!(!d.is_empty(), "changed text must differ"); |
| 253 | d.clear(); |
| 254 | json::diff("v", &want, &unlabelled, &mut d); |
| 255 | assert!(!d.is_empty(), "a lost label must differ"); |
| 256 | // And a dictionary, unlike content, is ordered. |
| 257 | let d1 = res!(json::parse(r#"{"a":1,"b":2}"#)); |
| 258 | let d2 = res!(json::parse(r#"{"b":2,"a":1}"#)); |
| 259 | d.clear(); |
| 260 | json::diff("v", &d1, &d2, &mut d); |
| 261 | assert!(!d.is_empty(), "dictionary order is significant"); |
| 262 | Ok(()) |
| 263 | } |
| 264 | |
| 265 | #[test] |
| 266 | fn layout_reader_recovers_known_baselines() -> Outcome<()> { |
| 267 | let o = match res!(oracle()) { |
| 268 | Some(o) => o, |
| 269 | None => return Ok(()), |
| 270 | }; |
| 271 | let fx = res!(harness_fixture("baselines")); |
| 272 | let pages = match res!(o.layout(&fx)) { |
| 273 | Ok(p) => p, |
| 274 | Err(e) => return Err(err!("the oracle could not lay out harness/baselines: {}", e; Invalid)), |
| 275 | }; |
| 276 | assert_eq!(pages.len(), 1); |
| 277 | let p = &pages[0]; |
| 278 | assert!((p.width - 200.0).abs() < 0.01 && (p.height - 220.0).abs() < 0.01, "page {}x{}", p.width, p.height); |
| 279 | // Placed with baseline edges at these offsets, so these are the baselines, whatever the face. |
| 280 | let expect = [("Alpha beta", 40.0), ("Gamma", 100.0), ("Delta", 150.0), ("Epsilon2 zeta", 190.0)]; |
| 281 | assert_eq!(p.lines.len(), expect.len(), "lines: {:?}", p.lines); |
| 282 | for (l, (text, base)) in p.lines.iter().zip(expect.iter()) { |
| 283 | assert_eq!(l.text.replace(' ', ""), text.replace(' ', ""), "line text"); |
| 284 | assert!(l.measured, "{:?}: baseline estimated, not measured", l.text); |
| 285 | assert!((l.base - base).abs() < 0.01, "{:?}: baseline {} not {}", l.text, l.base, base); |
| 286 | assert!((l.x0 - 10.0).abs() < 0.5, "{:?}: x {}", l.text, l.x0); |
| 287 | } |
| 288 | |
| 289 | // The comparator: Typst's own lines as Austenite runs pass; a 2pt drop, a lost line, changed |
| 290 | // text, an extra run and an extra page each fail. |
| 291 | let as_runs = |shift: f64| -> Vec<APage> { |
| 292 | vec![APage { |
| 293 | width: p.width, |
| 294 | height: p.height, |
| 295 | runs: p.lines.iter().map(|l| ARun { text: l.text.clone(), x0: l.x0, x1: l.x1, base: l.base + shift }).collect(), |
| 296 | }] |
| 297 | }; |
| 298 | let mut d = Vec::new(); |
| 299 | layout::compare(&pages, &as_runs(0.0), &mut d); |
| 300 | assert!(d.is_empty(), "identity differs: {:?}", d); |
| 301 | layout::compare(&pages, &as_runs(0.8), &mut d); |
| 302 | assert!(d.is_empty(), "0.8pt is inside the tolerance: {:?}", d); |
| 303 | layout::compare(&pages, &as_runs(2.0), &mut d); |
| 304 | assert!(!d.is_empty(), "a 2pt drop must fail"); |
| 305 | let mut lost = as_runs(0.0); |
| 306 | lost[0].runs.remove(1); |
| 307 | d.clear(); |
| 308 | layout::compare(&pages, &lost, &mut d); |
| 309 | assert!(!d.is_empty(), "a lost line must fail"); |
| 310 | let mut changed = as_runs(0.0); |
| 311 | changed[0].runs[0].text = "Alpha beat".to_string(); |
| 312 | d.clear(); |
| 313 | layout::compare(&pages, &changed, &mut d); |
| 314 | assert!(!d.is_empty(), "changed text must fail"); |
| 315 | let mut extra = as_runs(0.0); |
| 316 | extra[0].runs.push(ARun { text: "stray".to_string(), x0: 100.0, x1: 120.0, base: 70.0 }); |
| 317 | d.clear(); |
| 318 | layout::compare(&pages, &extra, &mut d); |
| 319 | assert!(!d.is_empty(), "stray text must fail"); |
| 320 | let mut more = as_runs(0.0); |
| 321 | more.push(APage { width: p.width, height: p.height, runs: Vec::new() }); |
| 322 | d.clear(); |
| 323 | layout::compare(&pages, &more, &mut d); |
| 324 | assert!(!d.is_empty(), "an extra page must fail"); |
| 325 | Ok(()) |
| 326 | } |
| 327 | |
| 328 | #[test] |
| 329 | fn structure_reader_matches_the_html_export() -> Outcome<()> { |
| 330 | let o = match res!(oracle()) { |
| 331 | Some(o) => o, |
| 332 | None => return Ok(()), |
| 333 | }; |
| 334 | let fx = res!(harness_fixture("structure")); |
| 335 | let html = match res!(o.html(&fx)) { |
| 336 | Ok(h) => h, |
| 337 | Err(e) => return Err(err!("the oracle could not export harness/structure: {}", e; Invalid)), |
| 338 | }; |
| 339 | assert!(html.ignored.is_empty(), "ignored: {:?}", html.ignored); |
| 340 | let want = concat!( |
| 341 | r#"h2["Title words"] p["A paragraph with " strong["strong"] ", " em["emph"] " and " code["code"] "."] "#, |
| 342 | r#"ul[li["one"] li["two"]] ol[li["first"] li["second"]] dl[dt["Term"] dd["its description"]] "#, |
| 343 | r#"blockquote["Quoted"] p[a["a link"]]"#, |
| 344 | ); |
| 345 | assert_eq!(structure::render(&html.skeleton), want); |
| 346 | |
| 347 | let mut d = Vec::new(); |
| 348 | structure::compare(&html.skeleton, &html.skeleton.clone(), &mut d); |
| 349 | assert!(d.is_empty()); |
| 350 | let t = |s: &str| Sk::Text(s.to_string()); |
| 351 | let tag = |n: &str, k: Vec<Sk>| Sk::Tag(n.to_string(), k); |
| 352 | // Whitespace and quote glyphs normalise away; a tag change or lost element does not. |
| 353 | let a = structure::normalise(vec![tag("p", vec![t(" a “b”\n c ")])]); |
| 354 | let b = structure::normalise(vec![tag("p", vec![t("a \"b\" c")])]); |
| 355 | structure::compare(&a, &b, &mut d); |
| 356 | assert!(d.is_empty(), "{:?}", d); |
| 357 | let c = structure::normalise(vec![tag("h3", vec![t("a \"b\" c")])]); |
| 358 | structure::compare(&a, &c, &mut d); |
| 359 | assert!(!d.is_empty(), "a changed tag must fail"); |
| 360 | d.clear(); |
| 361 | let mut lost = html.skeleton.clone(); |
| 362 | lost.pop(); |
| 363 | structure::compare(&html.skeleton, &lost, &mut d); |
| 364 | assert!(!d.is_empty(), "a lost element must fail"); |
| 365 | // And an export Typst mutilates is recognised, so it is never compared. |
| 366 | assert_eq!(structure::ignored_elements("warning: align was ignored during HTML export\nwarning: x"), vec!["align"]); |
| 367 | Ok(()) |
| 368 | } |
| 369 | |
| 370 | #[test] |
| 371 | fn position_reader_reads_known_positions() -> Outcome<()> { |
| 372 | let o = match res!(oracle()) { |
| 373 | Some(o) => o, |
| 374 | None => return Ok(()), |
| 375 | }; |
| 376 | let fx = res!(harness_fixture("positions")); |
| 377 | let rows = match res!(o.positions(&fx)) { |
| 378 | Ok(r) => r, |
| 379 | Err(e) => return Err(err!("the oracle could not locate harness/positions: {}", e; Invalid)), |
| 380 | }; |
| 381 | let funcs: Vec<&str> = rows.iter().map(|r| r.func.as_str()).collect(); |
| 382 | assert_eq!(funcs, vec!["heading", "metadata", "heading"]); |
| 383 | // Both headings sit at the top-left margin corner, one on each page. |
| 384 | assert_eq!((rows[0].page, rows[2].page), (1, 2)); |
| 385 | for r in [&rows[0], &rows[2]] { |
| 386 | assert!((r.x - 20.0).abs() < 1e-6 && (r.y - 20.0).abs() < 1e-6, "{:?}", r); |
| 387 | } |
| 388 | assert!(harness::compare_positions(&rows, &rows).is_empty()); |
| 389 | let moved: Vec<PosRow> = rows.iter().map(|r| PosRow { y: r.y + 0.6, ..r.clone() }).collect(); |
| 390 | assert!(!harness::compare_positions(&rows, &moved).is_empty(), "0.6pt is outside the tolerance"); |
| 391 | let within: Vec<PosRow> = rows.iter().map(|r| PosRow { x: r.x - 0.4, ..r.clone() }).collect(); |
| 392 | assert!(harness::compare_positions(&rows, &within).is_empty(), "0.4pt is inside the tolerance"); |
| 393 | let paged: Vec<PosRow> = rows.iter().map(|r| PosRow { page: 1, ..r.clone() }).collect(); |
| 394 | assert!(!harness::compare_positions(&rows, &paged).is_empty(), "a page change must fail"); |
| 395 | assert!(!harness::compare_positions(&rows, &rows[..2]).is_empty(), "a lost element must fail"); |
| 396 | Ok(()) |
| 397 | } |
| 398 | |
| 399 | // A laid-out document built by hand in Austenite's own page and introspector types, placed where |
| 400 | // `harness/baselines.typ` and `harness/positions.typ` put things, so Austenite's side of levels 2 and |
| 401 | // 4 is read through the same code the conformance run uses and checked against the oracle. |
| 402 | fn hand_laid(lines: &[(&str, f64)], located: &[(ElemKind, bool, u32, f64, f64)], shift: f64) |
| 403 | -> Outcome<(Laid, Vec<Page>)> |
| 404 | { |
| 405 | let fonts = Arc::new(res!(fonts::libertinus())); |
| 406 | let mut frame = Frame::new(); |
| 407 | for (text, base) in lines { |
| 408 | let st = res!(ShapedText::new(fonts.clone(), Role::Body, Dir::Ltr, Sp::from_pt(11.0), text)); |
| 409 | let dims = st.dims(); |
| 410 | // A placed box's top-left is its position; its baseline sits `height` below. |
| 411 | let y = Sp::from_pt(base + shift) - dims.height; |
| 412 | frame.push(Placed::new(Sp::from_pt(10.0), y, dims, PlacedKind::Text(st))); |
| 413 | } |
| 414 | let mut builder = Builder::new(); |
| 415 | for (i, (kind, probe, page, x, y)) in located.iter().enumerate() { |
| 416 | let loc = Location(0x1000 + i as u64); |
| 417 | let label = if *probe { Some(Label::new("probe")) } else { None }; |
| 418 | let mut e = Elem::new(*kind, Vec::new(), Span::detached()); |
| 419 | e.label = label; |
| 420 | e.location = Some(loc); |
| 421 | e.prepared = true; |
| 422 | let elem = Content::from_elem(e); |
| 423 | builder.record(&elem, Position::new(*page, Sp::from_pt(*x), Sp::from_pt(*y + shift)), None); |
| 424 | } |
| 425 | builder.page(1, &Value::None); |
| 426 | let geom = PageGeometry::new(Sp::from_pt(200.0), Sp::from_pt(220.0), Sp::ZERO); |
| 427 | Ok((Laid { |
| 428 | intro: Arc::new(builder.finish()), |
| 429 | pages: 1, |
| 430 | passes: 1, |
| 431 | converged: true, |
| 432 | }, vec![Page::new(1, geom, frame)])) |
| 433 | } |
| 434 | |
| 435 | #[test] |
| 436 | fn austenite_side_readers_agree_with_the_oracle_on_hand_built_layout() -> Outcome<()> { |
| 437 | let o = match res!(oracle()) { |
| 438 | Some(o) => o, |
| 439 | None => return Ok(()), |
| 440 | }; |
| 441 | let lines = [("Alpha beta", 40.0), ("Gamma", 100.0), ("Delta", 150.0), ("Epsilon2 zeta", 190.0)]; |
| 442 | let want_pages = match res!(o.layout(&res!(harness_fixture("baselines")))) { |
| 443 | Ok(p) => p, |
| 444 | Err(e) => return Err(err!("the oracle could not lay out harness/baselines: {}", e; Invalid)), |
| 445 | }; |
| 446 | let want_pos = match res!(o.positions(&res!(harness_fixture("positions")))) { |
| 447 | Ok(p) => p, |
| 448 | Err(e) => return Err(err!("the oracle could not locate harness/positions: {}", e; Invalid)), |
| 449 | }; |
| 450 | let located = [ |
| 451 | (ElemKind::Heading, false, 1, 20.0, 20.0), |
| 452 | (ElemKind::Metadata, true, 1, 20.0, want_pos[1].y), |
| 453 | (ElemKind::Heading, false, 2, 20.0, 20.0), |
| 454 | ]; |
| 455 | let (laid, laid_pages) = res!(hand_laid(&lines, &located, 0.0)); |
| 456 | let mut d = Vec::new(); |
| 457 | layout::compare(&want_pages, &austenite::pages(&laid_pages), &mut d); |
| 458 | assert!(d.is_empty(), "hand-built layout differs from typst: {:?}", d); |
| 459 | let pos = match austenite::positions(&laid) { |
| 460 | Ok(p) => p, |
| 461 | Err(e) => return Err(err!("positions: {}", e; Invalid)), |
| 462 | }; |
| 463 | let d = harness::compare_positions(&want_pos, &pos); |
| 464 | assert!(d.is_empty(), "hand-built positions differ from typst: {:?}", d); |
| 465 | |
| 466 | // Moved by 1.5pt, both go red. |
| 467 | let (moved, moved_pages) = res!(hand_laid(&lines, &located, 1.5)); |
| 468 | let mut d = Vec::new(); |
| 469 | layout::compare(&want_pages, &austenite::pages(&moved_pages), &mut d); |
| 470 | assert!(!d.is_empty(), "a 1.5pt shift of every baseline must fail level 4"); |
| 471 | let pos = match austenite::positions(&moved) { |
| 472 | Ok(p) => p, |
| 473 | Err(e) => return Err(err!("positions: {}", e; Invalid)), |
| 474 | }; |
| 475 | assert!(!harness::compare_positions(&want_pos, &pos).is_empty(), "a 1.5pt shift must fail level 2"); |
| 476 | // An unlabelled metadata element is not a probe, so the rows no longer line up. |
| 477 | let unlabelled = [located[0], (ElemKind::Metadata, false, 1, 20.0, want_pos[1].y), located[2]]; |
| 478 | let pos = match austenite::positions(&res!(hand_laid(&lines, &unlabelled, 0.0)).0) { |
| 479 | Ok(p) => p, |
| 480 | Err(e) => return Err(err!("positions: {}", e; Invalid)), |
| 481 | }; |
| 482 | assert!(!harness::compare_positions(&want_pos, &pos).is_empty(), "a lost probe label must fail level 2"); |
| 483 | Ok(()) |
| 484 | } |
| 485 | |
| 486 | // `harness/rejects.typ` fails at a place fixed by construction: line 2, column 4 -- the `p` of `panic`, |
| 487 | // since an embedded expression's span leaves out its `#`, and counted in characters, so the two-byte |
| 488 | // `é` before it is one column. |
| 489 | #[test] |
| 490 | fn first_error_reader_and_comparator_agree_with_the_oracle_and_go_red_on_a_difference() -> Outcome<()> { |
| 491 | let o = match res!(oracle()) { |
| 492 | Some(o) => o, |
| 493 | None => return Ok(()), |
| 494 | }; |
| 495 | let fx = res!(harness_fixture("rejects")); |
| 496 | assert_eq!(fx.expect, Expect::Rejects, "harness/rejects.typ must carry `oracle: rejects`"); |
| 497 | let said = match res!(o.compiles(&fx)) { |
| 498 | Ok(()) => return Err(err!("typst accepts harness/rejects.typ"; Invalid)), |
| 499 | Err(e) => e, |
| 500 | }; |
| 501 | let (msg, pos) = harness::split_first_error(&said); |
| 502 | assert!(msg.contains("rejected on purpose"), "message read as `{}`", msg); |
| 503 | assert_eq!(pos, Some((2, 4)), "position read from `{}`", said); |
| 504 | let same = (msg.clone(), pos); |
| 505 | assert!(harness::compare_first_error(&msg, pos, Some(&same)).is_pass()); |
| 506 | let moved = (msg.clone(), Some((2, 5))); |
| 507 | assert!(!harness::compare_first_error(&msg, pos, Some(&moved)).is_pass(), "a moved error must fail"); |
| 508 | let reworded = ("rejected".to_string(), pos); |
| 509 | assert!(!harness::compare_first_error(&msg, pos, Some(&reworded)).is_pass(), "a reworded error must fail"); |
| 510 | assert!(!harness::compare_first_error(&msg, pos, None).is_pass(), "accepting it must fail"); |
| 511 | Ok(()) |
| 512 | } |
| 513 | |
| 514 | #[test] |
| 515 | fn evaluator_conformance_against_the_oracle() -> Outcome<()> { |
| 516 | let o = match res!(oracle()) { |
| 517 | Some(o) => o, |
| 518 | None => return Ok(()), |
| 519 | }; |
| 520 | let fixtures = res!(corpus::discover(&Filter::from_env())); |
| 521 | let limit = harness::timeout(); |
| 522 | let mut board = Board { reports: Vec::new() }; |
| 523 | for fx in &fixtures { |
| 524 | board.reports.push(res!(harness::check(fx, &o, limit))); |
| 525 | } |
| 526 | board.print(); |
| 527 | let report = o.work.join("report.json"); |
| 528 | res!(std::fs::write(&report, board.to_json().render()).map_err(|e| err!( |
| 529 | "Cannot write {}: {}", report.display(), e; IO, File, Write))); |
| 530 | println!("[eval-oracle] scoreboard written to {}", report.display()); |
| 531 | let failures = board.failures(); |
| 532 | assert!(failures.is_empty(), "{} failure(s):\n{}", failures.len(), failures.join("\n")); |
| 533 | Ok(()) |
| 534 | } |
| 535 | |
| 536 | // The parse-and-evaluate corpus: every `.typ` of five popular packages. There is no oracle for a |
| 537 | // library file, so the claims are Austenite's own: the tree is lossless, and evaluation neither |
| 538 | // panics nor runs away. Error nodes are counted; under `EVAL_ORACLE_STRICT` naming `packages` (or |
| 539 | // `all`) any error node or lossy tree fails. |
| 540 | const PACKAGES: [&str; 5] = ["cetz", "cetz-plot", "fletcher", "tablex", "oxifmt"]; |
| 541 | |
| 542 | fn package_root(p: &std::path::Path) -> PathBuf { |
| 543 | let mut d = p.parent(); |
| 544 | while let Some(dir) = d { |
| 545 | if dir.join("typst.toml").exists() { |
| 546 | return dir.to_path_buf(); |
| 547 | } |
| 548 | d = dir.parent(); |
| 549 | } |
| 550 | p.parent().map(|x| x.to_path_buf()).unwrap_or_default() |
| 551 | } |
| 552 | |
| 553 | fn typ_files(dir: &std::path::Path, out: &mut Vec<PathBuf>, depth: usize) { |
| 554 | if depth > 12 { |
| 555 | return; |
| 556 | } |
| 557 | if let Ok(rd) = std::fs::read_dir(dir) { |
| 558 | let mut v: Vec<PathBuf> = rd.filter_map(|e| e.ok().map(|e| e.path())).collect(); |
| 559 | v.sort(); |
| 560 | for p in v { |
| 561 | if p.is_dir() { |
| 562 | typ_files(&p, out, depth + 1); |
| 563 | } else if p.extension().map(|e| e == "typ").unwrap_or(false) { |
| 564 | out.push(p); |
| 565 | } |
| 566 | } |
| 567 | } |
| 568 | } |
| 569 | |
| 570 | #[test] |
| 571 | fn package_corpus_parses_losslessly_and_evaluates_without_panic() -> Outcome<()> { |
| 572 | let home = std::env::var("HOME").unwrap_or_default(); |
| 573 | let base = PathBuf::from(home).join(".cache/typst/packages/preview"); |
| 574 | let mut files = Vec::new(); |
| 575 | for p in PACKAGES { |
| 576 | typ_files(&base.join(p), &mut files, 0); |
| 577 | } |
| 578 | if files.is_empty() { |
| 579 | let skip = std::env::var("EVAL_ORACLE_SKIP").map(|v| v == "1").unwrap_or(false); |
| 580 | assert!(skip, "no package corpus under {} (set EVAL_ORACLE_SKIP=1 to skip explicitly)", base.display()); |
| 581 | println!("[eval-oracle] SKIPPED: no package corpus under {}", base.display()); |
| 582 | return Ok(()); |
| 583 | } |
| 584 | let strict = harness::strict("packages"); |
| 585 | let limit = harness::timeout(); |
| 586 | let mut faults = Vec::new(); |
| 587 | let mut shortfalls = Vec::new(); |
| 588 | let (mut errors, mut lossy, mut evaluated) = (0usize, 0usize, 0usize); |
| 589 | for p in &files { |
| 590 | let text = match std::fs::read_to_string(p) { |
| 591 | Ok(t) => t, |
| 592 | Err(_) => continue, |
| 593 | }; |
| 594 | let rel = p.strip_prefix(&base).map(|r| r.display().to_string()).unwrap_or_default(); |
| 595 | let src = std::panic::catch_unwind(|| Source::new(FileId(0), p.clone(), text.clone())); |
| 596 | let src = match src { |
| 597 | Ok(s) => s, |
| 598 | Err(_) => { |
| 599 | faults.push(fmt!("{}: the parser panicked", rel)); |
| 600 | continue; |
| 601 | } |
| 602 | }; |
| 603 | let errs = src.root.errors().len(); |
| 604 | if errs > 0 { |
| 605 | errors += 1; |
| 606 | shortfalls.push(fmt!("{}: {} error node(s)", rel, errs)); |
| 607 | } |
| 608 | if src.root.full_text() != text { |
| 609 | lossy += 1; |
| 610 | shortfalls.push(fmt!("{}: the tree does not reproduce the source", rel)); |
| 611 | } |
| 612 | match austenite::eval_only(p, &package_root(p), limit) { |
| 613 | RunEnd::Done(out) => if out.eval.is_none() { |
| 614 | evaluated += 1; |
| 615 | }, |
| 616 | RunEnd::Panic(m) => faults.push(fmt!("{}: evaluation panicked: {}", rel, m)), |
| 617 | RunEnd::Timeout(s) => faults.push(fmt!("{}: evaluation ran past {}s", rel, s)), |
| 618 | } |
| 619 | } |
| 620 | println!("[eval-oracle] packages: {} file(s); {} with error nodes, {} lossy, {} evaluated without error", |
| 621 | files.len(), errors, lossy, evaluated); |
| 622 | for s in shortfalls.iter().take(10) { |
| 623 | println!("[eval-oracle] {}", s); |
| 624 | } |
| 625 | assert!(faults.is_empty(), "{} fault(s):\n{}", faults.len(), faults.join("\n")); |
| 626 | if strict { |
| 627 | assert!(shortfalls.is_empty(), "{} shortfall(s) in strict mode:\n{}", shortfalls.len(), shortfalls.join("\n")); |
| 628 | } |
| 629 | Ok(()) |
| 630 | } |