Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tests/eval_oracle.rs

24.2 KiB, 7 runs

created by r1870400018:63610, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! The evaluator's structure oracle (U12): Austenite against the host's `typst` 0.15.x on a synthetic
2//! fixture corpus, at four levels -- values, positions, realised structure, layout. The driver is
3//! `tests/eval_oracle/mod.rs`; this file states what is asserted.
4//!
5//! Two kinds of test live here. The harness's own tests prove each reader and comparator against the
6//! oracle on fixtures whose answer is fixed by construction, and prove each comparator goes red on a
7//! difference -- a harness that cannot fail measures nothing. The conformance test runs every
8//! fixture and prints the scoreboard; it fails on any Austenite panic or unbounded run, on any
9//! fixture Typst rejects (or accepts, when it is marked `oracle: rejects`), and, in the areas `EVAL_ORACLE_STRICT` names, on any shortfall.
10//!
11//! cargo test -p oxedyne_fe2o3_austenite --test eval_oracle -- --nocapture
12//! EVAL_ORACLE_AREA=general EVAL_ORACLE_STRICT=general cargo test ... --test eval_oracle
13
14// `#[path]` because this test crate's root is itself `eval_oracle.rs`; a plain `mod eval_oracle;`
15// would find the module at two paths.
16#[path = "eval_oracle/mod.rs"]
17mod harness;
18
19use harness::austenite::{
20 self,
21 RunEnd,
22};
23use harness::corpus::{
24 self,
25 Expect,
26 Filter,
27};
28use harness::json::{
29 self,
30 J,
31};
32use harness::layout::{
33 self,
34 APage,
35 ARun,
36};
37use harness::oracle::Oracle;
38use harness::structure::{
39 self,
40 Sk,
41};
42use harness::{
43 Board,
44 PosRow,
45};
46
47use oxedyne_fe2o3_austenite::eval::content::{
48 Content,
49 Elem,
50 ElemKind,
51};
52use oxedyne_fe2o3_austenite::eval::fixpoint::Laid;
53use oxedyne_fe2o3_austenite::eval::intro::Builder;
54use oxedyne_fe2o3_austenite::eval::locate::Location;
55use oxedyne_fe2o3_austenite::eval::value::{
56 Dict,
57 Label,
58 Value,
59};
60use oxedyne_fe2o3_austenite::font::ShapedText;
61use oxedyne_fe2o3_austenite::fonts;
62use oxedyne_fe2o3_austenite::ir::Sp;
63use oxedyne_fe2o3_austenite::ledger::Position;
64use oxedyne_fe2o3_austenite::page::{
65 Frame,
66 Page,
67 PageGeometry,
68 Placed,
69 PlacedKind,
70};
71use oxedyne_fe2o3_austenite::syntax::{
72 FileId,
73 Source,
74 Span,
75};
76
77use oxedyne_fe2o3_core::prelude::*;
78use oxedyne_fe2o3_font::face::Role;
79use oxedyne_fe2o3_font::shape::Dir;
80
81use std::path::PathBuf;
82use std::process::Command;
83use std::sync::Arc;
84
85/// The oracle, or `None` after saying loudly that it was skipped on request.
86fn oracle() -> Outcome<Option<Oracle>> {
87 let o = res!(Oracle::find());
88 if o.is_none() {
89 println!("[eval-oracle] SKIPPED: EVAL_ORACLE_SKIP=1 and no typst 0.15.x -- nothing was compared");
90 }
91 Ok(o)
92}
93
94// An account whose own font folder holds a face Typst does not carry: Libertinus Mono, copied beside a fake
95// home. Typst finds it unless the run hides the account's fonts, and every run of the harness must.
96#[test]
97fn typst_runs_hide_the_accounts_own_fonts_and_go_red_when_they_do_not() -> Outcome<()> {
98 let o = match res!(oracle()) {
99 Some(o) => o,
100 None => return Ok(()),
101 };
102 let home = o.work.join("account-home");
103 let fonts = home.join(".local/share/fonts");
104 res!(std::fs::create_dir_all(&fonts));
105 let face = "LibertinusMono-Regular.otf";
106 res!(std::fs::copy(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("fonts").join(face), fonts.join(face)));
107 let typ = o.work.join("account-home.typ");
108 res!(std::fs::write(&typ, "#set text(font: \"Libertinus Mono\")\nMono\n"));
109 let listed = |mut c: Command| -> Outcome<String> {
110 let out = res!(c.arg("fonts").env("HOME", &home).output());
111 Ok(String::from_utf8_lossy(&out.stdout).to_string())
112 };
113 let embedded = |mut c: Command, name: &str| -> Outcome<Vec<String>> {
114 let pdf = home.join(name);
115 let status = res!(c.env("HOME", &home).status());
116 assert!(status.success(), "typst did not compile {}", typ.display());
117 harness::pdf::font_names(&pdf)
118 };
119 // The premise: with nothing hidden, the face is there to be found.
120 let mut open = Command::new(&o.bin);
121 open.env_remove("XDG_DATA_HOME");
122 assert!(res!(listed(open)).lines().any(|l| l == "Libertinus Mono"), "the fake home's face was not found by an open run");
123 assert!(!res!(listed(harness::typst_command::typst_command(&o.bin, None, &o.work))).lines().any(|l| l == "Libertinus Mono"),
124 "the oracle's own run saw the account's face");
125 // The PDF builder the comparators use.
126 let mut open = Command::new(harness::pdf::TYPST);
127 open.env_remove("XDG_DATA_HOME").arg("compile").arg(&typ).arg(home.join("open.pdf"));
128 assert!(res!(embedded(open, "open.pdf")).iter().any(|n| n.starts_with("LibertinusMono")), "an open run did not embed the face");
129 let hidden = res!(harness::pdf::typst_pdf_command(&typ, &home.join("hidden.pdf"), &[]));
130 assert!(!res!(embedded(hidden, "hidden.pdf")).iter().any(|n| n.starts_with("LibertinusMono")),
131 "the PDF builder's run embedded the account's face");
132 Ok(())
133}
134
135fn harness_fixture(name: &str) -> Outcome<corpus::Fixture> {
136 let all = res!(corpus::discover(&Filter::only("harness")));
137 match all.into_iter().find(|f| f.name == name) {
138 Some(f) => Ok(f),
139 None => Err(err!("The harness fixture {} is missing from tests/fixtures/eval/harness", name; Missing)),
140 }
141}
142
143#[test]
144fn oracle_is_present_and_pinned() -> Outcome<()> {
145 if let Some(o) = res!(oracle()) {
146 assert!(o.version.starts_with(harness::oracle::PINNED), "oracle version {}", o.version);
147 println!("[eval-oracle] {} (cache {})", o.version, o.work.display());
148 }
149 Ok(())
150}
151
152#[test]
153fn fixture_corpus_is_synthetic_and_accepted_by_typst() -> Outcome<()> {
154 let o = match res!(oracle()) {
155 Some(o) => o,
156 None => return Ok(()),
157 };
158 let all = res!(corpus::discover(&Filter::from_env()));
159 assert!(!all.is_empty(), "no fixtures under {}", corpus::fixtures_dir().display());
160 let mut bad = Vec::new();
161 for fx in &all {
162 if let Some(f) = corpus::provenance_fault(fx) {
163 bad.push(f);
164 continue;
165 }
166 match (res!(o.compiles(fx)), fx.expect) {
167 (Err(e), Expect::Accepts) => bad.push(fmt!("{}: typst rejects it: {}", fx.id(), e)),
168 (Ok(()), Expect::Rejects) => bad.push(fmt!("{}: marked `oracle: rejects`, but typst accepts it", fx.id())),
169 _ => (),
170 }
171 }
172 assert!(bad.is_empty(), "corpus faults:\n{}", bad.join("\n"));
173 Ok(())
174}
175
176// The values of `harness/values.typ`, built by hand from Austenite's own value types.
177fn hand_values() -> Vec<Value> {
178 let mut inner = Dict::new();
179 inner.insert("c", Value::str("x"));
180 let mut d = Dict::new();
181 d.insert("b", Value::Int(1));
182 d.insert("a", Value::dict(inner));
183 vec![
184 Value::Int(1),
185 Value::Int(-7),
186 Value::Float(1.5),
187 Value::Float(1.0),
188 Value::str("quoted \"text\", tab\tand ünïcode →"),
189 Value::Bool(true),
190 Value::None,
191 Value::array(vec![Value::Int(1), Value::array(vec![Value::Int(2), Value::Int(3)]), Value::array(vec![])]),
192 Value::dict(d),
193 Value::Float(f64::INFINITY),
194 Value::Int(9_007_199_254_740_993),
195 ]
196}
197
198#[test]
199fn value_serialisation_agrees_with_the_oracle_and_goes_red_on_a_difference() -> Outcome<()> {
200 let o = match res!(oracle()) {
201 Some(o) => o,
202 None => return Ok(()),
203 };
204 let fx = res!(harness_fixture("values"));
205 let want = match res!(o.probes(&fx)) {
206 Ok(v) => J::Arr(v),
207 Err(e) => return Err(err!("the oracle could not read harness/values: {}", e; Invalid)),
208 };
209 let vals = hand_values();
210 let got = J::Arr(vals.iter().map(austenite::to_json).collect());
211 let mut d = Vec::new();
212 json::diff("probes", &want, &got, &mut d);
213 assert!(d.is_empty(), "hand-built values differ from typst:\n{}", d.join("\n"));
214
215 // Each perturbation is a distinction Typst makes and the comparison must keep.
216 let perturb: [(&str, fn(&mut Vec<Value>)); 7] = [
217 ("int for float", |v| v[3] = Value::Int(1)),
218 ("float for int", |v| v[0] = Value::Float(1.0)),
219 ("changed string", |v| v[4] = Value::str("quoted")),
220 ("dictionary order", |v| {
221 let mut d = Dict::new();
222 d.insert("a", Value::Int(0));
223 d.insert("b", Value::Int(1));
224 v[8] = Value::dict(d);
225 }),
226 ("missing item", |v| { v.pop(); }),
227 ("none for false", |v| v[5] = Value::None),
228 ("near integer", |v| v[10] = Value::Int(9_007_199_254_740_992)),
229 ];
230 for (what, f) in perturb {
231 let mut vals = hand_values();
232 f(&mut vals);
233 let got = J::Arr(vals.iter().map(austenite::to_json).collect());
234 let mut d = Vec::new();
235 json::diff("probes", &want, &got, &mut d);
236 assert!(!d.is_empty(), "the comparison missed a {}", what);
237 }
238 Ok(())
239}
240
241#[test]
242fn content_comparison_ignores_field_order_but_not_text() -> Outcome<()> {
243 // Content in the shape the oracle writes it (see `markup_content.typ`), and variations of it.
244 let want = res!(json::parse(r#"{"func":"strong","body":{"func":"text","text":"b"},"label":"<x>"}"#));
245 let same = res!(json::parse(r#"{"label":"<x>","body":{"text":"b","func":"text"},"func":"strong"}"#));
246 let other = res!(json::parse(r#"{"func":"strong","body":{"func":"text","text":"c"},"label":"<x>"}"#));
247 let unlabelled = res!(json::parse(r#"{"func":"strong","body":{"func":"text","text":"b"}}"#));
248 let mut d = Vec::new();
249 json::diff("v", &want, &same, &mut d);
250 assert!(d.is_empty(), "field order is not significant in content: {:?}", d);
251 json::diff("v", &want, &other, &mut d);
252 assert!(!d.is_empty(), "changed text must differ");
253 d.clear();
254 json::diff("v", &want, &unlabelled, &mut d);
255 assert!(!d.is_empty(), "a lost label must differ");
256 // And a dictionary, unlike content, is ordered.
257 let d1 = res!(json::parse(r#"{"a":1,"b":2}"#));
258 let d2 = res!(json::parse(r#"{"b":2,"a":1}"#));
259 d.clear();
260 json::diff("v", &d1, &d2, &mut d);
261 assert!(!d.is_empty(), "dictionary order is significant");
262 Ok(())
263}
264
265#[test]
266fn layout_reader_recovers_known_baselines() -> Outcome<()> {
267 let o = match res!(oracle()) {
268 Some(o) => o,
269 None => return Ok(()),
270 };
271 let fx = res!(harness_fixture("baselines"));
272 let pages = match res!(o.layout(&fx)) {
273 Ok(p) => p,
274 Err(e) => return Err(err!("the oracle could not lay out harness/baselines: {}", e; Invalid)),
275 };
276 assert_eq!(pages.len(), 1);
277 let p = &pages[0];
278 assert!((p.width - 200.0).abs() < 0.01 && (p.height - 220.0).abs() < 0.01, "page {}x{}", p.width, p.height);
279 // Placed with baseline edges at these offsets, so these are the baselines, whatever the face.
280 let expect = [("Alpha beta", 40.0), ("Gamma", 100.0), ("Delta", 150.0), ("Epsilon2 zeta", 190.0)];
281 assert_eq!(p.lines.len(), expect.len(), "lines: {:?}", p.lines);
282 for (l, (text, base)) in p.lines.iter().zip(expect.iter()) {
283 assert_eq!(l.text.replace(' ', ""), text.replace(' ', ""), "line text");
284 assert!(l.measured, "{:?}: baseline estimated, not measured", l.text);
285 assert!((l.base - base).abs() < 0.01, "{:?}: baseline {} not {}", l.text, l.base, base);
286 assert!((l.x0 - 10.0).abs() < 0.5, "{:?}: x {}", l.text, l.x0);
287 }
288
289 // The comparator: Typst's own lines as Austenite runs pass; a 2pt drop, a lost line, changed
290 // text, an extra run and an extra page each fail.
291 let as_runs = |shift: f64| -> Vec<APage> {
292 vec![APage {
293 width: p.width,
294 height: p.height,
295 runs: p.lines.iter().map(|l| ARun { text: l.text.clone(), x0: l.x0, x1: l.x1, base: l.base + shift }).collect(),
296 }]
297 };
298 let mut d = Vec::new();
299 layout::compare(&pages, &as_runs(0.0), &mut d);
300 assert!(d.is_empty(), "identity differs: {:?}", d);
301 layout::compare(&pages, &as_runs(0.8), &mut d);
302 assert!(d.is_empty(), "0.8pt is inside the tolerance: {:?}", d);
303 layout::compare(&pages, &as_runs(2.0), &mut d);
304 assert!(!d.is_empty(), "a 2pt drop must fail");
305 let mut lost = as_runs(0.0);
306 lost[0].runs.remove(1);
307 d.clear();
308 layout::compare(&pages, &lost, &mut d);
309 assert!(!d.is_empty(), "a lost line must fail");
310 let mut changed = as_runs(0.0);
311 changed[0].runs[0].text = "Alpha beat".to_string();
312 d.clear();
313 layout::compare(&pages, &changed, &mut d);
314 assert!(!d.is_empty(), "changed text must fail");
315 let mut extra = as_runs(0.0);
316 extra[0].runs.push(ARun { text: "stray".to_string(), x0: 100.0, x1: 120.0, base: 70.0 });
317 d.clear();
318 layout::compare(&pages, &extra, &mut d);
319 assert!(!d.is_empty(), "stray text must fail");
320 let mut more = as_runs(0.0);
321 more.push(APage { width: p.width, height: p.height, runs: Vec::new() });
322 d.clear();
323 layout::compare(&pages, &more, &mut d);
324 assert!(!d.is_empty(), "an extra page must fail");
325 Ok(())
326}
327
328#[test]
329fn structure_reader_matches_the_html_export() -> Outcome<()> {
330 let o = match res!(oracle()) {
331 Some(o) => o,
332 None => return Ok(()),
333 };
334 let fx = res!(harness_fixture("structure"));
335 let html = match res!(o.html(&fx)) {
336 Ok(h) => h,
337 Err(e) => return Err(err!("the oracle could not export harness/structure: {}", e; Invalid)),
338 };
339 assert!(html.ignored.is_empty(), "ignored: {:?}", html.ignored);
340 let want = concat!(
341 r#"h2["Title words"] p["A paragraph with " strong["strong"] ", " em["emph"] " and " code["code"] "."] "#,
342 r#"ul[li["one"] li["two"]] ol[li["first"] li["second"]] dl[dt["Term"] dd["its description"]] "#,
343 r#"blockquote["Quoted"] p[a["a link"]]"#,
344 );
345 assert_eq!(structure::render(&html.skeleton), want);
346
347 let mut d = Vec::new();
348 structure::compare(&html.skeleton, &html.skeleton.clone(), &mut d);
349 assert!(d.is_empty());
350 let t = |s: &str| Sk::Text(s.to_string());
351 let tag = |n: &str, k: Vec<Sk>| Sk::Tag(n.to_string(), k);
352 // Whitespace and quote glyphs normalise away; a tag change or lost element does not.
353 let a = structure::normalise(vec![tag("p", vec![t(" a “b”\n c ")])]);
354 let b = structure::normalise(vec![tag("p", vec![t("a \"b\" c")])]);
355 structure::compare(&a, &b, &mut d);
356 assert!(d.is_empty(), "{:?}", d);
357 let c = structure::normalise(vec![tag("h3", vec![t("a \"b\" c")])]);
358 structure::compare(&a, &c, &mut d);
359 assert!(!d.is_empty(), "a changed tag must fail");
360 d.clear();
361 let mut lost = html.skeleton.clone();
362 lost.pop();
363 structure::compare(&html.skeleton, &lost, &mut d);
364 assert!(!d.is_empty(), "a lost element must fail");
365 // And an export Typst mutilates is recognised, so it is never compared.
366 assert_eq!(structure::ignored_elements("warning: align was ignored during HTML export\nwarning: x"), vec!["align"]);
367 Ok(())
368}
369
370#[test]
371fn position_reader_reads_known_positions() -> Outcome<()> {
372 let o = match res!(oracle()) {
373 Some(o) => o,
374 None => return Ok(()),
375 };
376 let fx = res!(harness_fixture("positions"));
377 let rows = match res!(o.positions(&fx)) {
378 Ok(r) => r,
379 Err(e) => return Err(err!("the oracle could not locate harness/positions: {}", e; Invalid)),
380 };
381 let funcs: Vec<&str> = rows.iter().map(|r| r.func.as_str()).collect();
382 assert_eq!(funcs, vec!["heading", "metadata", "heading"]);
383 // Both headings sit at the top-left margin corner, one on each page.
384 assert_eq!((rows[0].page, rows[2].page), (1, 2));
385 for r in [&rows[0], &rows[2]] {
386 assert!((r.x - 20.0).abs() < 1e-6 && (r.y - 20.0).abs() < 1e-6, "{:?}", r);
387 }
388 assert!(harness::compare_positions(&rows, &rows).is_empty());
389 let moved: Vec<PosRow> = rows.iter().map(|r| PosRow { y: r.y + 0.6, ..r.clone() }).collect();
390 assert!(!harness::compare_positions(&rows, &moved).is_empty(), "0.6pt is outside the tolerance");
391 let within: Vec<PosRow> = rows.iter().map(|r| PosRow { x: r.x - 0.4, ..r.clone() }).collect();
392 assert!(harness::compare_positions(&rows, &within).is_empty(), "0.4pt is inside the tolerance");
393 let paged: Vec<PosRow> = rows.iter().map(|r| PosRow { page: 1, ..r.clone() }).collect();
394 assert!(!harness::compare_positions(&rows, &paged).is_empty(), "a page change must fail");
395 assert!(!harness::compare_positions(&rows, &rows[..2]).is_empty(), "a lost element must fail");
396 Ok(())
397}
398
399// A laid-out document built by hand in Austenite's own page and introspector types, placed where
400// `harness/baselines.typ` and `harness/positions.typ` put things, so Austenite's side of levels 2 and
401// 4 is read through the same code the conformance run uses and checked against the oracle.
402fn hand_laid(lines: &[(&str, f64)], located: &[(ElemKind, bool, u32, f64, f64)], shift: f64)
403 -> Outcome<(Laid, Vec<Page>)>
404{
405 let fonts = Arc::new(res!(fonts::libertinus()));
406 let mut frame = Frame::new();
407 for (text, base) in lines {
408 let st = res!(ShapedText::new(fonts.clone(), Role::Body, Dir::Ltr, Sp::from_pt(11.0), text));
409 let dims = st.dims();
410 // A placed box's top-left is its position; its baseline sits `height` below.
411 let y = Sp::from_pt(base + shift) - dims.height;
412 frame.push(Placed::new(Sp::from_pt(10.0), y, dims, PlacedKind::Text(st)));
413 }
414 let mut builder = Builder::new();
415 for (i, (kind, probe, page, x, y)) in located.iter().enumerate() {
416 let loc = Location(0x1000 + i as u64);
417 let label = if *probe { Some(Label::new("probe")) } else { None };
418 let mut e = Elem::new(*kind, Vec::new(), Span::detached());
419 e.label = label;
420 e.location = Some(loc);
421 e.prepared = true;
422 let elem = Content::from_elem(e);
423 builder.record(&elem, Position::new(*page, Sp::from_pt(*x), Sp::from_pt(*y + shift)), None);
424 }
425 builder.page(1, &Value::None);
426 let geom = PageGeometry::new(Sp::from_pt(200.0), Sp::from_pt(220.0), Sp::ZERO);
427 Ok((Laid {
428 intro: Arc::new(builder.finish()),
429 pages: 1,
430 passes: 1,
431 converged: true,
432 }, vec![Page::new(1, geom, frame)]))
433}
434
435#[test]
436fn austenite_side_readers_agree_with_the_oracle_on_hand_built_layout() -> Outcome<()> {
437 let o = match res!(oracle()) {
438 Some(o) => o,
439 None => return Ok(()),
440 };
441 let lines = [("Alpha beta", 40.0), ("Gamma", 100.0), ("Delta", 150.0), ("Epsilon2 zeta", 190.0)];
442 let want_pages = match res!(o.layout(&res!(harness_fixture("baselines")))) {
443 Ok(p) => p,
444 Err(e) => return Err(err!("the oracle could not lay out harness/baselines: {}", e; Invalid)),
445 };
446 let want_pos = match res!(o.positions(&res!(harness_fixture("positions")))) {
447 Ok(p) => p,
448 Err(e) => return Err(err!("the oracle could not locate harness/positions: {}", e; Invalid)),
449 };
450 let located = [
451 (ElemKind::Heading, false, 1, 20.0, 20.0),
452 (ElemKind::Metadata, true, 1, 20.0, want_pos[1].y),
453 (ElemKind::Heading, false, 2, 20.0, 20.0),
454 ];
455 let (laid, laid_pages) = res!(hand_laid(&lines, &located, 0.0));
456 let mut d = Vec::new();
457 layout::compare(&want_pages, &austenite::pages(&laid_pages), &mut d);
458 assert!(d.is_empty(), "hand-built layout differs from typst: {:?}", d);
459 let pos = match austenite::positions(&laid) {
460 Ok(p) => p,
461 Err(e) => return Err(err!("positions: {}", e; Invalid)),
462 };
463 let d = harness::compare_positions(&want_pos, &pos);
464 assert!(d.is_empty(), "hand-built positions differ from typst: {:?}", d);
465
466 // Moved by 1.5pt, both go red.
467 let (moved, moved_pages) = res!(hand_laid(&lines, &located, 1.5));
468 let mut d = Vec::new();
469 layout::compare(&want_pages, &austenite::pages(&moved_pages), &mut d);
470 assert!(!d.is_empty(), "a 1.5pt shift of every baseline must fail level 4");
471 let pos = match austenite::positions(&moved) {
472 Ok(p) => p,
473 Err(e) => return Err(err!("positions: {}", e; Invalid)),
474 };
475 assert!(!harness::compare_positions(&want_pos, &pos).is_empty(), "a 1.5pt shift must fail level 2");
476 // An unlabelled metadata element is not a probe, so the rows no longer line up.
477 let unlabelled = [located[0], (ElemKind::Metadata, false, 1, 20.0, want_pos[1].y), located[2]];
478 let pos = match austenite::positions(&res!(hand_laid(&lines, &unlabelled, 0.0)).0) {
479 Ok(p) => p,
480 Err(e) => return Err(err!("positions: {}", e; Invalid)),
481 };
482 assert!(!harness::compare_positions(&want_pos, &pos).is_empty(), "a lost probe label must fail level 2");
483 Ok(())
484}
485
486// `harness/rejects.typ` fails at a place fixed by construction: line 2, column 4 -- the `p` of `panic`,
487// since an embedded expression's span leaves out its `#`, and counted in characters, so the two-byte
488// `é` before it is one column.
489#[test]
490fn first_error_reader_and_comparator_agree_with_the_oracle_and_go_red_on_a_difference() -> Outcome<()> {
491 let o = match res!(oracle()) {
492 Some(o) => o,
493 None => return Ok(()),
494 };
495 let fx = res!(harness_fixture("rejects"));
496 assert_eq!(fx.expect, Expect::Rejects, "harness/rejects.typ must carry `oracle: rejects`");
497 let said = match res!(o.compiles(&fx)) {
498 Ok(()) => return Err(err!("typst accepts harness/rejects.typ"; Invalid)),
499 Err(e) => e,
500 };
501 let (msg, pos) = harness::split_first_error(&said);
502 assert!(msg.contains("rejected on purpose"), "message read as `{}`", msg);
503 assert_eq!(pos, Some((2, 4)), "position read from `{}`", said);
504 let same = (msg.clone(), pos);
505 assert!(harness::compare_first_error(&msg, pos, Some(&same)).is_pass());
506 let moved = (msg.clone(), Some((2, 5)));
507 assert!(!harness::compare_first_error(&msg, pos, Some(&moved)).is_pass(), "a moved error must fail");
508 let reworded = ("rejected".to_string(), pos);
509 assert!(!harness::compare_first_error(&msg, pos, Some(&reworded)).is_pass(), "a reworded error must fail");
510 assert!(!harness::compare_first_error(&msg, pos, None).is_pass(), "accepting it must fail");
511 Ok(())
512}
513
514#[test]
515fn evaluator_conformance_against_the_oracle() -> Outcome<()> {
516 let o = match res!(oracle()) {
517 Some(o) => o,
518 None => return Ok(()),
519 };
520 let fixtures = res!(corpus::discover(&Filter::from_env()));
521 let limit = harness::timeout();
522 let mut board = Board { reports: Vec::new() };
523 for fx in &fixtures {
524 board.reports.push(res!(harness::check(fx, &o, limit)));
525 }
526 board.print();
527 let report = o.work.join("report.json");
528 res!(std::fs::write(&report, board.to_json().render()).map_err(|e| err!(
529 "Cannot write {}: {}", report.display(), e; IO, File, Write)));
530 println!("[eval-oracle] scoreboard written to {}", report.display());
531 let failures = board.failures();
532 assert!(failures.is_empty(), "{} failure(s):\n{}", failures.len(), failures.join("\n"));
533 Ok(())
534}
535
536// The parse-and-evaluate corpus: every `.typ` of five popular packages. There is no oracle for a
537// library file, so the claims are Austenite's own: the tree is lossless, and evaluation neither
538// panics nor runs away. Error nodes are counted; under `EVAL_ORACLE_STRICT` naming `packages` (or
539// `all`) any error node or lossy tree fails.
540const PACKAGES: [&str; 5] = ["cetz", "cetz-plot", "fletcher", "tablex", "oxifmt"];
541
542fn package_root(p: &std::path::Path) -> PathBuf {
543 let mut d = p.parent();
544 while let Some(dir) = d {
545 if dir.join("typst.toml").exists() {
546 return dir.to_path_buf();
547 }
548 d = dir.parent();
549 }
550 p.parent().map(|x| x.to_path_buf()).unwrap_or_default()
551}
552
553fn typ_files(dir: &std::path::Path, out: &mut Vec<PathBuf>, depth: usize) {
554 if depth > 12 {
555 return;
556 }
557 if let Ok(rd) = std::fs::read_dir(dir) {
558 let mut v: Vec<PathBuf> = rd.filter_map(|e| e.ok().map(|e| e.path())).collect();
559 v.sort();
560 for p in v {
561 if p.is_dir() {
562 typ_files(&p, out, depth + 1);
563 } else if p.extension().map(|e| e == "typ").unwrap_or(false) {
564 out.push(p);
565 }
566 }
567 }
568}
569
570#[test]
571fn package_corpus_parses_losslessly_and_evaluates_without_panic() -> Outcome<()> {
572 let home = std::env::var("HOME").unwrap_or_default();
573 let base = PathBuf::from(home).join(".cache/typst/packages/preview");
574 let mut files = Vec::new();
575 for p in PACKAGES {
576 typ_files(&base.join(p), &mut files, 0);
577 }
578 if files.is_empty() {
579 let skip = std::env::var("EVAL_ORACLE_SKIP").map(|v| v == "1").unwrap_or(false);
580 assert!(skip, "no package corpus under {} (set EVAL_ORACLE_SKIP=1 to skip explicitly)", base.display());
581 println!("[eval-oracle] SKIPPED: no package corpus under {}", base.display());
582 return Ok(());
583 }
584 let strict = harness::strict("packages");
585 let limit = harness::timeout();
586 let mut faults = Vec::new();
587 let mut shortfalls = Vec::new();
588 let (mut errors, mut lossy, mut evaluated) = (0usize, 0usize, 0usize);
589 for p in &files {
590 let text = match std::fs::read_to_string(p) {
591 Ok(t) => t,
592 Err(_) => continue,
593 };
594 let rel = p.strip_prefix(&base).map(|r| r.display().to_string()).unwrap_or_default();
595 let src = std::panic::catch_unwind(|| Source::new(FileId(0), p.clone(), text.clone()));
596 let src = match src {
597 Ok(s) => s,
598 Err(_) => {
599 faults.push(fmt!("{}: the parser panicked", rel));
600 continue;
601 }
602 };
603 let errs = src.root.errors().len();
604 if errs > 0 {
605 errors += 1;
606 shortfalls.push(fmt!("{}: {} error node(s)", rel, errs));
607 }
608 if src.root.full_text() != text {
609 lossy += 1;
610 shortfalls.push(fmt!("{}: the tree does not reproduce the source", rel));
611 }
612 match austenite::eval_only(p, &package_root(p), limit) {
613 RunEnd::Done(out) => if out.eval.is_none() {
614 evaluated += 1;
615 },
616 RunEnd::Panic(m) => faults.push(fmt!("{}: evaluation panicked: {}", rel, m)),
617 RunEnd::Timeout(s) => faults.push(fmt!("{}: evaluation ran past {}s", rel, s)),
618 }
619 }
620 println!("[eval-oracle] packages: {} file(s); {} with error nodes, {} lossy, {} evaluated without error",
621 files.len(), errors, lossy, evaluated);
622 for s in shortfalls.iter().take(10) {
623 println!("[eval-oracle] {}", s);
624 }
625 assert!(faults.is_empty(), "{} fault(s):\n{}", faults.len(), faults.join("\n"));
626 if strict {
627 assert!(shortfalls.is_empty(), "{} shortfall(s) in strict mode:\n{}", shortfalls.len(), shortfalls.join("\n"));
628 }
629 Ok(())
630}