Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tests/eval_oracle/mod.rs

16.7 KiB, 7 runs

created by r1870400018:63622, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// The structure oracle harness (U12): each synthetic fixture under `tests/fixtures/eval/<area>/` is
2// run through the host's `typst` 0.15.x and through Austenite's evaluator, and the two are compared
3// at four levels -- values, positions, realised structure and layout (see `corpus.rs` for the
4// levels and the fixture directives). Conformance is reported as numbers, per area and level, so
5// "as faithfully as it can" is measurable; `EVAL_ORACLE_STRICT` turns any shortfall in the named
6// areas into a failure, which is how a unit gates its own area and how U11 gates the whole.
7//
8// Environment:
9//
10// EVAL_ORACLE_AREA=a,b only these areas
11// EVAL_ORACLE_FIXTURE=s only fixtures whose `area/name` contains `s`
12// EVAL_ORACLE_STRICT=a,b|all every compared level must pass in these areas
13// EVAL_ORACLE_TIMEOUT=secs per fixture on Austenite's side (default 60)
14// EVAL_ORACLE_SKIP=1 skip, explicitly, when no typst 0.15.x exists
15// EVAL_ORACLE_TYPST=path the typst binary (default `typst` on the path)
16// EVAL_ORACLE_CAP=3G|none the memory cap each typst run is given
17//
18// A panic or an unbounded run on Austenite's side, and a fixture Typst itself rejects, fail the
19// suite whatever the strictness: the first two are engine faults, the last a corpus fault. A fixture
20// marked `oracle: rejects` inverts the last: Typst must reject it, and it is compared on the first
21// error alone (the `Err` column), message and line:column both.
22
23pub mod austenite;
24pub mod corpus;
25pub mod json;
26pub mod layout;
27pub mod markup;
28pub mod oracle;
29pub mod pdf;
30pub mod structure;
31#[path = "../support/typst_command.rs"]
32pub mod typst_command;
33
34use austenite::{
35 ProbeSource,
36 RunEnd,
37 Want,
38};
39use corpus::{
40 Expect,
41 Fixture,
42};
43use json::J;
44use oracle::Oracle;
45
46use oxedyne_fe2o3_core::prelude::*;
47
48use std::time::Duration;
49
50pub const POS_TOL: f64 = 0.5; // points, the design's level-2 tolerance
51
52/// One located element: `func` is the element's name as `repr(it.func())` prints it.
53#[derive(Clone, Debug)]
54pub struct PosRow {
55 pub func: String,
56 pub page: u32,
57 pub x: f64,
58 pub y: f64,
59}
60
61#[derive(Clone, Debug)]
62pub enum Verdict {
63 Pass,
64 Fail(Vec<String>),
65 NotApplicable(String), // nothing to compare: no probes, or Typst's HTML export drops the construct
66 NotRequested,
67}
68
69impl Verdict {
70 pub fn is_pass(&self) -> bool { matches!(self, Verdict::Pass) }
71
72 pub fn counts(&self) -> bool { matches!(self, Verdict::Pass | Verdict::Fail(_)) }
73
74 fn tag(&self) -> &'static str {
75 match self {
76 Verdict::Pass => "pass",
77 Verdict::Fail(_) => "FAIL",
78 Verdict::NotApplicable(_) => "n/a",
79 Verdict::NotRequested => "-",
80 }
81 }
82}
83
84#[derive(Clone, Debug)]
85pub struct FixtureReport {
86 pub id: String,
87 pub area: String,
88 pub levels: [Verdict; 4],
89 pub rejection: Verdict, // first error against Typst's, for an `oracle: rejects` fixture
90 pub fault: Option<String>, // panic, timeout or an oracle rejection: fails the suite always
91 pub notes: Vec<String>, // Austenite's evaluation and layout errors, and its diagnostics
92 pub probe_src: Option<ProbeSource>,
93}
94
95pub fn timeout() -> Duration {
96 let s = std::env::var("EVAL_ORACLE_TIMEOUT").ok().and_then(|v| v.parse::<u64>().ok()).unwrap_or(60);
97 Duration::from_secs(s.max(1))
98}
99
100/// Compares one fixture at every level it asks for.
101pub fn check(fx: &Fixture, oracle: &Oracle, limit: Duration) -> Outcome<FixtureReport> {
102 let mut rep = FixtureReport {
103 id: fx.id(),
104 area: fx.area.clone(),
105 levels: [Verdict::NotRequested, Verdict::NotRequested, Verdict::NotRequested, Verdict::NotRequested],
106 rejection: Verdict::NotRequested,
107 fault: None,
108 notes: Vec::new(),
109 probe_src: None,
110 };
111 if let Some(f) = corpus::provenance_fault(fx) {
112 rep.fault = Some(f);
113 return Ok(rep);
114 }
115 match (res!(oracle.compiles(fx)), fx.expect) {
116 (Err(e), Expect::Accepts) => {
117 rep.fault = Some(fmt!("typst rejects the fixture: {}", e));
118 return Ok(rep);
119 }
120 (Ok(()), Expect::Rejects) => {
121 rep.fault = Some("typst accepts a fixture marked `oracle: rejects`".to_string());
122 return Ok(rep);
123 }
124 (Err(e), Expect::Rejects) => {
125 rep.rejection = check_rejection(fx, &e, limit, &mut rep);
126 return Ok(rep);
127 }
128 (Ok(()), Expect::Accepts) => (),
129 }
130
131 // The oracle first: what it has nothing to say about, Austenite is not asked.
132 let o1 = if fx.wants(1) { Some(res!(oracle.probes(fx))) } else { None };
133 let o2 = if fx.wants(2) { Some(res!(oracle.positions(fx))) } else { None };
134 let o3 = if fx.wants(3) { Some(res!(oracle.html(fx))) } else { None };
135 let o4 = if fx.wants(4) { Some(res!(oracle.layout(fx))) } else { None };
136 let mut want = Want { levels: [false; 4] };
137 match &o1 {
138 Some(Ok(v)) if v.is_empty() => rep.levels[0] = Verdict::NotApplicable("no <probe> values".to_string()),
139 Some(Ok(_)) => want.levels[0] = true,
140 Some(Err(e)) => rep.fault = Some(fmt!("typst cannot answer the probe query: {}", e)),
141 None => (),
142 }
143 match &o2 {
144 Some(Ok(v)) if v.is_empty() => rep.levels[1] = Verdict::NotApplicable("no located elements".to_string()),
145 Some(Ok(_)) => want.levels[1] = true,
146 Some(Err(e)) => rep.fault = Some(fmt!("typst cannot answer the position query: {}", e)),
147 None => (),
148 }
149 let lossy: Vec<String> = match &o3 {
150 Some(Ok(h)) => h.ignored.iter().filter(|n| !structure::HARMLESS.contains(&n.as_str())).cloned().collect(),
151 _ => Vec::new(),
152 };
153 match &o3 {
154 Some(Ok(_)) if !lossy.is_empty() => rep.levels[2] = Verdict::NotApplicable(fmt!(
155 "typst's HTML export drops {}", lossy.join(", "))),
156 Some(Ok(_)) => want.levels[2] = true,
157 Some(Err(e)) => rep.levels[2] = Verdict::NotApplicable(fmt!("typst's HTML export fails: {}", e)),
158 None => (),
159 }
160 match &o4 {
161 Some(Ok(_)) => want.levels[3] = true,
162 Some(Err(e)) => rep.fault = Some(fmt!("typst cannot lay the fixture out: {}", e)),
163 None => (),
164 }
165 if rep.fault.is_some() || !want.levels.iter().any(|w| *w) {
166 return Ok(rep);
167 }
168
169 let out = match austenite::run(fx, want, limit) {
170 RunEnd::Done(out) => out,
171 RunEnd::Panic(p) => {
172 rep.fault = Some(fmt!("austenite panicked: {}", p));
173 return Ok(rep);
174 }
175 RunEnd::Timeout(s) => {
176 rep.fault = Some(fmt!("austenite did not finish within {}s", s));
177 return Ok(rep);
178 }
179 };
180 if let Some(e) = &out.layout {
181 rep.notes.push(fmt!("layout: {}", e));
182 }
183 rep.notes.extend(out.diags.iter().map(|d| fmt!("diagnostic: {}", d)));
184
185 let failed = |why: &str| Verdict::Fail(vec![why.to_string()]);
186 let eval_failed = out.eval.as_ref().map(|e| fmt!("evaluation failed: {}", json::clip(e)));
187
188 if want.levels[0] {
189 rep.levels[0] = match (&o1, &out.probes, &eval_failed) {
190 (_, _, Some(e)) => failed(e),
191 (Some(Ok(w)), Some(Ok((g, src))), _) => {
192 rep.probe_src = Some(*src);
193 let mut d = Vec::new();
194 json::diff("probes", &J::Arr(w.clone()), &J::Arr(g.clone()), &mut d);
195 if d.is_empty() { Verdict::Pass } else { Verdict::Fail(d) }
196 }
197 (_, Some(Err(e)), _) => failed(e),
198 _ => failed("no probe values"),
199 };
200 }
201 if want.levels[1] {
202 rep.levels[1] = match (&o2, &out.positions, &eval_failed) {
203 (_, _, Some(e)) => failed(e),
204 (Some(Ok(w)), Some(Ok(g)), _) => {
205 let d = compare_positions(w, g);
206 if d.is_empty() { Verdict::Pass } else { Verdict::Fail(d) }
207 }
208 (_, Some(Err(e)), _) => failed(e),
209 _ => failed("no positions"),
210 };
211 }
212 if want.levels[2] {
213 rep.levels[2] = match (&o3, &out.skeleton, &eval_failed) {
214 (_, _, Some(e)) => failed(e),
215 (Some(Ok(w)), Some(Ok(g)), _) => {
216 let mut d = Vec::new();
217 structure::compare(&w.skeleton, g, &mut d);
218 if d.is_empty() { Verdict::Pass } else { Verdict::Fail(d) }
219 }
220 (_, Some(Err(e)), _) => failed(e),
221 _ => failed("no realised structure"),
222 };
223 }
224 if want.levels[3] {
225 rep.levels[3] = match (&o4, &out.pages, &eval_failed) {
226 (_, _, Some(e)) => failed(e),
227 (Some(Ok(w)), Some(Ok(g)), _) => {
228 let mut d = Vec::new();
229 layout::compare(w, g, &mut d);
230 if d.is_empty() { Verdict::Pass } else { Verdict::Fail(d) }
231 }
232 (_, Some(Err(e)), _) => failed(e),
233 _ => failed("no pages"),
234 };
235 }
236 Ok(rep)
237}
238
239// Typst's first error, as `oracle::first_error` keeps it (`error: <message> ┌─ <file>:<line>:<col>`),
240// against Austenite's first error diagnostic.
241fn check_rejection(fx: &Fixture, typst: &str, limit: Duration, rep: &mut FixtureReport) -> Verdict {
242 let (want_msg, want_pos) = split_first_error(typst);
243 let out = match austenite::run(fx, Want { levels: [false; 4] }, limit) {
244 RunEnd::Done(out) => out,
245 RunEnd::Panic(p) => {
246 rep.fault = Some(fmt!("austenite panicked: {}", p));
247 return Verdict::NotRequested;
248 }
249 RunEnd::Timeout(s) => {
250 rep.fault = Some(fmt!("austenite did not finish within {}s", s));
251 return Verdict::NotRequested;
252 }
253 };
254 rep.notes.extend(out.diags.iter().map(|d| fmt!("diagnostic: {}", d)));
255 let verdict = compare_first_error(&want_msg, want_pos, out.first_error.as_ref());
256 // An error raised inside an imported file or a package is placed in that file, so the file is compared
257 // too, where Typst names one.
258 match (split_first_error_file(typst), &out.first_file) {
259 (Some(w), Some(g)) if w != *g => {
260 let mut d = match verdict {
261 Verdict::Fail(d) => d,
262 _ => Vec::new(),
263 };
264 d.push(fmt!("file: typst `{}`, austenite `{}`", w, g));
265 Verdict::Fail(d)
266 }
267 _ => verdict,
268 }
269}
270
271/// The file Typst's first error is placed in, as the CLI prints it.
272pub fn split_first_error_file(s: &str) -> Option<String> {
273 let (_, loc) = s.trim().split_once(" ┌─ ")?;
274 let mut parts = loc.trim().rsplitn(3, ':');
275 let _col = parts.next()?;
276 let _line = parts.next()?;
277 parts.next().map(|f| f.to_string())
278}
279
280/// Austenite's first error against Typst's: the same message and, where Typst gives one, the same
281/// line and column.
282pub fn compare_first_error(
283 want_msg: &str,
284 want_pos: Option<(usize, usize)>,
285 got: Option<&(String, Option<(usize, usize)>)>,
286)
287 -> Verdict
288{
289 match got {
290 None => Verdict::Fail(vec![fmt!("austenite accepts it; typst: {}", want_msg)]),
291 Some((msg, pos)) => {
292 let mut d = Vec::new();
293 // The oracle reads Typst's message a line at a time and trims each, so a message that runs over
294 // several lines is compared the same way on both sides.
295 if lines_trimmed(msg) != lines_trimmed(want_msg) {
296 d.push(fmt!("message: typst `{}`, austenite `{}`", want_msg, msg));
297 }
298 if want_pos.is_some() && *pos != want_pos {
299 d.push(fmt!("position: typst {:?}, austenite {:?}", want_pos, pos));
300 }
301 if d.is_empty() { Verdict::Pass } else { Verdict::Fail(d) }
302 }
303 }
304}
305
306fn lines_trimmed(s: &str) -> String {
307 s.lines().map(str::trim).collect::<Vec<_>>().join("\n")
308}
309
310/// Typst's first error split into its message and, when it has one, its line and column, both 1-based.
311/// The CLI prints the line 1-based but the column as a 0-based character index (`ab #panic()` fails
312/// at `1:4`, the `p`), so the column is shifted to match `Source::line_col`.
313pub fn split_first_error(s: &str) -> (String, Option<(usize, usize)>) {
314 let s = s.trim().trim_start_matches("error:").trim();
315 let (msg, loc) = match s.split_once(" ┌─ ") {
316 Some((m, l)) => (m.trim(), Some(l.trim())),
317 None => (s, None),
318 };
319 let pos = loc.and_then(|l| {
320 let mut parts = l.rsplitn(3, ':');
321 let col = parts.next().and_then(|c| c.parse::<usize>().ok());
322 let line = parts.next().and_then(|c| c.parse::<usize>().ok());
323 match (line, col) {
324 (Some(l), Some(c)) => Some((l, c + 1)),
325 _ => None,
326 }
327 });
328 (msg.to_string(), pos)
329}
330
331/// Level-2 differences: the same elements in the same order, on the same page, within [`POS_TOL`].
332pub fn compare_positions(want: &[PosRow], got: &[PosRow]) -> Vec<String> {
333 let mut out = Vec::new();
334 if want.len() != got.len() {
335 out.push(fmt!("typst locates {} element(s), austenite {}: typst [{}], austenite [{}]",
336 want.len(), got.len(),
337 want.iter().map(|r| r.func.as_str()).collect::<Vec<_>>().join(", "),
338 got.iter().map(|r| r.func.as_str()).collect::<Vec<_>>().join(", ")));
339 }
340 for (i, (w, g)) in want.iter().zip(got.iter()).enumerate() {
341 if w.func != g.func {
342 out.push(fmt!("#{}: typst {}, austenite {}", i, w.func, g.func));
343 continue;
344 }
345 if w.page != g.page {
346 out.push(fmt!("#{} {}: page typst {}, austenite {}", i, w.func, w.page, g.page));
347 continue;
348 }
349 if !((w.x - g.x).abs() <= POS_TOL && (w.y - g.y).abs() <= POS_TOL) {
350 out.push(fmt!("#{} {} on page {}: typst ({:.2}, {:.2})pt, austenite ({:.2}, {:.2})pt",
351 i, w.func, w.page, w.x, w.y, g.x, g.y));
352 }
353 }
354 out.truncate(20);
355 out
356}
357
358// Scoreboard
359
360/// Which areas must pass every compared level: `EVAL_ORACLE_STRICT=all` or a list.
361pub fn strict(area: &str) -> bool {
362 match std::env::var("EVAL_ORACLE_STRICT") {
363 Ok(v) if v.trim() == "all" || v.trim() == "1" => true,
364 Ok(v) => v.split(',').any(|a| a.trim() == area),
365 Err(_) => false,
366 }
367}
368
369pub struct Board {
370 pub reports: Vec<FixtureReport>,
371}
372
373impl Board {
374 /// Passes and compared fixtures for one column (0-3 the levels, 4 the first error), over the
375 /// reports `pick` admits.
376 pub fn score<F: Fn(&FixtureReport) -> bool>(&self, level: usize, pick: F) -> (usize, usize) {
377 let mut pass = 0;
378 let mut n = 0;
379 for r in self.reports.iter().filter(|r| pick(r)) {
380 let v = match r.levels.get(level) {
381 Some(v) => v,
382 None => &r.rejection,
383 };
384 if v.counts() {
385 n += 1;
386 if v.is_pass() {
387 pass += 1;
388 }
389 }
390 }
391 (pass, n)
392 }
393
394 pub fn areas(&self) -> Vec<String> {
395 let mut a: Vec<String> = self.reports.iter().map(|r| r.area.clone()).collect();
396 a.dedup();
397 a
398 }
399
400 pub fn print(&self) {
401 println!("[eval-oracle] {:<36} {:>5} {:>5} {:>5} {:>5} {:>5} notes", "fixture", "L1", "L2", "L3", "L4", "Err");
402 for r in &self.reports {
403 let src = match r.probe_src {
404 Some(ProbeSource::Content) => " (L1 read from evaluated content)",
405 _ => "",
406 };
407 println!("[eval-oracle] {:<36} {:>5} {:>5} {:>5} {:>5} {:>5} {}{}", r.id,
408 r.levels[0].tag(), r.levels[1].tag(), r.levels[2].tag(), r.levels[3].tag(), r.rejection.tag(),
409 r.fault.clone().unwrap_or_default(), src);
410 for (i, v) in r.levels.iter().enumerate() {
411 match v {
412 Verdict::Fail(d) => for m in d.iter().take(6) {
413 println!("[eval-oracle] L{}: {}", i + 1, m);
414 },
415 Verdict::NotApplicable(w) => println!("[eval-oracle] L{} n/a: {}", i + 1, w),
416 _ => (),
417 }
418 }
419 if let Verdict::Fail(d) = &r.rejection {
420 for m in d.iter().take(6) {
421 println!("[eval-oracle] Err: {}", m);
422 }
423 }
424 for n in r.notes.iter().take(4) {
425 println!("[eval-oracle] {}", json::clip(n));
426 }
427 }
428 for area in self.areas() {
429 let cells: Vec<String> = (0..5).map(|l| {
430 let (p, n) = self.score(l, |r| r.area == area);
431 fmt!("{} {}/{}", column(l), p, n)
432 }).collect();
433 println!("[eval-oracle] area {:<16} {}{}", area, cells.join(" "),
434 if strict(&area) { " (strict)" } else { "" });
435 }
436 let cells: Vec<String> = (0..5).map(|l| {
437 let (p, n) = self.score(l, |_| true);
438 fmt!("{} {}/{}", column(l), p, n)
439 }).collect();
440 println!("[eval-oracle] conformance {}", cells.join(" "));
441 }
442
443 /// The machine-readable scoreboard, for the orchestrator and the U11 gate.
444 pub fn to_json(&self) -> J {
445 let fixtures = self.reports.iter().map(|r| {
446 let levels = r.levels.iter().chain(std::iter::once(&r.rejection)).map(|v| match v {
447 Verdict::Fail(d) => J::Obj(vec![
448 ("verdict".to_string(), J::str("fail")),
449 ("why".to_string(), J::Arr(d.iter().map(|m| J::Str(m.clone())).collect())),
450 ]),
451 Verdict::NotApplicable(w) => J::Obj(vec![
452 ("verdict".to_string(), J::str("n/a")),
453 ("why".to_string(), J::Arr(vec![J::Str(w.clone())])),
454 ]),
455 other => J::Obj(vec![("verdict".to_string(), J::str(other.tag()))]),
456 }).collect();
457 J::Obj(vec![
458 ("id".to_string(), J::Str(r.id.clone())),
459 ("levels".to_string(), J::Arr(levels)),
460 ("fault".to_string(), r.fault.clone().map(J::Str).unwrap_or(J::Null)),
461 ("notes".to_string(), J::Arr(r.notes.iter().map(|n| J::Str(n.clone())).collect())),
462 ])
463 }).collect();
464 let totals = (0..5).map(|l| {
465 let (p, n) = self.score(l, |_| true);
466 J::Obj(vec![("pass".to_string(), J::Int(p as i64)), ("compared".to_string(), J::Int(n as i64))])
467 }).collect();
468 J::Obj(vec![("totals".to_string(), J::Arr(totals)), ("fixtures".to_string(), J::Arr(fixtures))])
469 }
470
471 /// Why the run fails: every fault, and every shortfall in a strict area.
472 pub fn failures(&self) -> Vec<String> {
473 let mut out = Vec::new();
474 for r in &self.reports {
475 if let Some(f) = &r.fault {
476 out.push(fmt!("{}: {}", r.id, f));
477 }
478 if strict(&r.area) {
479 for (i, v) in r.levels.iter().chain(std::iter::once(&r.rejection)).enumerate() {
480 if let Verdict::Fail(d) = v {
481 out.push(fmt!("{} {}: {}", r.id, column(i), d.first().cloned().unwrap_or_default()));
482 }
483 }
484 }
485 }
486 out
487 }
488}
489
490// A scoreboard column's name: `L1`-`L4` for the levels, `Err` for the first error.
491fn column(i: usize) -> String {
492 if i < 4 { fmt!("L{}", i + 1) } else { "Err".to_string() }
493}