oxedyne/fe2o3/fe2o3_austenite/tests/eval_oracle/mod.rs
16.7 KiB, 7 runs
created by r1870400018:63622, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | // The structure oracle harness (U12): each synthetic fixture under `tests/fixtures/eval/<area>/` is |
| 2 | // run through the host's `typst` 0.15.x and through Austenite's evaluator, and the two are compared |
| 3 | // at four levels -- values, positions, realised structure and layout (see `corpus.rs` for the |
| 4 | // levels and the fixture directives). Conformance is reported as numbers, per area and level, so |
| 5 | // "as faithfully as it can" is measurable; `EVAL_ORACLE_STRICT` turns any shortfall in the named |
| 6 | // areas into a failure, which is how a unit gates its own area and how U11 gates the whole. |
| 7 | // |
| 8 | // Environment: |
| 9 | // |
| 10 | // EVAL_ORACLE_AREA=a,b only these areas |
| 11 | // EVAL_ORACLE_FIXTURE=s only fixtures whose `area/name` contains `s` |
| 12 | // EVAL_ORACLE_STRICT=a,b|all every compared level must pass in these areas |
| 13 | // EVAL_ORACLE_TIMEOUT=secs per fixture on Austenite's side (default 60) |
| 14 | // EVAL_ORACLE_SKIP=1 skip, explicitly, when no typst 0.15.x exists |
| 15 | // EVAL_ORACLE_TYPST=path the typst binary (default `typst` on the path) |
| 16 | // EVAL_ORACLE_CAP=3G|none the memory cap each typst run is given |
| 17 | // |
| 18 | // A panic or an unbounded run on Austenite's side, and a fixture Typst itself rejects, fail the |
| 19 | // suite whatever the strictness: the first two are engine faults, the last a corpus fault. A fixture |
| 20 | // marked `oracle: rejects` inverts the last: Typst must reject it, and it is compared on the first |
| 21 | // error alone (the `Err` column), message and line:column both. |
| 22 | |
| 23 | pub mod austenite; |
| 24 | pub mod corpus; |
| 25 | pub mod json; |
| 26 | pub mod layout; |
| 27 | pub mod markup; |
| 28 | pub mod oracle; |
| 29 | pub mod pdf; |
| 30 | pub mod structure; |
| 31 | #[path = "../support/typst_command.rs"] |
| 32 | pub mod typst_command; |
| 33 | |
| 34 | use austenite::{ |
| 35 | ProbeSource, |
| 36 | RunEnd, |
| 37 | Want, |
| 38 | }; |
| 39 | use corpus::{ |
| 40 | Expect, |
| 41 | Fixture, |
| 42 | }; |
| 43 | use json::J; |
| 44 | use oracle::Oracle; |
| 45 | |
| 46 | use oxedyne_fe2o3_core::prelude::*; |
| 47 | |
| 48 | use std::time::Duration; |
| 49 | |
| 50 | pub const POS_TOL: f64 = 0.5; // points, the design's level-2 tolerance |
| 51 | |
| 52 | /// One located element: `func` is the element's name as `repr(it.func())` prints it. |
| 53 | #[derive(Clone, Debug)] |
| 54 | pub struct PosRow { |
| 55 | pub func: String, |
| 56 | pub page: u32, |
| 57 | pub x: f64, |
| 58 | pub y: f64, |
| 59 | } |
| 60 | |
| 61 | #[derive(Clone, Debug)] |
| 62 | pub enum Verdict { |
| 63 | Pass, |
| 64 | Fail(Vec<String>), |
| 65 | NotApplicable(String), // nothing to compare: no probes, or Typst's HTML export drops the construct |
| 66 | NotRequested, |
| 67 | } |
| 68 | |
| 69 | impl Verdict { |
| 70 | pub fn is_pass(&self) -> bool { matches!(self, Verdict::Pass) } |
| 71 | |
| 72 | pub fn counts(&self) -> bool { matches!(self, Verdict::Pass | Verdict::Fail(_)) } |
| 73 | |
| 74 | fn tag(&self) -> &'static str { |
| 75 | match self { |
| 76 | Verdict::Pass => "pass", |
| 77 | Verdict::Fail(_) => "FAIL", |
| 78 | Verdict::NotApplicable(_) => "n/a", |
| 79 | Verdict::NotRequested => "-", |
| 80 | } |
| 81 | } |
| 82 | } |
| 83 | |
| 84 | #[derive(Clone, Debug)] |
| 85 | pub struct FixtureReport { |
| 86 | pub id: String, |
| 87 | pub area: String, |
| 88 | pub levels: [Verdict; 4], |
| 89 | pub rejection: Verdict, // first error against Typst's, for an `oracle: rejects` fixture |
| 90 | pub fault: Option<String>, // panic, timeout or an oracle rejection: fails the suite always |
| 91 | pub notes: Vec<String>, // Austenite's evaluation and layout errors, and its diagnostics |
| 92 | pub probe_src: Option<ProbeSource>, |
| 93 | } |
| 94 | |
| 95 | pub fn timeout() -> Duration { |
| 96 | let s = std::env::var("EVAL_ORACLE_TIMEOUT").ok().and_then(|v| v.parse::<u64>().ok()).unwrap_or(60); |
| 97 | Duration::from_secs(s.max(1)) |
| 98 | } |
| 99 | |
| 100 | /// Compares one fixture at every level it asks for. |
| 101 | pub fn check(fx: &Fixture, oracle: &Oracle, limit: Duration) -> Outcome<FixtureReport> { |
| 102 | let mut rep = FixtureReport { |
| 103 | id: fx.id(), |
| 104 | area: fx.area.clone(), |
| 105 | levels: [Verdict::NotRequested, Verdict::NotRequested, Verdict::NotRequested, Verdict::NotRequested], |
| 106 | rejection: Verdict::NotRequested, |
| 107 | fault: None, |
| 108 | notes: Vec::new(), |
| 109 | probe_src: None, |
| 110 | }; |
| 111 | if let Some(f) = corpus::provenance_fault(fx) { |
| 112 | rep.fault = Some(f); |
| 113 | return Ok(rep); |
| 114 | } |
| 115 | match (res!(oracle.compiles(fx)), fx.expect) { |
| 116 | (Err(e), Expect::Accepts) => { |
| 117 | rep.fault = Some(fmt!("typst rejects the fixture: {}", e)); |
| 118 | return Ok(rep); |
| 119 | } |
| 120 | (Ok(()), Expect::Rejects) => { |
| 121 | rep.fault = Some("typst accepts a fixture marked `oracle: rejects`".to_string()); |
| 122 | return Ok(rep); |
| 123 | } |
| 124 | (Err(e), Expect::Rejects) => { |
| 125 | rep.rejection = check_rejection(fx, &e, limit, &mut rep); |
| 126 | return Ok(rep); |
| 127 | } |
| 128 | (Ok(()), Expect::Accepts) => (), |
| 129 | } |
| 130 | |
| 131 | // The oracle first: what it has nothing to say about, Austenite is not asked. |
| 132 | let o1 = if fx.wants(1) { Some(res!(oracle.probes(fx))) } else { None }; |
| 133 | let o2 = if fx.wants(2) { Some(res!(oracle.positions(fx))) } else { None }; |
| 134 | let o3 = if fx.wants(3) { Some(res!(oracle.html(fx))) } else { None }; |
| 135 | let o4 = if fx.wants(4) { Some(res!(oracle.layout(fx))) } else { None }; |
| 136 | let mut want = Want { levels: [false; 4] }; |
| 137 | match &o1 { |
| 138 | Some(Ok(v)) if v.is_empty() => rep.levels[0] = Verdict::NotApplicable("no <probe> values".to_string()), |
| 139 | Some(Ok(_)) => want.levels[0] = true, |
| 140 | Some(Err(e)) => rep.fault = Some(fmt!("typst cannot answer the probe query: {}", e)), |
| 141 | None => (), |
| 142 | } |
| 143 | match &o2 { |
| 144 | Some(Ok(v)) if v.is_empty() => rep.levels[1] = Verdict::NotApplicable("no located elements".to_string()), |
| 145 | Some(Ok(_)) => want.levels[1] = true, |
| 146 | Some(Err(e)) => rep.fault = Some(fmt!("typst cannot answer the position query: {}", e)), |
| 147 | None => (), |
| 148 | } |
| 149 | let lossy: Vec<String> = match &o3 { |
| 150 | Some(Ok(h)) => h.ignored.iter().filter(|n| !structure::HARMLESS.contains(&n.as_str())).cloned().collect(), |
| 151 | _ => Vec::new(), |
| 152 | }; |
| 153 | match &o3 { |
| 154 | Some(Ok(_)) if !lossy.is_empty() => rep.levels[2] = Verdict::NotApplicable(fmt!( |
| 155 | "typst's HTML export drops {}", lossy.join(", "))), |
| 156 | Some(Ok(_)) => want.levels[2] = true, |
| 157 | Some(Err(e)) => rep.levels[2] = Verdict::NotApplicable(fmt!("typst's HTML export fails: {}", e)), |
| 158 | None => (), |
| 159 | } |
| 160 | match &o4 { |
| 161 | Some(Ok(_)) => want.levels[3] = true, |
| 162 | Some(Err(e)) => rep.fault = Some(fmt!("typst cannot lay the fixture out: {}", e)), |
| 163 | None => (), |
| 164 | } |
| 165 | if rep.fault.is_some() || !want.levels.iter().any(|w| *w) { |
| 166 | return Ok(rep); |
| 167 | } |
| 168 | |
| 169 | let out = match austenite::run(fx, want, limit) { |
| 170 | RunEnd::Done(out) => out, |
| 171 | RunEnd::Panic(p) => { |
| 172 | rep.fault = Some(fmt!("austenite panicked: {}", p)); |
| 173 | return Ok(rep); |
| 174 | } |
| 175 | RunEnd::Timeout(s) => { |
| 176 | rep.fault = Some(fmt!("austenite did not finish within {}s", s)); |
| 177 | return Ok(rep); |
| 178 | } |
| 179 | }; |
| 180 | if let Some(e) = &out.layout { |
| 181 | rep.notes.push(fmt!("layout: {}", e)); |
| 182 | } |
| 183 | rep.notes.extend(out.diags.iter().map(|d| fmt!("diagnostic: {}", d))); |
| 184 | |
| 185 | let failed = |why: &str| Verdict::Fail(vec![why.to_string()]); |
| 186 | let eval_failed = out.eval.as_ref().map(|e| fmt!("evaluation failed: {}", json::clip(e))); |
| 187 | |
| 188 | if want.levels[0] { |
| 189 | rep.levels[0] = match (&o1, &out.probes, &eval_failed) { |
| 190 | (_, _, Some(e)) => failed(e), |
| 191 | (Some(Ok(w)), Some(Ok((g, src))), _) => { |
| 192 | rep.probe_src = Some(*src); |
| 193 | let mut d = Vec::new(); |
| 194 | json::diff("probes", &J::Arr(w.clone()), &J::Arr(g.clone()), &mut d); |
| 195 | if d.is_empty() { Verdict::Pass } else { Verdict::Fail(d) } |
| 196 | } |
| 197 | (_, Some(Err(e)), _) => failed(e), |
| 198 | _ => failed("no probe values"), |
| 199 | }; |
| 200 | } |
| 201 | if want.levels[1] { |
| 202 | rep.levels[1] = match (&o2, &out.positions, &eval_failed) { |
| 203 | (_, _, Some(e)) => failed(e), |
| 204 | (Some(Ok(w)), Some(Ok(g)), _) => { |
| 205 | let d = compare_positions(w, g); |
| 206 | if d.is_empty() { Verdict::Pass } else { Verdict::Fail(d) } |
| 207 | } |
| 208 | (_, Some(Err(e)), _) => failed(e), |
| 209 | _ => failed("no positions"), |
| 210 | }; |
| 211 | } |
| 212 | if want.levels[2] { |
| 213 | rep.levels[2] = match (&o3, &out.skeleton, &eval_failed) { |
| 214 | (_, _, Some(e)) => failed(e), |
| 215 | (Some(Ok(w)), Some(Ok(g)), _) => { |
| 216 | let mut d = Vec::new(); |
| 217 | structure::compare(&w.skeleton, g, &mut d); |
| 218 | if d.is_empty() { Verdict::Pass } else { Verdict::Fail(d) } |
| 219 | } |
| 220 | (_, Some(Err(e)), _) => failed(e), |
| 221 | _ => failed("no realised structure"), |
| 222 | }; |
| 223 | } |
| 224 | if want.levels[3] { |
| 225 | rep.levels[3] = match (&o4, &out.pages, &eval_failed) { |
| 226 | (_, _, Some(e)) => failed(e), |
| 227 | (Some(Ok(w)), Some(Ok(g)), _) => { |
| 228 | let mut d = Vec::new(); |
| 229 | layout::compare(w, g, &mut d); |
| 230 | if d.is_empty() { Verdict::Pass } else { Verdict::Fail(d) } |
| 231 | } |
| 232 | (_, Some(Err(e)), _) => failed(e), |
| 233 | _ => failed("no pages"), |
| 234 | }; |
| 235 | } |
| 236 | Ok(rep) |
| 237 | } |
| 238 | |
| 239 | // Typst's first error, as `oracle::first_error` keeps it (`error: <message> ┌─ <file>:<line>:<col>`), |
| 240 | // against Austenite's first error diagnostic. |
| 241 | fn check_rejection(fx: &Fixture, typst: &str, limit: Duration, rep: &mut FixtureReport) -> Verdict { |
| 242 | let (want_msg, want_pos) = split_first_error(typst); |
| 243 | let out = match austenite::run(fx, Want { levels: [false; 4] }, limit) { |
| 244 | RunEnd::Done(out) => out, |
| 245 | RunEnd::Panic(p) => { |
| 246 | rep.fault = Some(fmt!("austenite panicked: {}", p)); |
| 247 | return Verdict::NotRequested; |
| 248 | } |
| 249 | RunEnd::Timeout(s) => { |
| 250 | rep.fault = Some(fmt!("austenite did not finish within {}s", s)); |
| 251 | return Verdict::NotRequested; |
| 252 | } |
| 253 | }; |
| 254 | rep.notes.extend(out.diags.iter().map(|d| fmt!("diagnostic: {}", d))); |
| 255 | let verdict = compare_first_error(&want_msg, want_pos, out.first_error.as_ref()); |
| 256 | // An error raised inside an imported file or a package is placed in that file, so the file is compared |
| 257 | // too, where Typst names one. |
| 258 | match (split_first_error_file(typst), &out.first_file) { |
| 259 | (Some(w), Some(g)) if w != *g => { |
| 260 | let mut d = match verdict { |
| 261 | Verdict::Fail(d) => d, |
| 262 | _ => Vec::new(), |
| 263 | }; |
| 264 | d.push(fmt!("file: typst `{}`, austenite `{}`", w, g)); |
| 265 | Verdict::Fail(d) |
| 266 | } |
| 267 | _ => verdict, |
| 268 | } |
| 269 | } |
| 270 | |
| 271 | /// The file Typst's first error is placed in, as the CLI prints it. |
| 272 | pub fn split_first_error_file(s: &str) -> Option<String> { |
| 273 | let (_, loc) = s.trim().split_once(" ┌─ ")?; |
| 274 | let mut parts = loc.trim().rsplitn(3, ':'); |
| 275 | let _col = parts.next()?; |
| 276 | let _line = parts.next()?; |
| 277 | parts.next().map(|f| f.to_string()) |
| 278 | } |
| 279 | |
| 280 | /// Austenite's first error against Typst's: the same message and, where Typst gives one, the same |
| 281 | /// line and column. |
| 282 | pub fn compare_first_error( |
| 283 | want_msg: &str, |
| 284 | want_pos: Option<(usize, usize)>, |
| 285 | got: Option<&(String, Option<(usize, usize)>)>, |
| 286 | ) |
| 287 | -> Verdict |
| 288 | { |
| 289 | match got { |
| 290 | None => Verdict::Fail(vec![fmt!("austenite accepts it; typst: {}", want_msg)]), |
| 291 | Some((msg, pos)) => { |
| 292 | let mut d = Vec::new(); |
| 293 | // The oracle reads Typst's message a line at a time and trims each, so a message that runs over |
| 294 | // several lines is compared the same way on both sides. |
| 295 | if lines_trimmed(msg) != lines_trimmed(want_msg) { |
| 296 | d.push(fmt!("message: typst `{}`, austenite `{}`", want_msg, msg)); |
| 297 | } |
| 298 | if want_pos.is_some() && *pos != want_pos { |
| 299 | d.push(fmt!("position: typst {:?}, austenite {:?}", want_pos, pos)); |
| 300 | } |
| 301 | if d.is_empty() { Verdict::Pass } else { Verdict::Fail(d) } |
| 302 | } |
| 303 | } |
| 304 | } |
| 305 | |
| 306 | fn lines_trimmed(s: &str) -> String { |
| 307 | s.lines().map(str::trim).collect::<Vec<_>>().join("\n") |
| 308 | } |
| 309 | |
| 310 | /// Typst's first error split into its message and, when it has one, its line and column, both 1-based. |
| 311 | /// The CLI prints the line 1-based but the column as a 0-based character index (`ab #panic()` fails |
| 312 | /// at `1:4`, the `p`), so the column is shifted to match `Source::line_col`. |
| 313 | pub fn split_first_error(s: &str) -> (String, Option<(usize, usize)>) { |
| 314 | let s = s.trim().trim_start_matches("error:").trim(); |
| 315 | let (msg, loc) = match s.split_once(" ┌─ ") { |
| 316 | Some((m, l)) => (m.trim(), Some(l.trim())), |
| 317 | None => (s, None), |
| 318 | }; |
| 319 | let pos = loc.and_then(|l| { |
| 320 | let mut parts = l.rsplitn(3, ':'); |
| 321 | let col = parts.next().and_then(|c| c.parse::<usize>().ok()); |
| 322 | let line = parts.next().and_then(|c| c.parse::<usize>().ok()); |
| 323 | match (line, col) { |
| 324 | (Some(l), Some(c)) => Some((l, c + 1)), |
| 325 | _ => None, |
| 326 | } |
| 327 | }); |
| 328 | (msg.to_string(), pos) |
| 329 | } |
| 330 | |
| 331 | /// Level-2 differences: the same elements in the same order, on the same page, within [`POS_TOL`]. |
| 332 | pub fn compare_positions(want: &[PosRow], got: &[PosRow]) -> Vec<String> { |
| 333 | let mut out = Vec::new(); |
| 334 | if want.len() != got.len() { |
| 335 | out.push(fmt!("typst locates {} element(s), austenite {}: typst [{}], austenite [{}]", |
| 336 | want.len(), got.len(), |
| 337 | want.iter().map(|r| r.func.as_str()).collect::<Vec<_>>().join(", "), |
| 338 | got.iter().map(|r| r.func.as_str()).collect::<Vec<_>>().join(", "))); |
| 339 | } |
| 340 | for (i, (w, g)) in want.iter().zip(got.iter()).enumerate() { |
| 341 | if w.func != g.func { |
| 342 | out.push(fmt!("#{}: typst {}, austenite {}", i, w.func, g.func)); |
| 343 | continue; |
| 344 | } |
| 345 | if w.page != g.page { |
| 346 | out.push(fmt!("#{} {}: page typst {}, austenite {}", i, w.func, w.page, g.page)); |
| 347 | continue; |
| 348 | } |
| 349 | if !((w.x - g.x).abs() <= POS_TOL && (w.y - g.y).abs() <= POS_TOL) { |
| 350 | out.push(fmt!("#{} {} on page {}: typst ({:.2}, {:.2})pt, austenite ({:.2}, {:.2})pt", |
| 351 | i, w.func, w.page, w.x, w.y, g.x, g.y)); |
| 352 | } |
| 353 | } |
| 354 | out.truncate(20); |
| 355 | out |
| 356 | } |
| 357 | |
| 358 | // Scoreboard |
| 359 | |
| 360 | /// Which areas must pass every compared level: `EVAL_ORACLE_STRICT=all` or a list. |
| 361 | pub fn strict(area: &str) -> bool { |
| 362 | match std::env::var("EVAL_ORACLE_STRICT") { |
| 363 | Ok(v) if v.trim() == "all" || v.trim() == "1" => true, |
| 364 | Ok(v) => v.split(',').any(|a| a.trim() == area), |
| 365 | Err(_) => false, |
| 366 | } |
| 367 | } |
| 368 | |
| 369 | pub struct Board { |
| 370 | pub reports: Vec<FixtureReport>, |
| 371 | } |
| 372 | |
| 373 | impl Board { |
| 374 | /// Passes and compared fixtures for one column (0-3 the levels, 4 the first error), over the |
| 375 | /// reports `pick` admits. |
| 376 | pub fn score<F: Fn(&FixtureReport) -> bool>(&self, level: usize, pick: F) -> (usize, usize) { |
| 377 | let mut pass = 0; |
| 378 | let mut n = 0; |
| 379 | for r in self.reports.iter().filter(|r| pick(r)) { |
| 380 | let v = match r.levels.get(level) { |
| 381 | Some(v) => v, |
| 382 | None => &r.rejection, |
| 383 | }; |
| 384 | if v.counts() { |
| 385 | n += 1; |
| 386 | if v.is_pass() { |
| 387 | pass += 1; |
| 388 | } |
| 389 | } |
| 390 | } |
| 391 | (pass, n) |
| 392 | } |
| 393 | |
| 394 | pub fn areas(&self) -> Vec<String> { |
| 395 | let mut a: Vec<String> = self.reports.iter().map(|r| r.area.clone()).collect(); |
| 396 | a.dedup(); |
| 397 | a |
| 398 | } |
| 399 | |
| 400 | pub fn print(&self) { |
| 401 | println!("[eval-oracle] {:<36} {:>5} {:>5} {:>5} {:>5} {:>5} notes", "fixture", "L1", "L2", "L3", "L4", "Err"); |
| 402 | for r in &self.reports { |
| 403 | let src = match r.probe_src { |
| 404 | Some(ProbeSource::Content) => " (L1 read from evaluated content)", |
| 405 | _ => "", |
| 406 | }; |
| 407 | println!("[eval-oracle] {:<36} {:>5} {:>5} {:>5} {:>5} {:>5} {}{}", r.id, |
| 408 | r.levels[0].tag(), r.levels[1].tag(), r.levels[2].tag(), r.levels[3].tag(), r.rejection.tag(), |
| 409 | r.fault.clone().unwrap_or_default(), src); |
| 410 | for (i, v) in r.levels.iter().enumerate() { |
| 411 | match v { |
| 412 | Verdict::Fail(d) => for m in d.iter().take(6) { |
| 413 | println!("[eval-oracle] L{}: {}", i + 1, m); |
| 414 | }, |
| 415 | Verdict::NotApplicable(w) => println!("[eval-oracle] L{} n/a: {}", i + 1, w), |
| 416 | _ => (), |
| 417 | } |
| 418 | } |
| 419 | if let Verdict::Fail(d) = &r.rejection { |
| 420 | for m in d.iter().take(6) { |
| 421 | println!("[eval-oracle] Err: {}", m); |
| 422 | } |
| 423 | } |
| 424 | for n in r.notes.iter().take(4) { |
| 425 | println!("[eval-oracle] {}", json::clip(n)); |
| 426 | } |
| 427 | } |
| 428 | for area in self.areas() { |
| 429 | let cells: Vec<String> = (0..5).map(|l| { |
| 430 | let (p, n) = self.score(l, |r| r.area == area); |
| 431 | fmt!("{} {}/{}", column(l), p, n) |
| 432 | }).collect(); |
| 433 | println!("[eval-oracle] area {:<16} {}{}", area, cells.join(" "), |
| 434 | if strict(&area) { " (strict)" } else { "" }); |
| 435 | } |
| 436 | let cells: Vec<String> = (0..5).map(|l| { |
| 437 | let (p, n) = self.score(l, |_| true); |
| 438 | fmt!("{} {}/{}", column(l), p, n) |
| 439 | }).collect(); |
| 440 | println!("[eval-oracle] conformance {}", cells.join(" ")); |
| 441 | } |
| 442 | |
| 443 | /// The machine-readable scoreboard, for the orchestrator and the U11 gate. |
| 444 | pub fn to_json(&self) -> J { |
| 445 | let fixtures = self.reports.iter().map(|r| { |
| 446 | let levels = r.levels.iter().chain(std::iter::once(&r.rejection)).map(|v| match v { |
| 447 | Verdict::Fail(d) => J::Obj(vec![ |
| 448 | ("verdict".to_string(), J::str("fail")), |
| 449 | ("why".to_string(), J::Arr(d.iter().map(|m| J::Str(m.clone())).collect())), |
| 450 | ]), |
| 451 | Verdict::NotApplicable(w) => J::Obj(vec![ |
| 452 | ("verdict".to_string(), J::str("n/a")), |
| 453 | ("why".to_string(), J::Arr(vec![J::Str(w.clone())])), |
| 454 | ]), |
| 455 | other => J::Obj(vec![("verdict".to_string(), J::str(other.tag()))]), |
| 456 | }).collect(); |
| 457 | J::Obj(vec![ |
| 458 | ("id".to_string(), J::Str(r.id.clone())), |
| 459 | ("levels".to_string(), J::Arr(levels)), |
| 460 | ("fault".to_string(), r.fault.clone().map(J::Str).unwrap_or(J::Null)), |
| 461 | ("notes".to_string(), J::Arr(r.notes.iter().map(|n| J::Str(n.clone())).collect())), |
| 462 | ]) |
| 463 | }).collect(); |
| 464 | let totals = (0..5).map(|l| { |
| 465 | let (p, n) = self.score(l, |_| true); |
| 466 | J::Obj(vec![("pass".to_string(), J::Int(p as i64)), ("compared".to_string(), J::Int(n as i64))]) |
| 467 | }).collect(); |
| 468 | J::Obj(vec![("totals".to_string(), J::Arr(totals)), ("fixtures".to_string(), J::Arr(fixtures))]) |
| 469 | } |
| 470 | |
| 471 | /// Why the run fails: every fault, and every shortfall in a strict area. |
| 472 | pub fn failures(&self) -> Vec<String> { |
| 473 | let mut out = Vec::new(); |
| 474 | for r in &self.reports { |
| 475 | if let Some(f) = &r.fault { |
| 476 | out.push(fmt!("{}: {}", r.id, f)); |
| 477 | } |
| 478 | if strict(&r.area) { |
| 479 | for (i, v) in r.levels.iter().chain(std::iter::once(&r.rejection)).enumerate() { |
| 480 | if let Verdict::Fail(d) = v { |
| 481 | out.push(fmt!("{} {}: {}", r.id, column(i), d.first().cloned().unwrap_or_default())); |
| 482 | } |
| 483 | } |
| 484 | } |
| 485 | } |
| 486 | out |
| 487 | } |
| 488 | } |
| 489 | |
| 490 | // A scoreboard column's name: `L1`-`L4` for the levels, `Err` for the first error. |
| 491 | fn column(i: usize) -> String { |
| 492 | if i < 4 { fmt!("L{}", i + 1) } else { "Err".to_string() } |
| 493 | } |