oxedyne/fe2o3/fe2o3_austenite/tests/eval_oracle/oracle.rs
13.9 KiB, 11 runs
created by r1870400018:63624, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | // The external oracle: the host's `typst` 0.15.x, run as a process, never linked. Every run is |
| 2 | // capped with `systemd-run --user --scope -p MemoryMax=<cap>` where systemd-run exists |
| 3 | // (`EVAL_ORACLE_CAP`, default 3G, `none` to run uncapped); a missing or wrongly versioned binary is |
| 4 | // an error unless `EVAL_ORACLE_SKIP=1` says, explicitly, to skip -- so absence never reads as green. |
| 5 | // |
| 6 | // Outputs are cached under the test binary's target directory (never `/tmp`, never the source |
| 7 | // tree), keyed by the typst version and every byte of the fixture's area, so an edit to a fixture or |
| 8 | // a helper it imports re-runs the oracle and nothing else does. Expected values are always the |
| 9 | // oracle's own output; nothing Austenite produced is ever written here. |
| 10 | |
| 11 | use crate::harness::corpus::{ |
| 12 | area_bytes, |
| 13 | Fixture, |
| 14 | }; |
| 15 | use crate::harness::json::{ |
| 16 | self, |
| 17 | J, |
| 18 | }; |
| 19 | use crate::harness::layout::{ |
| 20 | self, |
| 21 | OPage, |
| 22 | }; |
| 23 | use crate::harness::structure::{ |
| 24 | self, |
| 25 | Sk, |
| 26 | }; |
| 27 | use crate::harness::typst_command::typst_command; |
| 28 | use crate::harness::PosRow; |
| 29 | |
| 30 | use oxedyne_fe2o3_core::prelude::*; |
| 31 | |
| 32 | use std::path::{ |
| 33 | Path, |
| 34 | PathBuf, |
| 35 | }; |
| 36 | use std::process::Command; |
| 37 | |
| 38 | pub const PINNED: &str = "typst 0.15."; |
| 39 | |
| 40 | // Part of every cache key: which faces Typst was given, so that a product made with the account's own fonts is |
| 41 | // never read as made without them. |
| 42 | pub const FACES: &str = "fonts=system"; |
| 43 | |
| 44 | pub const PROBES_EXPR: &str = "query(<probe>).map(it => it.value)"; |
| 45 | pub const POSITIONS_EXPR: &str = "query(selector(heading).or(figure, math.equation, footnote, <probe>))\ |
| 46 | .map(it => (func: repr(it.func()), pos: it.location().position()))"; |
| 47 | |
| 48 | /// What the oracle said about one fixture at one level: its answer, or the reason it has none. |
| 49 | pub type Said<T> = std::result::Result<T, String>; |
| 50 | |
| 51 | #[derive(Clone, Debug)] |
| 52 | pub struct Oracle { |
| 53 | pub bin: String, |
| 54 | pub version: String, |
| 55 | cap: Option<String>, |
| 56 | pub work: PathBuf, |
| 57 | } |
| 58 | |
| 59 | pub struct Html { |
| 60 | pub skeleton: Vec<Sk>, |
| 61 | pub ignored: Vec<String>, // elements the HTML export dropped |
| 62 | } |
| 63 | |
| 64 | impl Oracle { |
| 65 | /// The oracle, `Ok(None)` only when `EVAL_ORACLE_SKIP=1` is set and no usable `typst` exists. |
| 66 | pub fn find() -> Outcome<Option<Oracle>> { |
| 67 | let skip = std::env::var("EVAL_ORACLE_SKIP").map(|v| v == "1").unwrap_or(false); |
| 68 | let bin = std::env::var("EVAL_ORACLE_TYPST").unwrap_or_else(|_| "typst".to_string()); |
| 69 | let version = match Command::new(&bin).arg("--version").output() { |
| 70 | Ok(o) if o.status.success() => String::from_utf8_lossy(&o.stdout).trim().to_string(), |
| 71 | Ok(o) => { |
| 72 | if skip { |
| 73 | return Ok(None); |
| 74 | } |
| 75 | return Err(err!("`{} --version` failed: {}", bin, String::from_utf8_lossy(&o.stderr).trim(); IO, Missing)); |
| 76 | } |
| 77 | Err(e) => { |
| 78 | if skip { |
| 79 | return Ok(None); |
| 80 | } |
| 81 | return Err(err!("The typst oracle `{}` cannot be run ({}). Install typst 0.15.x, point \ |
| 82 | EVAL_ORACLE_TYPST at it, or set EVAL_ORACLE_SKIP=1 to skip the oracle explicitly.", bin, e; IO, Missing)); |
| 83 | } |
| 84 | }; |
| 85 | if !version.starts_with(PINNED) { |
| 86 | if skip { |
| 87 | return Ok(None); |
| 88 | } |
| 89 | return Err(err!("The oracle is pinned to {}x; `{}` reports {:?}.", PINNED, bin, version; Invalid, Mismatch)); |
| 90 | } |
| 91 | let cap = match std::env::var("EVAL_ORACLE_CAP") { |
| 92 | Ok(v) if v == "none" => None, |
| 93 | Ok(v) if !v.is_empty() => Some(v), |
| 94 | _ => Some("3G".to_string()), |
| 95 | }; |
| 96 | let cap = match cap { |
| 97 | Some(c) if Command::new("systemd-run").arg("--version").output().map(|o| o.status.success()).unwrap_or(false) => Some(c), |
| 98 | _ => None, |
| 99 | }; |
| 100 | let work = res!(work_dir()); |
| 101 | Ok(Some(Oracle { bin, version, cap, work })) |
| 102 | } |
| 103 | |
| 104 | fn command(&self, args: &[String], cwd: &Path) -> Command { |
| 105 | let mut c = typst_command(&self.bin, self.cap.as_deref(), &self.work); |
| 106 | c.args(args); |
| 107 | c.current_dir(cwd); |
| 108 | // An area that keeps packages of its own beside its fixtures lays them out as Typst's package |
| 109 | // path does, `packages/<namespace>/<name>/<version>`. |
| 110 | let own = cwd.join("packages"); |
| 111 | if own.is_dir() { |
| 112 | c.env("TYPST_PACKAGE_PATH", &own); |
| 113 | } |
| 114 | c |
| 115 | } |
| 116 | |
| 117 | /// Runs typst; `Ok(Err(stderr))` when typst itself reported failure. |
| 118 | fn run(&self, args: &[String], cwd: &Path) -> Outcome<Said<(String, String)>> { |
| 119 | let out = match self.command(args, cwd).output() { |
| 120 | Ok(o) => o, |
| 121 | Err(e) => return Err(err!("Could not run the typst oracle: {}", e; IO)), |
| 122 | }; |
| 123 | let stdout = String::from_utf8_lossy(&out.stdout).to_string(); |
| 124 | let stderr = String::from_utf8_lossy(&out.stderr).to_string(); |
| 125 | if out.status.success() { |
| 126 | Ok(Ok((stdout, stderr))) |
| 127 | } else { |
| 128 | Ok(Err(first_error(&stderr))) |
| 129 | } |
| 130 | } |
| 131 | |
| 132 | /// The fixture's cache directory, created on first use. |
| 133 | pub fn dir(&self, fx: &Fixture) -> Outcome<PathBuf> { |
| 134 | let files = res!(area_bytes(&fx.root)); |
| 135 | let mut h = Fnv::new(); |
| 136 | h.feed(self.version.as_bytes()); |
| 137 | h.feed(FACES.as_bytes()); |
| 138 | h.feed(fx.name.as_bytes()); |
| 139 | // Typst's answers name the fixture by its absolute path, so a target directory shared by two |
| 140 | // checkouts must not hand one checkout's answers to the other. |
| 141 | let root = fx.root.canonicalize().unwrap_or_else(|_| fx.root.clone()); |
| 142 | h.feed(root.to_string_lossy().as_bytes()); |
| 143 | h.feed(&[0]); |
| 144 | for (rel, bytes) in &files { |
| 145 | h.feed(rel.as_bytes()); |
| 146 | h.feed(&[0]); |
| 147 | h.feed(bytes); |
| 148 | } |
| 149 | let d = self.work.join(&fx.area).join(fmt!("{}-{:016x}", fx.name, h.0)); |
| 150 | res!(std::fs::create_dir_all(&d).map_err(|e| err!("Cannot create {}: {}", d.display(), e; IO, File, Write))); |
| 151 | Ok(d) |
| 152 | } |
| 153 | |
| 154 | /// A cached text product: `<what>.out` holds a success, `<what>.err` a typst failure. |
| 155 | fn cached<F>(&self, fx: &Fixture, what: &str, make: F) -> Outcome<Said<String>> |
| 156 | where F: FnOnce(&Oracle, &Path) -> Outcome<Said<String>> |
| 157 | { |
| 158 | let d = res!(self.dir(fx)); |
| 159 | let ok = d.join(fmt!("{}.out", what)); |
| 160 | let bad = d.join(fmt!("{}.err", what)); |
| 161 | // Tests run in parallel and share fixtures, so one of them makes a product at a time. |
| 162 | let _guard = lock_mutex!(ORACLE_LOCK); |
| 163 | if let Ok(s) = std::fs::read_to_string(&ok) { |
| 164 | return Ok(Ok(s)); |
| 165 | } |
| 166 | if let Ok(s) = std::fs::read_to_string(&bad) { |
| 167 | return Ok(Err(s)); |
| 168 | } |
| 169 | let said = res!(make(self, &d)); |
| 170 | let (path, text) = match &said { |
| 171 | Ok(s) => (ok, s.clone()), |
| 172 | Err(s) => (bad, s.clone()), |
| 173 | }; |
| 174 | res!(write_whole(&path, text.as_bytes())); |
| 175 | Ok(said) |
| 176 | } |
| 177 | |
| 178 | fn eval(&self, fx: &Fixture, what: &str, expr: &str) -> Outcome<Said<J>> { |
| 179 | let said = res!(self.cached(fx, what, |o, _| { |
| 180 | let args = vec![ |
| 181 | "eval".to_string(), expr.to_string(), |
| 182 | "--in".to_string(), fx.path.display().to_string(), |
| 183 | "--root".to_string(), fx.root.display().to_string(), |
| 184 | ]; |
| 185 | Ok(res!(o.run(&args, &fx.root)).map(|(out, _)| out)) |
| 186 | })); |
| 187 | match said { |
| 188 | Ok(text) => Ok(Ok(res!(json::parse(&text)))), |
| 189 | Err(e) => Ok(Err(e)), |
| 190 | } |
| 191 | } |
| 192 | |
| 193 | /// The value of a code expression, `typst eval` in the fixture's context with the area as its root, so |
| 194 | /// an `import` or a `read` in the expression resolves there. Cached like every other product. |
| 195 | #[allow(dead_code)] // the import corpus asks for values; the other test crates do not |
| 196 | pub fn value(&self, fx: &Fixture, expr: &str) -> Outcome<Said<J>> { |
| 197 | let mut h = Fnv::new(); |
| 198 | h.feed(expr.as_bytes()); |
| 199 | self.eval(fx, &fmt!("value-{:016x}", h.0), expr) |
| 200 | } |
| 201 | |
| 202 | /// Does the fixture compile without error? `Ok(Err(message))` names the first error. |
| 203 | pub fn compiles(&self, fx: &Fixture) -> Outcome<Said<()>> { |
| 204 | let pdf = res!(self.pdf(fx)); |
| 205 | Ok(pdf.map(|_| ())) |
| 206 | } |
| 207 | |
| 208 | fn pdf(&self, fx: &Fixture) -> Outcome<Said<PathBuf>> { |
| 209 | let said = res!(self.cached(fx, "pdf", |o, d| { |
| 210 | let pdf = d.join("typst.pdf"); |
| 211 | let args = vec![ |
| 212 | "compile".to_string(), fx.path.display().to_string(), pdf.display().to_string(), |
| 213 | "--root".to_string(), fx.root.display().to_string(), |
| 214 | ]; |
| 215 | Ok(res!(o.run(&args, &fx.root)).map(|_| pdf.display().to_string())) |
| 216 | })); |
| 217 | Ok(said.map(PathBuf::from)) |
| 218 | } |
| 219 | |
| 220 | /// Level 1: every `<probe>` metadata value, in document order. |
| 221 | pub fn probes(&self, fx: &Fixture) -> Outcome<Said<Vec<J>>> { |
| 222 | match res!(self.eval(fx, "probes", PROBES_EXPR)) { |
| 223 | Ok(J::Arr(v)) => Ok(Ok(v)), |
| 224 | Ok(other) => Ok(Err(fmt!("the probe query returned {}", other.render()))), |
| 225 | Err(e) => Ok(Err(e)), |
| 226 | } |
| 227 | } |
| 228 | |
| 229 | /// Level 2: headings, figures, equations, footnotes and probes with their positions. |
| 230 | pub fn positions(&self, fx: &Fixture) -> Outcome<Said<Vec<PosRow>>> { |
| 231 | let v = match res!(self.eval(fx, "positions", POSITIONS_EXPR)) { |
| 232 | Ok(J::Arr(v)) => v, |
| 233 | Ok(other) => return Ok(Err(fmt!("the position query returned {}", other.render()))), |
| 234 | Err(e) => return Ok(Err(e)), |
| 235 | }; |
| 236 | let mut rows = Vec::new(); |
| 237 | for r in v { |
| 238 | let func = r.get("func").and_then(|f| f.as_str()).unwrap_or("").to_string(); |
| 239 | let pos = match r.get("pos") { |
| 240 | Some(p) => p, |
| 241 | None => return Ok(Err(fmt!("a position row has no `pos`: {}", r.render()))), |
| 242 | }; |
| 243 | rows.push(PosRow { |
| 244 | func, |
| 245 | page: pos.get("page").and_then(|p| p.as_i64()).unwrap_or(0) as u32, |
| 246 | x: pt(pos.get("x")), |
| 247 | y: pt(pos.get("y")), |
| 248 | }); |
| 249 | } |
| 250 | Ok(Ok(rows)) |
| 251 | } |
| 252 | |
| 253 | /// Level 3: the HTML export's skeleton and the elements it ignored. |
| 254 | pub fn html(&self, fx: &Fixture) -> Outcome<Said<Html>> { |
| 255 | let said = res!(self.cached(fx, "html", |o, d| { |
| 256 | let out = d.join("typst.html"); |
| 257 | let args = vec![ |
| 258 | "compile".to_string(), "--features".to_string(), "html".to_string(), |
| 259 | "--format".to_string(), "html".to_string(), |
| 260 | fx.path.display().to_string(), out.display().to_string(), |
| 261 | "--root".to_string(), fx.root.display().to_string(), |
| 262 | ]; |
| 263 | match res!(o.run(&args, &fx.root)) { |
| 264 | Ok((_, stderr)) => { |
| 265 | let html = res!(std::fs::read_to_string(&out).map_err(|e| err!( |
| 266 | "Cannot read {}: {}", out.display(), e; IO, File, Read))); |
| 267 | // The ignored-element warnings ride along, one per line, ahead of a marker. |
| 268 | let mut s = String::new(); |
| 269 | for n in structure::ignored_elements(&stderr) { |
| 270 | s.push_str(&n); |
| 271 | s.push('\n'); |
| 272 | } |
| 273 | s.push_str("\u{0}\n"); |
| 274 | s.push_str(&html); |
| 275 | Ok(Ok(s)) |
| 276 | } |
| 277 | Err(e) => Ok(Err(e)), |
| 278 | } |
| 279 | })); |
| 280 | let text = match said { |
| 281 | Ok(t) => t, |
| 282 | Err(e) => return Ok(Err(e)), |
| 283 | }; |
| 284 | let (head, html) = match text.split_once("\u{0}\n") { |
| 285 | Some(p) => p, |
| 286 | None => ("", text.as_str()), |
| 287 | }; |
| 288 | let ignored = head.lines().filter(|l| !l.is_empty()).map(|l| l.to_string()).collect(); |
| 289 | Ok(Ok(Html { skeleton: structure::from_html(html), ignored })) |
| 290 | } |
| 291 | |
| 292 | /// Level 4: pages with their lines and baselines. |
| 293 | pub fn layout(&self, fx: &Fixture) -> Outcome<Said<Vec<OPage>>> { |
| 294 | let pdf = match res!(self.pdf(fx)) { |
| 295 | Ok(p) => p, |
| 296 | Err(e) => return Ok(Err(e)), |
| 297 | }; |
| 298 | let d = res!(self.dir(fx)); |
| 299 | let bbox = d.join("typst.bbox.html"); |
| 300 | let _guard = lock_mutex!(ORACLE_LOCK); |
| 301 | if !bbox.exists() { |
| 302 | let st = Command::new("pdftotext").arg("-bbox-layout").arg(&pdf).arg(&bbox).output(); |
| 303 | match st { |
| 304 | Ok(o) if o.status.success() => (), |
| 305 | Ok(o) => return Err(err!("pdftotext failed on {}: {}", pdf.display(), |
| 306 | String::from_utf8_lossy(&o.stderr).trim(); IO)), |
| 307 | Err(e) => return Err(err!("pdftotext (poppler-utils) is needed for level 4: {}", e; IO, Missing)), |
| 308 | } |
| 309 | } |
| 310 | let xml = res!(std::fs::read_to_string(&bbox).map_err(|e| err!("Cannot read {}: {}", bbox.display(), e; IO, File, Read))); |
| 311 | let mut pages = layout::read_bbox(&xml); |
| 312 | let svg1 = d.join("typst-1.svg"); |
| 313 | if !svg1.exists() { |
| 314 | let pattern = d.join("typst-{p}.svg"); |
| 315 | let args = vec![ |
| 316 | "compile".to_string(), fx.path.display().to_string(), pattern.display().to_string(), |
| 317 | "--root".to_string(), fx.root.display().to_string(), |
| 318 | ]; |
| 319 | if let Err(e) = res!(self.run(&args, &fx.root)) { |
| 320 | return Ok(Err(e)); |
| 321 | } |
| 322 | } |
| 323 | for (i, p) in pages.iter_mut().enumerate() { |
| 324 | let svg = d.join(fmt!("typst-{}.svg", i + 1)); |
| 325 | let text = res!(std::fs::read_to_string(&svg).map_err(|e| err!("Cannot read {}: {}", svg.display(), e; IO, File, Read))); |
| 326 | layout::attach_baselines(p, &layout::glyph_origins(&text)); |
| 327 | layout::attach_starts(p, &layout::run_starts(&text)); |
| 328 | } |
| 329 | Ok(Ok(pages)) |
| 330 | } |
| 331 | } |
| 332 | |
| 333 | fn pt(v: Option<&J>) -> f64 { |
| 334 | match v { |
| 335 | Some(J::Str(s)) => s.trim_end_matches("pt").parse::<f64>().unwrap_or(f64::NAN), |
| 336 | Some(J::Float(f)) => *f, |
| 337 | Some(J::Int(i)) => *i as f64, |
| 338 | _ => f64::NAN, |
| 339 | } |
| 340 | } |
| 341 | |
| 342 | /// The first `error:` line of typst's diagnostics, with its location line when there is one. |
| 343 | fn first_error(stderr: &str) -> String { |
| 344 | let mut lines = stderr.lines(); |
| 345 | while let Some(l) = lines.next() { |
| 346 | if l.starts_with("error:") { |
| 347 | // A message may run over several lines; the place follows it, on the line that opens `┌─`. |
| 348 | let mut message = l.trim().to_string(); |
| 349 | let mut loc = String::new(); |
| 350 | for n in lines.by_ref() { |
| 351 | let t = n.trim(); |
| 352 | if t.starts_with('┌') { |
| 353 | loc = t.to_string(); |
| 354 | break; |
| 355 | } |
| 356 | if t.starts_with('│') || t.starts_with('=') || t.is_empty() { |
| 357 | break; |
| 358 | } |
| 359 | message.push('\n'); |
| 360 | message.push_str(t); |
| 361 | } |
| 362 | return fmt!("{} {}", message, loc); |
| 363 | } |
| 364 | } |
| 365 | stderr.trim().lines().take(3).collect::<Vec<_>>().join(" | ") |
| 366 | } |
| 367 | |
| 368 | static ORACLE_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); |
| 369 | |
| 370 | /// Writes beside the target and renames over it, so a reader never sees half a file. |
| 371 | fn write_whole(path: &Path, bytes: &[u8]) -> Outcome<()> { |
| 372 | let tmp = path.with_extension(fmt!("part{}", std::process::id())); |
| 373 | res!(std::fs::write(&tmp, bytes).map_err(|e| err!("Cannot write {}: {}", tmp.display(), e; IO, File, Write))); |
| 374 | res!(std::fs::rename(&tmp, path).map_err(|e| err!("Cannot rename {}: {}", tmp.display(), e; IO, File, Write))); |
| 375 | Ok(()) |
| 376 | } |
| 377 | |
| 378 | /// `<target>/<profile>/eval_oracle`, beside the test binary that is running. |
| 379 | pub fn work_dir() -> Outcome<PathBuf> { |
| 380 | let exe = res!(std::env::current_exe().map_err(|e| err!("Cannot locate the test binary: {}", e; IO))); |
| 381 | // `<target>/<profile>/deps/<binary>` |
| 382 | let profile = match exe.parent().and_then(|deps| deps.parent()) { |
| 383 | Some(p) => p.to_path_buf(), |
| 384 | None => return Err(err!("The test binary {} has no target directory above it", exe.display(); Missing)), |
| 385 | }; |
| 386 | let d = profile.join("eval_oracle"); |
| 387 | res!(std::fs::create_dir_all(&d).map_err(|e| err!("Cannot create {}: {}", d.display(), e; IO, File, Write))); |
| 388 | Ok(d) |
| 389 | } |
| 390 | |
| 391 | pub struct Fnv(pub u64); |
| 392 | |
| 393 | impl Fnv { |
| 394 | pub fn new() -> Self { Fnv(0xcbf2_9ce4_8422_2325) } |
| 395 | |
| 396 | pub fn feed(&mut self, b: &[u8]) { |
| 397 | for x in b { |
| 398 | self.0 ^= *x as u64; |
| 399 | self.0 = self.0.wrapping_mul(0x0000_0100_0000_01b3); |
| 400 | } |
| 401 | } |
| 402 | } |