Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tests/eval_oracle/oracle.rs

13.9 KiB, 11 runs

created by r1870400018:63624, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// The external oracle: the host's `typst` 0.15.x, run as a process, never linked. Every run is
2// capped with `systemd-run --user --scope -p MemoryMax=<cap>` where systemd-run exists
3// (`EVAL_ORACLE_CAP`, default 3G, `none` to run uncapped); a missing or wrongly versioned binary is
4// an error unless `EVAL_ORACLE_SKIP=1` says, explicitly, to skip -- so absence never reads as green.
5//
6// Outputs are cached under the test binary's target directory (never `/tmp`, never the source
7// tree), keyed by the typst version and every byte of the fixture's area, so an edit to a fixture or
8// a helper it imports re-runs the oracle and nothing else does. Expected values are always the
9// oracle's own output; nothing Austenite produced is ever written here.
10
11use crate::harness::corpus::{
12 area_bytes,
13 Fixture,
14};
15use crate::harness::json::{
16 self,
17 J,
18};
19use crate::harness::layout::{
20 self,
21 OPage,
22};
23use crate::harness::structure::{
24 self,
25 Sk,
26};
27use crate::harness::typst_command::typst_command;
28use crate::harness::PosRow;
29
30use oxedyne_fe2o3_core::prelude::*;
31
32use std::path::{
33 Path,
34 PathBuf,
35};
36use std::process::Command;
37
38pub const PINNED: &str = "typst 0.15.";
39
40// Part of every cache key: which faces Typst was given, so that a product made with the account's own fonts is
41// never read as made without them.
42pub const FACES: &str = "fonts=system";
43
44pub const PROBES_EXPR: &str = "query(<probe>).map(it => it.value)";
45pub const POSITIONS_EXPR: &str = "query(selector(heading).or(figure, math.equation, footnote, <probe>))\
46 .map(it => (func: repr(it.func()), pos: it.location().position()))";
47
48/// What the oracle said about one fixture at one level: its answer, or the reason it has none.
49pub type Said<T> = std::result::Result<T, String>;
50
51#[derive(Clone, Debug)]
52pub struct Oracle {
53 pub bin: String,
54 pub version: String,
55 cap: Option<String>,
56 pub work: PathBuf,
57}
58
59pub struct Html {
60 pub skeleton: Vec<Sk>,
61 pub ignored: Vec<String>, // elements the HTML export dropped
62}
63
64impl Oracle {
65 /// The oracle, `Ok(None)` only when `EVAL_ORACLE_SKIP=1` is set and no usable `typst` exists.
66 pub fn find() -> Outcome<Option<Oracle>> {
67 let skip = std::env::var("EVAL_ORACLE_SKIP").map(|v| v == "1").unwrap_or(false);
68 let bin = std::env::var("EVAL_ORACLE_TYPST").unwrap_or_else(|_| "typst".to_string());
69 let version = match Command::new(&bin).arg("--version").output() {
70 Ok(o) if o.status.success() => String::from_utf8_lossy(&o.stdout).trim().to_string(),
71 Ok(o) => {
72 if skip {
73 return Ok(None);
74 }
75 return Err(err!("`{} --version` failed: {}", bin, String::from_utf8_lossy(&o.stderr).trim(); IO, Missing));
76 }
77 Err(e) => {
78 if skip {
79 return Ok(None);
80 }
81 return Err(err!("The typst oracle `{}` cannot be run ({}). Install typst 0.15.x, point \
82 EVAL_ORACLE_TYPST at it, or set EVAL_ORACLE_SKIP=1 to skip the oracle explicitly.", bin, e; IO, Missing));
83 }
84 };
85 if !version.starts_with(PINNED) {
86 if skip {
87 return Ok(None);
88 }
89 return Err(err!("The oracle is pinned to {}x; `{}` reports {:?}.", PINNED, bin, version; Invalid, Mismatch));
90 }
91 let cap = match std::env::var("EVAL_ORACLE_CAP") {
92 Ok(v) if v == "none" => None,
93 Ok(v) if !v.is_empty() => Some(v),
94 _ => Some("3G".to_string()),
95 };
96 let cap = match cap {
97 Some(c) if Command::new("systemd-run").arg("--version").output().map(|o| o.status.success()).unwrap_or(false) => Some(c),
98 _ => None,
99 };
100 let work = res!(work_dir());
101 Ok(Some(Oracle { bin, version, cap, work }))
102 }
103
104 fn command(&self, args: &[String], cwd: &Path) -> Command {
105 let mut c = typst_command(&self.bin, self.cap.as_deref(), &self.work);
106 c.args(args);
107 c.current_dir(cwd);
108 // An area that keeps packages of its own beside its fixtures lays them out as Typst's package
109 // path does, `packages/<namespace>/<name>/<version>`.
110 let own = cwd.join("packages");
111 if own.is_dir() {
112 c.env("TYPST_PACKAGE_PATH", &own);
113 }
114 c
115 }
116
117 /// Runs typst; `Ok(Err(stderr))` when typst itself reported failure.
118 fn run(&self, args: &[String], cwd: &Path) -> Outcome<Said<(String, String)>> {
119 let out = match self.command(args, cwd).output() {
120 Ok(o) => o,
121 Err(e) => return Err(err!("Could not run the typst oracle: {}", e; IO)),
122 };
123 let stdout = String::from_utf8_lossy(&out.stdout).to_string();
124 let stderr = String::from_utf8_lossy(&out.stderr).to_string();
125 if out.status.success() {
126 Ok(Ok((stdout, stderr)))
127 } else {
128 Ok(Err(first_error(&stderr)))
129 }
130 }
131
132 /// The fixture's cache directory, created on first use.
133 pub fn dir(&self, fx: &Fixture) -> Outcome<PathBuf> {
134 let files = res!(area_bytes(&fx.root));
135 let mut h = Fnv::new();
136 h.feed(self.version.as_bytes());
137 h.feed(FACES.as_bytes());
138 h.feed(fx.name.as_bytes());
139 // Typst's answers name the fixture by its absolute path, so a target directory shared by two
140 // checkouts must not hand one checkout's answers to the other.
141 let root = fx.root.canonicalize().unwrap_or_else(|_| fx.root.clone());
142 h.feed(root.to_string_lossy().as_bytes());
143 h.feed(&[0]);
144 for (rel, bytes) in &files {
145 h.feed(rel.as_bytes());
146 h.feed(&[0]);
147 h.feed(bytes);
148 }
149 let d = self.work.join(&fx.area).join(fmt!("{}-{:016x}", fx.name, h.0));
150 res!(std::fs::create_dir_all(&d).map_err(|e| err!("Cannot create {}: {}", d.display(), e; IO, File, Write)));
151 Ok(d)
152 }
153
154 /// A cached text product: `<what>.out` holds a success, `<what>.err` a typst failure.
155 fn cached<F>(&self, fx: &Fixture, what: &str, make: F) -> Outcome<Said<String>>
156 where F: FnOnce(&Oracle, &Path) -> Outcome<Said<String>>
157 {
158 let d = res!(self.dir(fx));
159 let ok = d.join(fmt!("{}.out", what));
160 let bad = d.join(fmt!("{}.err", what));
161 // Tests run in parallel and share fixtures, so one of them makes a product at a time.
162 let _guard = lock_mutex!(ORACLE_LOCK);
163 if let Ok(s) = std::fs::read_to_string(&ok) {
164 return Ok(Ok(s));
165 }
166 if let Ok(s) = std::fs::read_to_string(&bad) {
167 return Ok(Err(s));
168 }
169 let said = res!(make(self, &d));
170 let (path, text) = match &said {
171 Ok(s) => (ok, s.clone()),
172 Err(s) => (bad, s.clone()),
173 };
174 res!(write_whole(&path, text.as_bytes()));
175 Ok(said)
176 }
177
178 fn eval(&self, fx: &Fixture, what: &str, expr: &str) -> Outcome<Said<J>> {
179 let said = res!(self.cached(fx, what, |o, _| {
180 let args = vec![
181 "eval".to_string(), expr.to_string(),
182 "--in".to_string(), fx.path.display().to_string(),
183 "--root".to_string(), fx.root.display().to_string(),
184 ];
185 Ok(res!(o.run(&args, &fx.root)).map(|(out, _)| out))
186 }));
187 match said {
188 Ok(text) => Ok(Ok(res!(json::parse(&text)))),
189 Err(e) => Ok(Err(e)),
190 }
191 }
192
193 /// The value of a code expression, `typst eval` in the fixture's context with the area as its root, so
194 /// an `import` or a `read` in the expression resolves there. Cached like every other product.
195 #[allow(dead_code)] // the import corpus asks for values; the other test crates do not
196 pub fn value(&self, fx: &Fixture, expr: &str) -> Outcome<Said<J>> {
197 let mut h = Fnv::new();
198 h.feed(expr.as_bytes());
199 self.eval(fx, &fmt!("value-{:016x}", h.0), expr)
200 }
201
202 /// Does the fixture compile without error? `Ok(Err(message))` names the first error.
203 pub fn compiles(&self, fx: &Fixture) -> Outcome<Said<()>> {
204 let pdf = res!(self.pdf(fx));
205 Ok(pdf.map(|_| ()))
206 }
207
208 fn pdf(&self, fx: &Fixture) -> Outcome<Said<PathBuf>> {
209 let said = res!(self.cached(fx, "pdf", |o, d| {
210 let pdf = d.join("typst.pdf");
211 let args = vec![
212 "compile".to_string(), fx.path.display().to_string(), pdf.display().to_string(),
213 "--root".to_string(), fx.root.display().to_string(),
214 ];
215 Ok(res!(o.run(&args, &fx.root)).map(|_| pdf.display().to_string()))
216 }));
217 Ok(said.map(PathBuf::from))
218 }
219
220 /// Level 1: every `<probe>` metadata value, in document order.
221 pub fn probes(&self, fx: &Fixture) -> Outcome<Said<Vec<J>>> {
222 match res!(self.eval(fx, "probes", PROBES_EXPR)) {
223 Ok(J::Arr(v)) => Ok(Ok(v)),
224 Ok(other) => Ok(Err(fmt!("the probe query returned {}", other.render()))),
225 Err(e) => Ok(Err(e)),
226 }
227 }
228
229 /// Level 2: headings, figures, equations, footnotes and probes with their positions.
230 pub fn positions(&self, fx: &Fixture) -> Outcome<Said<Vec<PosRow>>> {
231 let v = match res!(self.eval(fx, "positions", POSITIONS_EXPR)) {
232 Ok(J::Arr(v)) => v,
233 Ok(other) => return Ok(Err(fmt!("the position query returned {}", other.render()))),
234 Err(e) => return Ok(Err(e)),
235 };
236 let mut rows = Vec::new();
237 for r in v {
238 let func = r.get("func").and_then(|f| f.as_str()).unwrap_or("").to_string();
239 let pos = match r.get("pos") {
240 Some(p) => p,
241 None => return Ok(Err(fmt!("a position row has no `pos`: {}", r.render()))),
242 };
243 rows.push(PosRow {
244 func,
245 page: pos.get("page").and_then(|p| p.as_i64()).unwrap_or(0) as u32,
246 x: pt(pos.get("x")),
247 y: pt(pos.get("y")),
248 });
249 }
250 Ok(Ok(rows))
251 }
252
253 /// Level 3: the HTML export's skeleton and the elements it ignored.
254 pub fn html(&self, fx: &Fixture) -> Outcome<Said<Html>> {
255 let said = res!(self.cached(fx, "html", |o, d| {
256 let out = d.join("typst.html");
257 let args = vec![
258 "compile".to_string(), "--features".to_string(), "html".to_string(),
259 "--format".to_string(), "html".to_string(),
260 fx.path.display().to_string(), out.display().to_string(),
261 "--root".to_string(), fx.root.display().to_string(),
262 ];
263 match res!(o.run(&args, &fx.root)) {
264 Ok((_, stderr)) => {
265 let html = res!(std::fs::read_to_string(&out).map_err(|e| err!(
266 "Cannot read {}: {}", out.display(), e; IO, File, Read)));
267 // The ignored-element warnings ride along, one per line, ahead of a marker.
268 let mut s = String::new();
269 for n in structure::ignored_elements(&stderr) {
270 s.push_str(&n);
271 s.push('\n');
272 }
273 s.push_str("\u{0}\n");
274 s.push_str(&html);
275 Ok(Ok(s))
276 }
277 Err(e) => Ok(Err(e)),
278 }
279 }));
280 let text = match said {
281 Ok(t) => t,
282 Err(e) => return Ok(Err(e)),
283 };
284 let (head, html) = match text.split_once("\u{0}\n") {
285 Some(p) => p,
286 None => ("", text.as_str()),
287 };
288 let ignored = head.lines().filter(|l| !l.is_empty()).map(|l| l.to_string()).collect();
289 Ok(Ok(Html { skeleton: structure::from_html(html), ignored }))
290 }
291
292 /// Level 4: pages with their lines and baselines.
293 pub fn layout(&self, fx: &Fixture) -> Outcome<Said<Vec<OPage>>> {
294 let pdf = match res!(self.pdf(fx)) {
295 Ok(p) => p,
296 Err(e) => return Ok(Err(e)),
297 };
298 let d = res!(self.dir(fx));
299 let bbox = d.join("typst.bbox.html");
300 let _guard = lock_mutex!(ORACLE_LOCK);
301 if !bbox.exists() {
302 let st = Command::new("pdftotext").arg("-bbox-layout").arg(&pdf).arg(&bbox).output();
303 match st {
304 Ok(o) if o.status.success() => (),
305 Ok(o) => return Err(err!("pdftotext failed on {}: {}", pdf.display(),
306 String::from_utf8_lossy(&o.stderr).trim(); IO)),
307 Err(e) => return Err(err!("pdftotext (poppler-utils) is needed for level 4: {}", e; IO, Missing)),
308 }
309 }
310 let xml = res!(std::fs::read_to_string(&bbox).map_err(|e| err!("Cannot read {}: {}", bbox.display(), e; IO, File, Read)));
311 let mut pages = layout::read_bbox(&xml);
312 let svg1 = d.join("typst-1.svg");
313 if !svg1.exists() {
314 let pattern = d.join("typst-{p}.svg");
315 let args = vec![
316 "compile".to_string(), fx.path.display().to_string(), pattern.display().to_string(),
317 "--root".to_string(), fx.root.display().to_string(),
318 ];
319 if let Err(e) = res!(self.run(&args, &fx.root)) {
320 return Ok(Err(e));
321 }
322 }
323 for (i, p) in pages.iter_mut().enumerate() {
324 let svg = d.join(fmt!("typst-{}.svg", i + 1));
325 let text = res!(std::fs::read_to_string(&svg).map_err(|e| err!("Cannot read {}: {}", svg.display(), e; IO, File, Read)));
326 layout::attach_baselines(p, &layout::glyph_origins(&text));
327 layout::attach_starts(p, &layout::run_starts(&text));
328 }
329 Ok(Ok(pages))
330 }
331}
332
333fn pt(v: Option<&J>) -> f64 {
334 match v {
335 Some(J::Str(s)) => s.trim_end_matches("pt").parse::<f64>().unwrap_or(f64::NAN),
336 Some(J::Float(f)) => *f,
337 Some(J::Int(i)) => *i as f64,
338 _ => f64::NAN,
339 }
340}
341
342/// The first `error:` line of typst's diagnostics, with its location line when there is one.
343fn first_error(stderr: &str) -> String {
344 let mut lines = stderr.lines();
345 while let Some(l) = lines.next() {
346 if l.starts_with("error:") {
347 // A message may run over several lines; the place follows it, on the line that opens `┌─`.
348 let mut message = l.trim().to_string();
349 let mut loc = String::new();
350 for n in lines.by_ref() {
351 let t = n.trim();
352 if t.starts_with('┌') {
353 loc = t.to_string();
354 break;
355 }
356 if t.starts_with('│') || t.starts_with('=') || t.is_empty() {
357 break;
358 }
359 message.push('\n');
360 message.push_str(t);
361 }
362 return fmt!("{} {}", message, loc);
363 }
364 }
365 stderr.trim().lines().take(3).collect::<Vec<_>>().join(" | ")
366}
367
368static ORACLE_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
369
370/// Writes beside the target and renames over it, so a reader never sees half a file.
371fn write_whole(path: &Path, bytes: &[u8]) -> Outcome<()> {
372 let tmp = path.with_extension(fmt!("part{}", std::process::id()));
373 res!(std::fs::write(&tmp, bytes).map_err(|e| err!("Cannot write {}: {}", tmp.display(), e; IO, File, Write)));
374 res!(std::fs::rename(&tmp, path).map_err(|e| err!("Cannot rename {}: {}", tmp.display(), e; IO, File, Write)));
375 Ok(())
376}
377
378/// `<target>/<profile>/eval_oracle`, beside the test binary that is running.
379pub fn work_dir() -> Outcome<PathBuf> {
380 let exe = res!(std::env::current_exe().map_err(|e| err!("Cannot locate the test binary: {}", e; IO)));
381 // `<target>/<profile>/deps/<binary>`
382 let profile = match exe.parent().and_then(|deps| deps.parent()) {
383 Some(p) => p.to_path_buf(),
384 None => return Err(err!("The test binary {} has no target directory above it", exe.display(); Missing)),
385 };
386 let d = profile.join("eval_oracle");
387 res!(std::fs::create_dir_all(&d).map_err(|e| err!("Cannot create {}: {}", d.display(), e; IO, File, Write)));
388 Ok(d)
389}
390
391pub struct Fnv(pub u64);
392
393impl Fnv {
394 pub fn new() -> Self { Fnv(0xcbf2_9ce4_8422_2325) }
395
396 pub fn feed(&mut self, b: &[u8]) {
397 for x in b {
398 self.0 ^= *x as u64;
399 self.0 = self.0.wrapping_mul(0x0000_0100_0000_01b3);
400 }
401 }
402}