Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tests/oracle.rs

14.7 KiB, 66 runs

created by r1870400018:38843, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! The oracle harness -- reproduction as instrument.
2//!
3//! Before this unit, the Typst-vs-Austenite comparison lived only in a person's hands: compile both,
4//! eyeball the PDFs, remember what to check next time. These tests make that repeatable: the bounded
5//! corpus (see `tests/oracle/mod.rs`'s [`driver::corpus`]) is compiled with both engines and compared, a
6//! fixed trio of fixtures guards three fixes already on `main`, and a baseline file under the harness's
7//! own working directory (never `/tmp`, never the crate's `tests/` tree) lets a later run -- this
8//! session's or a future one's -- diff against what this one found.
9//!
10//! The baseline now asserts, not just reports: austenite's own rendered PDF hash and the worst of three
11//! sampled pages' raster diff against Typst are compared to what was recorded, and a run that finds
12//! either moved fails -- unless `ORACLE_ACCEPT=1` is set in the environment, in which case the new
13//! values are re-recorded and printed as an accepted change rather than silently passing. See
14//! `tests/oracle/mod.rs`'s `record_and_diff` and `BaselineOutcome`.
15//!
16//! The reference for the crate-owned corpus roots is now checked in at `tests/oracle/expected.json`, and
17//! the harness seeds a fresh (or cleared) cache from it: a pinned root can no longer bootstrap on whatever
18//! renders, so a regression on a fresh box fails instead of self-blessing -- the milestone audit's first
19//! finding. austenite-doc is deferred (its source is under active authoring), so it still bootstraps.
20//!
21//! `tests/oracle/mod.rs` (loaded below as `mod driver`, see its own comment for why) is the driver;
22//! everything that shells out to `typst`, `pdfinfo`, `pdftoppm`, ImageMagick or `sha256sum` lives there,
23//! so this file stays a short statement of what is being asserted and why.
24
25// `#[path]` rather than a plain `mod oracle;`: the crate root of this integration test is itself
26// named `oracle` (the file is `tests/oracle.rs`), and a plain `mod oracle;` pointing at the sibling
27// `tests/oracle/mod.rs` collides with that -- rustc reports the same module found at both paths. Naming
28// the module `driver` here sidesteps the collision; the directory stays `tests/oracle/`, matching the
29// unit's own file layout.
30#[path = "oracle/mod.rs"]
31mod driver;
32
33use driver::{
34 BaselineOutcome,
35 baseline_path,
36 compare_root,
37 corpus,
38 qc_dir,
39 record_and_diff,
40 trio,
41};
42
43use oxedyne_fe2o3_core::prelude::*;
44
45/// Compiles every corpus root with both `austenite` and the installed `typst`, and asserts:
46///
47/// - Austenite itself produced at least one page for every root (the one hard stop: every other
48/// comparison depends on that page existing).
49/// - Where the Typst oracle could run, its page count and its heading/figure pages agree with
50/// Austenite's own ledger, order-matched (see `tests/oracle/mod.rs` for why order rather than name).
51/// - The corpus's numbers, PDF hash and raster fraction do not drift against the recorded baseline
52/// within this run -- a bootstrapping first run for a root always passes, laying the baseline a later
53/// run diffs against; a later run that DOES drift fails unless `ORACLE_ACCEPT=1` is set, in which case
54/// the drift is printed and the baseline is re-recorded rather than the run failing.
55///
56/// A root whose oracle comparison could not run at all (a missing `typst` binary, or a template/helper
57/// incompatibility this crate does not own -- see [`driver::corpus`]'s doc comment for the one such case
58/// found and the re-check that closed it) is reported, not failed: the Austenite-only checks above still
59/// ran for it. No root in the corpus is in that state as of 2026-09-23.
60#[test]
61fn corpus_roots_compile_and_match_the_typst_oracle() -> Outcome<()> {
62 let work_dir = res!(qc_dir());
63 let baseline = baseline_path(&work_dir);
64 let mut problems: Vec<String> = Vec::new();
65 // `ORACLE_ACCEPT=1` gates every baseline drift this run finds, across every root -- an accepted
66 // styling change is expected to move more than one root's PDF hash at once, so this is read once
67 // rather than per root.
68 let accept = std::env::var("ORACLE_ACCEPT").map(|v| v == "1").unwrap_or(false);
69
70 for root in corpus() {
71 let report = match compare_root(&root, &work_dir) {
72 Ok(r) => r,
73 Err(e) => {
74 problems.push(fmt!("{}: austenite could not compile it at all: {}", root.name, e));
75 continue;
76 },
77 };
78 println!("[oracle] {}", report.summary());
79
80 if let Some(note) = &report.oracle_note {
81 println!("[oracle] {}: typst oracle unavailable -- {}", report.name, note);
82 }
83 // Each sampled page's own raster fraction, beyond the worst-of-three figure already folded into
84 // `report.summary()` -- visible under `--nocapture` so a failure's root cause (WHICH page moved)
85 // does not need a re-run with extra flags to see.
86 for (page, pct) in &report.raster_samples {
87 println!("[oracle] {}: page {} raster diff {:.2}%", report.name, page, pct);
88 }
89 for m in &report.mismatches {
90 problems.push(fmt!("{}: {} (see {:?})", report.name, m, report.pdf_path));
91 }
92 match res!(record_and_diff(&baseline, &report, accept)) {
93 BaselineOutcome::Bootstrapped => println!("[oracle] {}: baseline recorded (first run for this root)", report.name),
94 BaselineOutcome::Unchanged => {},
95 BaselineOutcome::Accepted(msg) => println!("[oracle] {}: ACCEPTED (ORACLE_ACCEPT=1), baseline re-recorded", msg),
96 BaselineOutcome::Rejected(msg) => problems.push(msg),
97 }
98 }
99
100 if !problems.is_empty() {
101 return Err(err!("The oracle harness found {} problem(s):\n{}",
102 problems.len(), problems.join("\n"); Test, Mismatch));
103 }
104 Ok(())
105}
106
107/// The three fixed no-regression fixtures -- see `tests/oracle/trio.rs` for what each guards.
108#[test]
109fn no_regression_trio_holds() -> Outcome<()> {
110 let work_dir = res!(qc_dir());
111 res!(trio::inline_maths_gallery_renders(&work_dir));
112 res!(trio::display_equation_cases_render());
113 res!(trio::pdf_stays_compact_and_extractable(&work_dir));
114 Ok(())
115}
116
117/// The harness's own self-test: a [`driver::Baseline`] written to the working directory reads back
118/// exactly what was written, including its PDF hash and raster fields, and a changed entry persists
119/// through a rewrite -- the round-trip the real regression check above depends on.
120#[test]
121fn baseline_records_and_diffs_correctly() -> Outcome<()> {
122 use driver::{Baseline, BaselineEntry};
123 use std::collections::BTreeMap;
124
125 let work_dir = res!(qc_dir());
126 let path = work_dir.join("baseline-selftest.json");
127
128 let entry = |pages: usize, hash: &str, raster: Option<f64>| BaselineEntry {
129 pages, anchors: 7, pdf_sha256: hash.to_string(), raster_worst_pct: raster,
130 };
131
132 let mut entries = BTreeMap::new();
133 entries.insert("fixture-root".to_string(), entry(3, "aaaa1111", Some(1.25)));
134 let baseline = Baseline::from_entries(entries);
135 res!(baseline.write_to_file(&path));
136
137 let read_back = res!(Baseline::read_from_file(&path));
138 if read_back.get("fixture-root") != Some(entry(3, "aaaa1111", Some(1.25))) {
139 return Err(err!("A baseline did not round-trip through {:?}: got {:?}",
140 path, read_back.get("fixture-root"); Test, Mismatch));
141 }
142
143 // `record_and_diff` works off a `RootReport`, which only `compare_root` builds (see
144 // `oracle_accept_gates_and_rerecords_a_changed_baseline` below for that path); this self-test
145 // exercises the underlying read-compare-write round trip directly on `Baseline`, including a root
146 // whose raster field was never measured (`None`) -- a box with no ImageMagick's own shape.
147 let mut moved = read_back;
148 moved.set("fixture-root", entry(4, "bbbb2222", None));
149 res!(moved.write_to_file(&path));
150 let after = res!(Baseline::read_from_file(&path));
151 if after.get("fixture-root") != Some(entry(4, "bbbb2222", None)) {
152 return Err(err!("A changed baseline entry did not persist through {:?}.", path; Test, Mismatch));
153 }
154
155 let _ = std::fs::remove_file(&path);
156 Ok(())
157}
158
159/// The accept-gate flow [`driver::record_and_diff`] implements, driven directly off two synthetic
160/// [`driver::RootReport`]s (building a real one means compiling something, which the harness-level tests
161/// above already cover): a first report bootstraps the baseline; a second report with a different PDF
162/// hash and a moved raster fraction is REJECTED, unchanged on disk, when `accept` is `false`, and
163/// ACCEPTED and re-recorded when `accept` is `true` -- the exact two branches item 1 of this unit's brief
164/// asks be provable, not just asserted by inspection.
165#[test]
166fn oracle_accept_gates_and_rerecords_a_changed_baseline() -> Outcome<()> {
167 use driver::{Baseline, BaselineOutcome, RootReport, record_and_diff};
168 use std::path::PathBuf;
169
170 let work_dir = res!(qc_dir());
171 let path = work_dir.join("baseline-accept-selftest.json");
172 let _ = std::fs::remove_file(&path); // a stale file from an earlier aborted run must not leak in
173
174 let report = |hash: &str, raster: Option<f64>| RootReport {
175 name: "accept-fixture-root",
176 austenite_pages: 3,
177 austenite_anchors: 5,
178 pdf_sha256: hash.to_string(),
179 raster_samples: Vec::new(),
180 raster_worst_pct: raster,
181 skip_line: None,
182 typst_pages: Some(3),
183 oracle_note: None,
184 mismatches: Vec::new(),
185 raster_note: None,
186 pdf_path: PathBuf::from("/nonexistent/accept-fixture.pdf"), // never read by record_and_diff
187 };
188
189 // First run: nothing recorded yet, so this bootstraps regardless of `accept`.
190 match res!(record_and_diff(&path, &report("hash-one", Some(2.0)), false)) {
191 BaselineOutcome::Bootstrapped => {},
192 _ => return Err(err!("The first run for a new root was not a bootstrap."; Test, Mismatch)),
193 }
194
195 // Second run, same values: unchanged, whichever way `accept` is set.
196 match res!(record_and_diff(&path, &report("hash-one", Some(2.0)), false)) {
197 BaselineOutcome::Unchanged => {},
198 _ => return Err(err!("An unchanged report was not reported unchanged."; Test, Mismatch)),
199 }
200
201 // A changed hash and raster fraction, NOT accepted: rejected, and the baseline on disk stays at the
202 // first run's values.
203 match res!(record_and_diff(&path, &report("hash-two", Some(9.0)), false)) {
204 BaselineOutcome::Rejected(msg) => {
205 if !msg.contains("sha256") || !msg.contains("raster") {
206 return Err(err!("A rejected drift's message named neither hash nor raster: {:?}", msg; Test, Mismatch));
207 }
208 },
209 _ => return Err(err!("A changed, unaccepted report was not rejected."; Test, Mismatch)),
210 }
211 let baseline = res!(Baseline::read_from_file(&path));
212 let still_first = baseline.get("accept-fixture-root");
213 if still_first.as_ref().map(|e| e.pdf_sha256.as_str()) != Some("hash-one") {
214 return Err(err!("A rejected drift's baseline was rewritten anyway: {:?}", still_first; Test, Mismatch));
215 }
216
217 // The same changed report, NOW accepted: re-recorded, and the baseline on disk moves to it.
218 match res!(record_and_diff(&path, &report("hash-two", Some(9.0)), true)) {
219 BaselineOutcome::Accepted(msg) => {
220 if !msg.contains("sha256") || !msg.contains("raster") {
221 return Err(err!("An accepted drift's message named neither hash nor raster: {:?}", msg; Test, Mismatch));
222 }
223 },
224 _ => return Err(err!("A changed, accepted report was not accepted."; Test, Mismatch)),
225 }
226 let baseline = res!(Baseline::read_from_file(&path));
227 let now_second = baseline.get("accept-fixture-root");
228 if now_second.as_ref().map(|e| e.pdf_sha256.as_str()) != Some("hash-two") {
229 return Err(err!("An accepted drift's baseline was not re-recorded: {:?}", now_second; Test, Mismatch));
230 }
231
232 let _ = std::fs::remove_file(&path);
233 Ok(())
234}
235
236/// The in-tree pin: a root listed in `tests/oracle/expected.json` seeds its reference from that tracked
237/// file when the cache has no entry, so on a fresh (or cleared) cache a matching render is `Unchanged` and
238/// a regressed one is `Rejected` -- never the always-pass `Bootstrapped`. A root absent from the file
239/// (austenite-doc, under active authoring) still bootstraps. This is the milestone audit's first finding
240/// fixed: a fresh box can no longer silently re-baseline on whatever it renders.
241#[test]
242fn expected_json_pin_blocks_bootstrap_for_crate_owned_roots() -> Outcome<()> {
243 use driver::{BaselineOutcome, RootReport, record_and_diff};
244 use std::path::PathBuf;
245
246 let work_dir = res!(qc_dir());
247 let path = work_dir.join("baseline-expected-selftest.json");
248
249 let report = |name: &'static str, hash: &str| RootReport {
250 name,
251 austenite_pages: 1,
252 austenite_anchors: 4,
253 pdf_sha256: hash.to_string(),
254 raster_samples: Vec::new(),
255 raster_worst_pct: None,
256 skip_line: None,
257 typst_pages: Some(1),
258 oracle_note: None,
259 mismatches: Vec::new(),
260 raster_note: None,
261 pdf_path: PathBuf::from("/nonexistent/expected-selftest.pdf"),
262 };
263
264 // styling-fixture is pinned; this is exactly its hash in tests/oracle/expected.json.
265 const FIXTURE_HASH: &str = "131cb8eee106280131d58083e969df1519dcb9b88096e93fec64019599764daa";
266
267 // A matching render on a fresh cache is Unchanged (the reference came from expected.json), not Bootstrapped.
268 let _ = std::fs::remove_file(&path);
269 match res!(record_and_diff(&path, &report("styling-fixture", FIXTURE_HASH), false)) {
270 BaselineOutcome::Unchanged => {},
271 _ => return Err(err!(
272 "a pinned root matching expected.json was not Unchanged -- it bootstrapped or rejected instead";
273 Test, Mismatch)),
274 }
275
276 // A regressed render on a fresh cache is Rejected, not Bootstrapped: the pin blocks self-blessing.
277 let _ = std::fs::remove_file(&path);
278 match res!(record_and_diff(&path, &report("styling-fixture", "deadbeefdeadbeef"), false)) {
279 BaselineOutcome::Rejected(_) => {},
280 _ => return Err(err!(
281 "a regressed pinned root was not Rejected -- the in-tree pin did not block the bootstrap";
282 Test, Mismatch)),
283 }
284
285 // A root absent from expected.json (austenite-doc's shape) still bootstraps.
286 let _ = std::fs::remove_file(&path);
287 match res!(record_and_diff(&path, &report("not-a-pinned-root", "abc123"), false)) {
288 BaselineOutcome::Bootstrapped => {},
289 _ => return Err(err!("a non-pinned root did not bootstrap"; Test, Mismatch)),
290 }
291
292 // Q6: a cache that self-blessed a WRONG hash for a pinned root does not govern it -- the tracked
293 // expected.json is authoritative even when a cache entry exists, so the correct render is still
294 // Unchanged (matched against expected, not the poisoned cache) rather than the cache masking a drift.
295 {
296 use driver::{Baseline, BaselineEntry};
297 use std::collections::BTreeMap;
298 let mut seeded = BTreeMap::new();
299 seeded.insert("styling-fixture".to_string(), BaselineEntry {
300 pages: 1,
301 anchors: 4,
302 pdf_sha256: "cache-self-blessed-wrong-hash".to_string(),
303 raster_worst_pct: None,
304 });
305 res!(Baseline::from_entries(seeded).write_to_file(&path));
306 }
307 match res!(record_and_diff(&path, &report("styling-fixture", FIXTURE_HASH), false)) {
308 BaselineOutcome::Unchanged => {},
309 _ => return Err(err!(
310 "a pinned root was governed by a poisoned cache instead of expected.json";
311 Test, Mismatch)),
312 }
313
314 let _ = std::fs::remove_file(&path);
315 Ok(())
316}