Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tests/eval_timings.rs

18.4 KiB, 54 runs

created by r1870400018:66082, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! `austenite --eval --timings FILE.json`: the wall time of each phase of a compile.
2//!
3//! Each case compiles a synthetic document with the built binary. The record is judged on what it must
4//! always hold: the PDF is the same bytes with the recorder on or off, every phase of every fixpoint pass
5//! was entered, and the phases add up to the run's wall time within a tenth. The realisation's split is
6//! held to its own sums: the sub-phases and the part outside them are the `realise` cell, and the calls by
7//! cause are its entries.
8
9#[allow(dead_code)]
10#[path = "eval_oracle/json.rs"]
11mod json;
12
13use json::J;
14
15use oxedyne_fe2o3_austenite::compile;
16use oxedyne_fe2o3_austenite::emit::sinks::PdfSink;
17use oxedyne_fe2o3_austenite::flow::text::FontStore;
18use oxedyne_fe2o3_austenite::timings::Phase;
19use oxedyne_fe2o3_austenite::timings::Timings;
20
21use std::path::PathBuf;
22use std::process::Command;
23use std::sync::OnceLock;
24
25// A forward read, the page total on its first page, so the fixpoint takes more than one pass; furniture on
26// every page; a show rule on an element and one on a word, so realisation runs user code and searches text;
27// an outline and a reference with no answer until the first pass, so the second pass realises new bodies;
28// and enough pages for each phase to be measurable.
29const DOC: &str = "\
30#set page(width: 220pt, height: 260pt, margin: 24pt, header: [Header], footer: context counter(page).display())
31#show heading: it => block(fill: luma(235), inset: 4pt, it.body)
32#show \"Sentence\": strong
33#set heading(numbering: \"1.\")
34#context [Pages: #counter(page).final().first()]
35
36#outline()
37
38#for i in range(1, 15) [
39= Section #i
40#for j in range(1, 16) [Sentence #j of section #i sets some words in a paragraph. ]
41]
42
43#heading[Last]<sec>
44
45See @sec.
46";
47
48struct Runs {
49 plain: Vec<u8>, // the PDF with no recorder
50 timed: Vec<u8>, // the PDF with it
51 record: String, // the record's JSON
52 fine: Vec<u8>, // the PDF with the fine recorder
53 fine_rec: String, // the fine record's JSON
54}
55
56fn case_dir(name: &str) -> PathBuf {
57 let dir = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join("eval_timings").join(name);
58 let _ = std::fs::remove_dir_all(&dir);
59 std::fs::create_dir_all(&dir).expect("the case's directory");
60 dir
61}
62
63fn austenite(args: &[&str]) -> std::process::Output {
64 Command::new(env!("CARGO_BIN_EXE_austenite"))
65 .args(args)
66 .output()
67 .expect("the built austenite binary")
68}
69
70// The document compiled once without `--timings` and once with, shared by the cases that read the pair.
71fn runs() -> &'static Runs {
72 static RUNS: OnceLock<Runs> = OnceLock::new();
73 RUNS.get_or_init(|| {
74 let dir = case_dir("pair");
75 let main = dir.join("main.typ");
76 std::fs::write(&main, DOC).expect("the case's source");
77 let (plain, timed, record) = (dir.join("plain"), dir.join("timed"), dir.join("record.json"));
78 let a = austenite(&["--eval", main.to_str().expect("a path"), plain.to_str().expect("a path")]);
79 assert!(a.status.success(), "the plain compile succeeds:\n{}", String::from_utf8_lossy(&a.stderr));
80 let b = austenite(&["--eval", "--timings", record.to_str().expect("a path"),
81 main.to_str().expect("a path"), timed.to_str().expect("a path")]);
82 assert!(b.status.success(), "the timed compile succeeds:\n{}", String::from_utf8_lossy(&b.stderr));
83 let (fine, fine_rec) = (dir.join("fine"), dir.join("fine.json"));
84 let c = austenite(&["--eval", "--timings", fine_rec.to_str().expect("a path"), "--timings-fine",
85 main.to_str().expect("a path"), fine.to_str().expect("a path")]);
86 assert!(c.status.success(), "the fine compile succeeds:\n{}", String::from_utf8_lossy(&c.stderr));
87 Runs {
88 plain: std::fs::read(plain.join("document.pdf")).expect("the plain PDF"),
89 timed: std::fs::read(timed.join("document.pdf")).expect("the timed PDF"),
90 record: std::fs::read_to_string(&record).expect("the record"),
91 fine: std::fs::read(fine.join("document.pdf")).expect("the fine PDF"),
92 fine_rec: std::fs::read_to_string(&fine_rec).expect("the fine record"),
93 }
94 })
95}
96
97fn num(j: &J, path: &[&str]) -> i64 {
98 let mut at = j;
99 for key in path {
100 at = at.get(key).unwrap_or_else(|| panic!("the record has no {:?} (looking for {})", key, path.join(".")));
101 }
102 at.as_i64().unwrap_or_else(|| panic!("{} is not an integer", path.join(".")))
103}
104
105// What is wrong with a record: a phase of a pass with no entry, a pass count that disagrees with the
106// array, or phases that do not add up to the total within a tenth. Empty when the record is sound.
107fn faults(record: &J) -> Vec<String> {
108 let mut out = Vec::new();
109 let total = num(record, &["total"]);
110 let passes = num(record, &["passes"]);
111 let mut sum = 0i64;
112 for p in Phase::RUN {
113 let (ns, n) = (num(record, &["run", p.name(), "ns"]), num(record, &["run", p.name(), "n"]));
114 if n < 1 {
115 out.push(format!("run phase {} was never entered", p.name()));
116 }
117 sum += ns;
118 }
119 let each = match record.get("pass") {
120 Some(J::Arr(v)) => v.as_slice(),
121 _ => {
122 out.push("the record has no pass array".to_string());
123 &[]
124 },
125 };
126 if each.len() as i64 != passes {
127 out.push(format!("the record says {} passes and holds {}", passes, each.len()));
128 }
129 for (i, pass) in each.iter().enumerate() {
130 for p in Phase::PASS {
131 let (ns, n) = (num(pass, &[p.name(), "ns"]), num(pass, &[p.name(), "n"]));
132 if n < 1 {
133 out.push(format!("pass {} never entered {}", i + 1, p.name()));
134 }
135 sum += ns;
136 }
137 }
138 if (sum - total).abs() * 10 > total {
139 out.push(format!("the phases sum to {} ns against a total of {} ns", sum, total));
140 }
141 // The shaped-run cache was used, and holds no more than its budget.
142 let (hits, misses) = (num(record, &["shape", "hits"]), num(record, &["shape", "misses"]));
143 if hits + misses < 1 || misses < 1 {
144 out.push(format!("the shaped-run cache shows {} hits and {} misses", hits, misses));
145 }
146 if num(record, &["shape", "bytes"]) > num(record, &["shape", "budget"]) {
147 out.push("the shaped-run cache holds more than its budget".to_string());
148 }
149 out
150}
151
152#[test]
153fn the_pdf_is_the_same_bytes_with_the_recorder_on_or_off() {
154 let r = runs();
155 assert!(r.plain.starts_with(b"%PDF-"), "the plain output is a PDF");
156 assert!(r.plain.len() > 4_000, "the document made real pages: {} bytes", r.plain.len());
157 assert!(r.plain == r.timed, "--timings changed the PDF ({} bytes against {})", r.plain.len(), r.timed.len());
158 assert!(r.plain == r.fine, "--timings-fine changed the PDF ({} bytes against {})", r.plain.len(), r.fine.len());
159}
160
161#[test]
162fn the_record_holds_every_phase_of_every_pass_and_the_phases_sum_to_the_wall_time() {
163 let record = json::parse(&runs().record).expect("the record is JSON");
164 assert!(num(&record, &["passes"]) >= 2, "the forward read takes a second pass: {}", runs().record);
165 let found = faults(&record);
166 assert!(found.is_empty(), "the record is unsound: {:#?}\n{}", found, runs().record);
167 let fine = json::parse(&runs().fine_rec).expect("the fine record is JSON");
168 let found = faults(&fine);
169 assert!(found.is_empty(), "the fine record is unsound: {:#?}\n{}", found, runs().fine_rec);
170 // A page record for every page of every pass, and the probe entered for each.
171 let lists = match record.get("pages") {
172 Some(J::Arr(v)) => v.as_slice(),
173 _ => panic!("the record has no pages array"),
174 };
175 assert_eq!(lists.len() as i64, num(&record, &["passes"]), "one list of page records to a pass");
176 for (i, l) in lists.iter().enumerate() {
177 let n = match l {
178 J::Arr(v) => v.len(),
179 _ => panic!("a pass's page records are not a list"),
180 };
181 assert!(n >= 2, "pass {} holds the records of its pages: {}", i + 1, n);
182 }
183}
184
185// Sets the time and the count of every cell named `phase`, in the run or in any pass, to zero.
186fn never_recorded(j: &mut J, phase: &str, cells: &mut usize) {
187 match j {
188 J::Obj(kv) => {
189 for (k, v) in kv.iter_mut() {
190 if k == phase {
191 *v = J::Obj(vec![("ns".to_string(), J::Int(0)), ("n".to_string(), J::Int(0))]);
192 *cells += 1;
193 } else {
194 never_recorded(v, phase, cells);
195 }
196 }
197 },
198 J::Arr(v) => for x in v.iter_mut() {
199 never_recorded(x, phase, cells);
200 },
201 _ => (),
202 }
203}
204
205#[test]
206fn a_record_with_a_phase_never_recorded_is_refused() {
207 // The record as the engine would write it were flow never entered: every `flow` cell is empty.
208 let mut record = json::parse(&runs().record).expect("the record is JSON");
209 let mut cells = 0;
210 never_recorded(&mut record, "flow", &mut cells);
211 assert!(cells >= 2, "at least two passes had a flow cell to empty, found {}", cells);
212 let found = faults(&record);
213 assert!(found.iter().any(|f| f.contains("never entered flow")), "the missing phase is named: {:#?}", found);
214 assert!(found.iter().any(|f| f.contains("sum to")), "the missing time breaks the sum: {:#?}", found);
215}
216
217#[test]
218fn timings_is_refused_on_the_curated_reader_and_needs_a_file() {
219 let dir = case_dir("flags");
220 let main = dir.join("main.typ");
221 std::fs::write(&main, "Text.\n").expect("the case's source");
222 let m = main.to_str().expect("a path");
223 let rec = dir.join("t.json");
224 let out = austenite(&["--engine", "curated", "--timings", rec.to_str().expect("a path"), m]);
225 assert!(!out.status.success(), "--timings on the curated reader is refused");
226 let err = String::from_utf8_lossy(&out.stderr);
227 assert!(err.contains("--timings is not read on the curated reader path"), "{}", err);
228 let out = austenite(&["--eval", m, "--timings"]);
229 assert!(!out.status.success(), "--timings with no file is refused");
230 assert!(String::from_utf8_lossy(&out.stderr).contains("--timings needs a file"), "{}", String::from_utf8_lossy(&out.stderr));
231 let out = austenite(&["--eval", "--timings-fine", m]);
232 assert!(!out.status.success(), "--timings-fine without --timings is refused");
233 assert!(String::from_utf8_lossy(&out.stderr).contains("--timings-fine needs --timings"), "{}", String::from_utf8_lossy(&out.stderr));
234 assert!(!rec.exists(), "a refused run writes no record");
235}
236
237// The realisation's split, as a list of what is wrong with it. Each pass must hold: the `realise` cell equal
238// to the sub-phases and `self` added up, in time and in entries; the calls by cause adding to the entries;
239// no call in pass one but a first one or a repeat within the pass; no first call after pass one; and the
240// work counters present. Empty when the split is sound.
241fn split_faults(record: &J, fine: bool) -> Vec<String> {
242 let mut out = Vec::new();
243 let each = match record.get("pass") {
244 Some(J::Arr(v)) => v.as_slice(),
245 _ => return vec!["the record has no pass array".to_string()],
246 };
247 for (i, pass) in each.iter().enumerate() {
248 let at = i + 1;
249 let (ns, n) = (num(pass, &["realise", "ns"]), num(pass, &["realise", "n"]));
250 let (mut sub_ns, mut sub_n) = (0i64, 0i64);
251 for key in ["rules", "regex", "show", "repack", "styles", "intro", "self"] {
252 sub_ns += num(pass, &["sub", key, "ns"]);
253 sub_n += if key == "self" { num(pass, &["sub", key, "n"]) } else { 0 };
254 }
255 if sub_ns != ns {
256 out.push(format!("pass {} realise holds {} ns and its sub-phases sum to {} ns", at, ns, sub_ns));
257 }
258 if sub_n != n {
259 out.push(format!("pass {} realise was entered {} times and `self` says {}", at, n, sub_n));
260 }
261 // The phases entered for every element are counted always and clocked only in a fine record.
262 for key in ["rules", "show", "styles"] {
263 if num(pass, &["sub", key, "n"]) < 1 {
264 out.push(format!("pass {} never entered {}", at, key));
265 }
266 if !fine && num(pass, &["sub", key, "ns"]) != 0 {
267 out.push(format!("pass {} clocked {} in a coarse record", at, key));
268 }
269 if fine && num(pass, &["sub", key, "ns"]) < 1 {
270 out.push(format!("pass {} left {} unclocked in a fine record", at, key));
271 }
272 }
273 let mut by_cause = [0i64; 4];
274 match pass.get("calls") {
275 Some(J::Obj(kv)) => for (label, causes) in kv {
276 for (c, cause) in ["first", "new", "repass", "again"].iter().enumerate() {
277 by_cause[c] += num(causes, &[cause]);
278 }
279 if label.is_empty() {
280 out.push(format!("pass {} has a call with no label", at));
281 }
282 },
283 _ => out.push(format!("pass {} has no calls object", at)),
284 }
285 let calls: i64 = by_cause.iter().sum();
286 if calls != n {
287 out.push(format!("pass {} counts {} realise calls and realise was entered {} times", at, calls, n));
288 }
289 if i == 0 && (by_cause[1] != 0 || by_cause[2] != 0) {
290 out.push(format!("pass 1 has {} new and {} repass calls", by_cause[1], by_cause[2]));
291 }
292 if i > 0 && by_cause[0] != 0 {
293 out.push(format!("pass {} has {} first calls", at, by_cause[0]));
294 }
295 for c in ["visits", "leaves", "shown", "recipes", "matched", "prepared", "copied", "textual", "finished"] {
296 if pass.get("counts").and_then(|k| k.get(c)).is_none() {
297 out.push(format!("pass {} has no {} counter", at, c));
298 }
299 }
300 }
301 out
302}
303
304#[test]
305fn the_realise_cell_is_its_sub_phases_and_its_calls_are_its_entries() {
306 for (fine, text) in [(false, &runs().record), (true, &runs().fine_rec)] {
307 check_split(text, fine);
308 }
309}
310
311fn check_split(text: &str, fine: bool) {
312 let record = json::parse(text).expect("the record is JSON");
313 assert!(matches!(record.get("fine"), Some(J::Bool(b)) if *b == fine), "the record says whether it is fine:\n{}", text);
314 let found = split_faults(&record, fine);
315 assert!(found.is_empty(), "the split is unsound: {:#?}\n{}", found, text);
316 // The document reaches every part of realisation in every pass, so a sub-phase that stays empty is not
317 // being opened.
318 let each = match record.get("pass") {
319 Some(J::Arr(v)) => v,
320 _ => panic!("the record has no pass array"),
321 };
322 for (i, pass) in each.iter().enumerate() {
323 for key in ["show", "repack", "styles", "regex", "intro"] {
324 assert!(num(pass, &["sub", key, "n"]) >= 1, "pass {} never entered the {} sub-phase:\n{}", i + 1, key, text);
325 }
326 for c in ["visits", "shown", "recipes", "matched", "prepared", "textual", "finished"] {
327 assert!(num(pass, &["counts", c]) >= 1, "pass {} counted no {}:\n{}", i + 1, c, text);
328 }
329 assert!(num(pass, &["counts", "recipes"]) >= num(pass, &["counts", "shown"]),
330 "pass {} has fewer recipes in force than elements shown:\n{}", i + 1, text);
331 }
332 // A later pass realises some bodies that the pass before it realised too.
333 let later: i64 = each.iter().skip(1).map(|p| match p.get("calls") {
334 Some(J::Obj(kv)) => kv.iter().map(|(_, c)| num(c, &["repass"])).sum::<i64>(),
335 _ => 0,
336 }).sum();
337 assert!(later >= 1, "a second pass realised nothing it had realised before:\n{}", text);
338 // The outline and the reference have no answer until the first pass has laid the document out, so the
339 // second pass realises bodies that did not exist before it.
340 let fresh: i64 = each.iter().skip(1).map(|p| match p.get("calls") {
341 Some(J::Obj(kv)) => kv.iter().map(|(_, c)| num(c, &["new"])).sum::<i64>(),
342 _ => 0,
343 }).sum();
344 assert!(fresh >= 1, "a later pass realised no body that was not there before:\n{}", text);
345}
346
347// The value at `path` below `j`, if the objects on the way hold it.
348fn at_mut<'a>(j: &'a mut J, path: &[&str]) -> Option<&'a mut J> {
349 let mut at = j;
350 for key in path {
351 let J::Obj(kv) = at else { return None };
352 match kv.iter_mut().find(|(k, _)| k == key) {
353 Some((_, v)) => at = v,
354 None => return None,
355 }
356 }
357 Some(at)
358}
359
360// Adds `by` to the integer at `path` in pass number `pass` (from zero) of the record.
361fn add_in_pass(record: &mut J, pass: usize, path: &[&str], by: i64) {
362 let Some(J::Arr(passes)) = at_mut(record, &["pass"]) else { panic!("the record has no pass array") };
363 match at_mut(&mut passes[pass], path) {
364 Some(J::Int(x)) => *x += by,
365 _ => panic!("pass {} has no integer at {}", pass + 1, path.join(".")),
366 }
367}
368
369#[test]
370fn a_record_whose_sub_phases_or_calls_do_not_add_up_is_refused() {
371 let sound = json::parse(&runs().record).expect("the record is JSON");
372 assert!(split_faults(&sound, false).is_empty(), "the unmutated record is sound");
373 // A sub-phase holding a thousand nanoseconds that `realise` does not.
374 let mut r = sound.clone();
375 add_in_pass(&mut r, 0, &["sub", "show", "ns"], 1_000);
376 let found = split_faults(&r, false);
377 assert!(found.iter().any(|f| f.contains("pass 1 realise holds") && f.contains("sub-phases sum to")), "{:#?}", found);
378 // An entry that `self` counts and `realise` does not.
379 let mut r = sound.clone();
380 add_in_pass(&mut r, 1, &["sub", "self", "n"], 1);
381 let found = split_faults(&r, false);
382 assert!(found.iter().any(|f| f.contains("pass 2") && f.contains("`self` says")), "{:#?}", found);
383 // A coarse record that clocked a phase entered for every element, and a fine one that did not.
384 let mut r = sound.clone();
385 add_in_pass(&mut r, 0, &["sub", "styles", "ns"], 5);
386 let found = split_faults(&r, false);
387 assert!(found.iter().any(|f| f.contains("pass 1 clocked styles in a coarse record")), "{:#?}", found);
388 let found = split_faults(&sound, true);
389 assert!(found.iter().any(|f| f.contains("left rules unclocked in a fine record")), "{:#?}", found);
390 // A call in pass one with a cause that only a later pass can give it, and no entry behind it.
391 let mut r = sound.clone();
392 let label = match sound.get("pass") {
393 Some(J::Arr(v)) => match v[0].get("calls") {
394 Some(J::Obj(kv)) => kv[0].0.clone(),
395 _ => panic!("pass 1 has no calls"),
396 },
397 _ => panic!("the record has no pass array"),
398 };
399 add_in_pass(&mut r, 0, &["calls", &label, "repass"], 1);
400 let found = split_faults(&r, false);
401 assert!(found.iter().any(|f| f.contains("pass 1 has") && f.contains("repass calls")), "{:#?}", found);
402 assert!(found.iter().any(|f| f.contains("pass 1 counts") && f.contains("realise calls")), "{:#?}", found);
403 // A first call after pass one.
404 let mut r = sound.clone();
405 add_in_pass(&mut r, 1, &["calls", &label, "first"], 1);
406 let found = split_faults(&r, false);
407 assert!(found.iter().any(|f| f.contains("pass 2 has") && f.contains("first calls")), "{:#?}", found);
408}
409
410// An engine given no recorder holds none after a whole compile, so the sub-phases and counters cost a test
411// of the option and nothing more; one given a recorder holds it, with its passes.
412#[test]
413fn the_engine_holds_no_recorder_unless_it_was_given_one() {
414 let dir = case_dir("lib");
415 let main = dir.join("main.typ");
416 std::fs::write(&main, DOC).expect("the case's source");
417 let mut sink = PdfSink::new().expect("a PDF sink");
418 let plain = compile::assemble_eval(&main, &dir, FontStore::default(), &mut sink).expect("the plain compile");
419 assert!(plain.engine.timings.is_none(), "a compile without --timings made a recorder");
420 let mut sink = PdfSink::new().expect("a PDF sink");
421 let timed = compile::assemble_eval_timed(&main, &dir, FontStore::default(), &mut sink, Some(Timings::start()), None)
422 .expect("the timed compile");
423 let t = timed.engine.timings.as_ref().expect("the recorder stays in the engine");
424 assert!(t.passes() >= 2, "the forward read takes a second pass: {}", t.passes());
425}