Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tests/eval_oracle/corpus.rs

7.1 KiB, 1 run

created by r1870400018:63614, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// The synthetic fixture corpus: `tests/fixtures/eval/<area>/<name>.typ`, one area per unit. A file
2// whose name starts with `_` is a helper a fixture imports, not a fixture; so is anything in a
3// subdirectory. The area directory is the project root on both sides, so a fixture reaches nothing
4// outside its own area.
5//
6// A fixture may narrow what is compared with a directive among its leading comment lines:
7//
8// // oracle: levels 1 2 compare values and positions only
9// // oracle: no-html skip level 3 (Typst's HTML export cannot express the construct)
10// // oracle: rejects Typst must reject it; Austenite must fail with the same first error
11// // oracle: none not a document (a unit's case file), so not in the corpus
12//
13// Levels: 1 values (`#metadata(..) <probe>`), 2 positions of headings, figures, equations,
14// footnotes and probes, 3 realised structure against Typst's HTML export, 4 layout (page count,
15// line text and baselines).
16
17use oxedyne_fe2o3_core::prelude::*;
18
19use std::path::{
20 Path,
21 PathBuf,
22};
23
24#[derive(Clone, Debug)]
25pub struct Fixture {
26 pub area: String,
27 pub name: String,
28 pub path: PathBuf,
29 pub root: PathBuf, // the area directory: `--root` for typst, `World::root` for Austenite
30 pub text: String,
31 pub levels: [bool; 4],
32 pub expect: Expect,
33}
34
35/// What Typst is expected to do with a fixture.
36#[derive(Clone, Copy, Debug, PartialEq, Eq)]
37pub enum Expect {
38 Accepts,
39 Rejects, // compared on the first error's message and position, not on any level
40}
41
42impl Fixture {
43 pub fn id(&self) -> String { fmt!("{}/{}", self.area, self.name) }
44
45 /// Does the fixture ask for level `n` (1-4)?
46 pub fn wants(&self, n: usize) -> bool { n >= 1 && n <= 4 && self.levels[n - 1] }
47
48 /// Does anything in it depend on introspection, so its probes can only be read after layout?
49 pub fn contextual(&self) -> bool {
50 self.text.contains("context") || self.text.contains("show") || self.text.contains("query")
51 }
52}
53
54pub fn fixtures_dir() -> PathBuf {
55 Path::new(env!("CARGO_MANIFEST_DIR")).join("tests").join("fixtures").join("eval")
56}
57
58/// Which fixtures a run covers: `EVAL_ORACLE_AREA=a,b` limits the areas, `EVAL_ORACLE_FIXTURE=s` the
59/// fixtures whose `area/name` contains `s`.
60#[derive(Clone, Debug, Default)]
61pub struct Filter {
62 pub areas: Option<Vec<String>>,
63 pub name: Option<String>,
64}
65
66impl Filter {
67 pub fn from_env() -> Self {
68 let areas = std::env::var("EVAL_ORACLE_AREA").ok()
69 .filter(|s| !s.trim().is_empty())
70 .map(|s| s.split(',').map(|a| a.trim().to_string()).collect());
71 let name = std::env::var("EVAL_ORACLE_FIXTURE").ok().filter(|s| !s.trim().is_empty());
72 Self { areas, name }
73 }
74
75 pub fn only(area: &str) -> Self {
76 Self { areas: Some(vec![area.to_string()]), name: None }
77 }
78
79 fn admits(&self, area: &str, id: &str) -> bool {
80 if let Some(a) = &self.areas {
81 if !a.iter().any(|x| x == area) {
82 return false;
83 }
84 }
85 match &self.name {
86 Some(n) => id.contains(n.as_str()),
87 None => true,
88 }
89 }
90}
91
92pub fn discover(filter: &Filter) -> Outcome<Vec<Fixture>> {
93 let base = fixtures_dir();
94 let mut areas: Vec<PathBuf> = Vec::new();
95 let rd = res!(std::fs::read_dir(&base).map_err(|e| err!(
96 "Cannot read the fixture corpus at {}: {}", base.display(), e; IO, File, Read)));
97 for entry in rd {
98 let entry = res!(entry.map_err(|e| err!("Cannot list {}: {}", base.display(), e; IO, File, Read)));
99 let p = entry.path();
100 if p.is_dir() {
101 areas.push(p);
102 }
103 }
104 areas.sort();
105 let mut out = Vec::new();
106 for dir in areas {
107 let area = dir.file_name().map(|s| s.to_string_lossy().to_string()).unwrap_or_default();
108 let mut files: Vec<PathBuf> = Vec::new();
109 let rd = res!(std::fs::read_dir(&dir).map_err(|e| err!(
110 "Cannot read the fixture area {}: {}", dir.display(), e; IO, File, Read)));
111 for entry in rd {
112 let entry = res!(entry.map_err(|e| err!("Cannot list {}: {}", dir.display(), e; IO, File, Read)));
113 let p = entry.path();
114 let fname = p.file_name().map(|s| s.to_string_lossy().to_string()).unwrap_or_default();
115 if p.is_file() && fname.ends_with(".typ") && !fname.starts_with('_') {
116 files.push(p);
117 }
118 }
119 files.sort();
120 for path in files {
121 let name = path.file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or_default();
122 if !filter.admits(&area, &fmt!("{}/{}", area, name)) {
123 continue;
124 }
125 let text = res!(std::fs::read_to_string(&path).map_err(|e| err!(
126 "Cannot read fixture {}: {}", path.display(), e; IO, File, Read)));
127 let (levels, expect) = match directives(&text) {
128 Some(d) => d,
129 None => continue,
130 };
131 out.push(Fixture { area: area.clone(), name, path, root: dir.clone(), text, levels, expect });
132 }
133 }
134 Ok(out)
135}
136
137// The levels and expectation a fixture's leading directives ask for, or `None` for `oracle: none`.
138fn directives(text: &str) -> Option<([bool; 4], Expect)> {
139 let mut levels = [true; 4];
140 let mut expect = Expect::Accepts;
141 for line in text.lines() {
142 let t = line.trim();
143 if !t.starts_with("//") {
144 if t.is_empty() {
145 continue;
146 }
147 break;
148 }
149 let d = t.trim_start_matches('/').trim();
150 let d = match d.strip_prefix("oracle:") {
151 Some(d) => d.trim(),
152 None => continue,
153 };
154 if let Some(list) = d.strip_prefix("levels") {
155 levels = [false; 4];
156 for n in list.split(|c: char| c == ',' || c.is_whitespace()) {
157 if let Ok(n) = n.parse::<usize>() {
158 if n >= 1 && n <= 4 {
159 levels[n - 1] = true;
160 }
161 }
162 }
163 } else if d == "no-html" {
164 levels[2] = false;
165 } else if d == "rejects" {
166 expect = Expect::Rejects;
167 } else if d == "none" {
168 return None;
169 }
170 }
171 Some((levels, expect))
172}
173
174/// Why a fixture cannot be in the corpus, if it cannot: it must be synthetic, so it may not name a
175/// book or reach outside its area.
176pub fn provenance_fault(f: &Fixture) -> Option<String> {
177 // A `../` is not a fault in itself: Typst, run with the area as its root, and the evaluator's own
178 // resolver both refuse a path that leaves the root, so a fixture that climbs either stays inside or is
179 // rejected by both, and the import corpus tests that refusal.
180 for bad in ["usr/books", "/books/"] {
181 if f.text.contains(bad) {
182 return Some(fmt!("{} names `{}`: fixtures are synthetic and stay inside their area", f.id(), bad));
183 }
184 }
185 None
186}
187
188/// Every file in a fixture's area, for the oracle cache key: a helper edit invalidates the area.
189pub fn area_bytes(root: &Path) -> Outcome<Vec<(String, Vec<u8>)>> {
190 let mut out = Vec::new();
191 res!(walk(root, root, &mut out, 0));
192 out.sort_by(|a, b| a.0.cmp(&b.0));
193 Ok(out)
194}
195
196fn walk(base: &Path, dir: &Path, out: &mut Vec<(String, Vec<u8>)>, depth: usize) -> Outcome<()> {
197 if depth > 8 {
198 return Ok(());
199 }
200 let rd = res!(std::fs::read_dir(dir).map_err(|e| err!("Cannot list {}: {}", dir.display(), e; IO, File, Read)));
201 for entry in rd {
202 let entry = res!(entry.map_err(|e| err!("Cannot list {}: {}", dir.display(), e; IO, File, Read)));
203 let p = entry.path();
204 if p.is_dir() {
205 res!(walk(base, &p, out, depth + 1));
206 } else if p.is_file() {
207 let rel = p.strip_prefix(base).map(|r| r.to_string_lossy().to_string()).unwrap_or_default();
208 let bytes = res!(std::fs::read(&p).map_err(|e| err!("Cannot read {}: {}", p.display(), e; IO, File, Read)));
209 out.push((rel, bytes));
210 }
211 }
212 Ok(())
213}