Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tests/eval_oracle/structure.rs

8.4 KiB, 1 run

created by r1870400018:63628, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// Level 3: realised structure, compared as a skeleton both engines can be brought to.
2//
3// The oracle is `typst compile --features html --format html`, which realises the document with its
4// show rules and writes the model elements as HTML. The skeleton keeps the structural tags and the
5// text, and drops what is presentation (`div`, `span`, attributes, the style sheet). Three
6// normalisations keep it to structure: whitespace collapses; smart quotes read as plain ones (the
7// glyph a quote resolves to is level 4's business); and footnotes are left to levels 2 and 4,
8// because the HTML export moves them into an endnote section a paged document does not have.
9//
10// Typst's HTML export ignores much of the layout library (`align`, `pad`, `grid`, `place`, `h`,
11// `v`, `stack`, `columns`, shapes), dropping the element and its content with a warning. A fixture
12// whose export warns that way has no level-3 oracle; the harness reports it as not applicable
13// rather than comparing against a document Typst itself has mutilated.
14
15use crate::harness::markup::{
16 tokens,
17 Tok,
18};
19
20use oxedyne_fe2o3_core::prelude::*;
21
22#[derive(Clone, Debug, PartialEq, Eq)]
23pub enum Sk {
24 Tag(String, Vec<Sk>),
25 Text(String),
26}
27
28pub const TAGS: &[&str] = &[
29 "h1", "h2", "h3", "h4", "h5", "h6", "p", "ul", "ol", "li", "dl", "dt", "dd", "figure", "figcaption",
30 "table", "td", "th", "blockquote", "pre", "code", "strong", "em", "a", "sup", "sub", "mark", "math",
31 "img", "br",
32];
33
34const BLOCK: &[&str] = &[
35 "h1", "h2", "h3", "h4", "h5", "h6", "p", "ul", "ol", "li", "dl", "dt", "dd", "figure", "figcaption",
36 "table", "td", "th", "blockquote", "pre",
37];
38
39/// Ignored elements that carry no content, so the export that drops them still has the document's
40/// whole structure: page setup, breaks and spacing.
41pub const HARMLESS: &[&str] = &["page set rule", "pagebreak", "colbreak", "h", "v"];
42
43const VOID: &[&str] = &["br", "img", "hr", "meta", "link", "input", "col", "wbr"];
44
45/// The body of a Typst HTML export as a skeleton.
46pub fn from_html(html: &str) -> Vec<Sk> {
47 // Frames: (tag kept, children). A dropped subtree is counted, not framed.
48 let mut stack: Vec<(Option<String>, Vec<Sk>)> = vec![(None, Vec::new())];
49 let mut in_body = false;
50 let mut drop_depth = 0usize;
51 let mut open_names: Vec<String> = Vec::new();
52 for t in tokens(html) {
53 match t {
54 Tok::Open { name, attrs, closed } => {
55 if name == "body" {
56 in_body = true;
57 continue;
58 }
59 if !in_body {
60 continue;
61 }
62 let void = closed || VOID.contains(&name.as_str());
63 let role = Tok::attr(&attrs, "role").unwrap_or("");
64 let dropped = role == "doc-noteref" || role == "doc-endnotes" || role == "doc-backlink";
65 if drop_depth > 0 || dropped {
66 if !void {
67 drop_depth += 1;
68 open_names.push(name);
69 }
70 continue;
71 }
72 if void {
73 if TAGS.contains(&name.as_str()) {
74 if let Some(top) = stack.last_mut() {
75 top.1.push(Sk::Tag(name, Vec::new()));
76 }
77 }
78 continue;
79 }
80 if name == "math" {
81 // MathML is level 4's business; the equation is one leaf.
82 drop_depth += 1;
83 open_names.push(name.clone());
84 if let Some(top) = stack.last_mut() {
85 top.1.push(Sk::Tag(name, Vec::new()));
86 }
87 continue;
88 }
89 open_names.push(name.clone());
90 let keep = TAGS.contains(&name.as_str());
91 stack.push((if keep { Some(name) } else { None }, Vec::new()));
92 }
93 Tok::Close(name) => {
94 if name == "body" {
95 in_body = false;
96 continue;
97 }
98 if !in_body {
99 continue;
100 }
101 // Pop to the matching open tag; HTML here is well formed, so this is one pop.
102 let pos = open_names.iter().rposition(|n| *n == name);
103 let pos = match pos {
104 Some(p) => p,
105 None => continue,
106 };
107 while open_names.len() > pos {
108 open_names.pop();
109 if drop_depth > 0 {
110 drop_depth -= 1;
111 continue;
112 }
113 if stack.len() > 1 {
114 if let Some((tag, kids)) = stack.pop() {
115 if let Some(top) = stack.last_mut() {
116 match tag {
117 Some(t) => top.1.push(Sk::Tag(t, kids)),
118 None => top.1.extend(kids),
119 }
120 }
121 }
122 }
123 }
124 }
125 Tok::Text(s) => {
126 if in_body && drop_depth == 0 {
127 if let Some(top) = stack.last_mut() {
128 top.1.push(Sk::Text(s));
129 }
130 }
131 }
132 }
133 }
134 while stack.len() > 1 {
135 if let Some((tag, kids)) = stack.pop() {
136 if let Some(top) = stack.last_mut() {
137 match tag {
138 Some(t) => top.1.push(Sk::Tag(t, kids)),
139 None => top.1.extend(kids),
140 }
141 }
142 }
143 }
144 normalise(stack.pop().map(|f| f.1).unwrap_or_default())
145}
146
147/// The elements Typst's HTML export reports it ignored, from its warnings.
148pub fn ignored_elements(stderr: &str) -> Vec<String> {
149 let mut out: Vec<String> = Vec::new();
150 for line in stderr.lines() {
151 if let Some(rest) = line.trim().strip_prefix("warning: ") {
152 if let Some(name) = rest.strip_suffix(" was ignored during HTML export") {
153 if !out.iter().any(|n| n == name) {
154 out.push(name.to_string());
155 }
156 }
157 }
158 }
159 out
160}
161
162fn quotes(s: &str) -> String {
163 s.chars().map(|c| match c {
164 '\u{201c}' | '\u{201d}' | '\u{201e}' | '\u{00ab}' | '\u{00bb}' => '"',
165 '\u{2018}' | '\u{2019}' | '\u{201a}' | '\u{2039}' | '\u{203a}' => '\'',
166 '\u{a0}' => ' ',
167 c => c,
168 }).collect()
169}
170
171/// Merges adjacent text, collapses whitespace, trims it at block edges and drops empty blocks'
172/// paragraphs, so two skeletons differ only where the structure does.
173pub fn normalise(kids: Vec<Sk>) -> Vec<Sk> {
174 let mut out: Vec<Sk> = Vec::new();
175 for k in kids {
176 match k {
177 Sk::Text(t) => {
178 let t = quotes(&t);
179 let mut c = String::new();
180 let mut ws = false;
181 for ch in t.chars() {
182 if ch.is_whitespace() {
183 ws = true;
184 } else {
185 if ws {
186 c.push(' ');
187 }
188 ws = false;
189 c.push(ch);
190 }
191 }
192 if ws {
193 c.push(' ');
194 }
195 if c.is_empty() {
196 continue;
197 }
198 match out.last_mut() {
199 Some(Sk::Text(prev)) => {
200 if prev.ends_with(' ') && c.starts_with(' ') {
201 c.remove(0);
202 }
203 prev.push_str(&c);
204 }
205 _ => out.push(Sk::Text(c)),
206 }
207 }
208 Sk::Tag(name, inner) => {
209 let mut inner = normalise(inner);
210 if BLOCK.contains(&name.as_str()) {
211 trim_edges(&mut inner);
212 if name == "p" && inner.is_empty() {
213 continue;
214 }
215 // A block ends the text before it.
216 if let Some(Sk::Text(prev)) = out.last_mut() {
217 let t = prev.trim_end().to_string();
218 *prev = t;
219 }
220 if matches!(out.last(), Some(Sk::Text(p)) if p.is_empty()) {
221 out.pop();
222 }
223 }
224 out.push(Sk::Tag(name, inner));
225 }
226 }
227 }
228 // Text after a block starts afresh.
229 let mut i = 1;
230 while i < out.len() {
231 let after_block = matches!(&out[i - 1], Sk::Tag(n, _) if BLOCK.contains(&n.as_str()));
232 if after_block {
233 if let Sk::Text(t) = &mut out[i] {
234 let s = t.trim_start().to_string();
235 *t = s;
236 }
237 }
238 i += 1;
239 }
240 out.retain(|k| !matches!(k, Sk::Text(t) if t.is_empty()));
241 trim_edges_if_blocky(&mut out);
242 out
243}
244
245fn trim_edges(v: &mut Vec<Sk>) {
246 if let Some(Sk::Text(t)) = v.first_mut() {
247 let s = t.trim_start().to_string();
248 *t = s;
249 }
250 if let Some(Sk::Text(t)) = v.last_mut() {
251 let s = t.trim_end().to_string();
252 *t = s;
253 }
254 v.retain(|k| !matches!(k, Sk::Text(t) if t.is_empty()));
255}
256
257fn trim_edges_if_blocky(v: &mut Vec<Sk>) {
258 if v.iter().any(|k| matches!(k, Sk::Tag(n, _) if BLOCK.contains(&n.as_str()))) {
259 trim_edges(v);
260 }
261}
262
263pub fn render(v: &[Sk]) -> String {
264 let mut s = String::new();
265 for (i, k) in v.iter().enumerate() {
266 if i > 0 {
267 s.push(' ');
268 }
269 match k {
270 Sk::Text(t) => s.push_str(&fmt!("{:?}", t)),
271 Sk::Tag(n, inner) => {
272 s.push_str(n);
273 if !inner.is_empty() {
274 s.push('[');
275 s.push_str(&render(inner));
276 s.push(']');
277 }
278 }
279 }
280 }
281 s
282}
283
284/// Level-3 differences: the first place the skeletons part, with a little context either side.
285pub fn compare(want: &[Sk], got: &[Sk], out: &mut Vec<String>) {
286 let a = render(want);
287 let b = render(got);
288 if a == b {
289 return;
290 }
291 let ac: Vec<char> = a.chars().collect();
292 let bc: Vec<char> = b.chars().collect();
293 let mut i = 0;
294 while i < ac.len() && i < bc.len() && ac[i] == bc[i] {
295 i += 1;
296 }
297 let from = i.saturating_sub(40);
298 let ta: String = ac[from..(i + 60).min(ac.len())].iter().collect();
299 let tb: String = bc[from..(i + 60).min(bc.len())].iter().collect();
300 out.push(fmt!("structure parts at character {}: typst ...{}... austenite ...{}...", i, ta, tb));
301}