oxedyne/fe2o3/fe2o3_austenite/tests/eval_oracle/structure.rs
8.4 KiB, 1 run
created by r1870400018:63628, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | // Level 3: realised structure, compared as a skeleton both engines can be brought to. |
| 2 | // |
| 3 | // The oracle is `typst compile --features html --format html`, which realises the document with its |
| 4 | // show rules and writes the model elements as HTML. The skeleton keeps the structural tags and the |
| 5 | // text, and drops what is presentation (`div`, `span`, attributes, the style sheet). Three |
| 6 | // normalisations keep it to structure: whitespace collapses; smart quotes read as plain ones (the |
| 7 | // glyph a quote resolves to is level 4's business); and footnotes are left to levels 2 and 4, |
| 8 | // because the HTML export moves them into an endnote section a paged document does not have. |
| 9 | // |
| 10 | // Typst's HTML export ignores much of the layout library (`align`, `pad`, `grid`, `place`, `h`, |
| 11 | // `v`, `stack`, `columns`, shapes), dropping the element and its content with a warning. A fixture |
| 12 | // whose export warns that way has no level-3 oracle; the harness reports it as not applicable |
| 13 | // rather than comparing against a document Typst itself has mutilated. |
| 14 | |
| 15 | use crate::harness::markup::{ |
| 16 | tokens, |
| 17 | Tok, |
| 18 | }; |
| 19 | |
| 20 | use oxedyne_fe2o3_core::prelude::*; |
| 21 | |
| 22 | #[derive(Clone, Debug, PartialEq, Eq)] |
| 23 | pub enum Sk { |
| 24 | Tag(String, Vec<Sk>), |
| 25 | Text(String), |
| 26 | } |
| 27 | |
| 28 | pub const TAGS: &[&str] = &[ |
| 29 | "h1", "h2", "h3", "h4", "h5", "h6", "p", "ul", "ol", "li", "dl", "dt", "dd", "figure", "figcaption", |
| 30 | "table", "td", "th", "blockquote", "pre", "code", "strong", "em", "a", "sup", "sub", "mark", "math", |
| 31 | "img", "br", |
| 32 | ]; |
| 33 | |
| 34 | const BLOCK: &[&str] = &[ |
| 35 | "h1", "h2", "h3", "h4", "h5", "h6", "p", "ul", "ol", "li", "dl", "dt", "dd", "figure", "figcaption", |
| 36 | "table", "td", "th", "blockquote", "pre", |
| 37 | ]; |
| 38 | |
| 39 | /// Ignored elements that carry no content, so the export that drops them still has the document's |
| 40 | /// whole structure: page setup, breaks and spacing. |
| 41 | pub const HARMLESS: &[&str] = &["page set rule", "pagebreak", "colbreak", "h", "v"]; |
| 42 | |
| 43 | const VOID: &[&str] = &["br", "img", "hr", "meta", "link", "input", "col", "wbr"]; |
| 44 | |
| 45 | /// The body of a Typst HTML export as a skeleton. |
| 46 | pub fn from_html(html: &str) -> Vec<Sk> { |
| 47 | // Frames: (tag kept, children). A dropped subtree is counted, not framed. |
| 48 | let mut stack: Vec<(Option<String>, Vec<Sk>)> = vec![(None, Vec::new())]; |
| 49 | let mut in_body = false; |
| 50 | let mut drop_depth = 0usize; |
| 51 | let mut open_names: Vec<String> = Vec::new(); |
| 52 | for t in tokens(html) { |
| 53 | match t { |
| 54 | Tok::Open { name, attrs, closed } => { |
| 55 | if name == "body" { |
| 56 | in_body = true; |
| 57 | continue; |
| 58 | } |
| 59 | if !in_body { |
| 60 | continue; |
| 61 | } |
| 62 | let void = closed || VOID.contains(&name.as_str()); |
| 63 | let role = Tok::attr(&attrs, "role").unwrap_or(""); |
| 64 | let dropped = role == "doc-noteref" || role == "doc-endnotes" || role == "doc-backlink"; |
| 65 | if drop_depth > 0 || dropped { |
| 66 | if !void { |
| 67 | drop_depth += 1; |
| 68 | open_names.push(name); |
| 69 | } |
| 70 | continue; |
| 71 | } |
| 72 | if void { |
| 73 | if TAGS.contains(&name.as_str()) { |
| 74 | if let Some(top) = stack.last_mut() { |
| 75 | top.1.push(Sk::Tag(name, Vec::new())); |
| 76 | } |
| 77 | } |
| 78 | continue; |
| 79 | } |
| 80 | if name == "math" { |
| 81 | // MathML is level 4's business; the equation is one leaf. |
| 82 | drop_depth += 1; |
| 83 | open_names.push(name.clone()); |
| 84 | if let Some(top) = stack.last_mut() { |
| 85 | top.1.push(Sk::Tag(name, Vec::new())); |
| 86 | } |
| 87 | continue; |
| 88 | } |
| 89 | open_names.push(name.clone()); |
| 90 | let keep = TAGS.contains(&name.as_str()); |
| 91 | stack.push((if keep { Some(name) } else { None }, Vec::new())); |
| 92 | } |
| 93 | Tok::Close(name) => { |
| 94 | if name == "body" { |
| 95 | in_body = false; |
| 96 | continue; |
| 97 | } |
| 98 | if !in_body { |
| 99 | continue; |
| 100 | } |
| 101 | // Pop to the matching open tag; HTML here is well formed, so this is one pop. |
| 102 | let pos = open_names.iter().rposition(|n| *n == name); |
| 103 | let pos = match pos { |
| 104 | Some(p) => p, |
| 105 | None => continue, |
| 106 | }; |
| 107 | while open_names.len() > pos { |
| 108 | open_names.pop(); |
| 109 | if drop_depth > 0 { |
| 110 | drop_depth -= 1; |
| 111 | continue; |
| 112 | } |
| 113 | if stack.len() > 1 { |
| 114 | if let Some((tag, kids)) = stack.pop() { |
| 115 | if let Some(top) = stack.last_mut() { |
| 116 | match tag { |
| 117 | Some(t) => top.1.push(Sk::Tag(t, kids)), |
| 118 | None => top.1.extend(kids), |
| 119 | } |
| 120 | } |
| 121 | } |
| 122 | } |
| 123 | } |
| 124 | } |
| 125 | Tok::Text(s) => { |
| 126 | if in_body && drop_depth == 0 { |
| 127 | if let Some(top) = stack.last_mut() { |
| 128 | top.1.push(Sk::Text(s)); |
| 129 | } |
| 130 | } |
| 131 | } |
| 132 | } |
| 133 | } |
| 134 | while stack.len() > 1 { |
| 135 | if let Some((tag, kids)) = stack.pop() { |
| 136 | if let Some(top) = stack.last_mut() { |
| 137 | match tag { |
| 138 | Some(t) => top.1.push(Sk::Tag(t, kids)), |
| 139 | None => top.1.extend(kids), |
| 140 | } |
| 141 | } |
| 142 | } |
| 143 | } |
| 144 | normalise(stack.pop().map(|f| f.1).unwrap_or_default()) |
| 145 | } |
| 146 | |
| 147 | /// The elements Typst's HTML export reports it ignored, from its warnings. |
| 148 | pub fn ignored_elements(stderr: &str) -> Vec<String> { |
| 149 | let mut out: Vec<String> = Vec::new(); |
| 150 | for line in stderr.lines() { |
| 151 | if let Some(rest) = line.trim().strip_prefix("warning: ") { |
| 152 | if let Some(name) = rest.strip_suffix(" was ignored during HTML export") { |
| 153 | if !out.iter().any(|n| n == name) { |
| 154 | out.push(name.to_string()); |
| 155 | } |
| 156 | } |
| 157 | } |
| 158 | } |
| 159 | out |
| 160 | } |
| 161 | |
| 162 | fn quotes(s: &str) -> String { |
| 163 | s.chars().map(|c| match c { |
| 164 | '\u{201c}' | '\u{201d}' | '\u{201e}' | '\u{00ab}' | '\u{00bb}' => '"', |
| 165 | '\u{2018}' | '\u{2019}' | '\u{201a}' | '\u{2039}' | '\u{203a}' => '\'', |
| 166 | '\u{a0}' => ' ', |
| 167 | c => c, |
| 168 | }).collect() |
| 169 | } |
| 170 | |
| 171 | /// Merges adjacent text, collapses whitespace, trims it at block edges and drops empty blocks' |
| 172 | /// paragraphs, so two skeletons differ only where the structure does. |
| 173 | pub fn normalise(kids: Vec<Sk>) -> Vec<Sk> { |
| 174 | let mut out: Vec<Sk> = Vec::new(); |
| 175 | for k in kids { |
| 176 | match k { |
| 177 | Sk::Text(t) => { |
| 178 | let t = quotes(&t); |
| 179 | let mut c = String::new(); |
| 180 | let mut ws = false; |
| 181 | for ch in t.chars() { |
| 182 | if ch.is_whitespace() { |
| 183 | ws = true; |
| 184 | } else { |
| 185 | if ws { |
| 186 | c.push(' '); |
| 187 | } |
| 188 | ws = false; |
| 189 | c.push(ch); |
| 190 | } |
| 191 | } |
| 192 | if ws { |
| 193 | c.push(' '); |
| 194 | } |
| 195 | if c.is_empty() { |
| 196 | continue; |
| 197 | } |
| 198 | match out.last_mut() { |
| 199 | Some(Sk::Text(prev)) => { |
| 200 | if prev.ends_with(' ') && c.starts_with(' ') { |
| 201 | c.remove(0); |
| 202 | } |
| 203 | prev.push_str(&c); |
| 204 | } |
| 205 | _ => out.push(Sk::Text(c)), |
| 206 | } |
| 207 | } |
| 208 | Sk::Tag(name, inner) => { |
| 209 | let mut inner = normalise(inner); |
| 210 | if BLOCK.contains(&name.as_str()) { |
| 211 | trim_edges(&mut inner); |
| 212 | if name == "p" && inner.is_empty() { |
| 213 | continue; |
| 214 | } |
| 215 | // A block ends the text before it. |
| 216 | if let Some(Sk::Text(prev)) = out.last_mut() { |
| 217 | let t = prev.trim_end().to_string(); |
| 218 | *prev = t; |
| 219 | } |
| 220 | if matches!(out.last(), Some(Sk::Text(p)) if p.is_empty()) { |
| 221 | out.pop(); |
| 222 | } |
| 223 | } |
| 224 | out.push(Sk::Tag(name, inner)); |
| 225 | } |
| 226 | } |
| 227 | } |
| 228 | // Text after a block starts afresh. |
| 229 | let mut i = 1; |
| 230 | while i < out.len() { |
| 231 | let after_block = matches!(&out[i - 1], Sk::Tag(n, _) if BLOCK.contains(&n.as_str())); |
| 232 | if after_block { |
| 233 | if let Sk::Text(t) = &mut out[i] { |
| 234 | let s = t.trim_start().to_string(); |
| 235 | *t = s; |
| 236 | } |
| 237 | } |
| 238 | i += 1; |
| 239 | } |
| 240 | out.retain(|k| !matches!(k, Sk::Text(t) if t.is_empty())); |
| 241 | trim_edges_if_blocky(&mut out); |
| 242 | out |
| 243 | } |
| 244 | |
| 245 | fn trim_edges(v: &mut Vec<Sk>) { |
| 246 | if let Some(Sk::Text(t)) = v.first_mut() { |
| 247 | let s = t.trim_start().to_string(); |
| 248 | *t = s; |
| 249 | } |
| 250 | if let Some(Sk::Text(t)) = v.last_mut() { |
| 251 | let s = t.trim_end().to_string(); |
| 252 | *t = s; |
| 253 | } |
| 254 | v.retain(|k| !matches!(k, Sk::Text(t) if t.is_empty())); |
| 255 | } |
| 256 | |
| 257 | fn trim_edges_if_blocky(v: &mut Vec<Sk>) { |
| 258 | if v.iter().any(|k| matches!(k, Sk::Tag(n, _) if BLOCK.contains(&n.as_str()))) { |
| 259 | trim_edges(v); |
| 260 | } |
| 261 | } |
| 262 | |
| 263 | pub fn render(v: &[Sk]) -> String { |
| 264 | let mut s = String::new(); |
| 265 | for (i, k) in v.iter().enumerate() { |
| 266 | if i > 0 { |
| 267 | s.push(' '); |
| 268 | } |
| 269 | match k { |
| 270 | Sk::Text(t) => s.push_str(&fmt!("{:?}", t)), |
| 271 | Sk::Tag(n, inner) => { |
| 272 | s.push_str(n); |
| 273 | if !inner.is_empty() { |
| 274 | s.push('['); |
| 275 | s.push_str(&render(inner)); |
| 276 | s.push(']'); |
| 277 | } |
| 278 | } |
| 279 | } |
| 280 | } |
| 281 | s |
| 282 | } |
| 283 | |
| 284 | /// Level-3 differences: the first place the skeletons part, with a little context either side. |
| 285 | pub fn compare(want: &[Sk], got: &[Sk], out: &mut Vec<String>) { |
| 286 | let a = render(want); |
| 287 | let b = render(got); |
| 288 | if a == b { |
| 289 | return; |
| 290 | } |
| 291 | let ac: Vec<char> = a.chars().collect(); |
| 292 | let bc: Vec<char> = b.chars().collect(); |
| 293 | let mut i = 0; |
| 294 | while i < ac.len() && i < bc.len() && ac[i] == bc[i] { |
| 295 | i += 1; |
| 296 | } |
| 297 | let from = i.saturating_sub(40); |
| 298 | let ta: String = ac[from..(i + 60).min(ac.len())].iter().collect(); |
| 299 | let tb: String = bc[from..(i + 60).min(bc.len())].iter().collect(); |
| 300 | out.push(fmt!("structure parts at character {}: typst ...{}... austenite ...{}...", i, ta, tb)); |
| 301 | } |