oxedyne/fe2o3/fe2o3_austenite/tests/eval_oracle/layout.rs
13.2 KiB, 1 run
created by r1870400018:63618, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | // Level 4: pages, line text and baselines. |
| 2 | // |
| 3 | // The oracle side comes from two of Typst's outputs, because neither alone has both halves. |
| 4 | // `pdftotext -bbox-layout` on Typst's PDF gives each line's text and box, but a box, not a |
| 5 | // baseline: poppler sizes it from the font descriptor's ascent and descent, which differ per face. |
| 6 | // Typst's SVG pages give every glyph's origin exactly -- a text run is a group translated to its |
| 7 | // baseline -- but no text. So each pdftotext line takes its baseline from the glyph origins that |
| 8 | // fall inside its box, the most common y among them. |
| 9 | // |
| 10 | // Austenite's side is its placed text runs, read straight from the laid-out pages: a run's top-left |
| 11 | // is `(x, y)` and its baseline `y + height`. |
| 12 | // |
| 13 | // A line is held to its text, its baseline and the x where it starts. The start is where a first-line |
| 14 | // or hanging indent, a margin and an alignment all show, none of which moves a break or a baseline, so |
| 15 | // a line set sideways passes the first two checks and fails the third. |
| 16 | |
| 17 | use crate::harness::markup::{ |
| 18 | tokens, |
| 19 | Tok, |
| 20 | }; |
| 21 | |
| 22 | use oxedyne_fe2o3_core::prelude::*; |
| 23 | |
| 24 | pub const BASELINE_TOL: f64 = 1.0; // points, the design's level-4 tolerance |
| 25 | pub const START_TOL: f64 = 0.05; // points, where a line starts: its indent or its alignment offset |
| 26 | pub const SPACE_REACH: f64 = 40.0; // points, the most a space opening a line can be wide |
| 27 | |
| 28 | #[derive(Clone, Debug)] |
| 29 | pub struct OLine { |
| 30 | pub text: String, |
| 31 | pub x0: f64, // where the first word starts |
| 32 | pub x1: f64, |
| 33 | pub start: f64, // where the first glyph drawn starts, a leading space included: its text run's start |
| 34 | pub y0: f64, |
| 35 | pub y1: f64, |
| 36 | pub base: f64, |
| 37 | pub measured: bool, // the baseline came from glyph origins, not the box |
| 38 | } |
| 39 | |
| 40 | #[derive(Clone, Debug, Default)] |
| 41 | pub struct OPage { |
| 42 | pub width: f64, |
| 43 | pub height: f64, |
| 44 | pub lines: Vec<OLine>, |
| 45 | } |
| 46 | |
| 47 | #[derive(Clone, Debug)] |
| 48 | pub struct ARun { |
| 49 | pub text: String, |
| 50 | pub x0: f64, |
| 51 | pub x1: f64, |
| 52 | pub base: f64, |
| 53 | } |
| 54 | |
| 55 | #[derive(Clone, Debug, Default)] |
| 56 | pub struct APage { |
| 57 | pub width: f64, |
| 58 | pub height: f64, |
| 59 | pub runs: Vec<ARun>, |
| 60 | } |
| 61 | |
| 62 | fn num(attrs: &[(String, String)], key: &str) -> f64 { |
| 63 | Tok::attr(attrs, key).and_then(|v| v.trim().trim_end_matches("pt").parse::<f64>().ok()).unwrap_or(0.0) |
| 64 | } |
| 65 | |
| 66 | /// The pages and lines of `pdftotext -bbox-layout` output, baselines not yet attached. |
| 67 | /// |
| 68 | /// A line's horizontal extent is its words', not its own `xMin` and `xMax`: pdftotext (poppler 26.01) |
| 69 | /// gives a line the left edge of its second word, so a line's start read from the line is wrong. And |
| 70 | /// a loosely justified line comes back from it as several, cut at its widest gaps, so the pieces of one |
| 71 | /// block that stand on one baseline are one line again. |
| 72 | pub fn read_bbox(xml: &str) -> Vec<OPage> { |
| 73 | let mut pages: Vec<OPage> = Vec::new(); |
| 74 | let mut block_from = 0usize; // where the current block's lines begin on the page |
| 75 | let mut line: Option<OLine> = None; |
| 76 | let mut words_seen = false; |
| 77 | let mut in_word = false; |
| 78 | for t in tokens(xml) { |
| 79 | match t { |
| 80 | Tok::Open { name, attrs, .. } => match name.as_str() { |
| 81 | "page" => { |
| 82 | block_from = 0; |
| 83 | pages.push(OPage { width: num(&attrs, "width"), height: num(&attrs, "height"), lines: Vec::new() }); |
| 84 | } |
| 85 | "block" => block_from = pages.last().map_or(0, |p| p.lines.len()), |
| 86 | "line" => { |
| 87 | words_seen = false; |
| 88 | line = Some(OLine { |
| 89 | text: String::new(), |
| 90 | x0: num(&attrs, "xmin"), |
| 91 | x1: num(&attrs, "xmax"), |
| 92 | start: num(&attrs, "xmin"), |
| 93 | y0: num(&attrs, "ymin"), |
| 94 | y1: num(&attrs, "ymax"), |
| 95 | base: 0.0, |
| 96 | measured: false, |
| 97 | }); |
| 98 | } |
| 99 | "word" => { |
| 100 | in_word = true; |
| 101 | if let Some(l) = line.as_mut() { |
| 102 | let (w0, w1) = (num(&attrs, "xmin"), num(&attrs, "xmax")); |
| 103 | if words_seen { |
| 104 | l.x0 = l.x0.min(w0); |
| 105 | l.x1 = l.x1.max(w1); |
| 106 | } else { |
| 107 | l.x0 = w0; |
| 108 | l.x1 = w1; |
| 109 | words_seen = true; |
| 110 | } |
| 111 | l.start = l.x0; |
| 112 | } |
| 113 | } |
| 114 | _ => (), |
| 115 | }, |
| 116 | Tok::Close(name) => match name.as_str() { |
| 117 | "word" => { |
| 118 | in_word = false; |
| 119 | if let Some(l) = line.as_mut() { |
| 120 | l.text.push(' '); |
| 121 | } |
| 122 | } |
| 123 | "line" => if let Some(mut l) = line.take() { |
| 124 | l.text = l.text.trim().to_string(); |
| 125 | if let Some(p) = pages.last_mut() { |
| 126 | let same = |o: &OLine| (o.y0 - l.y0).abs() < 0.5 && (o.y1 - l.y1).abs() < 0.5; |
| 127 | match p.lines.iter_mut().skip(block_from).find(|o| same(o)) { |
| 128 | Some(o) => { |
| 129 | o.text = fmt!("{} {}", o.text, l.text); |
| 130 | o.x0 = o.x0.min(l.x0); |
| 131 | o.x1 = o.x1.max(l.x1); |
| 132 | o.start = o.x0; |
| 133 | } |
| 134 | None => p.lines.push(l), |
| 135 | } |
| 136 | } |
| 137 | }, |
| 138 | _ => (), |
| 139 | }, |
| 140 | Tok::Text(s) => if in_word { |
| 141 | if let Some(l) = line.as_mut() { |
| 142 | l.text.push_str(&s); |
| 143 | } |
| 144 | }, |
| 145 | } |
| 146 | } |
| 147 | pages |
| 148 | } |
| 149 | |
| 150 | // An affine map (x, y) -> (a x + c y + e, b x + d y + f). |
| 151 | #[derive(Clone, Copy, Debug)] |
| 152 | struct M([f64; 6]); |
| 153 | |
| 154 | impl M { |
| 155 | const ID: M = M([1.0, 0.0, 0.0, 1.0, 0.0, 0.0]); |
| 156 | |
| 157 | fn then(self, n: M) -> M { |
| 158 | let (m, n) = (self.0, n.0); |
| 159 | M([ |
| 160 | m[0] * n[0] + m[2] * n[1], |
| 161 | m[1] * n[0] + m[3] * n[1], |
| 162 | m[0] * n[2] + m[2] * n[3], |
| 163 | m[1] * n[2] + m[3] * n[3], |
| 164 | m[0] * n[4] + m[2] * n[5] + m[4], |
| 165 | m[1] * n[4] + m[3] * n[5] + m[5], |
| 166 | ]) |
| 167 | } |
| 168 | |
| 169 | fn apply(self, x: f64, y: f64) -> (f64, f64) { |
| 170 | let m = self.0; |
| 171 | (m[0] * x + m[2] * y + m[4], m[1] * x + m[3] * y + m[5]) |
| 172 | } |
| 173 | } |
| 174 | |
| 175 | fn parse_transform(s: &str) -> M { |
| 176 | let mut m = M::ID; |
| 177 | let mut rest = s; |
| 178 | while let Some(open) = rest.find('(') { |
| 179 | let name = rest[..open].trim().trim_start_matches(',').trim(); |
| 180 | let close = match rest[open..].find(')') { |
| 181 | Some(c) => open + c, |
| 182 | None => break, |
| 183 | }; |
| 184 | let args: Vec<f64> = rest[open + 1..close] |
| 185 | .split(|c: char| c == ',' || c.is_whitespace()) |
| 186 | .filter(|t| !t.is_empty()) |
| 187 | .filter_map(|t| t.parse::<f64>().ok()) |
| 188 | .collect(); |
| 189 | let a = |i: usize| args.get(i).copied().unwrap_or(0.0); |
| 190 | let own = match name { |
| 191 | "matrix" if args.len() == 6 => M([a(0), a(1), a(2), a(3), a(4), a(5)]), |
| 192 | "translate" => M([1.0, 0.0, 0.0, 1.0, a(0), a(1)]), |
| 193 | "scale" => { |
| 194 | let sy = if args.len() > 1 { a(1) } else { a(0) }; |
| 195 | M([a(0), 0.0, 0.0, sy, 0.0, 0.0]) |
| 196 | } |
| 197 | "rotate" => { |
| 198 | let r = a(0).to_radians(); |
| 199 | let rot = M([r.cos(), r.sin(), -r.sin(), r.cos(), 0.0, 0.0]); |
| 200 | if args.len() == 3 { |
| 201 | M([1.0, 0.0, 0.0, 1.0, a(1), a(2)]).then(rot).then(M([1.0, 0.0, 0.0, 1.0, -a(1), -a(2)])) |
| 202 | } else { |
| 203 | rot |
| 204 | } |
| 205 | } |
| 206 | "skewX" => M([1.0, 0.0, a(0).to_radians().tan(), 1.0, 0.0, 0.0]), |
| 207 | "skewY" => M([1.0, a(0).to_radians().tan(), 0.0, 1.0, 0.0, 0.0]), |
| 208 | _ => M::ID, |
| 209 | }; |
| 210 | m = m.then(own); |
| 211 | rest = &rest[close + 1..]; |
| 212 | } |
| 213 | m |
| 214 | } |
| 215 | |
| 216 | /// The page-space origin of every glyph a Typst SVG page draws, and the origin of each text run: the |
| 217 | /// group that holds its glyphs, which Typst draws at the run's start, a space it leaves out included. |
| 218 | fn walk_glyphs(svg: &str) -> (Vec<(f64, f64)>, Vec<(f64, f64)>) { |
| 219 | // Per open element: its transform, whether it is inside defs, and its origin while no glyph has been |
| 220 | // drawn in it. |
| 221 | let mut stack: Vec<(M, bool, Option<(f64, f64)>)> = vec![(M::ID, false, None)]; |
| 222 | let (mut out, mut starts) = (Vec::new(), Vec::new()); |
| 223 | for t in tokens(svg) { |
| 224 | match t { |
| 225 | Tok::Open { name, attrs, closed } => { |
| 226 | let (top, in_defs, _) = stack.last().copied().unwrap_or((M::ID, false, None)); |
| 227 | let own = Tok::attr(&attrs, "transform").map(parse_transform).unwrap_or(M::ID); |
| 228 | let cur = top.then(own); |
| 229 | let defs = in_defs || name == "defs" || name == "symbol"; |
| 230 | if name == "use" && !defs { |
| 231 | let href = Tok::attr(&attrs, "xlink:href").or_else(|| Tok::attr(&attrs, "href")).unwrap_or(""); |
| 232 | if href.starts_with("#g") { |
| 233 | out.push(cur.apply(num(&attrs, "x"), num(&attrs, "y"))); |
| 234 | if let Some(last) = stack.last_mut() { |
| 235 | if let Some(origin) = last.2.take() { |
| 236 | starts.push(origin); |
| 237 | } |
| 238 | } |
| 239 | } |
| 240 | } |
| 241 | if !closed { |
| 242 | stack.push((cur, defs, if name == "g" { Some(cur.apply(0.0, 0.0)) } else { None })); |
| 243 | } |
| 244 | } |
| 245 | Tok::Close(_) => { |
| 246 | if stack.len() > 1 { |
| 247 | stack.pop(); |
| 248 | } |
| 249 | } |
| 250 | Tok::Text(_) => (), |
| 251 | } |
| 252 | } |
| 253 | (out, starts) |
| 254 | } |
| 255 | |
| 256 | /// The page-space origin of every glyph a Typst SVG page draws. |
| 257 | pub fn glyph_origins(svg: &str) -> Vec<(f64, f64)> { |
| 258 | walk_glyphs(svg).0 |
| 259 | } |
| 260 | |
| 261 | /// The page-space origin of every text run a Typst SVG page draws. |
| 262 | pub fn run_starts(svg: &str) -> Vec<(f64, f64)> { |
| 263 | walk_glyphs(svg).1 |
| 264 | } |
| 265 | |
| 266 | /// Gives each line the baseline its glyphs sit on; a line with no glyph origin inside its box keeps |
| 267 | /// an estimate from the box (Libertinus Serif's descent) and is marked unmeasured. |
| 268 | pub fn attach_baselines(page: &mut OPage, origins: &[(f64, f64)]) { |
| 269 | for l in &mut page.lines { |
| 270 | let mut ys: Vec<i64> = origins.iter() |
| 271 | .filter(|(x, y)| *x >= l.x0 - 1.0 && *x <= l.x1 + 1.0 && *y >= l.y0 - 0.5 && *y <= l.y1 + 0.5) |
| 272 | .map(|(_, y)| (y * 100.0).round() as i64) |
| 273 | .collect(); |
| 274 | if ys.is_empty() { |
| 275 | l.base = l.y1 - 0.2157 * (l.y1 - l.y0); |
| 276 | l.measured = false; |
| 277 | continue; |
| 278 | } |
| 279 | ys.sort(); |
| 280 | let mut best = (ys[0], 0usize); |
| 281 | let mut i = 0; |
| 282 | while i < ys.len() { |
| 283 | let mut j = i; |
| 284 | while j < ys.len() && ys[j] == ys[i] { |
| 285 | j += 1; |
| 286 | } |
| 287 | if j - i > best.1 { |
| 288 | best = (ys[i], j - i); |
| 289 | } |
| 290 | i = j; |
| 291 | } |
| 292 | l.base = best.0 as f64 / 100.0; |
| 293 | l.measured = true; |
| 294 | } |
| 295 | } |
| 296 | |
| 297 | /// Gives each line the start of the text run its first word is drawn in: the nearest run start on the |
| 298 | /// line's baseline at or before the word, within a space's reach. That run opens with a space when |
| 299 | /// Typst's line begins after an inline box or a space (pdftotext's words leave the space out), where |
| 300 | /// Austenite's run opens as well; a line whose run is not found starts where its first word does. |
| 301 | /// Call after [`attach_baselines`]. |
| 302 | pub fn attach_starts(page: &mut OPage, starts: &[(f64, f64)]) { |
| 303 | for l in &mut page.lines { |
| 304 | let found = starts.iter() |
| 305 | .filter(|(x, y)| (*y - l.base).abs() < 0.02 && *x <= l.x0 + 0.02 && *x >= l.x0 - SPACE_REACH) |
| 306 | .map(|(x, _)| *x) |
| 307 | .fold(f64::NEG_INFINITY, f64::max); |
| 308 | if found.is_finite() { |
| 309 | l.start = found; |
| 310 | } |
| 311 | } |
| 312 | } |
| 313 | |
| 314 | fn squash(s: &str) -> String { |
| 315 | s.chars().filter(|c| !c.is_whitespace() && *c != '\u{ad}').collect() |
| 316 | } |
| 317 | |
| 318 | /// Level-4 differences: page count exact, page size within half a point, every Typst line matched by |
| 319 | /// Austenite text on the same page with the same characters, a baseline within [`BASELINE_TOL`] and a |
| 320 | /// start within [`START_TOL`] of where Typst's line starts. |
| 321 | pub fn compare(want: &[OPage], got: &[APage], out: &mut Vec<String>) { |
| 322 | if want.len() != got.len() { |
| 323 | out.push(fmt!("page count: typst {}, austenite {}", want.len(), got.len())); |
| 324 | } |
| 325 | for (pi, (w, g)) in want.iter().zip(got.iter()).enumerate() { |
| 326 | let pno = pi + 1; |
| 327 | if (w.width - g.width).abs() > 0.5 || (w.height - g.height).abs() > 0.5 { |
| 328 | out.push(fmt!("page {}: size typst {:.2}x{:.2}pt, austenite {:.2}x{:.2}pt", |
| 329 | pno, w.width, w.height, g.width, g.height)); |
| 330 | } |
| 331 | // Each Austenite run joins the Typst line whose box holds its baseline and overlaps it |
| 332 | // horizontally, the nearest baseline winning. |
| 333 | let mut owner: Vec<Option<usize>> = vec![None; g.runs.len()]; |
| 334 | for (ri, r) in g.runs.iter().enumerate() { |
| 335 | let mut best: Option<(usize, f64)> = None; |
| 336 | for (li, l) in w.lines.iter().enumerate() { |
| 337 | let inside = r.base >= l.y0 - BASELINE_TOL && r.base <= l.y1 + BASELINE_TOL; |
| 338 | let overlap = r.x1 >= l.x0 - 2.0 && r.x0 <= l.x1 + 2.0; |
| 339 | if inside && overlap { |
| 340 | let d = (r.base - l.base).abs(); |
| 341 | if best.map(|(_, bd)| d < bd).unwrap_or(true) { |
| 342 | best = Some((li, d)); |
| 343 | } |
| 344 | } |
| 345 | } |
| 346 | owner[ri] = best.map(|(li, _)| li); |
| 347 | } |
| 348 | for (li, l) in w.lines.iter().enumerate() { |
| 349 | let mut mine: Vec<&ARun> = g.runs.iter().enumerate() |
| 350 | .filter(|(ri, _)| owner[*ri] == Some(li)) |
| 351 | .map(|(_, r)| r) |
| 352 | .collect(); |
| 353 | if mine.is_empty() { |
| 354 | let near = g.runs.iter().min_by(|a, b| { |
| 355 | let da = (a.base - l.base).abs(); |
| 356 | let db = (b.base - l.base).abs(); |
| 357 | da.partial_cmp(&db).unwrap_or(std::cmp::Ordering::Equal) |
| 358 | }); |
| 359 | out.push(match near { |
| 360 | Some(n) => fmt!("page {}: line {:?} at baseline {:.2} has no austenite text (nearest {:?} at {:.2})", |
| 361 | pno, l.text, l.base, n.text, n.base), |
| 362 | None => fmt!("page {}: line {:?} at baseline {:.2} has no austenite text (the page is empty)", |
| 363 | pno, l.text, l.base), |
| 364 | }); |
| 365 | continue; |
| 366 | } |
| 367 | mine.sort_by(|a, b| a.x0.partial_cmp(&b.x0).unwrap_or(std::cmp::Ordering::Equal)); |
| 368 | let text: String = mine.iter().map(|r| r.text.as_str()).collect::<Vec<_>>().join(" "); |
| 369 | if squash(&text) != squash(&l.text) { |
| 370 | out.push(fmt!("page {}: line at baseline {:.2}: typst {:?}, austenite {:?}", pno, l.base, l.text, text)); |
| 371 | } |
| 372 | // The line's baseline is the one most of its characters sit on. |
| 373 | let main = mine.iter().max_by_key(|r| r.text.chars().count()).map(|r| r.base).unwrap_or(l.base); |
| 374 | if (main - l.base).abs() > BASELINE_TOL { |
| 375 | out.push(fmt!("page {}: line {:?}: baseline typst {:.2}, austenite {:.2}", pno, l.text, l.base, main)); |
| 376 | } |
| 377 | // Where the line starts: its first run's left edge, to the left edge of Typst's first run. |
| 378 | if let Some(first) = mine.first() { |
| 379 | if (first.x0 - l.start).abs() > START_TOL { |
| 380 | out.push(fmt!("page {}: line {:?}: starts at x {:.2} in typst, {:.2} in austenite ({:+.2})", |
| 381 | pno, l.text, l.start, first.x0, first.x0 - l.start)); |
| 382 | } |
| 383 | } |
| 384 | } |
| 385 | for (ri, r) in g.runs.iter().enumerate() { |
| 386 | if owner[ri].is_none() && !r.text.trim().is_empty() { |
| 387 | out.push(fmt!("page {}: austenite text {:?} at baseline {:.2} lies on no typst line", pno, r.text, r.base)); |
| 388 | } |
| 389 | } |
| 390 | } |
| 391 | out.truncate(24); |
| 392 | } |