Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tests/eval_oracle/layout.rs

13.2 KiB, 1 run

created by r1870400018:63618, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// Level 4: pages, line text and baselines.
2//
3// The oracle side comes from two of Typst's outputs, because neither alone has both halves.
4// `pdftotext -bbox-layout` on Typst's PDF gives each line's text and box, but a box, not a
5// baseline: poppler sizes it from the font descriptor's ascent and descent, which differ per face.
6// Typst's SVG pages give every glyph's origin exactly -- a text run is a group translated to its
7// baseline -- but no text. So each pdftotext line takes its baseline from the glyph origins that
8// fall inside its box, the most common y among them.
9//
10// Austenite's side is its placed text runs, read straight from the laid-out pages: a run's top-left
11// is `(x, y)` and its baseline `y + height`.
12//
13// A line is held to its text, its baseline and the x where it starts. The start is where a first-line
14// or hanging indent, a margin and an alignment all show, none of which moves a break or a baseline, so
15// a line set sideways passes the first two checks and fails the third.
16
17use crate::harness::markup::{
18 tokens,
19 Tok,
20};
21
22use oxedyne_fe2o3_core::prelude::*;
23
24pub const BASELINE_TOL: f64 = 1.0; // points, the design's level-4 tolerance
25pub const START_TOL: f64 = 0.05; // points, where a line starts: its indent or its alignment offset
26pub const SPACE_REACH: f64 = 40.0; // points, the most a space opening a line can be wide
27
28#[derive(Clone, Debug)]
29pub struct OLine {
30 pub text: String,
31 pub x0: f64, // where the first word starts
32 pub x1: f64,
33 pub start: f64, // where the first glyph drawn starts, a leading space included: its text run's start
34 pub y0: f64,
35 pub y1: f64,
36 pub base: f64,
37 pub measured: bool, // the baseline came from glyph origins, not the box
38}
39
40#[derive(Clone, Debug, Default)]
41pub struct OPage {
42 pub width: f64,
43 pub height: f64,
44 pub lines: Vec<OLine>,
45}
46
47#[derive(Clone, Debug)]
48pub struct ARun {
49 pub text: String,
50 pub x0: f64,
51 pub x1: f64,
52 pub base: f64,
53}
54
55#[derive(Clone, Debug, Default)]
56pub struct APage {
57 pub width: f64,
58 pub height: f64,
59 pub runs: Vec<ARun>,
60}
61
62fn num(attrs: &[(String, String)], key: &str) -> f64 {
63 Tok::attr(attrs, key).and_then(|v| v.trim().trim_end_matches("pt").parse::<f64>().ok()).unwrap_or(0.0)
64}
65
66/// The pages and lines of `pdftotext -bbox-layout` output, baselines not yet attached.
67///
68/// A line's horizontal extent is its words', not its own `xMin` and `xMax`: pdftotext (poppler 26.01)
69/// gives a line the left edge of its second word, so a line's start read from the line is wrong. And
70/// a loosely justified line comes back from it as several, cut at its widest gaps, so the pieces of one
71/// block that stand on one baseline are one line again.
72pub fn read_bbox(xml: &str) -> Vec<OPage> {
73 let mut pages: Vec<OPage> = Vec::new();
74 let mut block_from = 0usize; // where the current block's lines begin on the page
75 let mut line: Option<OLine> = None;
76 let mut words_seen = false;
77 let mut in_word = false;
78 for t in tokens(xml) {
79 match t {
80 Tok::Open { name, attrs, .. } => match name.as_str() {
81 "page" => {
82 block_from = 0;
83 pages.push(OPage { width: num(&attrs, "width"), height: num(&attrs, "height"), lines: Vec::new() });
84 }
85 "block" => block_from = pages.last().map_or(0, |p| p.lines.len()),
86 "line" => {
87 words_seen = false;
88 line = Some(OLine {
89 text: String::new(),
90 x0: num(&attrs, "xmin"),
91 x1: num(&attrs, "xmax"),
92 start: num(&attrs, "xmin"),
93 y0: num(&attrs, "ymin"),
94 y1: num(&attrs, "ymax"),
95 base: 0.0,
96 measured: false,
97 });
98 }
99 "word" => {
100 in_word = true;
101 if let Some(l) = line.as_mut() {
102 let (w0, w1) = (num(&attrs, "xmin"), num(&attrs, "xmax"));
103 if words_seen {
104 l.x0 = l.x0.min(w0);
105 l.x1 = l.x1.max(w1);
106 } else {
107 l.x0 = w0;
108 l.x1 = w1;
109 words_seen = true;
110 }
111 l.start = l.x0;
112 }
113 }
114 _ => (),
115 },
116 Tok::Close(name) => match name.as_str() {
117 "word" => {
118 in_word = false;
119 if let Some(l) = line.as_mut() {
120 l.text.push(' ');
121 }
122 }
123 "line" => if let Some(mut l) = line.take() {
124 l.text = l.text.trim().to_string();
125 if let Some(p) = pages.last_mut() {
126 let same = |o: &OLine| (o.y0 - l.y0).abs() < 0.5 && (o.y1 - l.y1).abs() < 0.5;
127 match p.lines.iter_mut().skip(block_from).find(|o| same(o)) {
128 Some(o) => {
129 o.text = fmt!("{} {}", o.text, l.text);
130 o.x0 = o.x0.min(l.x0);
131 o.x1 = o.x1.max(l.x1);
132 o.start = o.x0;
133 }
134 None => p.lines.push(l),
135 }
136 }
137 },
138 _ => (),
139 },
140 Tok::Text(s) => if in_word {
141 if let Some(l) = line.as_mut() {
142 l.text.push_str(&s);
143 }
144 },
145 }
146 }
147 pages
148}
149
150// An affine map (x, y) -> (a x + c y + e, b x + d y + f).
151#[derive(Clone, Copy, Debug)]
152struct M([f64; 6]);
153
154impl M {
155 const ID: M = M([1.0, 0.0, 0.0, 1.0, 0.0, 0.0]);
156
157 fn then(self, n: M) -> M {
158 let (m, n) = (self.0, n.0);
159 M([
160 m[0] * n[0] + m[2] * n[1],
161 m[1] * n[0] + m[3] * n[1],
162 m[0] * n[2] + m[2] * n[3],
163 m[1] * n[2] + m[3] * n[3],
164 m[0] * n[4] + m[2] * n[5] + m[4],
165 m[1] * n[4] + m[3] * n[5] + m[5],
166 ])
167 }
168
169 fn apply(self, x: f64, y: f64) -> (f64, f64) {
170 let m = self.0;
171 (m[0] * x + m[2] * y + m[4], m[1] * x + m[3] * y + m[5])
172 }
173}
174
175fn parse_transform(s: &str) -> M {
176 let mut m = M::ID;
177 let mut rest = s;
178 while let Some(open) = rest.find('(') {
179 let name = rest[..open].trim().trim_start_matches(',').trim();
180 let close = match rest[open..].find(')') {
181 Some(c) => open + c,
182 None => break,
183 };
184 let args: Vec<f64> = rest[open + 1..close]
185 .split(|c: char| c == ',' || c.is_whitespace())
186 .filter(|t| !t.is_empty())
187 .filter_map(|t| t.parse::<f64>().ok())
188 .collect();
189 let a = |i: usize| args.get(i).copied().unwrap_or(0.0);
190 let own = match name {
191 "matrix" if args.len() == 6 => M([a(0), a(1), a(2), a(3), a(4), a(5)]),
192 "translate" => M([1.0, 0.0, 0.0, 1.0, a(0), a(1)]),
193 "scale" => {
194 let sy = if args.len() > 1 { a(1) } else { a(0) };
195 M([a(0), 0.0, 0.0, sy, 0.0, 0.0])
196 }
197 "rotate" => {
198 let r = a(0).to_radians();
199 let rot = M([r.cos(), r.sin(), -r.sin(), r.cos(), 0.0, 0.0]);
200 if args.len() == 3 {
201 M([1.0, 0.0, 0.0, 1.0, a(1), a(2)]).then(rot).then(M([1.0, 0.0, 0.0, 1.0, -a(1), -a(2)]))
202 } else {
203 rot
204 }
205 }
206 "skewX" => M([1.0, 0.0, a(0).to_radians().tan(), 1.0, 0.0, 0.0]),
207 "skewY" => M([1.0, a(0).to_radians().tan(), 0.0, 1.0, 0.0, 0.0]),
208 _ => M::ID,
209 };
210 m = m.then(own);
211 rest = &rest[close + 1..];
212 }
213 m
214}
215
216/// The page-space origin of every glyph a Typst SVG page draws, and the origin of each text run: the
217/// group that holds its glyphs, which Typst draws at the run's start, a space it leaves out included.
218fn walk_glyphs(svg: &str) -> (Vec<(f64, f64)>, Vec<(f64, f64)>) {
219 // Per open element: its transform, whether it is inside defs, and its origin while no glyph has been
220 // drawn in it.
221 let mut stack: Vec<(M, bool, Option<(f64, f64)>)> = vec![(M::ID, false, None)];
222 let (mut out, mut starts) = (Vec::new(), Vec::new());
223 for t in tokens(svg) {
224 match t {
225 Tok::Open { name, attrs, closed } => {
226 let (top, in_defs, _) = stack.last().copied().unwrap_or((M::ID, false, None));
227 let own = Tok::attr(&attrs, "transform").map(parse_transform).unwrap_or(M::ID);
228 let cur = top.then(own);
229 let defs = in_defs || name == "defs" || name == "symbol";
230 if name == "use" && !defs {
231 let href = Tok::attr(&attrs, "xlink:href").or_else(|| Tok::attr(&attrs, "href")).unwrap_or("");
232 if href.starts_with("#g") {
233 out.push(cur.apply(num(&attrs, "x"), num(&attrs, "y")));
234 if let Some(last) = stack.last_mut() {
235 if let Some(origin) = last.2.take() {
236 starts.push(origin);
237 }
238 }
239 }
240 }
241 if !closed {
242 stack.push((cur, defs, if name == "g" { Some(cur.apply(0.0, 0.0)) } else { None }));
243 }
244 }
245 Tok::Close(_) => {
246 if stack.len() > 1 {
247 stack.pop();
248 }
249 }
250 Tok::Text(_) => (),
251 }
252 }
253 (out, starts)
254}
255
256/// The page-space origin of every glyph a Typst SVG page draws.
257pub fn glyph_origins(svg: &str) -> Vec<(f64, f64)> {
258 walk_glyphs(svg).0
259}
260
261/// The page-space origin of every text run a Typst SVG page draws.
262pub fn run_starts(svg: &str) -> Vec<(f64, f64)> {
263 walk_glyphs(svg).1
264}
265
266/// Gives each line the baseline its glyphs sit on; a line with no glyph origin inside its box keeps
267/// an estimate from the box (Libertinus Serif's descent) and is marked unmeasured.
268pub fn attach_baselines(page: &mut OPage, origins: &[(f64, f64)]) {
269 for l in &mut page.lines {
270 let mut ys: Vec<i64> = origins.iter()
271 .filter(|(x, y)| *x >= l.x0 - 1.0 && *x <= l.x1 + 1.0 && *y >= l.y0 - 0.5 && *y <= l.y1 + 0.5)
272 .map(|(_, y)| (y * 100.0).round() as i64)
273 .collect();
274 if ys.is_empty() {
275 l.base = l.y1 - 0.2157 * (l.y1 - l.y0);
276 l.measured = false;
277 continue;
278 }
279 ys.sort();
280 let mut best = (ys[0], 0usize);
281 let mut i = 0;
282 while i < ys.len() {
283 let mut j = i;
284 while j < ys.len() && ys[j] == ys[i] {
285 j += 1;
286 }
287 if j - i > best.1 {
288 best = (ys[i], j - i);
289 }
290 i = j;
291 }
292 l.base = best.0 as f64 / 100.0;
293 l.measured = true;
294 }
295}
296
297/// Gives each line the start of the text run its first word is drawn in: the nearest run start on the
298/// line's baseline at or before the word, within a space's reach. That run opens with a space when
299/// Typst's line begins after an inline box or a space (pdftotext's words leave the space out), where
300/// Austenite's run opens as well; a line whose run is not found starts where its first word does.
301/// Call after [`attach_baselines`].
302pub fn attach_starts(page: &mut OPage, starts: &[(f64, f64)]) {
303 for l in &mut page.lines {
304 let found = starts.iter()
305 .filter(|(x, y)| (*y - l.base).abs() < 0.02 && *x <= l.x0 + 0.02 && *x >= l.x0 - SPACE_REACH)
306 .map(|(x, _)| *x)
307 .fold(f64::NEG_INFINITY, f64::max);
308 if found.is_finite() {
309 l.start = found;
310 }
311 }
312}
313
314fn squash(s: &str) -> String {
315 s.chars().filter(|c| !c.is_whitespace() && *c != '\u{ad}').collect()
316}
317
318/// Level-4 differences: page count exact, page size within half a point, every Typst line matched by
319/// Austenite text on the same page with the same characters, a baseline within [`BASELINE_TOL`] and a
320/// start within [`START_TOL`] of where Typst's line starts.
321pub fn compare(want: &[OPage], got: &[APage], out: &mut Vec<String>) {
322 if want.len() != got.len() {
323 out.push(fmt!("page count: typst {}, austenite {}", want.len(), got.len()));
324 }
325 for (pi, (w, g)) in want.iter().zip(got.iter()).enumerate() {
326 let pno = pi + 1;
327 if (w.width - g.width).abs() > 0.5 || (w.height - g.height).abs() > 0.5 {
328 out.push(fmt!("page {}: size typst {:.2}x{:.2}pt, austenite {:.2}x{:.2}pt",
329 pno, w.width, w.height, g.width, g.height));
330 }
331 // Each Austenite run joins the Typst line whose box holds its baseline and overlaps it
332 // horizontally, the nearest baseline winning.
333 let mut owner: Vec<Option<usize>> = vec![None; g.runs.len()];
334 for (ri, r) in g.runs.iter().enumerate() {
335 let mut best: Option<(usize, f64)> = None;
336 for (li, l) in w.lines.iter().enumerate() {
337 let inside = r.base >= l.y0 - BASELINE_TOL && r.base <= l.y1 + BASELINE_TOL;
338 let overlap = r.x1 >= l.x0 - 2.0 && r.x0 <= l.x1 + 2.0;
339 if inside && overlap {
340 let d = (r.base - l.base).abs();
341 if best.map(|(_, bd)| d < bd).unwrap_or(true) {
342 best = Some((li, d));
343 }
344 }
345 }
346 owner[ri] = best.map(|(li, _)| li);
347 }
348 for (li, l) in w.lines.iter().enumerate() {
349 let mut mine: Vec<&ARun> = g.runs.iter().enumerate()
350 .filter(|(ri, _)| owner[*ri] == Some(li))
351 .map(|(_, r)| r)
352 .collect();
353 if mine.is_empty() {
354 let near = g.runs.iter().min_by(|a, b| {
355 let da = (a.base - l.base).abs();
356 let db = (b.base - l.base).abs();
357 da.partial_cmp(&db).unwrap_or(std::cmp::Ordering::Equal)
358 });
359 out.push(match near {
360 Some(n) => fmt!("page {}: line {:?} at baseline {:.2} has no austenite text (nearest {:?} at {:.2})",
361 pno, l.text, l.base, n.text, n.base),
362 None => fmt!("page {}: line {:?} at baseline {:.2} has no austenite text (the page is empty)",
363 pno, l.text, l.base),
364 });
365 continue;
366 }
367 mine.sort_by(|a, b| a.x0.partial_cmp(&b.x0).unwrap_or(std::cmp::Ordering::Equal));
368 let text: String = mine.iter().map(|r| r.text.as_str()).collect::<Vec<_>>().join(" ");
369 if squash(&text) != squash(&l.text) {
370 out.push(fmt!("page {}: line at baseline {:.2}: typst {:?}, austenite {:?}", pno, l.base, l.text, text));
371 }
372 // The line's baseline is the one most of its characters sit on.
373 let main = mine.iter().max_by_key(|r| r.text.chars().count()).map(|r| r.base).unwrap_or(l.base);
374 if (main - l.base).abs() > BASELINE_TOL {
375 out.push(fmt!("page {}: line {:?}: baseline typst {:.2}, austenite {:.2}", pno, l.text, l.base, main));
376 }
377 // Where the line starts: its first run's left edge, to the left edge of Typst's first run.
378 if let Some(first) = mine.first() {
379 if (first.x0 - l.start).abs() > START_TOL {
380 out.push(fmt!("page {}: line {:?}: starts at x {:.2} in typst, {:.2} in austenite ({:+.2})",
381 pno, l.text, l.start, first.x0, first.x0 - l.start));
382 }
383 }
384 }
385 for (ri, r) in g.runs.iter().enumerate() {
386 if owner[ri].is_none() && !r.text.trim().is_empty() {
387 out.push(fmt!("page {}: austenite text {:?} at baseline {:.2} lies on no typst line", pno, r.text, r.base));
388 }
389 }
390 }
391 out.truncate(24);
392}