Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tests/eval_indent.rs

19.8 KiB, 1 run

created by r1870400018:63590, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! U6a-fix: a justified paragraph's first-line indent, against the `typst` 0.15.1 oracle at level 4.
2//!
3//! Level 4 is the page count and size and, for each line, its text, its baseline and now the x where
4//! it starts (`tests/eval_oracle/layout.rs`). The x is what an indent, a margin and an alignment move
5//! without moving a break or a baseline.
6//!
7//! - The indent corpus (`tests/fixtures/eval/indent/`) sets justified, hanging-indented, ragged and
8//! aligned paragraphs, each in the situation it is met in: after a paragraph, first, after a block.
9//! Each is held to Typst's lines.
10//! - The corpora of the line-break and hyphenation fixtures, which set a single paragraph with no
11//! indent, are held to the same x.
12//! - The comparison itself is tested: a line set sideways is reported, and a loosely justified line that
13//! pdftotext cut in pieces is read as the one line it is, starting at its first word.
14//!
15//! Austenite's side lays each paragraph out with `layout_par` in the situation a block flow gives it
16//! (Typst's `collect`: first, consecutive after a paragraph, other after a block) and stacks the lines
17//! a paragraph spacing apart; the pagination itself is U6b's. A fixture that sets anything else is
18//! refused, so no result rests on a layout this file did not do.
19//!
20//! The one block a fixture may hold is `#block(height: 0pt)` under `#set block(spacing: 0pt)`, which
21//! Typst lays out as nothing but a paragraph's `other` situation. Austenite's evaluator has no `block`
22//! yet, so this file takes both lines out of the source and keeps the situation; Typst's own lines hold
23//! the model to account. A fixture whose first lines say `// needs: u6b` (it sets an `align`, a `box` or
24//! an `h`, whose schemas are U6b's) is reported as pending while Austenite cannot evaluate it, and held
25//! to Typst once it can.
26
27#![allow(dead_code)]
28
29#[path = "eval_oracle/mod.rs"]
30mod harness;
31
32use harness::corpus::{
33 discover,
34 Filter,
35 Fixture,
36};
37use harness::layout::{
38 attach_baselines,
39 attach_starts,
40 compare,
41 glyph_origins,
42 read_bbox,
43 run_starts,
44 APage,
45 ARun,
46 OLine,
47 OPage,
48 START_TOL,
49};
50use harness::oracle::Oracle;
51
52use oxedyne_fe2o3_austenite::eval::content::ElemKind;
53use oxedyne_fe2o3_austenite::eval::realise::{
54 realise,
55 Pair,
56 RealiseMode,
57};
58use oxedyne_fe2o3_austenite::eval::styles::StyleChain;
59use oxedyne_fe2o3_austenite::eval::value::Value;
60use oxedyne_fe2o3_austenite::eval::{
61 eval_source,
62 Engine,
63 World,
64};
65use oxedyne_fe2o3_austenite::flow::inline::ParSituation;
66use oxedyne_fe2o3_austenite::flow::par::layout_par;
67use oxedyne_fe2o3_austenite::flow::Region;
68use oxedyne_fe2o3_austenite::ir::{
69 LeafKind,
70 Node,
71 Sp,
72};
73
74use oxedyne_fe2o3_core::prelude::*;
75
76// ┌───────────────────────────────────────────────────────────────────────────┐
77// │ AUSTENITE'S PAGE │
78// └───────────────────────────────────────────────────────────────────────────┘
79
80/// A fixture's `#set page(width: .., height: .., margin: ..)`, which the evaluator reads only once the
81/// page schema exists.
82struct Setup {
83 width: f64,
84 height: Option<f64>, // `None` for `auto`
85 margin: f64,
86}
87
88/// The number of points a `key: <n>pt` argument gives, `None` for `auto` or an absent key.
89fn points(args: &str, key: &str) -> Option<f64> {
90 let rest = args.split(&fmt!("{}:", key)).nth(1).map(|r| r.trim());
91 let word = rest.and_then(|r| r.split(|c: char| c == ',' || c == ')').next()).map(|w| w.trim());
92 match word {
93 Some("auto") | None => None,
94 Some(w) => w.trim_end_matches("pt").trim().parse::<f64>().ok(),
95 }
96}
97
98/// The page a fixture sets, its source with the lines Austenite does not evaluate taken out, and which
99/// paragraphs (counted from 0) stand right after a `#block(height: 0pt)`.
100fn split_source(text: &str) -> std::result::Result<(Setup, String, Vec<usize>), String> {
101 let mut setup = None;
102 let mut kept: Vec<&str> = Vec::new();
103 let (mut after_block, mut blank, mut paragraphs) = (false, true, 0usize);
104 let mut others = Vec::new();
105 for line in text.lines() {
106 if let Some(args) = line.strip_prefix("#set page(") {
107 let width = match points(args, "width") {
108 Some(w) => w,
109 None => return Err(fmt!("no page width in {:?}", line)),
110 };
111 setup = Some(Setup { width, height: points(args, "height"), margin: points(args, "margin").unwrap_or(0.0) });
112 } else if line == "#set block(spacing: 0pt)" {
113 // Typst's own, which Austenite has no block to set it on.
114 } else if line == "#block(height: 0pt)" {
115 after_block = true;
116 } else {
117 if !line.starts_with("//") && !line.starts_with("#set ") {
118 if line.trim().is_empty() {
119 blank = true;
120 } else if blank {
121 // The first line of a paragraph.
122 if after_block {
123 others.push(paragraphs);
124 after_block = false;
125 }
126 paragraphs += 1;
127 blank = false;
128 }
129 }
130 kept.push(line);
131 }
132 }
133 match setup {
134 Some(s) => Ok((s, kept.join("\n"), others)),
135 None => Err("the fixture sets no page".to_string()),
136 }
137}
138
139fn last_msg(e: &Error<ErrTag>) -> String {
140 match e.msgs().into_iter().last() {
141 Some(m) => m,
142 None => fmt!("{}", e),
143 }
144}
145
146/// The text runs of a line's box with the x each starts at.
147fn line_runs(list: &[Node], x0: f64, base: f64, out: &mut Vec<ARun>) {
148 let mut x = x0;
149 for n in list {
150 match n {
151 Node::Glue(g) => x += g.natural.to_pt(),
152 Node::Leaf(l) => {
153 let w = l.dims.width.to_pt();
154 if let LeafKind::Text(t) = &l.kind {
155 out.push(ARun { text: t.source().to_string(), x0: x, x1: x + w, base: base + l.shift.to_pt() });
156 }
157 x += w;
158 }
159 Node::HBox(b) | Node::VBox(b) => x += b.dims.width.to_pt(),
160 _ => (),
161 }
162 }
163}
164
165/// What a paragraph's `spacing` comes to.
166fn spacing_of(pair: &Pair) -> std::result::Result<f64, String> {
167 let size = pair.styles.font_size();
168 let id = match ElemKind::Par.field_id("spacing") {
169 Some(id) => id,
170 None => return Err("no par spacing field".to_string()),
171 };
172 match pair.styles.resolve(&pair.content, id) {
173 Ok(Some(Value::Length(l))) => Ok(l.resolve(size)),
174 Ok(_) => Ok(1.2 * size),
175 Err(e) => Err(fmt!("par spacing: {}", last_msg(&e))),
176 }
177}
178
179/// Austenite's one page for a fixture: its paragraphs laid out in the situation each is met in.
180fn austenite_page(fx: &Fixture) -> std::result::Result<APage, String> {
181 let (setup, source, others) = match split_source(&fx.text) {
182 Ok(t) => t,
183 Err(e) => return Err(e),
184 };
185 let mut world = World::new(fx.root.clone());
186 let id = match world.add_source(fx.path.clone(), source) {
187 Ok(id) => id,
188 Err(e) => return Err(fmt!("source: {}", last_msg(&e))),
189 };
190 let mut engine = Engine::new(world);
191 let module = match eval_source(&mut engine, id) {
192 Ok(m) => m,
193 Err(e) => return Err(fmt!("eval: {}", last_msg(&e))),
194 };
195 let pairs = match realise(&mut engine, &module.content, &StyleChain::root(), RealiseMode::Document) {
196 Ok(p) => p,
197 Err(e) => return Err(fmt!("realise: {}", last_msg(&e))),
198 };
199 let measure = setup.width - 2.0 * setup.margin;
200 let mut y = setup.margin;
201 let mut index = 0usize; // the paragraph in hand, counted from 0
202 let mut gap: Option<f64> = None; // the spacing a paragraph leaves after it, until the next one asks
203 let mut runs: Vec<ARun> = Vec::new();
204 for pair in &pairs {
205 if pair.is_tag() {
206 continue;
207 }
208 if pair.content.is(ElemKind::Par) {
209 let spacing = match spacing_of(pair) {
210 Ok(g) => g,
211 Err(e) => return Err(e),
212 };
213 if let Some(g) = gap.take() {
214 y += g.max(spacing);
215 }
216 let region = Region::new(Sp::from_pt(measure), Sp::from_pt(setup.height.unwrap_or(10_000.0)));
217 // Typst's `collect`: first at the top, consecutive after a paragraph, other after a block.
218 let situation = if others.contains(&index) {
219 ParSituation::Other
220 } else if index == 0 {
221 ParSituation::First
222 } else {
223 ParSituation::Consecutive
224 };
225 index += 1;
226 let nodes = match layout_par(&mut engine, &pair.content, &pair.styles, region, situation) {
227 Ok(n) => n,
228 Err(e) => return Err(fmt!("layout_par: {}", last_msg(&e))),
229 };
230 for n in &nodes {
231 match n {
232 Node::HBox(b) => {
233 line_runs(&b.list, setup.margin, y + b.dims.height.to_pt(), &mut runs);
234 y += b.dims.vextent().to_pt();
235 }
236 Node::Glue(g) => y += g.natural.to_pt(),
237 Node::Penalty(_) => (),
238 other => return Err(fmt!("a paragraph laid out to {:?}", other)),
239 }
240 }
241 gap = Some(spacing);
242 } else {
243 return Err(fmt!("the fixture sets {:?}, which this file does not lay out", pair.content.kind()));
244 }
245 }
246 Ok(APage {
247 width: setup.width,
248 height: setup.height.unwrap_or(y + setup.margin),
249 runs,
250 })
251}
252
253// ┌───────────────────────────────────────────────────────────────────────────┐
254// │ THE COMPARISON │
255// └───────────────────────────────────────────────────────────────────────────┘
256
257/// Typst's pages for a fixture, or the reason there are none.
258fn typst_pages(o: &Oracle, fx: &Fixture) -> std::result::Result<Vec<OPage>, String> {
259 match o.layout(fx) {
260 Ok(Ok(p)) => Ok(p),
261 Ok(Err(e)) => Err(fmt!("typst: {}", e)),
262 Err(e) => Err(fmt!("oracle: {}", last_msg(&e))),
263 }
264}
265
266/// What a fixture's `// indents: <n> <x>` header asks of Typst's own lines: at least `n` of them start
267/// at `x` points, so a fixture that says it sets indented lines cannot pass without any.
268fn indents_asked(text: &str) -> Option<(usize, f64)> {
269 for line in text.lines().take_while(|l| l.starts_with("//")) {
270 if let Some(rest) = line.strip_prefix("// indents:") {
271 let words: Vec<&str> = rest.split_whitespace().collect();
272 return match (words.first().and_then(|n| n.parse::<usize>().ok()), words.get(1).and_then(|x| x.parse::<f64>().ok())) {
273 (Some(n), Some(x)) => Some((n, x)),
274 _ => None,
275 };
276 }
277 }
278 None
279}
280
281/// Does the fixture say it needs a unit that may not be merged yet?
282fn needs_unit(text: &str) -> bool {
283 text.lines().take_while(|l| l.starts_with("//")).any(|l| l.starts_with("// needs:"))
284}
285
286/// The fault a fixture records as known: Austenite is expected to differ from Typst on it, and the test
287/// fails the day it does not, so the header cannot outlive the fault.
288fn known_fault(text: &str) -> Option<String> {
289 text.lines().take_while(|l| l.starts_with("//"))
290 .find_map(|l| l.strip_prefix("// known:").map(|r| r.trim().to_string()))
291}
292
293/// What comparing a directory of fixtures against Typst found.
294struct Compared {
295 fixtures: usize,
296 lines: usize,
297 indented: usize,
298 pending: Vec<String>,
299 known: Vec<String>, // each with the differences it was held to
300 failures: Vec<String>,
301}
302
303fn compare_area(o: &Oracle, area: &str, only: Option<&str>) -> Outcome<Compared> {
304 let filter = Filter { areas: Some(vec![area.to_string()]), name: only.map(|s| s.to_string()) };
305 let fxs = res!(discover(&filter));
306 let mut c = Compared { fixtures: fxs.len(), lines: 0, indented: 0, pending: Vec::new(), known: Vec::new(), failures: Vec::new() };
307 for fx in &fxs {
308 let want = match typst_pages(o, fx) {
309 Ok(w) => w,
310 Err(e) => {
311 c.failures.push(fmt!("{}: {}", fx.id(), e));
312 continue;
313 }
314 };
315 c.lines += want.iter().map(|p| p.lines.len()).sum::<usize>();
316 if let Some((n, x)) = indents_asked(&fx.text) {
317 let got = want.iter().flat_map(|p| p.lines.iter()).filter(|l| (l.x0 - x).abs() < 0.2).count();
318 c.indented += got;
319 if got < n {
320 c.failures.push(fmt!("{}: asks for at least {} typst lines starting at x {} but typst has {}", fx.id(), n, x, got));
321 continue;
322 }
323 }
324 let page = match austenite_page(fx) {
325 Ok(p) => p,
326 Err(e) => {
327 if needs_unit(&fx.text) {
328 c.pending.push(fx.id());
329 } else {
330 c.failures.push(fmt!("{}: austenite: {}", fx.id(), e));
331 }
332 continue;
333 }
334 };
335 let mut diffs = Vec::new();
336 compare(&want, &[page.clone()], &mut diffs);
337 if !diffs.is_empty() && std::env::var("INDENT_DUMP").is_ok() {
338 eprintln!("-- {}: austenite runs", fx.id());
339 for r in &page.runs {
340 eprintln!(" x {:8.2}..{:8.2} base {:8.2} {:?}", r.x0, r.x1, r.base, r.text);
341 }
342 eprintln!("-- {}: typst lines", fx.id());
343 for l in want.iter().flat_map(|p| p.lines.iter()) {
344 eprintln!(" x {:8.2}..{:8.2} base {:8.2} {:?}", l.x0, l.x1, l.base, l.text);
345 }
346 }
347 match (known_fault(&fx.text), diffs.is_empty()) {
348 (Some(why), true) => c.failures.push(fmt!("{} no longer differs from typst; drop its `// known:` header ({})", fx.id(), why)),
349 (Some(why), false) => c.known.push(fmt!("{}: {} differences, known: {}", fx.id(), diffs.len(), why)),
350 (None, false) => c.failures.push(fmt!("{} ({} differences):\n {}", fx.id(), diffs.len(), diffs.join("\n "))),
351 (None, true) => (),
352 }
353 }
354 Ok(c)
355}
356
357#[test]
358fn indented_paragraphs_match_typst_lines_and_starts_at_level_4() -> Outcome<()> {
359 let o = match res!(Oracle::find()) {
360 Some(o) => o,
361 None => return Ok(()),
362 };
363 let only = std::env::var("INDENT_FIXTURE").ok();
364 let c = res!(compare_area(&o, "indent", only.as_deref()));
365 if only.is_none() {
366 assert_eq!(c.fixtures, 27, "the indent corpus is twenty-seven fixtures");
367 assert!(c.indented >= 40, "typst starts only {} lines at an indent, too few to test indents with", c.indented);
368 assert!(c.pending.len() <= 5, "{} fixtures are pending, only the five that need U6b's schema may be: {:?}", c.pending.len(), c.pending);
369 assert_eq!(c.known.len(), 1, "one fixture records a known fault: {:?}", c.known);
370 }
371 if !c.pending.is_empty() {
372 eprintln!("pending until their unit is merged: {:?}", c.pending);
373 }
374 for k in &c.known {
375 eprintln!("known: {}", k);
376 }
377 assert!(c.failures.is_empty(), "{} of {} fixtures ({} typst lines) differ at level 4:\n{}",
378 c.failures.len(), c.fixtures, c.lines, c.failures.join("\n"));
379 Ok(())
380}
381
382#[test]
383fn the_line_break_and_hyphenation_corpora_start_their_lines_where_typst_does() -> Outcome<()> {
384 let o = match res!(Oracle::find()) {
385 Some(o) => o,
386 None => return Ok(()),
387 };
388 let (mut fixtures, mut lines) = (0usize, 0usize);
389 let (mut failures, mut pending) = (Vec::new(), Vec::new());
390 for area in ["linebreak", "hyphenation"] {
391 let c = res!(compare_area(&o, area, None));
392 fixtures += c.fixtures;
393 lines += c.lines;
394 failures.extend(c.failures);
395 pending.extend(c.pending);
396 }
397 assert_eq!(fixtures, 84 + 27, "the line-break corpus is 84 fixtures and the hyphenation corpus 27");
398 // A shorthand such as `--` is text inside its paragraph, as in Typst, so nothing here waits on a unit.
399 assert!(pending.is_empty(), "no fixture of these corpora may be pending: {:?}", pending);
400 assert!(failures.is_empty(), "{} of {} fixtures ({} typst lines) differ at level 4:\n{}",
401 failures.len(), fixtures, lines, failures.join("\n"));
402 Ok(())
403}
404
405fn oline(text: &str, x0: f64, x1: f64, base: f64) -> OLine {
406 OLine { text: text.to_string(), x0, x1, start: x0, y0: base - 8.0, y1: base + 3.0, base, measured: true }
407}
408
409fn arun(text: &str, x0: f64, x1: f64, base: f64) -> ARun {
410 ARun { text: text.to_string(), x0, x1, base }
411}
412
413/// The comparison holds a line to the x it starts at: a line set sideways, which no break and no
414/// baseline shows, is reported, and so is a line Typst starts later than Austenite does.
415#[test]
416fn level_4_reports_a_line_that_starts_in_the_wrong_place() {
417 let want = vec![OPage { width: 200.0, height: 100.0, lines: vec![oline("Lorem ipsum dolor", 12.0, 150.0, 20.0)] }];
418 let page = |x0: f64| vec![APage { width: 200.0, height: 100.0, runs: vec![arun("Lorem ipsum dolor", x0, x0 + 138.0, 20.0)] }];
419 let mut said = Vec::new();
420 compare(&want, &page(12.0), &mut said);
421 assert!(said.is_empty(), "an exact start was reported: {:?}", said);
422 compare(&want, &page(12.0 + START_TOL * 0.5), &mut said);
423 assert!(said.is_empty(), "a start within the tolerance was reported: {:?}", said);
424 for sideways in [0.0, 6.0, 24.0] {
425 let mut said = Vec::new();
426 compare(&want, &page(sideways), &mut said);
427 assert!(said.iter().any(|m| m.contains("starts at x 12.00 in typst")),
428 "a line starting at {} instead of 12 was not reported: {:?}", sideways, said);
429 }
430}
431
432/// pdftotext cuts a loosely justified line into pieces at its widest gaps, and gives a line the left
433/// edge of its second word. Read back, the pieces on one baseline are one line, starting at its first
434/// word; a line in another block on the same baseline (the next column) is not joined to it.
435#[test]
436fn a_line_pdftotext_cut_in_pieces_is_one_line_starting_at_its_first_word() {
437 let xml = r#"<doc>
438 <page width="200.000000" height="100.000000">
439 <flow>
440 <block xMin="0" yMin="0" xMax="200" yMax="40">
441 <line xMin="60.0" yMin="0.0" xMax="90.0" yMax="12.0">
442 <word xMin="12.0" yMin="0.0" xMax="50.0" yMax="12.0">Lorem</word>
443 <word xMin="60.0" yMin="0.0" xMax="90.0" yMax="12.0">ipsum</word>
444 </line>
445 <line xMin="150.0" yMin="0.0" xMax="190.0" yMax="12.0">
446 <word xMin="140.0" yMin="0.0" xMax="148.0" yMax="12.0">dolor</word>
447 <word xMin="150.0" yMin="0.0" xMax="190.0" yMax="12.0">sit</word>
448 </line>
449 <line xMin="30.0" yMin="14.0" xMax="60.0" yMax="26.0">
450 <word xMin="20.0" yMin="14.0" xMax="28.0" yMax="26.0">amet</word>
451 <word xMin="30.0" yMin="14.0" xMax="60.0" yMax="26.0">elit</word>
452 </line>
453 </block>
454 <block xMin="0" yMin="0" xMax="200" yMax="40">
455 <line xMin="5.0" yMin="0.0" xMax="9.0" yMax="12.0">
456 <word xMin="2.0" yMin="0.0" xMax="4.0" yMax="12.0">other</word>
457 <word xMin="5.0" yMin="0.0" xMax="9.0" yMax="12.0">block</word>
458 </line>
459 </block>
460 </flow>
461 </page>
462 </doc>"#;
463 let pages = read_bbox(xml);
464 assert_eq!(pages.len(), 1);
465 let lines = &pages[0].lines;
466 assert_eq!(lines.len(), 3, "two lines in the first block, one in the second: {:?}", lines);
467 assert_eq!(lines[0].text, "Lorem ipsum dolor sit", "the pieces of one baseline join in order");
468 assert_eq!((lines[0].x0, lines[0].x1), (12.0, 190.0), "the line spans its words, not the second word's left edge");
469 assert_eq!(lines[1].text, "amet elit");
470 assert_eq!(lines[1].x0, 20.0);
471 assert_eq!(lines[2].text, "other block", "a line in another block stays its own, though it shares a baseline");
472}
473
474/// pdftotext's words leave out a space, so a line that opens with one (after an inline box, say) is
475/// read to start at its first word. Typst's SVG draws the run from its start, the space it leaves out
476/// included, and Austenite's run opens with the space, so a line starts where the run holding its first
477/// word starts: here 152, not 159.8.
478#[test]
479fn a_line_that_opens_with_a_space_starts_where_its_text_run_does() {
480 let svg = r##"<svg><defs><g id="g1"><path d="M0 0"/></g></defs>
481 <g transform="matrix(1 0 0 -1 152 20)"><use xlink:href="#g1" x="7.8" y="0"/><use xlink:href="#g1" x="13" y="0"/></g>
482 <g transform="matrix(1 0 0 -1 12 40)"><use xlink:href="#g1" x="0" y="0"/><use xlink:href="#g1" x="6" y="0"/></g>
483 <g transform="matrix(1 0 0 -1 100 60)"><use xlink:href="#g1" x="0" y="0"/></g></svg>"##;
484 let line = |text: &str, x0: f64, y: f64| OLine {
485 text: text.to_string(), x0, x1: x0 + 30.0, start: x0, y0: y - 10.0, y1: y + 2.0, base: 0.0, measured: false,
486 };
487 let mut page = OPage {
488 width: 200.0,
489 height: 100.0,
490 lines: vec![line("Supercal", 159.8, 20.0), line("Alpha", 12.0, 40.0), line("Far", 140.0, 60.0)],
491 };
492 attach_baselines(&mut page, &glyph_origins(svg));
493 attach_starts(&mut page, &run_starts(svg));
494 assert_eq!(page.lines[0].start, 152.0, "the line opens with a space");
495 assert_eq!(page.lines[1].start, 12.0, "a line opening with its first word starts there");
496 assert_eq!(page.lines[2].start, 140.0, "a run more than a space's reach before the word is another line's");
497 assert_eq!(page.lines[0].x0, 159.8, "the first word is kept");
498}