oxedyne/fe2o3/fe2o3_austenite/tests/eval_inline.rs
49.3 KiB, 5 runs
created by r1870400018:63594, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! U6a: inline flow and fonts against the `typst` 0.15.1 oracle, on synthetic fixtures only. |
| 2 | //! |
| 3 | //! - `lorem` gives Typst's words. |
| 4 | //! - Shaped widths: a corpus of words in Libertinus Serif, four sizes and four styles, measured by |
| 5 | //! Typst's `measure` and by Austenite's shaper, agree to 0.01pt. |
| 6 | //! - Line breaking: every paragraph of `tests/fixtures/eval/linebreak/` breaks into the same lines, |
| 7 | //! read from Typst's PDF by `pdftotext -bbox-layout`. |
| 8 | //! - Hyphenation: English, German and French paragraphs set so narrow that most lines end in a hyphen |
| 9 | //! break where Typst breaks them (`tests/fixtures/eval/hyphenation/`). |
| 10 | //! - Overhang: a justified line's last punctuation mark hangs into the margin as far as Typst hangs it, |
| 11 | //! and never moves a break. |
| 12 | //! - The paragraph rule: `layout_par` keeps nothing, takes nothing but its inputs, tags a located element |
| 13 | //! where it stands, and shares one face between every line. |
| 14 | //! - The shaped-run cache holds its byte budget and evicts the least recently used. |
| 15 | //! - A family no font declares warns as Typst warns, with kind `missing_font`. |
| 16 | //! - The language and region a chain sets, which the document's language is read from. |
| 17 | //! |
| 18 | //! Every `typst` run is capped by `systemd-run` and ignores system fonts, so both sides shape from the |
| 19 | //! same embedded faces. A missing `typst` fails the suite unless `EVAL_ORACLE_SKIP=1` is set. |
| 20 | |
| 21 | use oxedyne_fe2o3_austenite::diag::DiagnosticKind; |
| 22 | use oxedyne_fe2o3_austenite::eval::content::ElemKind; |
| 23 | use oxedyne_fe2o3_austenite::eval::realise::{ |
| 24 | realise, |
| 25 | Pair, |
| 26 | RealiseMode, |
| 27 | }; |
| 28 | use oxedyne_fe2o3_austenite::eval::styles::{ |
| 29 | Property, |
| 30 | Style, |
| 31 | StyleChain, |
| 32 | Styles, |
| 33 | }; |
| 34 | use oxedyne_fe2o3_austenite::eval::value::{ |
| 35 | Length, |
| 36 | Value, |
| 37 | }; |
| 38 | use oxedyne_fe2o3_austenite::eval::{ |
| 39 | eval_source, |
| 40 | Engine, |
| 41 | World, |
| 42 | }; |
| 43 | use oxedyne_fe2o3_austenite::flow::inline::{ |
| 44 | LineBox, |
| 45 | Linebreaks, |
| 46 | ParSituation, |
| 47 | }; |
| 48 | use oxedyne_fe2o3_austenite::flow::par::{ |
| 49 | layout_par, |
| 50 | layout_with, |
| 51 | ConfigBase, |
| 52 | }; |
| 53 | use oxedyne_fe2o3_austenite::flow::text::{ |
| 54 | locale, |
| 55 | props, |
| 56 | shape_run, |
| 57 | }; |
| 58 | use oxedyne_fe2o3_austenite::flow::Region; |
| 59 | use oxedyne_fe2o3_austenite::fonts::{ |
| 60 | FaceVariant, |
| 61 | FontBook, |
| 62 | }; |
| 63 | use oxedyne_fe2o3_austenite::ir::{ |
| 64 | LeafKind, |
| 65 | Node, |
| 66 | Sp, |
| 67 | }; |
| 68 | use oxedyne_fe2o3_austenite::syntax::Span; |
| 69 | use oxedyne_fe2o3_font::shape::{ |
| 70 | Dir as ShapeDir, |
| 71 | ShapeSpec, |
| 72 | }; |
| 73 | |
| 74 | use oxedyne_fe2o3_core::prelude::*; |
| 75 | |
| 76 | use std::collections::BTreeMap; |
| 77 | use std::path::{ |
| 78 | Path, |
| 79 | PathBuf, |
| 80 | }; |
| 81 | use std::process::Command; |
| 82 | use std::sync::Arc; |
| 83 | |
| 84 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 85 | // │ ORACLE │ |
| 86 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 87 | |
| 88 | /// The typst binary to run, or `None` when absent and explicitly skipped. |
| 89 | fn oracle() -> Outcome<Option<String>> { |
| 90 | let bin = std::env::var("TYPST").unwrap_or_else(|_| "typst".to_string()); |
| 91 | match Command::new(&bin).arg("--version").output() { |
| 92 | Ok(o) if o.status.success() => { |
| 93 | let v = String::from_utf8_lossy(&o.stdout).to_string(); |
| 94 | if !v.starts_with("typst 0.15.") { |
| 95 | return Err(err!("The oracle is {}, not typst 0.15.x.", v.trim(); Test, Invalid)); |
| 96 | } |
| 97 | Ok(Some(bin)) |
| 98 | } |
| 99 | _ => { |
| 100 | if std::env::var("EVAL_ORACLE_SKIP").as_deref() == Ok("1") { |
| 101 | Ok(None) |
| 102 | } else { |
| 103 | Err(err!("typst 0.15.1 is not on the PATH; set EVAL_ORACLE_SKIP=1 to skip on purpose."; Test, Missing)) |
| 104 | } |
| 105 | } |
| 106 | } |
| 107 | } |
| 108 | |
| 109 | /// Runs typst under the memory cap, in `cwd`; its standard output, or an error with its standard error. |
| 110 | fn typst(bin: &str, args: &[&str], cwd: &Path) -> Outcome<String> { |
| 111 | let capped = Command::new("systemd-run").arg("--version").output().map(|o| o.status.success()).unwrap_or(false); |
| 112 | let mut cmd = if capped { |
| 113 | let mut c = Command::new("systemd-run"); |
| 114 | c.args(["--user", "--scope", "--quiet", "-p", "MemoryMax=3G", "--slice=claude-rc.slice", bin]); |
| 115 | c |
| 116 | } else { |
| 117 | Command::new(bin) |
| 118 | }; |
| 119 | cmd.args(args).current_dir(cwd); |
| 120 | let out = res!(cmd.output().map_err(|e| err!(e, "Could not run typst."; IO))); |
| 121 | if !out.status.success() { |
| 122 | return Err(err!("typst {:?} failed: {}", args, String::from_utf8_lossy(&out.stderr); Test, Invalid)); |
| 123 | } |
| 124 | Ok(String::from_utf8_lossy(&out.stdout).to_string()) |
| 125 | } |
| 126 | |
| 127 | /// A scratch directory under the test target, never `/tmp`. |
| 128 | fn work(name: &str) -> Outcome<PathBuf> { |
| 129 | let dir = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join("u6a").join(name); |
| 130 | res!(std::fs::create_dir_all(&dir).map_err(|e| err!(e, "Could not make {:?}.", dir; IO))); |
| 131 | Ok(dir) |
| 132 | } |
| 133 | |
| 134 | fn fixtures(area: &str) -> PathBuf { |
| 135 | PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests").join("fixtures").join("eval").join(area) |
| 136 | } |
| 137 | |
| 138 | /// The strings of a JSON array of strings, as `typst eval` prints one. |
| 139 | fn json_strings(s: &str) -> Vec<String> { |
| 140 | let mut out = Vec::new(); |
| 141 | let mut cur: Option<String> = None; |
| 142 | let mut chars = s.chars(); |
| 143 | while let Some(c) = chars.next() { |
| 144 | match (&mut cur, c) { |
| 145 | (None, '"') => cur = Some(String::new()), |
| 146 | (Some(buf), '"') => { |
| 147 | out.push(std::mem::take(buf)); |
| 148 | cur = None; |
| 149 | } |
| 150 | (Some(buf), '\\') => match chars.next() { |
| 151 | Some('n') => buf.push('\n'), |
| 152 | Some('u') => { |
| 153 | let hex: String = chars.by_ref().take(4).collect(); |
| 154 | if let Ok(n) = u32::from_str_radix(&hex, 16) { |
| 155 | if let Some(ch) = char::from_u32(n) { |
| 156 | buf.push(ch); |
| 157 | } |
| 158 | } |
| 159 | } |
| 160 | Some(x) => buf.push(x), |
| 161 | None => (), |
| 162 | }, |
| 163 | (Some(buf), x) => buf.push(x), |
| 164 | (None, _) => (), |
| 165 | } |
| 166 | } |
| 167 | out |
| 168 | } |
| 169 | |
| 170 | /// The numbers of a JSON array of numbers. |
| 171 | fn json_numbers(s: &str) -> Vec<f64> { |
| 172 | s.trim().trim_start_matches('[').trim_end_matches(']').split(',') |
| 173 | .filter_map(|x| x.trim().parse::<f64>().ok()).collect() |
| 174 | } |
| 175 | |
| 176 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 177 | // │ LOREM │ |
| 178 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 179 | |
| 180 | #[test] |
| 181 | fn lorem_gives_typsts_words() -> Outcome<()> { |
| 182 | let bin = match res!(oracle()) { |
| 183 | Some(b) => b, |
| 184 | None => return Ok(()), |
| 185 | }; |
| 186 | let counts = [0usize, 1, 2, 3, 5, 8, 13, 30, 57, 100, 250, 600]; |
| 187 | let expr = fmt!("({})", counts.iter().map(|n| fmt!("lorem({})", n)).collect::<Vec<_>>().join(", ")); |
| 188 | let dir = res!(work("lorem")); |
| 189 | let got = json_strings(&res!(typst(&bin, &["eval", &expr], &dir))); |
| 190 | assert_eq!(got.len(), counts.len(), "typst gave {} strings", got.len()); |
| 191 | for (n, want) in counts.iter().zip(got.iter()) { |
| 192 | let ours = oxedyne_fe2o3_austenite::eval::lib::text::lorem(*n); |
| 193 | assert_eq!(&ours, want, "lorem({}) differs", n); |
| 194 | } |
| 195 | Ok(()) |
| 196 | } |
| 197 | |
| 198 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 199 | // │ WIDTHS │ |
| 200 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 201 | |
| 202 | /// Words for the width corpus: ligatures, kerning pairs, figures, punctuation and ordinary prose. Every |
| 203 | /// character is one Libertinus Serif draws: a character it lacks falls back, in Typst, to New Computer |
| 204 | /// Modern, which Austenite does not embed (the precomposed `fi` in italic, say). |
| 205 | fn width_words() -> Vec<String> { |
| 206 | let mut w: Vec<String> = [ |
| 207 | "Hyphenation", "office", "affine", "fluffy", "shuffle", "efficient", "flight", "finish", "waffle", |
| 208 | "AVAST", "Tomorrow", "Water", "Yawning", "LTA", "P.", "T,", "Vo", "WA", "Ty", "Te", "Yo", "fjord", |
| 209 | "1234567890", "3.75", "(parenthesis)", "[bracket]", "quote\u{2019}s", "\u{201C}open\u{201D}", |
| 210 | "em\u{2014}dash", "en\u{2013}dash", "naïve", "façade", "Übermäßig", "Æsop", "œuvre", "Q.E.D.", |
| 211 | "http://example.org", "e.g.,", "i.e.;", "etc.:", "!?", "#42", "50%", "&", "x*y", "a/b", |
| 212 | ].iter().map(|s| s.to_string()).collect(); |
| 213 | let prose = oxedyne_fe2o3_austenite::eval::lib::text::lorem(160); |
| 214 | for p in prose.split_whitespace() { |
| 215 | if w.len() >= 125 { |
| 216 | break; |
| 217 | } |
| 218 | if !w.iter().any(|x| x == p) { |
| 219 | w.push(p.to_string()); |
| 220 | } |
| 221 | } |
| 222 | w |
| 223 | } |
| 224 | |
| 225 | fn typst_string(s: &str) -> String { |
| 226 | let mut out = String::from("\""); |
| 227 | for c in s.chars() { |
| 228 | match c { |
| 229 | '"' => out.push_str("\\\""), |
| 230 | '\\' => out.push_str("\\\\"), |
| 231 | c => out.push(c), |
| 232 | } |
| 233 | } |
| 234 | out.push('"'); |
| 235 | out |
| 236 | } |
| 237 | |
| 238 | const STYLES: &[(&str, &str)] = &[ |
| 239 | ("normal", "regular"), |
| 240 | ("italic", "regular"), |
| 241 | ("normal", "bold"), |
| 242 | ("italic", "bold"), |
| 243 | ]; |
| 244 | |
| 245 | const WIDTH_SIZES: &[f64] = &[9.0, 10.0, 11.0, 12.0]; |
| 246 | |
| 247 | fn text_chain(size: f64, style: &str, weight: &str) -> StyleChain { |
| 248 | let mut s = Styles::new(); |
| 249 | let mut put = |name: &str, v: Value| { |
| 250 | if let Some(id) = ElemKind::Text.field_id(name) { |
| 251 | s.push(Style::Property(Property::new(ElemKind::Text, id, v, Span::detached()))); |
| 252 | } |
| 253 | }; |
| 254 | put("size", Value::Length(Length::pt(size))); |
| 255 | put("style", Value::str(style)); |
| 256 | put("weight", Value::str(weight)); |
| 257 | StyleChain::root().chain(&s) |
| 258 | } |
| 259 | |
| 260 | #[test] |
| 261 | fn shaped_widths_match_typsts_measure() -> Outcome<()> { |
| 262 | let bin = match res!(oracle()) { |
| 263 | Some(b) => b, |
| 264 | None => return Ok(()), |
| 265 | }; |
| 266 | let words = width_words(); |
| 267 | let dir = res!(work("widths")); |
| 268 | let list = words.iter().map(|w| typst_string(w)).collect::<Vec<_>>().join(", "); |
| 269 | let src = fmt!( |
| 270 | "#let words = ({},)\n#context [#metadata({{\n let out = ()\n for size in (9pt, 10pt, 11pt, 12pt) {{\n \ |
| 271 | for (style, weight) in ((\"normal\", \"regular\"), (\"italic\", \"regular\"), (\"normal\", \"bold\"), (\"italic\", \"bold\")) {{\n \ |
| 272 | for w in words {{ out.push(measure(text(size: size, style: style, weight: weight, w)).width.pt()) }}\n }}\n }}\n out\n}}) <m>]\n", |
| 273 | list, |
| 274 | ); |
| 275 | let path = dir.join("widths.typ"); |
| 276 | res!(std::fs::write(&path, src).map_err(|e| err!(e, "write"; IO))); |
| 277 | let want = json_numbers(&res!(typst(&bin, &["query", "--ignore-system-fonts", "--field", "value", "--one", "widths.typ", "<m>"], &dir))); |
| 278 | let expect = WIDTH_SIZES.len() * STYLES.len() * words.len(); |
| 279 | assert_eq!(want.len(), expect, "typst measured {} words, expected {}", want.len(), expect); |
| 280 | |
| 281 | let mut engine = Engine::new(World::new(dir.clone())); |
| 282 | let mut i = 0; |
| 283 | let mut bad: Vec<String> = Vec::new(); |
| 284 | for size in WIDTH_SIZES { |
| 285 | for (style, weight) in STYLES { |
| 286 | let chain = text_chain(*size, style, weight); |
| 287 | for w in &words { |
| 288 | let shaped = res!(oxedyne_fe2o3_austenite::flow::text::shape(&mut engine, w, &chain)); |
| 289 | let ours = shaped.dims().width.to_pt(); |
| 290 | if (ours - want[i]).abs() > 0.01 { |
| 291 | bad.push(fmt!("{}pt {} {} {:?}: typst {:.4} austenite {:.4}", size, style, weight, w, want[i], ours)); |
| 292 | } |
| 293 | i += 1; |
| 294 | } |
| 295 | } |
| 296 | } |
| 297 | assert!(bad.is_empty(), "{} of {} widths differ by more than 0.01pt:\n{}", bad.len(), expect, bad.join("\n")); |
| 298 | Ok(()) |
| 299 | } |
| 300 | |
| 301 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 302 | // │ LINE BREAKING │ |
| 303 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 304 | |
| 305 | /// The lines of a `pdftotext -bbox-layout` page: its words grouped by their vertical middles (a loose |
| 306 | /// justified line can come back from pdftotext as several), each line's words in order across. |
| 307 | fn bbox_lines(html: &str) -> Vec<String> { |
| 308 | let attr = |tag: &str, key: &str| -> f64 { |
| 309 | tag.split(&fmt!("{}=\"", key)).nth(1).and_then(|r| r.split('"').next()) |
| 310 | .and_then(|v| v.parse::<f64>().ok()).unwrap_or(0.0) |
| 311 | }; |
| 312 | let mut words: Vec<(f64, f64, f64, String)> = Vec::new(); // (middle, height, x, text) |
| 313 | let mut rest = html; |
| 314 | while let Some(i) = rest.find("<word") { |
| 315 | let tail = &rest[i..]; |
| 316 | let (gt, end) = match (tail.find('>'), tail.find("</word>")) { |
| 317 | (Some(g), Some(e)) => (g, e), |
| 318 | _ => break, |
| 319 | }; |
| 320 | let tag = &tail[..gt]; |
| 321 | let (y0, y1) = (attr(tag, "yMin"), attr(tag, "yMax")); |
| 322 | words.push(((y0 + y1) / 2.0, y1 - y0, attr(tag, "xMin"), unescape(&tail[gt + 1..end]))); |
| 323 | rest = &tail[end..]; |
| 324 | } |
| 325 | words.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)); |
| 326 | let mut lines: Vec<(f64, f64, Vec<(f64, String)>)> = Vec::new(); |
| 327 | for (mid, h, x, t) in words { |
| 328 | match lines.last_mut() { |
| 329 | Some((lm, lh, ws)) if (mid - *lm).abs() <= 0.35 * lh.max(h) => ws.push((x, t)), |
| 330 | _ => lines.push((mid, h, vec![(x, t)])), |
| 331 | } |
| 332 | } |
| 333 | lines.into_iter().map(|(_, _, mut ws)| { |
| 334 | ws.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)); |
| 335 | ws.into_iter().map(|(_, t)| t).collect::<Vec<_>>().join(" ") |
| 336 | }).collect() |
| 337 | } |
| 338 | |
| 339 | fn unescape(s: &str) -> String { |
| 340 | s.replace("&", "&").replace("<", "<").replace(">", ">").replace(""", "\"").replace("'", "'") |
| 341 | } |
| 342 | |
| 343 | /// A line's text for comparison: soft hyphens as hyphens, white space collapsed. |
| 344 | fn norm(s: &str) -> String { |
| 345 | s.replace('\u{00AD}', "-").split_whitespace().collect::<Vec<_>>().join(" ") |
| 346 | } |
| 347 | |
| 348 | /// A fixture's page width and `par` settings, read from its `#set page` and `#set par` lines, and its |
| 349 | /// source with those lines removed while the page and paragraph schemas are not yet built. |
| 350 | struct Fixture { |
| 351 | width: f64, |
| 352 | base: ConfigBase, |
| 353 | source: String, |
| 354 | } |
| 355 | |
| 356 | fn read_fixture(text: &str) -> Outcome<Fixture> { |
| 357 | let mut width = None; |
| 358 | let mut base = ConfigBase { justify: false, linebreaks: None, first_line_indent: (0.0, false), hanging_indent: 0.0 }; |
| 359 | let par_schema = ElemKind::Par.field_id("justify").is_some(); |
| 360 | let mut kept = Vec::new(); |
| 361 | for line in text.lines() { |
| 362 | if let Some(args) = line.strip_prefix("#set page(") { |
| 363 | if let Some(w) = args.split("width:").nth(1).and_then(|r| r.trim().split("pt").next()) { |
| 364 | width = w.trim().parse::<f64>().ok(); |
| 365 | } |
| 366 | // The width goes to the line setter directly; a page rule would be refused in an inline body. |
| 367 | } else if let Some(args) = line.strip_prefix("#set par(") { |
| 368 | base.justify = args.contains("justify: true"); |
| 369 | if args.contains("\"simple\"") { |
| 370 | base.linebreaks = Some(Linebreaks::Simple); |
| 371 | } else if args.contains("\"optimized\"") { |
| 372 | base.linebreaks = Some(Linebreaks::Optimized); |
| 373 | } |
| 374 | if par_schema { |
| 375 | kept.push(line); |
| 376 | } |
| 377 | } else { |
| 378 | kept.push(line); |
| 379 | } |
| 380 | } |
| 381 | let width = match width { |
| 382 | Some(w) => w, |
| 383 | None => return Err(err!("A line-break fixture names no page width."; Test, Missing)), |
| 384 | }; |
| 385 | Ok(Fixture { width, base, source: kept.join("\n") }) |
| 386 | } |
| 387 | |
| 388 | /// Austenite's lines for a fixture: evaluated, realised, and set as one paragraph at the page width. |
| 389 | fn austenite_lines(path: &Path, fx: &Fixture) -> Outcome<Vec<String>> { |
| 390 | Ok(res!(austenite_boxes(path, fx)).into_iter().map(|l| l.text).collect()) |
| 391 | } |
| 392 | |
| 393 | /// As [`austenite_lines`], the committed line boxes themselves. |
| 394 | fn austenite_boxes(path: &Path, fx: &Fixture) -> Outcome<Vec<LineBox>> { |
| 395 | let root = path.parent().map(|p| p.to_path_buf()).unwrap_or_default(); |
| 396 | let mut world = World::new(root); |
| 397 | let id = res!(world.add_source(path.to_path_buf(), fx.source.clone())); |
| 398 | let mut engine = Engine::new(world); |
| 399 | let module = res!(eval_source(&mut engine, id)); |
| 400 | // One paragraph's worth of content, realised as a paragraph's body is. |
| 401 | let pairs = res!(realise(&mut engine, &module.content, &StyleChain::root(), RealiseMode::Inline)); |
| 402 | let trunk = StyleChain::trunk(pairs.iter().filter(|p| !p.is_tag()).map(|p| &p.styles)); |
| 403 | let region = Region::new(Sp::from_pt(fx.width), Sp::from_pt(10_000.0)); |
| 404 | layout_with(&mut engine, &pairs, &trunk, region, true, Some(ParSituation::First), &fx.base) |
| 405 | } |
| 406 | |
| 407 | /// Typst's lines for a fixture, cached by its bytes under the test target. |
| 408 | fn typst_lines(bin: &str, path: &Path) -> Outcome<Vec<String>> { |
| 409 | let bytes = res!(std::fs::read(path).map_err(|e| err!(e, "read {:?}", path; IO))); |
| 410 | let mut h: u64 = 0xcbf2_9ce4_8422_2325; |
| 411 | for b in &bytes { |
| 412 | h ^= *b as u64; |
| 413 | h = h.wrapping_mul(0x0000_0100_0000_01b3); |
| 414 | } |
| 415 | let dir = res!(work("linebreak")); |
| 416 | let cached = dir.join(fmt!("{:016x}.lines", h)); |
| 417 | if let Ok(s) = std::fs::read_to_string(&cached) { |
| 418 | return Ok(s.lines().map(|l| l.to_string()).collect()); |
| 419 | } |
| 420 | let typ = dir.join(fmt!("{:016x}.typ", h)); |
| 421 | res!(std::fs::write(&typ, &bytes).map_err(|e| err!(e, "write"; IO))); |
| 422 | let pdf = dir.join(fmt!("{:016x}.pdf", h)); |
| 423 | let (t, p) = (typ.to_string_lossy().to_string(), pdf.to_string_lossy().to_string()); |
| 424 | res!(typst(bin, &["compile", "--ignore-system-fonts", &t, &p], &dir)); |
| 425 | let out = res!(Command::new("pdftotext").args(["-bbox-layout", &p, "-"]).output() |
| 426 | .map_err(|e| err!(e, "Could not run pdftotext."; IO))); |
| 427 | let lines: Vec<String> = bbox_lines(&String::from_utf8_lossy(&out.stdout)).into_iter() |
| 428 | .map(|l| norm(&l)).filter(|l| !l.is_empty()).collect(); |
| 429 | res!(std::fs::write(&cached, lines.join("\n")).map_err(|e| err!(e, "write"; IO))); |
| 430 | Ok(lines) |
| 431 | } |
| 432 | |
| 433 | /// What comparing a directory of fixtures against Typst found. |
| 434 | struct Compared { |
| 435 | fixtures: usize, |
| 436 | lines: usize, |
| 437 | hyphens: BTreeMap<String, usize>, // lines of Typst's that end in a hyphen, by the fixture's language |
| 438 | failures: Vec<String>, |
| 439 | } |
| 440 | |
| 441 | /// Every fixture of `dir` (those whose path contains `filter`, when given) set by both engines. |
| 442 | fn compare_fixtures(bin: &str, dir: &Path, filter: Option<&str>) -> Outcome<Compared> { |
| 443 | let mut paths: Vec<PathBuf> = res!(std::fs::read_dir(dir).map_err(|e| err!(e, "read {:?}", dir; IO))) |
| 444 | .filter_map(|e| e.ok().map(|e| e.path())) |
| 445 | .filter(|p| p.extension().map_or(false, |x| x == "typ")) |
| 446 | .filter(|p| filter.map_or(true, |f| p.to_string_lossy().contains(f))) |
| 447 | .collect(); |
| 448 | paths.sort(); |
| 449 | assert!(!paths.is_empty(), "no line-break fixtures found under {:?}", dir); |
| 450 | let mut out = Compared { fixtures: paths.len(), lines: 0, hyphens: BTreeMap::new(), failures: Vec::new() }; |
| 451 | for path in &paths { |
| 452 | let text = res!(std::fs::read_to_string(path).map_err(|e| err!(e, "read"; IO))); |
| 453 | let fx = res!(read_fixture(&text)); |
| 454 | let want = res!(typst_lines(bin, path)); |
| 455 | let name = path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); |
| 456 | let got: Vec<String> = match austenite_lines(path, &fx) { |
| 457 | Ok(l) => l.iter().map(|x| norm(x)).collect(), |
| 458 | Err(e) => { |
| 459 | out.failures.push(fmt!("{}: austenite failed: {}", name, e)); |
| 460 | continue; |
| 461 | } |
| 462 | }; |
| 463 | out.lines += want.len(); |
| 464 | // The language is the letters after the fixture's first hyphen: `hy-fr2-90.typ` is `fr`. |
| 465 | let lang: String = name.split('-').nth(1).unwrap_or("").chars().take_while(|c| c.is_alphabetic()).collect(); |
| 466 | *out.hyphens.entry(lang).or_insert(0) += want.iter().filter(|l| l.ends_with('-')).count(); |
| 467 | let n = want.len().max(got.len()); |
| 468 | let mut diffs = Vec::new(); |
| 469 | for i in 0..n { |
| 470 | let (w, g) = (want.get(i).map(|s| s.as_str()).unwrap_or("<none>"), got.get(i).map(|s| s.as_str()).unwrap_or("<none>")); |
| 471 | if w != g { |
| 472 | diffs.push(fmt!(" line {}:\n typst: {}\n austenite: {}", i + 1, w, g)); |
| 473 | } |
| 474 | } |
| 475 | if !diffs.is_empty() { |
| 476 | out.failures.push(fmt!("{} ({} lines, {} differ):\n{}", name, want.len(), diffs.len(), diffs.join("\n"))); |
| 477 | } |
| 478 | } |
| 479 | Ok(out) |
| 480 | } |
| 481 | |
| 482 | #[test] |
| 483 | fn paragraphs_break_into_typsts_lines() -> Outcome<()> { |
| 484 | let bin = match res!(oracle()) { |
| 485 | Some(b) => b, |
| 486 | None => return Ok(()), |
| 487 | }; |
| 488 | let filter = std::env::var("U6A_FIXTURE").ok(); |
| 489 | let c = res!(compare_fixtures(&bin, &fixtures("linebreak"), filter.as_deref())); |
| 490 | assert!(c.failures.is_empty(), "{} of {} fixtures ({} typst lines) break differently:\n{}", |
| 491 | c.failures.len(), c.fixtures, c.lines, c.failures.join("\n")); |
| 492 | Ok(()) |
| 493 | } |
| 494 | |
| 495 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 496 | // │ HYPHENATION │ |
| 497 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 498 | |
| 499 | /// English, German and French paragraphs of long words in narrow columns, set under `justify: true` so |
| 500 | /// that `hyphenate` is on, break at the same hyphenation points as Typst's. The count of hyphenated lines |
| 501 | /// is asserted so the test cannot pass on paragraphs that never needed a hyphen. |
| 502 | #[test] |
| 503 | fn hyphenation_in_english_german_and_french_matches_typst() -> Outcome<()> { |
| 504 | let bin = match res!(oracle()) { |
| 505 | Some(b) => b, |
| 506 | None => return Ok(()), |
| 507 | }; |
| 508 | let c = res!(compare_fixtures(&bin, &fixtures("hyphenation"), None)); |
| 509 | assert_eq!(c.fixtures, 27, "the hyphenation corpus is nine fixtures in each of three languages"); |
| 510 | for lang in ["en", "de", "fr"] { |
| 511 | let n = c.hyphens.get(lang).copied().unwrap_or(0); |
| 512 | assert!(n >= 20, "typst hyphenates only {} {} lines, too few to test hyphenation with", n, lang); |
| 513 | } |
| 514 | assert!(c.failures.is_empty(), "{} of {} fixtures ({} typst lines) hyphenate differently:\n{}", |
| 515 | c.failures.len(), c.fixtures, c.lines, c.failures.join("\n")); |
| 516 | Ok(()) |
| 517 | } |
| 518 | |
| 519 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 520 | // │ OVERHANG │ |
| 521 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 522 | |
| 523 | /// Typst's lines of a source, each with the right edge of its last word, from `pdftotext -bbox-layout`. |
| 524 | fn typst_line_edges(bin: &str, name: &str, src: &str) -> Outcome<Vec<(String, f64)>> { |
| 525 | let dir = res!(work("overhang")); |
| 526 | let typ = dir.join(fmt!("{}.typ", name)); |
| 527 | let pdf = dir.join(fmt!("{}.pdf", name)); |
| 528 | res!(std::fs::write(&typ, src).map_err(|e| err!(e, "write"; IO))); |
| 529 | let (t, p) = (typ.to_string_lossy().to_string(), pdf.to_string_lossy().to_string()); |
| 530 | res!(typst(bin, &["compile", "--ignore-system-fonts", &t, &p], &dir)); |
| 531 | let out = res!(Command::new("pdftotext").args(["-bbox-layout", &p, "-"]).output() |
| 532 | .map_err(|e| err!(e, "Could not run pdftotext."; IO))); |
| 533 | let html = String::from_utf8_lossy(&out.stdout).to_string(); |
| 534 | let attr = |tag: &str, key: &str| -> f64 { |
| 535 | tag.split(&fmt!("{}=\"", key)).nth(1).and_then(|r| r.split('"').next()) |
| 536 | .and_then(|v| v.parse::<f64>().ok()).unwrap_or(0.0) |
| 537 | }; |
| 538 | let mut words: Vec<(f64, f64, f64, f64, String)> = Vec::new(); // (middle, height, x0, x1, text) |
| 539 | let mut rest = html.as_str(); |
| 540 | while let Some(i) = rest.find("<word") { |
| 541 | let tail = &rest[i..]; |
| 542 | let (gt, end) = match (tail.find('>'), tail.find("</word>")) { |
| 543 | (Some(g), Some(e)) => (g, e), |
| 544 | _ => break, |
| 545 | }; |
| 546 | let tag = &tail[..gt]; |
| 547 | let (y0, y1) = (attr(tag, "yMin"), attr(tag, "yMax")); |
| 548 | words.push(((y0 + y1) / 2.0, y1 - y0, attr(tag, "xMin"), attr(tag, "xMax"), unescape(&tail[gt + 1..end]))); |
| 549 | rest = &tail[end..]; |
| 550 | } |
| 551 | words.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)); |
| 552 | let mut lines: Vec<(f64, f64, Vec<(f64, f64, String)>)> = Vec::new(); |
| 553 | for (mid, h, x0, x1, t) in words { |
| 554 | match lines.last_mut() { |
| 555 | Some((lm, lh, ws)) if (mid - *lm).abs() <= 0.35 * lh.max(h) => ws.push((x0, x1, t)), |
| 556 | _ => lines.push((mid, h, vec![(x0, x1, t)])), |
| 557 | } |
| 558 | } |
| 559 | Ok(lines.into_iter().map(|(_, _, mut ws)| { |
| 560 | ws.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)); |
| 561 | let edge = ws.last().map_or(0.0, |w| w.1); |
| 562 | (norm(&ws.into_iter().map(|(_, _, t)| t).collect::<Vec<_>>().join(" ")), edge) |
| 563 | }).collect()) |
| 564 | } |
| 565 | |
| 566 | /// How wide a committed line's items run, in points: the box's width plus whatever hangs past it. |
| 567 | fn run_width(node: &Node) -> f64 { |
| 568 | match node { |
| 569 | Node::HBox(b) => b.list.iter().map(|n| match n { |
| 570 | Node::Leaf(l) => l.dims.width.to_pt(), |
| 571 | Node::Glue(g) => g.natural.to_pt(), |
| 572 | Node::HBox(x) | Node::VBox(x) => x.dims.width.to_pt(), |
| 573 | _ => 0.0, |
| 574 | }).sum(), |
| 575 | _ => 0.0, |
| 576 | } |
| 577 | } |
| 578 | |
| 579 | /// Overhang moves text only when a line is committed: the last punctuation mark of a justified line hangs |
| 580 | /// into the end margin by a fixed fraction of its advance, the line is justified into that much more |
| 581 | /// room, and the breaker, which measures lines without it, breaks where it breaks with overhang off. |
| 582 | /// Typst's lines and the right edge of each are the oracle. |
| 583 | #[test] |
| 584 | fn overhang_hangs_punctuation_at_commit_and_never_moves_a_break() -> Outcome<()> { |
| 585 | let bin = match res!(oracle()) { |
| 586 | Some(b) => b, |
| 587 | None => return Ok(()), |
| 588 | }; |
| 589 | // Every line of every run ends in a comma, full stop, hyphen, colon or dash often enough that some hang. |
| 590 | let body = "Rain, wind, fog; hail: sleet. Night, dusk, dawn; noon: haze. Salt, sand, kelp, stone, tide. \ |
| 591 | Boats, nets, ropes, oars, sails, masts, decks. Gulls cry, waves roll, bells ring, lamps burn low. \ |
| 592 | Ink, paper, thread, glue, wax, board, cloth, leather, gold. Slow carts creak; tired horses plod; dust rises."; |
| 593 | let mut hung = 0usize; |
| 594 | for (width, on) in [(150.0f64, true), (150.0, false), (190.0, true), (190.0, false), (230.0, true), (230.0, false)] { |
| 595 | let flag = if on { "true" } else { "false" }; |
| 596 | let src = fmt!( |
| 597 | "#set page(width: {}pt, height: auto, margin: 0pt)\n#set par(justify: true)\n\ |
| 598 | #set text(size: 10pt, lang: \"en\", hyphenate: false, overhang: {})\n{}\n", width, flag, body); |
| 599 | let name = fmt!("w{}-{}", width as u32, flag); |
| 600 | let want = res!(typst_line_edges(&bin, &name, &src)); |
| 601 | assert!(want.len() >= 5, "{}: typst set only {} lines", name, want.len()); |
| 602 | |
| 603 | let path = res!(work("overhang")).join(fmt!("{}.typ", name)); |
| 604 | let fx = res!(read_fixture(&src)); |
| 605 | let boxes = res!(austenite_boxes(&path, &fx)); |
| 606 | let got_lines: Vec<String> = boxes.iter().map(|l| norm(&l.text)).collect(); |
| 607 | let want_lines: Vec<String> = want.iter().map(|l| l.0.clone()).collect(); |
| 608 | assert_eq!(got_lines, want_lines, "{}: the lines differ", name); |
| 609 | for (i, (line, w)) in boxes.iter().zip(want.iter()).enumerate() { |
| 610 | let ours = run_width(&line.node) - width; |
| 611 | let theirs = w.1 - width; |
| 612 | assert!((ours - theirs).abs() < 0.06, |
| 613 | "{} line {} {:?}: the last mark hangs {:.3}pt in typst and {:.3}pt in austenite", name, i + 1, w.0, theirs, ours); |
| 614 | if on && theirs > 0.5 { |
| 615 | hung += 1; |
| 616 | } |
| 617 | } |
| 618 | } |
| 619 | assert!(hung >= 6, "only {} lines hang a mark in typst, too few to test overhang with", hung); |
| 620 | Ok(()) |
| 621 | } |
| 622 | |
| 623 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 624 | // │ THE PARAGRAPH RULE │ |
| 625 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 626 | |
| 627 | /// The paragraphs a source realises into as a document, as a block flow meets them, with the engine. |
| 628 | fn paragraphs(src: &str) -> Outcome<(Engine, Vec<Pair>)> { |
| 629 | let root = res!(work("pars")); |
| 630 | let mut world = World::new(root.clone()); |
| 631 | let id = res!(world.add_source(root.join("p.typ"), src.to_string())); |
| 632 | let mut engine = Engine::new(world); |
| 633 | let module = res!(eval_source(&mut engine, id)); |
| 634 | let pairs = res!(realise(&mut engine, &module.content, &StyleChain::root(), RealiseMode::Document)); |
| 635 | Ok((engine, pairs.into_iter().filter(|p| p.content.is(ElemKind::Par)).collect())) |
| 636 | } |
| 637 | |
| 638 | fn region(width: f64) -> Region { |
| 639 | Region::new(Sp::from_pt(width), Sp::from_pt(10_000.0)) |
| 640 | } |
| 641 | |
| 642 | /// Everything layout decides about nodes, to the bit: the kinds, the extents, every glyph's id and place, |
| 643 | /// the text and, when `addr`, the font's address. Two equal signatures are two identical layouts. |
| 644 | fn signature(nodes: &[Node], out: &mut String, addr: bool) { |
| 645 | for n in nodes { |
| 646 | match n { |
| 647 | Node::HBox(b) => { |
| 648 | out.push_str(&fmt!("[h {} {} {}|", b.dims.width.raw(), b.dims.height.raw(), b.dims.depth.raw())); |
| 649 | signature(&b.list, out, addr); |
| 650 | out.push(']'); |
| 651 | } |
| 652 | Node::VBox(b) => { |
| 653 | out.push_str(&fmt!("[v {} {} {}|", b.dims.width.raw(), b.dims.height.raw(), b.dims.depth.raw())); |
| 654 | signature(&b.list, out, addr); |
| 655 | out.push(']'); |
| 656 | } |
| 657 | Node::Leaf(l) => match &l.kind { |
| 658 | LeafKind::Text(t) => { |
| 659 | out.push_str(&fmt!("(t {:?} {} {}", t.source(), l.dims.width.raw(), l.shift.raw())); |
| 660 | if addr { |
| 661 | out.push_str(&fmt!(" @{:p}", t.font())); |
| 662 | } |
| 663 | for g in &t.run().glyphs { |
| 664 | out.push_str(&fmt!(" {}:{}:{}:{}", g.id, g.x.to_bits(), g.y.to_bits(), g.adv.to_bits())); |
| 665 | } |
| 666 | out.push(')'); |
| 667 | } |
| 668 | other => out.push_str(&fmt!("(leaf {:?} {})", std::mem::discriminant(other), l.dims.width.raw())), |
| 669 | }, |
| 670 | Node::Glue(g) => out.push_str(&fmt!("(g {} {} {})", g.natural.raw(), g.stretch.raw(), g.shrink.raw())), |
| 671 | Node::Penalty(p) => out.push_str(&fmt!("(p {:?})", p)), |
| 672 | Node::Anchor(a) => out.push_str(&fmt!("(a {:?})", a)), |
| 673 | other => out.push_str(&fmt!("({:?})", std::mem::discriminant(other))), |
| 674 | } |
| 675 | } |
| 676 | } |
| 677 | |
| 678 | /// The signature of `nodes` with the fonts' addresses, for layouts made from one book. |
| 679 | fn sig(nodes: &[Node]) -> String { |
| 680 | let mut s = String::new(); |
| 681 | signature(nodes, &mut s, true); |
| 682 | s |
| 683 | } |
| 684 | |
| 685 | /// The signature without addresses, for layouts made from two books. |
| 686 | fn shape_sig(nodes: &[Node]) -> String { |
| 687 | let mut s = String::new(); |
| 688 | signature(nodes, &mut s, false); |
| 689 | s |
| 690 | } |
| 691 | |
| 692 | fn lines_of(nodes: &[Node]) -> usize { |
| 693 | nodes.iter().filter(|n| matches!(n, Node::HBox(_))).count() |
| 694 | } |
| 695 | |
| 696 | /// `layout_par` keeps nothing and answers from its inputs alone: the same paragraph gives the same lines |
| 697 | /// before and after another is set, and no locator ordinal, diagnostic or read is left behind. |
| 698 | #[test] |
| 699 | fn layout_par_keeps_nothing_and_depends_only_on_its_inputs() -> Outcome<()> { |
| 700 | let src = "A first paragraph with #strong[some strong words] and #emph[slanted ones], long enough to wrap onto \ |
| 701 | several lines at a narrow measure, with #lorem(30)\n\nA second paragraph, quite different from the first, \ |
| 702 | set between the two settings of the first, and #lorem(45)\n"; |
| 703 | let (mut engine, pars) = res!(paragraphs(src)); |
| 704 | assert_eq!(pars.len(), 2, "the source realises into two paragraphs"); |
| 705 | let (a, b) = (&pars[0], &pars[1]); |
| 706 | |
| 707 | let locator_before = fmt!("{:?}", engine.locator); |
| 708 | let (diags_before, reads_before) = (engine.diags.len(), fmt!("{:?}", engine.reads)); |
| 709 | let one = res!(layout_par(&mut engine, &a.content, &a.styles, region(150.0), ParSituation::First)); |
| 710 | assert!(lines_of(&one) >= 4, "the first paragraph sets in {} lines at 150pt, too few to test with", lines_of(&one)); |
| 711 | assert_eq!(fmt!("{:?}", engine.locator), locator_before, "layout_par consumed locator ordinals"); |
| 712 | assert_eq!(engine.diags.len(), diags_before, "layout_par left diagnostics behind"); |
| 713 | assert_eq!(fmt!("{:?}", engine.reads), reads_before, "layout_par recorded reads it had no cause to"); |
| 714 | |
| 715 | let other = res!(layout_par(&mut engine, &b.content, &b.styles, region(200.0), ParSituation::Consecutive)); |
| 716 | assert!(lines_of(&other) >= 3); |
| 717 | let again = res!(layout_par(&mut engine, &a.content, &a.styles, region(150.0), ParSituation::First)); |
| 718 | assert!(sig(&one) == sig(&again), "a paragraph set again, after another, came out differently"); |
| 719 | |
| 720 | // A fresh engine, which has set nothing, sets it identically: nothing it needs was in the first. |
| 721 | let (mut fresh, fresh_pars) = res!(paragraphs(src)); |
| 722 | let from_fresh = res!(layout_par(&mut fresh, &fresh_pars[0].content, &fresh_pars[0].styles, region(150.0), ParSituation::First)); |
| 723 | assert!(shape_sig(&one) == shape_sig(&from_fresh), "a fresh engine set the paragraph differently"); |
| 724 | |
| 725 | // And the inputs do matter: the width and the situation each change the answer. |
| 726 | let wide = res!(layout_par(&mut engine, &a.content, &a.styles, region(400.0), ParSituation::First)); |
| 727 | assert!(lines_of(&wide) < lines_of(&one), "a wider measure should set fewer lines"); |
| 728 | Ok(()) |
| 729 | } |
| 730 | |
| 731 | /// A located element inside a paragraph is met where it stands in the text: `layout_par` puts its start |
| 732 | /// in the line as a tag, before the text it covers and after the text before it. |
| 733 | #[test] |
| 734 | fn a_located_element_is_tagged_where_it_stands_in_its_paragraph() -> Outcome<()> { |
| 735 | let (mut engine, pars) = res!(paragraphs("Alpha #strong[beta] gamma #emph[delta] epsilon\n")); |
| 736 | assert_eq!(pars.len(), 1); |
| 737 | let nodes = res!(layout_par(&mut engine, &pars[0].content, &pars[0].styles, region(400.0), ParSituation::First)); |
| 738 | assert_eq!(lines_of(&nodes), 1, "the paragraph fits one line"); |
| 739 | let mut flat: Vec<String> = Vec::new(); |
| 740 | fn walk(n: &Node, out: &mut Vec<String>) { |
| 741 | match n { |
| 742 | Node::HBox(b) | Node::VBox(b) => for c in &b.list { walk(c, out); }, |
| 743 | Node::Leaf(l) => if let LeafKind::Text(t) = &l.kind { out.push(fmt!("text:{}", t.source())); }, |
| 744 | Node::Tag(t) => if matches!(t, oxedyne_fe2o3_austenite::eval::realise::Tag::Start(_)) { |
| 745 | out.push("anchor".to_string()); |
| 746 | }, |
| 747 | _ => (), |
| 748 | } |
| 749 | } |
| 750 | for n in &nodes { |
| 751 | walk(n, &mut flat); |
| 752 | } |
| 753 | let joined = flat.join("|"); |
| 754 | let anchors = flat.iter().filter(|f| f.as_str() == "anchor").count(); |
| 755 | assert_eq!(anchors, 2, "strong and emph each start a tag: {}", joined); |
| 756 | let at = |needle: &str| flat.iter().position(|f| f.contains(needle)); |
| 757 | let (a1, a2) = { |
| 758 | let mut it = flat.iter().enumerate().filter(|(_, f)| f.as_str() == "anchor").map(|(i, _)| i); |
| 759 | (it.next().unwrap_or(usize::MAX), it.next().unwrap_or(usize::MAX)) |
| 760 | }; |
| 761 | assert!(at("Alpha").map_or(false, |i| i < a1), "the first tag precedes `Alpha`: {}", joined); |
| 762 | assert!(at("beta").map_or(false, |i| i > a1 && i < a2), "`beta` stands between the tags: {}", joined); |
| 763 | assert!(at("delta").map_or(false, |i| i > a2), "`delta` follows the second tag: {}", joined); |
| 764 | Ok(()) |
| 765 | } |
| 766 | |
| 767 | /// Lines share the face that drew them: every text leaf of a paragraph holds the book's one font, and |
| 768 | /// the same face, shaped twice, holds the one glyph list. |
| 769 | #[test] |
| 770 | fn lines_share_one_face_and_shaping_shares_one_glyph_list() -> Outcome<()> { |
| 771 | let (mut engine, pars) = res!(paragraphs("A paragraph of plain prose long enough to set over several lines in one face, #lorem(40)\n")); |
| 772 | let nodes = res!(layout_par(&mut engine, &pars[0].content, &pars[0].styles, region(120.0), ParSituation::First)); |
| 773 | assert!(lines_of(&nodes) >= 4); |
| 774 | let book = res!(engine.fonts.book()); |
| 775 | let face = res!(book.select("libertinus serif", FaceVariant::default()).ok_or_else(|| err!("no Libertinus"; Test, Missing))); |
| 776 | let want: *const _ = &*res!(book.face(face).ok_or_else(|| err!("no face"; Test, Missing))).font; |
| 777 | let mut leaves = 0usize; |
| 778 | fn walk(n: &Node, want: *const oxedyne_fe2o3_font::font::Font, leaves: &mut usize, bad: &mut usize) { |
| 779 | match n { |
| 780 | Node::HBox(b) | Node::VBox(b) => for c in &b.list { walk(c, want, leaves, bad); }, |
| 781 | Node::Leaf(l) => if let LeafKind::Text(t) = &l.kind { |
| 782 | *leaves += 1; |
| 783 | if !std::ptr::eq(t.font(), want) { |
| 784 | *bad += 1; |
| 785 | } |
| 786 | }, |
| 787 | _ => (), |
| 788 | } |
| 789 | } |
| 790 | let mut bad = 0usize; |
| 791 | for n in &nodes { |
| 792 | walk(n, want, &mut leaves, &mut bad); |
| 793 | } |
| 794 | assert!(leaves >= 8, "only {} text leaves", leaves); |
| 795 | assert_eq!(bad, 0, "{} of {} text leaves hold a copy of the font, not the book's", bad, leaves); |
| 796 | |
| 797 | let spec = ShapeSpec { remove_ignorables: true, ..ShapeSpec::default() }; |
| 798 | let x = res!(book.shape(face, "shared", ShapeDir::Ltr, &spec)); |
| 799 | let y = res!(book.shape(face, "shared", ShapeDir::Ltr, &spec)); |
| 800 | assert!(Arc::ptr_eq(&x, &y), "the same call returned two glyph lists, not one"); |
| 801 | Ok(()) |
| 802 | } |
| 803 | |
| 804 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 805 | // │ THE SHAPED-RUN CACHE │ |
| 806 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 807 | |
| 808 | /// The cache never holds more than its budget, evicts the least recently used entry first, and keeps |
| 809 | /// none that is larger than the budget alone. |
| 810 | #[test] |
| 811 | fn the_shape_cache_holds_its_budget_and_evicts_the_least_recently_used() -> Outcome<()> { |
| 812 | let book = res!(FontBook::embedded()); |
| 813 | let face = res!(book.select("libertinus serif", FaceVariant::default()).ok_or_else(|| err!("no Libertinus"; Test, Missing))); |
| 814 | let spec = ShapeSpec { remove_ignorables: true, ..ShapeSpec::default() }; |
| 815 | // Six letters and no ligature or kern pair between them, so every word costs the same. |
| 816 | let words = ["abcdef", "ghijkl", "mnopqr", "stuvwx", "yzabcd", "efghij"]; |
| 817 | |
| 818 | // What one entry costs: the cache's own count after one call. |
| 819 | res!(book.set_shape_budget(1 << 20)); |
| 820 | res!(book.shape(face, words[0], ShapeDir::Ltr, &spec)); |
| 821 | let one = res!(book.shape_stats()).bytes; |
| 822 | assert!(one > 0); |
| 823 | |
| 824 | // A budget for exactly three. |
| 825 | let book = res!(FontBook::embedded()); |
| 826 | res!(book.set_shape_budget(3 * one)); |
| 827 | let check = |label: &str| -> Outcome<oxedyne_fe2o3_austenite::fonts::ShapeStats> { |
| 828 | let s = res!(book.shape_stats()); |
| 829 | assert!(s.bytes <= s.budget, "{}: the cache holds {} bytes over a budget of {}", label, s.bytes, s.budget); |
| 830 | Ok(s) |
| 831 | }; |
| 832 | for w in &words[..3] { |
| 833 | res!(book.shape(face, w, ShapeDir::Ltr, &spec)); |
| 834 | res!(check(w)); |
| 835 | } |
| 836 | let s = res!(check("three words")); |
| 837 | assert_eq!((s.entries, s.hits, s.misses, s.evictions), (3, 0, 3, 0)); |
| 838 | |
| 839 | // A is used again, so B is now the least recently used. |
| 840 | res!(book.shape(face, words[0], ShapeDir::Ltr, &spec)); |
| 841 | assert_eq!(res!(check("A used again")).hits, 1); |
| 842 | // D arrives and B goes. |
| 843 | res!(book.shape(face, words[3], ShapeDir::Ltr, &spec)); |
| 844 | let s = res!(check("D arrives")); |
| 845 | assert_eq!((s.entries, s.evictions), (3, 1), "one entry should have gone"); |
| 846 | let (hits, misses) = (s.hits, s.misses); |
| 847 | res!(book.shape(face, words[0], ShapeDir::Ltr, &spec)); |
| 848 | res!(book.shape(face, words[2], ShapeDir::Ltr, &spec)); |
| 849 | let s = res!(check("A and C asked for")); |
| 850 | assert_eq!((s.hits, s.misses), (hits + 2, misses), "A and C should both have been kept"); |
| 851 | res!(book.shape(face, words[1], ShapeDir::Ltr, &spec)); |
| 852 | let s = res!(check("B asked for")); |
| 853 | assert_eq!(s.misses, misses + 1, "B should have been evicted, being the least recently used"); |
| 854 | |
| 855 | // A long run of words never takes the cache past its budget. |
| 856 | for i in 0..200 { |
| 857 | res!(book.shape(face, &fmt!("word{:03}", i), ShapeDir::Ltr, &spec)); |
| 858 | res!(check("a long run")); |
| 859 | } |
| 860 | assert!(res!(book.shape_stats()).evictions >= 190); |
| 861 | |
| 862 | // A result bigger than the whole budget is not kept, and the budget holds. |
| 863 | let tiny = res!(FontBook::embedded()); |
| 864 | res!(tiny.set_shape_budget(16)); |
| 865 | let glyphs = res!(tiny.shape(face, "far larger than sixteen bytes", ShapeDir::Ltr, &spec)); |
| 866 | assert!(!glyphs.is_empty()); |
| 867 | let s = res!(tiny.shape_stats()); |
| 868 | assert_eq!((s.entries, s.bytes), (0, 0), "an entry over the budget was kept"); |
| 869 | |
| 870 | // Lowering the budget evicts down to it at once. |
| 871 | res!(book.set_shape_budget(one)); |
| 872 | let s = res!(book.shape_stats()); |
| 873 | assert!(s.bytes <= one && s.entries <= 1, "{} entries, {} bytes after the budget fell to {}", s.entries, s.bytes, one); |
| 874 | Ok(()) |
| 875 | } |
| 876 | |
| 877 | /// The store's budget reaches the book it builds, and a paragraph set again is shaped from the cache. |
| 878 | #[test] |
| 879 | fn a_paragraph_set_again_is_shaped_from_the_cache() -> Outcome<()> { |
| 880 | let (mut engine, pars) = res!(paragraphs("Words that shape once and are then served again from the cache, #lorem(30)\n")); |
| 881 | res!(engine.fonts.set_shape_budget(1 << 20)); |
| 882 | let p = &pars[0]; |
| 883 | res!(layout_par(&mut engine, &p.content, &p.styles, region(200.0), ParSituation::First)); |
| 884 | let book = res!(engine.fonts.book()); |
| 885 | let first = res!(book.shape_stats()); |
| 886 | assert_eq!(first.budget, 1 << 20, "the store's budget did not reach the book"); |
| 887 | assert!(first.entries > 0 && first.misses > 0); |
| 888 | res!(layout_par(&mut engine, &p.content, &p.styles, region(200.0), ParSituation::First)); |
| 889 | let second = res!(book.shape_stats()); |
| 890 | assert_eq!(second.misses, first.misses, "the same paragraph, set again, was shaped again"); |
| 891 | assert!(second.hits > first.hits, "the second setting asked the cache for nothing"); |
| 892 | Ok(()) |
| 893 | } |
| 894 | |
| 895 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 896 | // │ A MISSING FAMILY │ |
| 897 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 898 | |
| 899 | /// Typst's warnings as `(message, line, column)`, in the order it prints them. Typst prints its columns |
| 900 | /// from zero, and Austenite's diagnostics from one, as the gate's do; the column here is from one. |
| 901 | fn typst_warnings(bin: &str, name: &str, src: &str) -> Outcome<Vec<(String, usize, usize)>> { |
| 902 | let dir = res!(work("fonts")); |
| 903 | let typ = dir.join(fmt!("{}.typ", name)); |
| 904 | let pdf = dir.join(fmt!("{}.pdf", name)); |
| 905 | res!(std::fs::write(&typ, src).map_err(|e| err!(e, "write"; IO))); |
| 906 | let capped = Command::new("systemd-run").arg("--version").output().map(|o| o.status.success()).unwrap_or(false); |
| 907 | let mut cmd = if capped { |
| 908 | let mut c = Command::new("systemd-run"); |
| 909 | c.args(["--user", "--scope", "--quiet", "-p", "MemoryMax=3G", "--slice=claude-rc.slice", bin]); |
| 910 | c |
| 911 | } else { |
| 912 | Command::new(bin) |
| 913 | }; |
| 914 | cmd.args(["compile", "--ignore-system-fonts"]).arg(&typ).arg(&pdf).current_dir(&dir); |
| 915 | let out = res!(cmd.output().map_err(|e| err!(e, "Could not run typst."; IO))); |
| 916 | let stderr = String::from_utf8_lossy(&out.stderr).to_string(); |
| 917 | let mut found = Vec::new(); |
| 918 | let mut message: Option<String> = None; |
| 919 | for line in stderr.lines() { |
| 920 | if let Some(m) = line.strip_prefix("warning: ") { |
| 921 | message = Some(m.to_string()); |
| 922 | } else if let (Some(m), Some(at)) = (&message, line.split("┌─ ").nth(1)) { |
| 923 | let mut parts = at.rsplit(':'); |
| 924 | let col = parts.next().and_then(|c| c.trim().parse::<usize>().ok()); |
| 925 | let ln = parts.next().and_then(|c| c.trim().parse::<usize>().ok()); |
| 926 | if let (Some(l), Some(c)) = (ln, col) { |
| 927 | found.push((m.clone(), l, c + 1)); |
| 928 | message = None; |
| 929 | } |
| 930 | } |
| 931 | } |
| 932 | Ok(found) |
| 933 | } |
| 934 | |
| 935 | /// Austenite's `missing_font` warnings of a source as `(message, line, column)`. |
| 936 | fn missing_fonts(src: &str) -> Outcome<(Vec<(String, usize, usize)>, Engine)> { |
| 937 | let root = res!(work("fonts")); |
| 938 | let mut world = World::new(root.clone()); |
| 939 | let id = res!(world.add_source(root.join("m.typ"), src.to_string())); |
| 940 | let mut engine = Engine::new(world); |
| 941 | res!(eval_source(&mut engine, id)); |
| 942 | let mut out = Vec::new(); |
| 943 | for d in &engine.diags { |
| 944 | if d.kind != DiagnosticKind::MissingFont { |
| 945 | continue; |
| 946 | } |
| 947 | assert!(!d.is_error(), "a missing family is a warning, not an error: {:?}", d); |
| 948 | let (l, c) = match engine.world.source(d.span.file) { |
| 949 | Some(s) => s.line_col(d.span.start), |
| 950 | None => return Err(err!("A diagnostic names a source the world does not hold."; Test, Missing)), |
| 951 | }; |
| 952 | out.push((d.message.clone(), l as usize, c as usize)); |
| 953 | } |
| 954 | Ok((out, engine)) |
| 955 | } |
| 956 | |
| 957 | /// A family no font declares is Typst's warning, `unknown font family: <name>`, at the `font` argument, |
| 958 | /// once for each family of a list, from a `set` rule and from a `text(..)` call, and once however often |
| 959 | /// the rule is met. Its kind refuses a strict compile. |
| 960 | #[test] |
| 961 | fn a_missing_family_warns_as_typst_warns_with_kind_missing_font() -> Outcome<()> { |
| 962 | let bin = match res!(oracle()) { |
| 963 | Some(b) => b, |
| 964 | None => return Ok(()), |
| 965 | }; |
| 966 | let cases: &[(&str, &str, usize)] = &[ |
| 967 | ("single", "#set text(font: \"Nonexistent Family\")\nHello world.\n", 1), |
| 968 | ("list", "#text(font: (\"Libertinus Serif\", \"Fooo\"))[Hi]\n\n#set text(font: (\"Bar\", \"Baz\"))\nTwo.\n", 3), |
| 969 | ("dict", "#set text(font: (name: \"Quux\", covers: \"latin-in-cjk\"))\nThree.\n", 1), |
| 970 | ("known", "#set text(font: (\"libertinus serif\", \"DejaVu Sans Mono\", \"New Computer Modern Math\"))\nFine.\n", 0), |
| 971 | ("again", "#for i in range(3) [#text(font: \"Nothing Like It\")[x]]\n", 1), |
| 972 | ]; |
| 973 | let mut total = 0usize; |
| 974 | for (name, src, count) in cases { |
| 975 | let want = res!(typst_warnings(&bin, name, src)); |
| 976 | let (got, engine) = res!(missing_fonts(src)); |
| 977 | assert_eq!(want.len(), *count, "{}: typst gave {} warnings, expected {}: {:?}", name, want.len(), count, want); |
| 978 | assert_eq!(got, want, "{}: austenite's warnings differ from typst's", name); |
| 979 | for d in engine.diags.iter().filter(|d| d.kind == DiagnosticKind::MissingFont) { |
| 980 | assert!(d.kind.refuses_strict(), "{}: a missing family should refuse a strict compile", name); |
| 981 | } |
| 982 | total += got.len(); |
| 983 | } |
| 984 | assert_eq!(total, 6); |
| 985 | Ok(()) |
| 986 | } |
| 987 | |
| 988 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 989 | // │ THE LANGUAGE A CHAIN SETS │ |
| 990 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 991 | |
| 992 | /// The language and region a document's text is set in, which the document's own language is read from, |
| 993 | /// follow `set text(lang:, region:)` as Typst reads them in a `context`, a later rule overriding an earlier. |
| 994 | #[test] |
| 995 | fn set_text_lang_reaches_the_language_a_chain_gives() -> Outcome<()> { |
| 996 | let bin = match res!(oracle()) { |
| 997 | Some(b) => b, |
| 998 | None => return Ok(()), |
| 999 | }; |
| 1000 | let cases: &[(&str, &str)] = &[ |
| 1001 | ("none", ""), |
| 1002 | ("lang", "#set text(lang: \"de\")\n"), |
| 1003 | ("both", "#set text(lang: \"de\", region: \"CH\")\n"), |
| 1004 | ("later", "#set text(lang: \"fr\")\n#set text(lang: \"pt\", region: \"BR\")\n"), |
| 1005 | ("region", "#set text(region: \"TW\", lang: \"zh\")\n"), |
| 1006 | ]; |
| 1007 | for (name, rules) in cases { |
| 1008 | let probe = fmt!("{}#context [#metadata((text.lang, text.region)) <m>]\n", rules); |
| 1009 | let dir = res!(work("lang")); |
| 1010 | let path = dir.join(fmt!("{}.typ", name)); |
| 1011 | res!(std::fs::write(&path, &probe).map_err(|e| err!(e, "write"; IO))); |
| 1012 | let p = path.to_string_lossy().to_string(); |
| 1013 | let json = res!(typst(&bin, &["query", "--ignore-system-fonts", "--field", "value", "--one", &p, "<m>"], &dir)); |
| 1014 | let mut want = json_strings(&json); |
| 1015 | let region_null = json.contains("null"); |
| 1016 | let (lang, region) = (want.remove(0), if region_null { None } else { want.pop() }); |
| 1017 | |
| 1018 | let (_, pars) = res!(paragraphs(&fmt!("{}Some text.\n", rules))); |
| 1019 | assert_eq!(pars.len(), 1, "{}: one paragraph", name); |
| 1020 | let (l, r) = res!(locale(&pars[0].styles)); |
| 1021 | assert_eq!((l, r), (lang, region), "{}: the chain's language and region differ from typst's ({})", name, json.trim()); |
| 1022 | } |
| 1023 | Ok(()) |
| 1024 | } |
| 1025 | |
| 1026 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 1027 | // │ SAFE-TO-BREAK SLICING │ |
| 1028 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 1029 | |
| 1030 | /// A line's piece of a run shares the run's glyphs when the shaper marked both its ends safe to break, |
| 1031 | /// and draws exactly what shaping the piece afresh draws; a cut inside a ligature or a kern pair is |
| 1032 | /// shaped afresh, and changing a shared piece copies its own glyphs alone. |
| 1033 | #[test] |
| 1034 | fn a_cut_run_shares_its_glyphs_at_safe_breaks_and_reshapes_inside_a_ligature() -> Outcome<()> { |
| 1035 | let book = res!(FontBook::embedded()); |
| 1036 | let styles = text_chain(10.0, "normal", "regular"); |
| 1037 | let tp = Arc::new(res!(props(&styles))); |
| 1038 | let text = "office affinity waffle"; |
| 1039 | let run = res!(shape_run(&book, 0, text, &tp, false)); |
| 1040 | |
| 1041 | // A kern pair is where the shaper says no: the flag reaches the run. |
| 1042 | let kerned = res!(shape_run(&book, 0, "AVAV", &tp, false)); |
| 1043 | assert!(kerned.all().iter().any(|g| !g.safe_to_break), "no glyph of a kerned AVAV is unsafe to break"); |
| 1044 | let half = res!(kerned.reshape(&book, 0, 2)); |
| 1045 | assert!(!Arc::ptr_eq(&half.glyphs, &kerned.glyphs), "a cut inside a kern pair shared the pair"); |
| 1046 | |
| 1047 | // At a word boundary: the same list, and the same glyphs a fresh shaping gives. |
| 1048 | let piece = res!(run.reshape(&book, 7, 22)); |
| 1049 | assert!(Arc::ptr_eq(&piece.glyphs, &run.glyphs), "a cut at a space was shaped afresh"); |
| 1050 | let fresh = res!(shape_run(&book, 7, &text[7..], &tp, false)); |
| 1051 | let ids = |r: &oxedyne_fe2o3_austenite::flow::text::ShapedRun| -> Vec<(u32, u64, (usize, usize))> { |
| 1052 | r.kept().iter().map(|g| (g.id, g.x_advance.to_bits(), g.range)).collect() |
| 1053 | }; |
| 1054 | assert_eq!(ids(&piece), ids(&fresh)); |
| 1055 | assert_eq!(piece.width().to_bits(), fresh.width().to_bits()); |
| 1056 | |
| 1057 | // Inside "ffi": shaped afresh, so the line ends in an "f" and not half a ligature. |
| 1058 | let cut = res!(run.reshape(&book, 0, 2)); |
| 1059 | assert!(!Arc::ptr_eq(&cut.glyphs, &run.glyphs), "a cut inside a ligature shared the ligature"); |
| 1060 | assert_eq!(ids(&cut), ids(&res!(shape_run(&book, 0, &text[..2], &tp, false)))); |
| 1061 | |
| 1062 | // Changing a shared piece copies its span, leaving the run whole. |
| 1063 | let mut owned = piece.clone(); |
| 1064 | if let Some(g) = owned.kept_mut().first_mut() { |
| 1065 | g.x_advance += 1.0; |
| 1066 | } |
| 1067 | assert_eq!(owned.glyphs.len(), piece.all().len()); |
| 1068 | let again = res!(run.reshape(&book, 7, 22)); |
| 1069 | assert_eq!(ids(&again), ids(&piece)); |
| 1070 | Ok(()) |
| 1071 | } |