oxedyne/fe2o3/fe2o3_austenite/tests/syntax.rs
35.7 KiB, 1 run
created by r1870400018:65168, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! U1's syntax tests. Every expectation comes from outside Austenite: the installed `typst` 0.15.1 (what |
| 2 | //! it evaluates markup and maths to, what values code evaluates to, which syntax errors and warnings it |
| 3 | //! reports and where) and the real package corpus in `~/.cache/typst/packages/preview`. A missing |
| 4 | //! `typst` or corpus fails loudly unless `EVAL_ORACLE_SKIP=1` is set. |
| 5 | |
| 6 | use oxedyne_fe2o3_austenite::syntax::ast::{ |
| 7 | self, |
| 8 | AstNode, |
| 9 | Expr, |
| 10 | }; |
| 11 | use oxedyne_fe2o3_austenite::syntax::parser; |
| 12 | use oxedyne_fe2o3_austenite::syntax::{ |
| 13 | FileId, |
| 14 | Span, |
| 15 | SyntaxKind, |
| 16 | SyntaxNode, |
| 17 | }; |
| 18 | |
| 19 | use oxedyne_fe2o3_core::prelude::*; |
| 20 | |
| 21 | use std::fs; |
| 22 | use std::path::{ |
| 23 | Path, |
| 24 | PathBuf, |
| 25 | }; |
| 26 | use std::process::Command; |
| 27 | |
| 28 | const FILE: FileId = FileId(0); |
| 29 | const CASE_SEP: &str = "\n// ---- case ----\n"; |
| 30 | |
| 31 | // ┌───────────────────────────────────────────────────────────────────────────────────────────┐ |
| 32 | // │ Harness │ |
| 33 | // └───────────────────────────────────────────────────────────────────────────────────────────┘ |
| 34 | |
| 35 | fn skip_allowed() -> bool { |
| 36 | std::env::var("EVAL_ORACLE_SKIP").map(|v| v == "1").unwrap_or(false) |
| 37 | } |
| 38 | |
| 39 | /// Is the 0.15 oracle on hand? `false` only when its absence is explicitly allowed. |
| 40 | fn typst_available() -> Outcome<bool> { |
| 41 | match Command::new("typst").arg("--version").output() { |
| 42 | Ok(out) => { |
| 43 | let v = String::from_utf8_lossy(&out.stdout).to_string(); |
| 44 | if !v.starts_with("typst 0.15") { |
| 45 | return Err(err!("The typst oracle is {:?}, not 0.15.x.", v.trim(); Invalid, Input)); |
| 46 | } |
| 47 | Ok(true) |
| 48 | }, |
| 49 | Err(e) => { |
| 50 | if skip_allowed() { |
| 51 | println!("[syntax] typst oracle absent, skipped by EVAL_ORACLE_SKIP=1: {}", e); |
| 52 | Ok(false) |
| 53 | } else { |
| 54 | Err(err!("The typst oracle is not on PATH ({}); set EVAL_ORACLE_SKIP=1 to skip.", e; |
| 55 | Missing, Input)) |
| 56 | } |
| 57 | }, |
| 58 | } |
| 59 | } |
| 60 | |
| 61 | fn fixture(name: &str) -> Outcome<Vec<String>> { |
| 62 | let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/eval/syntax").join(name); |
| 63 | let text = res!(fs::read_to_string(&path).map_err(|e| err!("Reading {:?}: {}", path, e; IO, File))); |
| 64 | let text = text.strip_suffix('\n').unwrap_or(&text).to_string(); |
| 65 | Ok(text.split(CASE_SEP).map(|s| s.to_string()).collect()) |
| 66 | } |
| 67 | |
| 68 | fn work_dir(name: &str) -> Outcome<PathBuf> { |
| 69 | let dir = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join("syntax_oracle").join(name); |
| 70 | let _ = fs::remove_dir_all(&dir); |
| 71 | res!(fs::create_dir_all(&dir).map_err(|e| err!("Creating {:?}: {}", dir, e; IO, File))); |
| 72 | Ok(dir) |
| 73 | } |
| 74 | |
| 75 | fn write(path: &Path, text: &str) -> Outcome<()> { |
| 76 | res!(fs::write(path, text).map_err(|e| err!("Writing {:?}: {}", path, e; IO, File))); |
| 77 | Ok(()) |
| 78 | } |
| 79 | |
| 80 | /// `typst eval 'query(<p>).map(it => it.value)'` over `main.typ` in `dir`, as JSON. |
| 81 | fn typst_probe_values(dir: &Path) -> Outcome<Json> { |
| 82 | let out = res!(Command::new("typst") |
| 83 | .args(["eval", "query(<p>).map(it => it.value)", "--in", "main.typ", "--root"]) |
| 84 | .arg(dir) |
| 85 | .current_dir(dir) |
| 86 | .output() |
| 87 | .map_err(|e| err!("Running typst eval: {}", e; IO))); |
| 88 | if !out.status.success() { |
| 89 | return Err(err!("typst eval failed: {}", String::from_utf8_lossy(&out.stderr); Invalid, Input)); |
| 90 | } |
| 91 | let text = String::from_utf8_lossy(&out.stdout).to_string(); |
| 92 | Json::parse(&text) |
| 93 | } |
| 94 | |
| 95 | /// A diagnostic as `typst compile` prints it: severity, 1-based line, 0-based column, message and the |
| 96 | /// hints that carry no span of their own. |
| 97 | #[derive(Clone, Debug, PartialEq)] |
| 98 | struct Diag { |
| 99 | error: bool, |
| 100 | line: usize, |
| 101 | col: usize, |
| 102 | message: String, |
| 103 | hints: Vec<String>, |
| 104 | } |
| 105 | |
| 106 | fn typst_diagnostics(dir: &Path, file: &str) -> Outcome<Vec<Diag>> { |
| 107 | let out = res!(Command::new("typst") |
| 108 | .args(["compile", "--root"]) |
| 109 | .arg(dir) |
| 110 | .arg(file) |
| 111 | .arg(format!("{}.pdf", file)) |
| 112 | .current_dir(dir) |
| 113 | .output() |
| 114 | .map_err(|e| err!("Running typst compile: {}", e; IO))); |
| 115 | let text = String::from_utf8_lossy(&out.stderr).to_string(); |
| 116 | let mut diags: Vec<Diag> = Vec::new(); |
| 117 | for line in text.lines() { |
| 118 | let t = line.trim_start(); |
| 119 | if let Some(m) = line.strip_prefix("error: ") { |
| 120 | diags.push(Diag { error: true, line: 0, col: 0, message: m.to_string(), hints: Vec::new() }); |
| 121 | } else if let Some(m) = line.strip_prefix("warning: ") { |
| 122 | diags.push(Diag { error: false, line: 0, col: 0, message: m.to_string(), hints: Vec::new() }); |
| 123 | } else if let Some(pos) = t.strip_prefix("┌─ ") { |
| 124 | if let Some(d) = diags.last_mut() { |
| 125 | if d.line == 0 { |
| 126 | let mut parts = pos.rsplit(':'); |
| 127 | let col = parts.next().and_then(|c| c.trim().parse().ok()).unwrap_or(0); |
| 128 | let ln = parts.next().and_then(|l| l.trim().parse().ok()).unwrap_or(0); |
| 129 | d.line = ln; |
| 130 | d.col = col; |
| 131 | } |
| 132 | } |
| 133 | } else if let Some(h) = t.strip_prefix("= hint: ") { |
| 134 | if let Some(d) = diags.last_mut() { |
| 135 | d.hints.push(h.to_string()); |
| 136 | } |
| 137 | } |
| 138 | } |
| 139 | Ok(diags) |
| 140 | } |
| 141 | |
| 142 | fn line_col(text: &str, offset: u32) -> (usize, usize) { |
| 143 | let offset = (offset as usize).min(text.len()); |
| 144 | let before = text.get(..offset).unwrap_or(""); |
| 145 | let line = before.matches('\n').count() + 1; |
| 146 | let col = match before.rfind('\n') { |
| 147 | Some(i) => before.get(i + 1..).unwrap_or("").chars().count(), |
| 148 | None => before.chars().count(), |
| 149 | }; |
| 150 | (line, col) |
| 151 | } |
| 152 | |
| 153 | fn our_diagnostics(text: &str, root: &SyntaxNode) -> Vec<Diag> { |
| 154 | let mut out = Vec::new(); |
| 155 | for (span, e) in root.errors() { |
| 156 | let (line, col) = line_col(text, span.start); |
| 157 | out.push(Diag { error: true, line, col, message: e.message.clone(), hints: e.hints.clone() }); |
| 158 | } |
| 159 | for (span, w) in root.warnings() { |
| 160 | let (line, col) = line_col(text, span.start); |
| 161 | out.push(Diag { error: false, line, col, message: w.message.clone(), hints: w.hints.clone() }); |
| 162 | } |
| 163 | out |
| 164 | } |
| 165 | |
| 166 | // Every node's span is its exact slice of the source, and the leaves rebuild the source. |
| 167 | fn assert_lossless(text: &str, root: &SyntaxNode, what: &str) { |
| 168 | assert_eq!(root.full_text(), text, "{}: the tree does not rebuild its source", what); |
| 169 | assert_eq!(root.span(), Span::new(FILE, 0, text.len() as u32), "{}: the root span", what); |
| 170 | fn walk(text: &str, n: &SyntaxNode, what: &str) { |
| 171 | let slice = text.get(n.span().start as usize..n.span().end as usize); |
| 172 | assert_eq!(slice, Some(n.full_text().as_str()), "{}: span of {:?} is not its text", what, n.kind()); |
| 173 | for c in n.children() { |
| 174 | walk(text, c, what); |
| 175 | } |
| 176 | } |
| 177 | walk(text, root, what); |
| 178 | } |
| 179 | |
| 180 | // ┌───────────────────────────────────────────────────────────────────────────────────────────┐ |
| 181 | // │ A small JSON reader, for the oracle's output │ |
| 182 | // └───────────────────────────────────────────────────────────────────────────────────────────┘ |
| 183 | |
| 184 | #[derive(Clone, Debug)] |
| 185 | enum Json { |
| 186 | Null, |
| 187 | Bool(bool), |
| 188 | Int(i64), |
| 189 | Float(f64), |
| 190 | Str(String), |
| 191 | Arr(Vec<Json>), |
| 192 | Obj(Vec<(String, Json)>), |
| 193 | } |
| 194 | |
| 195 | impl PartialEq for Json { |
| 196 | fn eq(&self, other: &Self) -> bool { |
| 197 | match (self, other) { |
| 198 | (Json::Null, Json::Null) => true, |
| 199 | (Json::Bool(a), Json::Bool(b)) => a == b, |
| 200 | (Json::Int(a), Json::Int(b)) => a == b, |
| 201 | (Json::Float(a), Json::Float(b)) => (a - b).abs() <= 1e-9 * a.abs().max(1.0), |
| 202 | (Json::Str(a), Json::Str(b)) => a == b, |
| 203 | (Json::Arr(a), Json::Arr(b)) => a == b, |
| 204 | // Element fields compare as sets; dictionary order is checked by `Json::keys_in_order`. |
| 205 | (Json::Obj(a), Json::Obj(b)) => a.len() == b.len() |
| 206 | && a.iter().all(|(k, v)| b.iter().any(|(k2, v2)| k == k2 && v == v2)), |
| 207 | _ => false, |
| 208 | } |
| 209 | } |
| 210 | } |
| 211 | |
| 212 | impl Json { |
| 213 | fn parse(s: &str) -> Outcome<Json> { |
| 214 | let mut p = JsonParser { s: s.as_bytes(), i: 0 }; |
| 215 | let v = res!(p.value()); |
| 216 | p.ws(); |
| 217 | if p.i != p.s.len() { |
| 218 | return Err(err!("Trailing JSON at byte {}.", p.i; Invalid, Input)); |
| 219 | } |
| 220 | Ok(v) |
| 221 | } |
| 222 | |
| 223 | fn obj(pairs: Vec<(&str, Json)>) -> Json { |
| 224 | Json::Obj(pairs.into_iter().map(|(k, v)| (k.to_string(), v)).collect()) |
| 225 | } |
| 226 | |
| 227 | fn elem(func: &str, mut fields: Vec<(&str, Json)>) -> Json { |
| 228 | fields.insert(0, ("func", Json::Str(func.to_string()))); |
| 229 | Json::obj(fields) |
| 230 | } |
| 231 | |
| 232 | fn str(s: &str) -> Json { Json::Str(s.to_string()) } |
| 233 | |
| 234 | fn get(&self, key: &str) -> Option<&Json> { |
| 235 | match self { |
| 236 | Json::Obj(pairs) => pairs.iter().find(|(k, _)| k == key).map(|(_, v)| v), |
| 237 | _ => None, |
| 238 | } |
| 239 | } |
| 240 | |
| 241 | fn keys_in_order(&self, other: &Json) -> bool { |
| 242 | match (self, other) { |
| 243 | (Json::Obj(a), Json::Obj(b)) => a.iter().map(|(k, _)| k).eq(b.iter().map(|(k, _)| k)) |
| 244 | && a.iter().zip(b.iter()).all(|((_, x), (_, y))| x.keys_in_order(y)), |
| 245 | (Json::Arr(a), Json::Arr(b)) => a.iter().zip(b.iter()).all(|(x, y)| x.keys_in_order(y)), |
| 246 | _ => true, |
| 247 | } |
| 248 | } |
| 249 | } |
| 250 | |
| 251 | struct JsonParser<'a> { |
| 252 | s: &'a [u8], |
| 253 | i: usize, |
| 254 | } |
| 255 | |
| 256 | impl JsonParser<'_> { |
| 257 | fn ws(&mut self) { |
| 258 | while self.i < self.s.len() && matches!(self.s[self.i], b' ' | b'\n' | b'\r' | b'\t') { |
| 259 | self.i += 1; |
| 260 | } |
| 261 | } |
| 262 | |
| 263 | fn value(&mut self) -> Outcome<Json> { |
| 264 | self.ws(); |
| 265 | match self.s.get(self.i) { |
| 266 | Some(b'n') => { self.i += 4; Ok(Json::Null) }, |
| 267 | Some(b't') => { self.i += 4; Ok(Json::Bool(true)) }, |
| 268 | Some(b'f') => { self.i += 5; Ok(Json::Bool(false)) }, |
| 269 | Some(b'"') => Ok(Json::Str(res!(self.string()))), |
| 270 | Some(b'[') => { |
| 271 | self.i += 1; |
| 272 | let mut v = Vec::new(); |
| 273 | loop { |
| 274 | self.ws(); |
| 275 | if self.s.get(self.i) == Some(&b']') { |
| 276 | self.i += 1; |
| 277 | break; |
| 278 | } |
| 279 | v.push(res!(self.value())); |
| 280 | self.ws(); |
| 281 | if self.s.get(self.i) == Some(&b',') { |
| 282 | self.i += 1; |
| 283 | } |
| 284 | } |
| 285 | Ok(Json::Arr(v)) |
| 286 | }, |
| 287 | Some(b'{') => { |
| 288 | self.i += 1; |
| 289 | let mut v = Vec::new(); |
| 290 | loop { |
| 291 | self.ws(); |
| 292 | if self.s.get(self.i) == Some(&b'}') { |
| 293 | self.i += 1; |
| 294 | break; |
| 295 | } |
| 296 | let k = res!(self.string()); |
| 297 | self.ws(); |
| 298 | self.i += 1; // colon |
| 299 | let val = res!(self.value()); |
| 300 | v.push((k, val)); |
| 301 | self.ws(); |
| 302 | if self.s.get(self.i) == Some(&b',') { |
| 303 | self.i += 1; |
| 304 | } |
| 305 | } |
| 306 | Ok(Json::Obj(v)) |
| 307 | }, |
| 308 | Some(_) => { |
| 309 | let start = self.i; |
| 310 | while self.i < self.s.len() && matches!(self.s[self.i], b'-' | b'+' | b'.' | b'e' | b'E' | b'0'..=b'9') { |
| 311 | self.i += 1; |
| 312 | } |
| 313 | let t = String::from_utf8_lossy(&self.s[start..self.i]).to_string(); |
| 314 | if t.contains(['.', 'e', 'E']) { |
| 315 | let f = res!(t.parse::<f64>().map_err(|e| err!("Bad float {:?}: {}", t, e; Invalid, Input))); |
| 316 | Ok(Json::Float(f)) |
| 317 | } else { |
| 318 | let n = res!(t.parse::<i64>().map_err(|e| err!("Bad int {:?}: {}", t, e; Invalid, Input))); |
| 319 | Ok(Json::Int(n)) |
| 320 | } |
| 321 | }, |
| 322 | None => Err(err!("Unexpected end of JSON."; Invalid, Input)), |
| 323 | } |
| 324 | } |
| 325 | |
| 326 | fn string(&mut self) -> Outcome<String> { |
| 327 | self.i += 1; |
| 328 | let mut out: Vec<u8> = Vec::new(); |
| 329 | while let Some(&b) = self.s.get(self.i) { |
| 330 | self.i += 1; |
| 331 | match b { |
| 332 | b'"' => return Ok(String::from_utf8_lossy(&out).to_string()), |
| 333 | b'\\' => { |
| 334 | let e = self.s.get(self.i).copied().unwrap_or(b'\\'); |
| 335 | self.i += 1; |
| 336 | match e { |
| 337 | b'n' => out.push(b'\n'), |
| 338 | b't' => out.push(b'\t'), |
| 339 | b'r' => out.push(b'\r'), |
| 340 | b'u' => { |
| 341 | let hex = String::from_utf8_lossy(&self.s[self.i..self.i + 4]).to_string(); |
| 342 | self.i += 4; |
| 343 | let mut cp = u32::from_str_radix(&hex, 16).unwrap_or(0xFFFD); |
| 344 | // A surrogate pair. |
| 345 | if (0xD800..0xDC00).contains(&cp) && self.s.get(self.i) == Some(&b'\\') { |
| 346 | let lo = String::from_utf8_lossy(&self.s[self.i + 2..self.i + 6]).to_string(); |
| 347 | self.i += 6; |
| 348 | let lo = u32::from_str_radix(&lo, 16).unwrap_or(0xDC00); |
| 349 | cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00); |
| 350 | } |
| 351 | let c = char::from_u32(cp).unwrap_or('\u{FFFD}'); |
| 352 | let mut buf = [0u8; 4]; |
| 353 | out.extend_from_slice(c.encode_utf8(&mut buf).as_bytes()); |
| 354 | }, |
| 355 | other => out.push(other), |
| 356 | } |
| 357 | }, |
| 358 | _ => out.push(b), |
| 359 | } |
| 360 | } |
| 361 | Err(err!("Unclosed JSON string."; Invalid, Input)) |
| 362 | } |
| 363 | } |
| 364 | |
| 365 | // ┌───────────────────────────────────────────────────────────────────────────────────────────┐ |
| 366 | // │ Projection: the tree read the way typst-eval reads markup and maths │ |
| 367 | // └───────────────────────────────────────────────────────────────────────────────────────────┘ |
| 368 | |
| 369 | // Only what syntax alone decides: the constructs the fixtures use. Anything else is a fixture error. |
| 370 | fn project_markup(m: ast::Markup) -> Outcome<Json> { |
| 371 | let mut seq: Vec<Json> = Vec::new(); |
| 372 | for e in m.exprs() { |
| 373 | match e { |
| 374 | Expr::Label(l) => { |
| 375 | let target = seq.iter_mut().rev().find(|j| { |
| 376 | !matches!(j.get("func"), Some(Json::Str(f)) if f == "space" || f == "parbreak") |
| 377 | }); |
| 378 | if let Some(Json::Obj(pairs)) = target { |
| 379 | pairs.retain(|(k, _)| k != "label"); |
| 380 | pairs.push(("label".to_string(), Json::Str(format!("<{}>", l.get())))); |
| 381 | } |
| 382 | }, |
| 383 | other => seq.push(res!(project_expr(other))), |
| 384 | } |
| 385 | } |
| 386 | Ok(sequence(seq)) |
| 387 | } |
| 388 | |
| 389 | fn sequence(mut seq: Vec<Json>) -> Json { |
| 390 | if seq.len() == 1 { |
| 391 | return seq.remove(0); |
| 392 | } |
| 393 | Json::elem("sequence", vec![("children", Json::Arr(seq))]) |
| 394 | } |
| 395 | |
| 396 | // `a + b` on content: sequences are spliced, one level. |
| 397 | fn concat(parts: Vec<Json>) -> Json { |
| 398 | let mut out = Vec::new(); |
| 399 | for p in parts { |
| 400 | let is_seq = matches!(p.get("func"), Some(Json::Str(f)) if f == "sequence"); |
| 401 | match (is_seq, p.get("children")) { |
| 402 | (true, Some(Json::Arr(c))) => out.extend(c.iter().cloned()), |
| 403 | _ => out.push(p), |
| 404 | } |
| 405 | } |
| 406 | Json::elem("sequence", vec![("children", Json::Arr(out))]) |
| 407 | } |
| 408 | |
| 409 | fn text(t: &str) -> Json { Json::elem("text", vec![("text", Json::str(t))]) } |
| 410 | fn symbol(t: &str) -> Json { Json::elem("symbol", vec![("text", Json::str(t))]) } |
| 411 | |
| 412 | fn project_expr(e: Expr) -> Outcome<Json> { |
| 413 | Ok(match e { |
| 414 | Expr::Text(t) => text(t.get()), |
| 415 | Expr::Space(_) => Json::elem("space", vec![]), |
| 416 | Expr::Linebreak(_) => Json::elem("linebreak", vec![]), |
| 417 | Expr::Parbreak(_) => Json::elem("parbreak", vec![]), |
| 418 | Expr::Escape(x) => symbol(&x.get().to_string()), |
| 419 | Expr::Shorthand(x) => symbol(&x.get().to_string()), |
| 420 | Expr::SmartQuote(q) => Json::elem("smartquote", vec![("double", Json::Bool(q.double()))]), |
| 421 | Expr::Strong(s) => Json::elem("strong", vec![("body", res!(project_markup(s.body())))]), |
| 422 | Expr::Emph(s) => Json::elem("emph", vec![("body", res!(project_markup(s.body())))]), |
| 423 | Expr::Raw(r) => { |
| 424 | let lines: Vec<&str> = r.lines().map(|l| l.get()).collect(); |
| 425 | let mut f = vec![("text", Json::Str(lines.join("\n"))), ("block", Json::Bool(r.block()))]; |
| 426 | if let Some(lang) = r.lang() { |
| 427 | f.push(("lang", Json::str(lang.get()))); |
| 428 | } |
| 429 | Json::elem("raw", f) |
| 430 | }, |
| 431 | Expr::Link(l) => Json::elem("link", vec![("dest", Json::str(l.get())), ("body", text(l.get()))]), |
| 432 | Expr::Ref(r) => { |
| 433 | let mut f = vec![("target", Json::Str(format!("<{}>", r.target())))]; |
| 434 | if let Some(s) = r.supplement() { |
| 435 | f.push(("supplement", res!(project_markup(s.body())))); |
| 436 | } |
| 437 | Json::elem("ref", f) |
| 438 | }, |
| 439 | Expr::Heading(h) => Json::elem("heading", vec![ |
| 440 | ("depth", Json::Int(h.depth() as i64)), |
| 441 | ("body", res!(project_markup(h.body()))), |
| 442 | ]), |
| 443 | Expr::ListItem(i) => Json::elem("item", vec![("body", res!(project_markup(i.body())))]), |
| 444 | Expr::EnumItem(i) => { |
| 445 | let mut f = Vec::new(); |
| 446 | if let Some(n) = i.number() { |
| 447 | f.push(("number", Json::Int(n as i64))); |
| 448 | } |
| 449 | f.push(("body", res!(project_markup(i.body())))); |
| 450 | Json::elem("item", f) |
| 451 | }, |
| 452 | Expr::TermItem(i) => Json::elem("item", vec![ |
| 453 | ("term", res!(project_markup(i.term()))), |
| 454 | ("description", res!(project_markup(i.description()))), |
| 455 | ]), |
| 456 | Expr::Equation(q) => Json::elem("equation", vec![ |
| 457 | ("block", Json::Bool(q.block())), |
| 458 | ("body", res!(project_math(q.body()))), |
| 459 | ]), |
| 460 | Expr::ContentBlock(b) => res!(project_markup(b.body())), |
| 461 | Expr::Str(s) => text(&s.get()), |
| 462 | Expr::Int(i) => text(&i.get().to_string()), |
| 463 | Expr::None(_) => Json::elem("sequence", vec![("children", Json::Arr(vec![]))]), |
| 464 | |
| 465 | // Maths. |
| 466 | Expr::Math(m) => res!(project_math(m)), |
| 467 | Expr::MathText(t) => match t.get() { |
| 468 | ast::MathTextKind::Number(n) => text(n), |
| 469 | ast::MathTextKind::Grapheme(g) => symbol(g), |
| 470 | }, |
| 471 | Expr::MathShorthand(s) => symbol(&s.get().to_string()), |
| 472 | Expr::MathAlignPoint(_) => Json::elem("align-point", vec![]), |
| 473 | Expr::MathDelimited(d) => Json::elem("lr", vec![("body", concat(vec![ |
| 474 | res!(project_expr(d.open())), |
| 475 | res!(project_math(d.body())), |
| 476 | res!(project_expr(d.close())), |
| 477 | ]))]), |
| 478 | Expr::MathAttach(a) => { |
| 479 | let mut f = vec![("base", res!(project_expr(a.base())))]; |
| 480 | if let Some(t) = a.top() { |
| 481 | f.push(("t", res!(project_expr(t)))); |
| 482 | } |
| 483 | if let Some(p) = a.primes() { |
| 484 | f.push(("tr", Json::elem("primes", vec![("count", Json::Int(p.count() as i64))]))); |
| 485 | } |
| 486 | if let Some(b) = a.bottom() { |
| 487 | f.push(("b", res!(project_expr(b)))); |
| 488 | } |
| 489 | Json::elem("attach", f) |
| 490 | }, |
| 491 | Expr::MathCall(c) => res!(project_call(c)), |
| 492 | Expr::MathPrimes(p) => Json::elem("primes", vec![("count", Json::Int(p.count() as i64))]), |
| 493 | Expr::MathFrac(f) => Json::elem("frac", vec![ |
| 494 | ("num", res!(project_expr(f.num()))), |
| 495 | ("denom", res!(project_expr(f.denom()))), |
| 496 | ]), |
| 497 | Expr::MathRoot(r) => { |
| 498 | let mut f = Vec::new(); |
| 499 | match r.index() { |
| 500 | Some(i) => f.push(("index", text(&i.to_string()))), |
| 501 | None => f.push(("index", Json::Null)), |
| 502 | } |
| 503 | f.push(("radicand", res!(project_expr(r.radicand())))); |
| 504 | Json::elem("root", f) |
| 505 | }, |
| 506 | other => return Err(err!( |
| 507 | "The fixture uses {:?}, which the projection does not read.", other.to_untyped().kind(); |
| 508 | Unimplemented)), |
| 509 | }) |
| 510 | } |
| 511 | |
| 512 | // The two maths functions the fixtures call, `sqrt` and `frac`, to check how arguments split. |
| 513 | fn project_call(c: ast::MathCall) -> Outcome<Json> { |
| 514 | let name = match c.callee() { |
| 515 | ast::MathAccess::MathIdent(i) => i.get().to_string(), |
| 516 | other => return Err(err!("Callee {:?}.", other; Unimplemented)), |
| 517 | }; |
| 518 | let mut args = Vec::new(); |
| 519 | for a in c.args().arg_items() { |
| 520 | match a.arg { |
| 521 | ast::Arg::Pos(e) => args.push(res!(project_expr(e))), |
| 522 | other => return Err(err!("Argument {:?}.", other; Unimplemented)), |
| 523 | } |
| 524 | } |
| 525 | Ok(match (name.as_str(), args.len()) { |
| 526 | ("sqrt", 1) => Json::elem("root", vec![("radicand", args.remove(0))]), |
| 527 | ("frac", 2) => { |
| 528 | let denom = args.remove(1); |
| 529 | Json::elem("frac", vec![("num", args.remove(0)), ("denom", denom)]) |
| 530 | }, |
| 531 | _ => return Err(err!("Call to {} with {} arguments.", name, args.len(); Unimplemented)), |
| 532 | }) |
| 533 | } |
| 534 | |
| 535 | fn project_math(m: ast::Math) -> Outcome<Json> { |
| 536 | let mut seq = Vec::new(); |
| 537 | for e in m.exprs() { |
| 538 | seq.push(res!(project_expr(e))); |
| 539 | } |
| 540 | Ok(sequence(seq)) |
| 541 | } |
| 542 | |
| 543 | // ┌───────────────────────────────────────────────────────────────────────────────────────────┐ |
| 544 | // │ A value reader for code, over the operators and literals the fixtures use │ |
| 545 | // └───────────────────────────────────────────────────────────────────────────────────────────┘ |
| 546 | |
| 547 | fn value(e: Expr) -> Outcome<Json> { |
| 548 | Ok(match e { |
| 549 | Expr::Int(i) => Json::Int(i.get()), |
| 550 | Expr::Float(f) => Json::Float(f.get()), |
| 551 | Expr::Bool(b) => Json::Bool(b.get()), |
| 552 | Expr::None(_) => Json::Null, |
| 553 | Expr::Str(s) => Json::Str(s.get()), |
| 554 | Expr::Parenthesized(p) => res!(value(p.expr())), |
| 555 | Expr::Array(a) => { |
| 556 | let mut v = Vec::new(); |
| 557 | for item in a.items() { |
| 558 | match item { |
| 559 | ast::ArrayItem::Pos(e) => v.push(res!(value(e))), |
| 560 | ast::ArrayItem::Spread(s) => match res!(value(s.expr())) { |
| 561 | Json::Arr(more) => v.extend(more), |
| 562 | other => return Err(err!("Spread of {:?}.", other; Invalid)), |
| 563 | }, |
| 564 | } |
| 565 | } |
| 566 | Json::Arr(v) |
| 567 | }, |
| 568 | Expr::Dict(d) => { |
| 569 | let mut v: Vec<(String, Json)> = Vec::new(); |
| 570 | let put = |k: String, x: Json, v: &mut Vec<(String, Json)>| { |
| 571 | v.retain(|(k2, _)| *k2 != k); |
| 572 | v.push((k, x)); |
| 573 | }; |
| 574 | for item in d.items() { |
| 575 | match item { |
| 576 | ast::DictItem::Named(n) => put(n.name().get().to_string(), res!(value(n.expr())), &mut v), |
| 577 | ast::DictItem::Keyed(k) => match res!(value(k.key())) { |
| 578 | Json::Str(key) => put(key, res!(value(k.expr())), &mut v), |
| 579 | other => return Err(err!("Key {:?}.", other; Invalid)), |
| 580 | }, |
| 581 | ast::DictItem::Spread(s) => match res!(value(s.expr())) { |
| 582 | Json::Obj(more) => for (k, x) in more { put(k, x, &mut v); }, |
| 583 | other => return Err(err!("Spread of {:?}.", other; Invalid)), |
| 584 | }, |
| 585 | } |
| 586 | } |
| 587 | Json::Obj(v) |
| 588 | }, |
| 589 | Expr::Unary(u) => { |
| 590 | let x = res!(value(u.expr())); |
| 591 | match (u.op(), x) { |
| 592 | (ast::UnOp::Neg, Json::Int(n)) => Json::Int(-n), |
| 593 | (ast::UnOp::Neg, Json::Float(f)) => Json::Float(-f), |
| 594 | (ast::UnOp::Pos, x) => x, |
| 595 | (ast::UnOp::Not, Json::Bool(b)) => Json::Bool(!b), |
| 596 | (op, x) => return Err(err!("{:?} {:?}.", op, x; Invalid)), |
| 597 | } |
| 598 | }, |
| 599 | Expr::Binary(b) => res!(binary(b)), |
| 600 | other => return Err(err!("The code fixture uses {:?}.", other.to_untyped().kind(); |
| 601 | Unimplemented)), |
| 602 | }) |
| 603 | } |
| 604 | |
| 605 | fn num(j: &Json) -> Option<f64> { |
| 606 | match j { |
| 607 | Json::Int(n) => Some(*n as f64), |
| 608 | Json::Float(f) => Some(*f), |
| 609 | _ => None, |
| 610 | } |
| 611 | } |
| 612 | |
| 613 | fn binary(b: ast::Binary) -> Outcome<Json> { |
| 614 | use ast::BinOp as B; |
| 615 | let op = b.op(); |
| 616 | let l = res!(value(b.lhs())); |
| 617 | // `and` and `or` short-circuit. |
| 618 | if let (B::And | B::Or, Json::Bool(x)) = (op, &l) { |
| 619 | if (op == B::And && !*x) || (op == B::Or && *x) { |
| 620 | return Ok(Json::Bool(*x)); |
| 621 | } |
| 622 | return value(b.rhs()); |
| 623 | } |
| 624 | let r = res!(value(b.rhs())); |
| 625 | Ok(match (op, &l, &r) { |
| 626 | (B::Add, Json::Int(x), Json::Int(y)) => Json::Int(x + y), |
| 627 | (B::Sub, Json::Int(x), Json::Int(y)) => Json::Int(x - y), |
| 628 | (B::Mul, Json::Int(x), Json::Int(y)) => Json::Int(x * y), |
| 629 | (B::Add, Json::Str(x), Json::Str(y)) => Json::Str(format!("{}{}", x, y)), |
| 630 | (B::Add, Json::Arr(x), Json::Arr(y)) => Json::Arr(x.iter().chain(y.iter()).cloned().collect()), |
| 631 | (B::Add, Json::Obj(x), Json::Obj(y)) => { |
| 632 | let mut v = x.clone(); |
| 633 | for (k, val) in y { |
| 634 | v.retain(|(k2, _)| k2 != k); |
| 635 | v.push((k.clone(), val.clone())); |
| 636 | } |
| 637 | Json::Obj(v) |
| 638 | }, |
| 639 | (B::Eq, _, _) => Json::Bool(l == r), |
| 640 | (B::Neq, _, _) => Json::Bool(l != r), |
| 641 | (B::In, _, Json::Arr(a)) => Json::Bool(a.contains(&l)), |
| 642 | (B::NotIn, _, Json::Arr(a)) => Json::Bool(!a.contains(&l)), |
| 643 | (B::Add | B::Sub | B::Mul | B::Div | B::Lt | B::Leq | B::Gt | B::Geq, _, _) => { |
| 644 | match (num(&l), num(&r)) { |
| 645 | (Some(x), Some(y)) => match op { |
| 646 | B::Add => Json::Float(x + y), |
| 647 | B::Sub => Json::Float(x - y), |
| 648 | B::Mul => Json::Float(x * y), |
| 649 | B::Div => Json::Float(x / y), |
| 650 | B::Lt => Json::Bool(x < y), |
| 651 | B::Leq => Json::Bool(x <= y), |
| 652 | B::Gt => Json::Bool(x > y), |
| 653 | _ => Json::Bool(x >= y), |
| 654 | }, |
| 655 | _ => return Err(err!("{:?} on {:?} and {:?}.", op, l, r; Invalid)), |
| 656 | } |
| 657 | }, |
| 658 | _ => return Err(err!("{:?} on {:?} and {:?}.", op, l, r; Invalid)), |
| 659 | }) |
| 660 | } |
| 661 | |
| 662 | // ┌───────────────────────────────────────────────────────────────────────────────────────────┐ |
| 663 | // │ Tests │ |
| 664 | // └───────────────────────────────────────────────────────────────────────────────────────────┘ |
| 665 | |
| 666 | // Markup and maths evaluate, in Typst, to the content the tree's shape implies. |
| 667 | fn content_oracle(fixture_name: &str, dir_name: &str) -> Outcome<()> { |
| 668 | if !res!(typst_available()) { |
| 669 | return Ok(()); |
| 670 | } |
| 671 | let cases = res!(fixture(fixture_name)); |
| 672 | assert!(cases.len() >= 20, "{} holds {} cases", fixture_name, cases.len()); |
| 673 | let dir = res!(work_dir(dir_name)); |
| 674 | let mut main = String::new(); |
| 675 | for (i, c) in cases.iter().enumerate() { |
| 676 | res!(write(&dir.join(format!("c{}.typ", i)), c)); |
| 677 | main.push_str(&format!("#metadata(include \"c{}.typ\") <p>\n", i)); |
| 678 | } |
| 679 | res!(write(&dir.join("main.typ"), &main)); |
| 680 | let expected = match res!(typst_probe_values(&dir)) { |
| 681 | Json::Arr(v) => v, |
| 682 | other => return Err(err!("Probe output {:?}.", other; Invalid)), |
| 683 | }; |
| 684 | assert_eq!(expected.len(), cases.len()); |
| 685 | |
| 686 | let mut failures = Vec::new(); |
| 687 | for (i, (case, want)) in cases.iter().zip(expected.iter()).enumerate() { |
| 688 | let root = parser::parse(case, FILE); |
| 689 | assert_lossless(case, &root, &format!("{} case {}", fixture_name, i)); |
| 690 | assert!(root.errors().is_empty(), "{} case {} has syntax errors: {:?}", fixture_name, i, root.errors()); |
| 691 | let markup = match ast::Markup::from_untyped(&root) { |
| 692 | Some(m) => m, |
| 693 | None => return Err(err!("The root is not markup."; Invalid)), |
| 694 | }; |
| 695 | let got = res!(project_markup(markup)); |
| 696 | if &got != want { |
| 697 | failures.push(format!("case {}: {:?}\n typst: {:?}\n ours: {:?}", i, case, want, got)); |
| 698 | } |
| 699 | } |
| 700 | assert!(failures.is_empty(), "{} of {} {} cases differ from typst:\n{}", |
| 701 | failures.len(), cases.len(), fixture_name, failures.join("\n")); |
| 702 | Ok(()) |
| 703 | } |
| 704 | |
| 705 | #[test] |
| 706 | fn markup_evaluates_as_typst_reads_it() -> Outcome<()> { |
| 707 | content_oracle("markup.typ", "markup") |
| 708 | } |
| 709 | |
| 710 | #[test] |
| 711 | fn maths_evaluates_as_typst_reads_it() -> Outcome<()> { |
| 712 | content_oracle("math.typ", "math") |
| 713 | } |
| 714 | |
| 715 | #[test] |
| 716 | fn code_precedence_and_literals_match_typst() -> Outcome<()> { |
| 717 | if !res!(typst_available()) { |
| 718 | return Ok(()); |
| 719 | } |
| 720 | let cases = res!(fixture("code.typ")); |
| 721 | assert!(cases.len() >= 30, "code.typ holds {} cases", cases.len()); |
| 722 | let dir = res!(work_dir("code")); |
| 723 | let mut main = String::new(); |
| 724 | for c in &cases { |
| 725 | main.push_str(&format!("#metadata({{\n{}\n}}) <p>\n", c)); |
| 726 | } |
| 727 | res!(write(&dir.join("main.typ"), &main)); |
| 728 | let expected = match res!(typst_probe_values(&dir)) { |
| 729 | Json::Arr(v) => v, |
| 730 | other => return Err(err!("Probe output {:?}.", other; Invalid)), |
| 731 | }; |
| 732 | assert_eq!(expected.len(), cases.len()); |
| 733 | |
| 734 | let mut failures = Vec::new(); |
| 735 | for (i, (case, want)) in cases.iter().zip(expected.iter()).enumerate() { |
| 736 | let root = parser::parse_code(case, FILE); |
| 737 | assert_lossless(case, &root, &format!("code case {}", i)); |
| 738 | assert!(root.errors().is_empty(), "code case {} has syntax errors: {:?}", i, root.errors()); |
| 739 | let code = match ast::Code::from_untyped(&root) { |
| 740 | Some(c) => c, |
| 741 | None => return Err(err!("The root is not code."; Invalid)), |
| 742 | }; |
| 743 | let exprs: Vec<Expr> = code.exprs().collect(); |
| 744 | assert_eq!(exprs.len(), 1, "code case {} parses to {} expressions", i, exprs.len()); |
| 745 | let got = res!(value(exprs[0])); |
| 746 | if &got != want || !got.keys_in_order(want) { |
| 747 | failures.push(format!("case {}: {:?}\n typst: {:?}\n ours: {:?}", i, case, want, got)); |
| 748 | } |
| 749 | } |
| 750 | assert!(failures.is_empty(), "{} of {} code cases differ from typst:\n{}", |
| 751 | failures.len(), cases.len(), failures.join("\n")); |
| 752 | Ok(()) |
| 753 | } |
| 754 | |
| 755 | fn diagnostics_oracle(fixture_name: &str, dir_name: &str, errors: bool) -> Outcome<usize> { |
| 756 | if !res!(typst_available()) { |
| 757 | return Ok(0); |
| 758 | } |
| 759 | let cases = res!(fixture(fixture_name)); |
| 760 | let dir = res!(work_dir(dir_name)); |
| 761 | let mut failures = Vec::new(); |
| 762 | let mut compared = 0; |
| 763 | for (i, case) in cases.iter().enumerate() { |
| 764 | let name = format!("e{}.typ", i); |
| 765 | res!(write(&dir.join(&name), case)); |
| 766 | let want = res!(typst_diagnostics(&dir, &name)); |
| 767 | let root = parser::parse(case, FILE); |
| 768 | assert_lossless(case, &root, &format!("{} case {}", fixture_name, i)); |
| 769 | let got: Vec<Diag> = our_diagnostics(case, &root).into_iter().filter(|d| d.error == errors).collect(); |
| 770 | let want: Vec<Diag> = want.into_iter().filter(|d| d.error == errors).collect(); |
| 771 | compared += want.len(); |
| 772 | if got != want { |
| 773 | failures.push(format!("case {}: {:?}\n typst: {:?}\n ours: {:?}", i, case, want, got)); |
| 774 | } |
| 775 | } |
| 776 | assert!(failures.is_empty(), "{} of {} {} cases differ from typst:\n{}", |
| 777 | failures.len(), cases.len(), fixture_name, failures.join("\n")); |
| 778 | Ok(compared) |
| 779 | } |
| 780 | |
| 781 | #[test] |
| 782 | fn syntax_errors_match_typst_in_message_position_and_hints() -> Outcome<()> { |
| 783 | let n = res!(diagnostics_oracle("errors.typ", "errors", true)); |
| 784 | assert!(n >= 50 || skip_allowed(), "only {} errors compared", n); |
| 785 | Ok(()) |
| 786 | } |
| 787 | |
| 788 | #[test] |
| 789 | fn syntax_warnings_match_typst() -> Outcome<()> { |
| 790 | let n = res!(diagnostics_oracle("warnings.typ", "warnings", false)); |
| 791 | assert!(n >= 5 || skip_allowed(), "only {} warnings compared", n); |
| 792 | Ok(()) |
| 793 | } |
| 794 | |
| 795 | // Random documents over Typst's significant characters: wherever Austenite finds a syntax error, Typst |
| 796 | // reports exactly the same ones; wherever it finds none, Typst reports no syntax error either. |
| 797 | #[test] |
| 798 | fn random_documents_draw_the_same_syntax_errors_as_typst() -> Outcome<()> { |
| 799 | if !res!(typst_available()) { |
| 800 | return Ok(()); |
| 801 | } |
| 802 | const ALPHABET: &[&str] = &[ |
| 803 | "*", "_", "=", "-", "+", "/", ":", "[", "]", "$", "`", "\"", "'", ".", "~", "<", ">", "@", "#", |
| 804 | "(", ")", "{", "}", ",", ";", " ", " ", "\n", "\n\n", "a", "b", "1", "x", "let", "if", "\\", |
| 805 | "^", "!", "&", "|", "..", "=>", "in", |
| 806 | ]; |
| 807 | const SYNTAX: &[&str] = &[ |
| 808 | "unclosed", "expected", "unexpected", "invalid", "duplicate", "the character", "label cannot", |
| 809 | "automatic links", "only one", "maximum parsing depth", "`&&`", "`||`", "`~=`", |
| 810 | ]; |
| 811 | let dir = res!(work_dir("fuzz")); |
| 812 | // `SYNTAX_FUZZ_N` and `SYNTAX_FUZZ_SEED` widen a local run; the default keeps the suite quick. |
| 813 | let n: usize = std::env::var("SYNTAX_FUZZ_N").ok().and_then(|v| v.parse().ok()).unwrap_or(150); |
| 814 | let mut seed: u64 = std::env::var("SYNTAX_FUZZ_SEED").ok().and_then(|v| v.parse::<u64>().ok()) |
| 815 | .unwrap_or(0x2545_F491_4F6C_DD1D).max(1); |
| 816 | let mut next = move || { |
| 817 | seed ^= seed << 13; |
| 818 | seed ^= seed >> 7; |
| 819 | seed ^= seed << 17; |
| 820 | seed |
| 821 | }; |
| 822 | let mut compared = 0; |
| 823 | let mut failures = Vec::new(); |
| 824 | for i in 0..n { |
| 825 | let len = 3 + (next() % 24) as usize; |
| 826 | let mut doc = String::new(); |
| 827 | for _ in 0..len { |
| 828 | doc.push_str(ALPHABET[(next() % ALPHABET.len() as u64) as usize]); |
| 829 | } |
| 830 | let root = parser::parse(&doc, FILE); |
| 831 | assert_lossless(&doc, &root, &format!("fuzz {}", i)); |
| 832 | let name = format!("f{}.typ", i); |
| 833 | res!(write(&dir.join(&name), &doc)); |
| 834 | let want: Vec<Diag> = res!(typst_diagnostics(&dir, &name)).into_iter().filter(|d| d.error).collect(); |
| 835 | let got: Vec<Diag> = our_diagnostics(&doc, &root).into_iter().filter(|d| d.error).collect(); |
| 836 | if !got.is_empty() { |
| 837 | compared += 1; |
| 838 | if got != want { |
| 839 | failures.push(format!("{:?}\n typst: {:?}\n ours: {:?}", doc, want, got)); |
| 840 | } |
| 841 | } else if let Some(d) = want.iter().find(|d| SYNTAX.iter().any(|s| d.message.starts_with(s))) { |
| 842 | failures.push(format!("{:?}\n typst reports {:?}; ours none", doc, d)); |
| 843 | } |
| 844 | } |
| 845 | assert!(compared >= 30, "only {} random documents had syntax errors", compared); |
| 846 | assert!(failures.is_empty(), "{} random documents differ:\n{}", failures.len(), failures.join("\n")); |
| 847 | Ok(()) |
| 848 | } |
| 849 | |
| 850 | // The contract gate: the preview packages Typst documents lean on parse without a single error node, |
| 851 | // losslessly, with exact spans. |
| 852 | #[test] |
| 853 | fn package_corpus_parses_without_error_nodes() -> Outcome<()> { |
| 854 | let home = std::env::var("HOME").unwrap_or_default(); |
| 855 | let base = PathBuf::from(home).join(".cache/typst/packages/preview"); |
| 856 | let mut files = Vec::new(); |
| 857 | for pkg in ["cetz", "cetz-plot", "fletcher", "tablex", "oxifmt"] { |
| 858 | collect_typ(&base.join(pkg), &mut files); |
| 859 | } |
| 860 | if files.is_empty() { |
| 861 | assert!(skip_allowed(), "no package corpus under {:?}; set EVAL_ORACLE_SKIP=1 to skip", base); |
| 862 | return Ok(()); |
| 863 | } |
| 864 | assert!(files.len() >= 150, "the corpus holds only {} files", files.len()); |
| 865 | let mut bad = Vec::new(); |
| 866 | let mut bytes = 0; |
| 867 | for f in &files { |
| 868 | let text = res!(fs::read_to_string(f).map_err(|e| err!("Reading {:?}: {}", f, e; IO, File))); |
| 869 | bytes += text.len(); |
| 870 | let root = parser::parse(&text, FILE); |
| 871 | assert_lossless(&text, &root, &f.display().to_string()); |
| 872 | let errs = root.errors(); |
| 873 | if let Some((span, e)) = errs.first() { |
| 874 | let (l, c) = line_col(&text, span.start); |
| 875 | bad.push(format!("{}:{}:{}: {} ({} errors)", f.display(), l, c, e.message, errs.len())); |
| 876 | } |
| 877 | } |
| 878 | println!("[syntax] {} package files, {} bytes, parsed", files.len(), bytes); |
| 879 | assert!(bad.is_empty(), "{} package files have error nodes:\n{}", bad.len(), bad.join("\n")); |
| 880 | Ok(()) |
| 881 | } |
| 882 | |
| 883 | fn collect_typ(dir: &Path, out: &mut Vec<PathBuf>) { |
| 884 | if let Ok(rd) = fs::read_dir(dir) { |
| 885 | let mut entries: Vec<PathBuf> = rd.filter_map(|e| e.ok().map(|e| e.path())).collect(); |
| 886 | entries.sort(); |
| 887 | for p in entries { |
| 888 | if p.is_dir() { |
| 889 | collect_typ(&p, out); |
| 890 | } else if p.extension().is_some_and(|x| x == "typ") { |
| 891 | out.push(p); |
| 892 | } |
| 893 | } |
| 894 | } |
| 895 | } |
| 896 | |
| 897 | // The three entry points return the root kinds the evaluator dispatches on, and a malformed input still |
| 898 | // yields a whole, lossless tree. |
| 899 | #[test] |
| 900 | fn entry_points_return_whole_trees() { |
| 901 | for (text, root) in [ |
| 902 | ("a *b* $c$ #d", parser::parse("a *b* $c$ #d", FILE)), |
| 903 | ("let x = 1; x + (2, 3)", parser::parse_code("let x = 1; x + (2, 3)", FILE)), |
| 904 | ("a_1^2 / (b)", parser::parse_math("a_1^2 / (b)", FILE)), |
| 905 | ("#{ ((] $ `", parser::parse("#{ ((] $ `", FILE)), |
| 906 | ] { |
| 907 | assert_lossless(text, &root, text); |
| 908 | } |
| 909 | assert_eq!(parser::parse("x", FILE).kind(), SyntaxKind::Markup); |
| 910 | assert_eq!(parser::parse_code("x", FILE).kind(), SyntaxKind::Code); |
| 911 | assert_eq!(parser::parse_math("x", FILE).kind(), SyntaxKind::Math); |
| 912 | assert!(parser::parse("#{ ((] $ `", FILE).erroneous()); |
| 913 | assert!(!parser::parse("a *b* $c$ #d", FILE).erroneous()); |
| 914 | } |
| 915 | |
| 916 | // Random error-free documents over markup and maths punctuation (no `#`, and letters only as a lone `x` |
| 917 | // so maths never names a variable): Typst evaluates each to the content its tree implies. |
| 918 | #[test] |
| 919 | fn random_documents_evaluate_as_typst_reads_them() -> Outcome<()> { |
| 920 | if !res!(typst_available()) { |
| 921 | return Ok(()); |
| 922 | } |
| 923 | const ALPHABET: &[&str] = &[ |
| 924 | "x ", "x ", "1", "2.5", " ", " ", "\n", "\n\n", "*", "_", "=", "-", "+", "/", ":", "[", "]", "$", |
| 925 | "$", "`", "\"", "'", ".", "~", "\\", "^", "!", "&", "|", "(", ")", "{", "}", ",", ";", "<", ">", |
| 926 | "@x ", "--", "...", "√", "⌈", "⌉", "[|", "|]", "->", "é", |
| 927 | ]; |
| 928 | let n: usize = std::env::var("SYNTAX_FUZZ_N").ok().and_then(|v| v.parse().ok()).unwrap_or(600); |
| 929 | let mut seed: u64 = std::env::var("SYNTAX_FUZZ_SEED").ok().and_then(|v| v.parse::<u64>().ok()) |
| 930 | .unwrap_or(0x9E37_79B9_7F4A_7C15).max(1); |
| 931 | let mut next = move || { |
| 932 | seed ^= seed << 13; |
| 933 | seed ^= seed >> 7; |
| 934 | seed ^= seed << 17; |
| 935 | seed |
| 936 | }; |
| 937 | let dir = res!(work_dir("fuzz_content")); |
| 938 | let mut docs = Vec::new(); |
| 939 | let mut main = String::new(); |
| 940 | for _ in 0..n { |
| 941 | let len = 2 + (next() % 20) as usize; |
| 942 | let mut doc = String::new(); |
| 943 | for _ in 0..len { |
| 944 | doc.push_str(ALPHABET[(next() % ALPHABET.len() as u64) as usize]); |
| 945 | } |
| 946 | let root = parser::parse(&doc, FILE); |
| 947 | if root.erroneous() { |
| 948 | continue; |
| 949 | } |
| 950 | res!(write(&dir.join(format!("c{}.typ", docs.len())), &doc)); |
| 951 | main.push_str(&format!("#metadata(include \"c{}.typ\") <p>\n", docs.len())); |
| 952 | docs.push(doc); |
| 953 | } |
| 954 | assert!(docs.len() >= n / 5, "only {} of {} random documents were error-free", docs.len(), n); |
| 955 | res!(write(&dir.join("main.typ"), &main)); |
| 956 | let expected = match res!(typst_probe_values(&dir)) { |
| 957 | Json::Arr(v) => v, |
| 958 | other => return Err(err!("Probe output {:?}.", other; Invalid)), |
| 959 | }; |
| 960 | assert_eq!(expected.len(), docs.len()); |
| 961 | let mut failures = Vec::new(); |
| 962 | for (doc, want) in docs.iter().zip(expected.iter()) { |
| 963 | let root = parser::parse(doc, FILE); |
| 964 | let markup = match ast::Markup::from_untyped(&root) { |
| 965 | Some(m) => m, |
| 966 | None => return Err(err!("The root is not markup."; Invalid)), |
| 967 | }; |
| 968 | let got = res!(project_markup(markup)); |
| 969 | if &got != want { |
| 970 | failures.push(format!("{:?}\n typst: {:?}\n ours: {:?}", doc, want, got)); |
| 971 | } |
| 972 | } |
| 973 | assert!(failures.is_empty(), "{} of {} random documents differ:\n{}", |
| 974 | failures.len(), docs.len(), failures.join("\n")); |
| 975 | Ok(()) |
| 976 | } |