Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tests/eval_core.rs

29.9 KiB, 5 runs

created by r1870400018:63578, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! U2's evaluator against the `typst` 0.15 oracle. Until U1's parser lands, each fixture pairs Typst code
2//! with the syntax tree the parser yields for it, written as an s-expression; the code goes to
3//! `typst eval`, the tree to Austenite's evaluator, and the two results are compared as JSON (values) or
4//! as the first error message (errors). Nothing expected is checked in: the oracle decides.
5//!
6//! Each case also runs through U1's parser, as `typst eval` itself does -- the code as a string to
7//! `eval_string` in code mode -- so the parser and the evaluator are checked together against the same
8//! oracle, and a hand tree that drifts from what the parser yields cannot hide a fault.
9//!
10//! `EVAL_ORACLE_SKIP=1` skips explicitly; a missing `typst` binary otherwise fails, so absence never
11//! reads as green.
12
13use oxedyne_fe2o3_austenite::diag::DiagnosticKind;
14use oxedyne_fe2o3_austenite::eval::content::{
15 Content,
16 ElemKind,
17};
18use oxedyne_fe2o3_austenite::eval::lib::foundations::repr;
19use oxedyne_fe2o3_austenite::eval::func::Func;
20use oxedyne_fe2o3_austenite::eval::value::{
21 Length,
22 Value,
23};
24use oxedyne_fe2o3_austenite::eval::eval::{
25 eval_string,
26 EvalMode,
27};
28use oxedyne_fe2o3_austenite::eval::scope::Scope;
29use oxedyne_fe2o3_austenite::eval::{
30 eval_source,
31 Engine,
32 World,
33};
34use oxedyne_fe2o3_austenite::syntax::{
35 FileId,
36 Source,
37 Span,
38 SyntaxKind,
39 SyntaxNode,
40};
41
42use oxedyne_fe2o3_core::prelude::*;
43
44use std::path::PathBuf;
45use std::process::Command;
46use std::sync::Arc;
47
48// S-expression trees
49
50enum Tok {
51 Open,
52 Close,
53 Quoted(String), // 'leaf text'
54 StrLit(String), // "a string literal", quotes included
55 Atom(String),
56}
57
58fn lex(src: &str) -> Outcome<Vec<Tok>> {
59 let mut out = Vec::new();
60 let cs: Vec<char> = src.chars().collect();
61 let mut i = 0;
62 while i < cs.len() {
63 let c = cs[i];
64 match c {
65 '(' => { out.push(Tok::Open); i += 1; }
66 ')' => { out.push(Tok::Close); i += 1; }
67 '\'' => {
68 let mut j = i + 1;
69 while j < cs.len() && cs[j] != '\'' {
70 j += 1;
71 }
72 if j >= cs.len() {
73 return Err(err!("Unterminated quoted leaf in {}", src; Input, Invalid));
74 }
75 out.push(Tok::Quoted(cs[i + 1..j].iter().collect()));
76 i = j + 1;
77 }
78 '"' => {
79 let mut j = i + 1;
80 while j < cs.len() && cs[j] != '"' {
81 if cs[j] == '\\' {
82 j += 1;
83 }
84 j += 1;
85 }
86 if j >= cs.len() {
87 return Err(err!("Unterminated string literal in {}", src; Input, Invalid));
88 }
89 out.push(Tok::StrLit(cs[i..=j].iter().collect()));
90 i = j + 1;
91 }
92 c if c.is_whitespace() => i += 1,
93 _ => {
94 let mut j = i;
95 while j < cs.len() && !cs[j].is_whitespace() && cs[j] != '(' && cs[j] != ')' {
96 j += 1;
97 }
98 out.push(Tok::Atom(cs[i..j].iter().collect()));
99 i = j;
100 }
101 }
102 }
103 Ok(out)
104}
105
106fn kind_of(name: &str) -> Option<(SyntaxKind, &'static str)> {
107 use SyntaxKind as K;
108 Some(match name {
109 "Markup" => (K::Markup, ""),
110 "Text" => (K::Text, ""),
111 "Space" => (K::Space, " "),
112 "Linebreak" => (K::Linebreak, "\\"),
113 "Parbreak" => (K::Parbreak, "\n\n"),
114 "Escape" => (K::Escape, ""),
115 "Shorthand" => (K::Shorthand, ""),
116 "Label" => (K::Label, ""),
117 "Hash" => (K::Hash, "#"),
118 "LeftBrace" => (K::LeftBrace, "{"),
119 "RightBrace" => (K::RightBrace, "}"),
120 "LeftBracket" => (K::LeftBracket, "["),
121 "RightBracket" => (K::RightBracket, "]"),
122 "LeftParen" => (K::LeftParen, "("),
123 "RightParen" => (K::RightParen, ")"),
124 "Comma" => (K::Comma, ","),
125 "Semicolon" => (K::Semicolon, ";"),
126 "Colon" => (K::Colon, ":"),
127 "Underscore" => (K::Underscore, "_"),
128 "Plus" => (K::Plus, "+"),
129 "Minus" => (K::Minus, "-"),
130 "Star" => (K::Star, "*"),
131 "Slash" => (K::Slash, "/"),
132 "Dot" => (K::Dot, "."),
133 "Eq" => (K::Eq, "="),
134 "EqEq" => (K::EqEq, "=="),
135 "ExclEq" => (K::ExclEq, "!="),
136 "Lt" => (K::Lt, "<"),
137 "LtEq" => (K::LtEq, "<="),
138 "Gt" => (K::Gt, ">"),
139 "GtEq" => (K::GtEq, ">="),
140 "PlusEq" => (K::PlusEq, "+="),
141 "HyphEq" => (K::HyphEq, "-="),
142 "StarEq" => (K::StarEq, "*="),
143 "SlashEq" => (K::SlashEq, "/="),
144 "Dots" => (K::Dots, ".."),
145 "Arrow" => (K::Arrow, "=>"),
146 "Not" => (K::Not, "not"),
147 "And" => (K::And, "and"),
148 "Or" => (K::Or, "or"),
149 "Let" => (K::Let, "let"),
150 "Set" => (K::Set, "set"),
151 "Show" => (K::Show, "show"),
152 "If" => (K::If, "if"),
153 "Else" => (K::Else, "else"),
154 "For" => (K::For, "for"),
155 "In" => (K::In, "in"),
156 "While" => (K::While, "while"),
157 "Break" => (K::Break, "break"),
158 "Continue" => (K::Continue, "continue"),
159 "Return" => (K::Return, "return"),
160 "Code" => (K::Code, ""),
161 "Strong" => (K::Strong, ""),
162 "Emph" => (K::Emph, ""),
163 "Raw" => (K::Raw, ""),
164 "RawLang" => (K::RawLang, ""),
165 "RawDelim" => (K::RawDelim, ""),
166 "RawTrimmed" => (K::RawTrimmed, ""),
167 "Link" => (K::Link, ""),
168 "Ref" => (K::Ref, ""),
169 "RefMarker" => (K::RefMarker, ""),
170 "Heading" => (K::Heading, ""),
171 "HeadingMarker" => (K::HeadingMarker, ""),
172 "ListItem" => (K::ListItem, ""),
173 "ListMarker" => (K::ListMarker, "-"),
174 "EnumItem" => (K::EnumItem, ""),
175 "EnumMarker" => (K::EnumMarker, ""),
176 "TermItem" => (K::TermItem, ""),
177 "TermMarker" => (K::TermMarker, "/"),
178 "SmartQuote" => (K::SmartQuote, ""),
179 "Equation" => (K::Equation, ""),
180 "Math" => (K::Math, ""),
181 "MathText" => (K::MathText, ""),
182 "MathIdent" => (K::MathIdent, ""),
183 "MathAttach" => (K::MathAttach, ""),
184 "MathPrimes" => (K::MathPrimes, "'"), // the parser lexes a run of primes as one leaf
185 "MathFrac" => (K::MathFrac, ""),
186 "MathDelimited" => (K::MathDelimited, ""),
187 "Dollar" => (K::Dollar, "$"),
188 "Hat" => (K::Hat, "^"),
189 "Prime" => (K::Prime, "'"),
190 "Contextual" => (K::Contextual, ""),
191 "ModuleImport" => (K::ModuleImport, ""),
192 "ImportItems" => (K::ImportItems, ""),
193 "ImportItemPath" => (K::ImportItemPath, ""),
194 "RenamedImportItem" => (K::RenamedImportItem, ""),
195 "Import" => (K::Import, "import"),
196 "As" => (K::As, "as"),
197 "Context" => (K::Context, "context"),
198 "CodeBlock" => (K::CodeBlock, ""),
199 "ContentBlock" => (K::ContentBlock, ""),
200 "Parenthesized" => (K::Parenthesized, ""),
201 "Array" => (K::Array, ""),
202 "Dict" => (K::Dict, ""),
203 "Named" => (K::Named, ""),
204 "Keyed" => (K::Keyed, ""),
205 "Unary" => (K::Unary, ""),
206 "Binary" => (K::Binary, ""),
207 "FieldAccess" => (K::FieldAccess, ""),
208 "FuncCall" => (K::FuncCall, ""),
209 "Args" => (K::Args, ""),
210 "Spread" => (K::Spread, ""),
211 "Closure" => (K::Closure, ""),
212 "Params" => (K::Params, ""),
213 "LetBinding" => (K::LetBinding, ""),
214 "SetRule" => (K::SetRule, ""),
215 "ShowRule" => (K::ShowRule, ""),
216 "Conditional" => (K::Conditional, ""),
217 "WhileLoop" => (K::WhileLoop, ""),
218 "ForLoop" => (K::ForLoop, ""),
219 "LoopBreak" => (K::LoopBreak, ""),
220 "LoopContinue" => (K::LoopContinue, ""),
221 "FuncReturn" => (K::FuncReturn, ""),
222 "Destructuring" => (K::Destructuring, ""),
223 "DestructAssignment" => (K::DestructAssignment, ""),
224 _ => return None,
225 })
226}
227
228const UNITS: &[&str] = &["pt", "mm", "cm", "in", "deg", "rad", "em", "fr", "%"];
229
230struct Builder {
231 offset: u32,
232}
233
234impl Builder {
235 fn leaf(&mut self, kind: SyntaxKind, text: &str) -> SyntaxNode {
236 let start = self.offset;
237 self.offset += text.len() as u32;
238 SyntaxNode::leaf(kind, text, Span::new(FileId(0), start, self.offset))
239 }
240
241 fn atom(&mut self, a: &str) -> Outcome<SyntaxNode> {
242 if let Some((k, text)) = kind_of(a) {
243 return Ok(self.leaf(k, text));
244 }
245 let first = a.chars().next().unwrap_or(' ');
246 let kind = if first.is_ascii_digit() {
247 if a.starts_with("0x") || a.starts_with("0o") || a.starts_with("0b") {
248 SyntaxKind::Int
249 } else if UNITS.iter().any(|u| a.ends_with(u)) {
250 SyntaxKind::Numeric
251 } else if a.contains('.') || a.contains('e') {
252 SyntaxKind::Float
253 } else {
254 SyntaxKind::Int
255 }
256 } else {
257 match a {
258 "true" | "false" => SyntaxKind::Bool,
259 "none" => SyntaxKind::None,
260 "auto" => SyntaxKind::Auto,
261 _ if first.is_ascii_uppercase() => return Err(err!("Unknown kind {}", a; Input, Invalid)),
262 _ => SyntaxKind::Ident,
263 }
264 };
265 Ok(self.leaf(kind, a))
266 }
267
268 fn node(&mut self, toks: &[Tok], i: &mut usize) -> Outcome<SyntaxNode> {
269 let t = match toks.get(*i) {
270 Some(t) => t,
271 None => return Err(err!("Unexpected end of tree"; Input, Invalid)),
272 };
273 *i += 1;
274 match t {
275 Tok::Atom(a) => self.atom(a),
276 Tok::StrLit(s) => Ok(self.leaf(SyntaxKind::Str, s)),
277 Tok::Quoted(_) => Err(err!("A quoted leaf must follow a kind"; Input, Invalid)),
278 Tok::Close => Err(err!("Unbalanced )"; Input, Invalid)),
279 Tok::Open => {
280 let name = match toks.get(*i) {
281 Some(Tok::Atom(a)) => a.clone(),
282 _ => return Err(err!("Expected a kind after ("; Input, Invalid)),
283 };
284 *i += 1;
285 let (kind, _) = res!(kind_of(&name).ok_or_else(|| err!("Unknown kind {}", name; Input, Invalid)));
286 if let (Some(Tok::Quoted(text)), Some(Tok::Close)) = (toks.get(*i), toks.get(*i + 1)) {
287 *i += 2;
288 return Ok(self.leaf(kind, text));
289 }
290 let mut kids = Vec::new();
291 loop {
292 match toks.get(*i) {
293 Some(Tok::Close) => {
294 *i += 1;
295 break;
296 }
297 Some(_) => kids.push(res!(self.node(toks, i))),
298 None => return Err(err!("Unclosed ( in {}", name; Input, Invalid)),
299 }
300 }
301 Ok(SyntaxNode::inner(kind, kids))
302 }
303 }
304 }
305}
306
307// The module `#let result = <expr>`, so the evaluated module's scope holds the value.
308fn module_for(sexpr: &str) -> Outcome<SyntaxNode> {
309 let toks = res!(lex(sexpr));
310 let mut b = Builder { offset: 0 };
311 let hash = b.leaf(SyntaxKind::Hash, "#");
312 let kw = b.leaf(SyntaxKind::Let, "let");
313 let name = b.leaf(SyntaxKind::Ident, "result");
314 let eq = b.leaf(SyntaxKind::Eq, "=");
315 let mut i = 0;
316 let expr = res!(b.node(&toks, &mut i));
317 if i != toks.len() {
318 return Err(err!("Trailing tokens after the tree {}", sexpr; Input, Invalid));
319 }
320 let bind = SyntaxNode::inner(SyntaxKind::LetBinding, vec![kw, name, eq, expr]);
321 Ok(SyntaxNode::inner(SyntaxKind::Markup, vec![hash, bind]))
322}
323
324fn evaluate(sexpr: &str) -> Outcome<(Engine, Outcome<Value>)> {
325 let root = res!(module_for(sexpr));
326 let mut world = World::new(PathBuf::from("/"));
327 world.sources.push(Source {
328 id: FileId(0),
329 path: PathBuf::from("/case.typ"),
330 text: Arc::new(root.full_text()),
331 root,
332 });
333 let mut engine = Engine::new(world);
334 let out = eval_source(&mut engine, FileId(0)).map(|m| m.scope.get("result").cloned().unwrap_or(Value::None));
335 Ok((engine, out))
336}
337
338// The case's code through U1's parser, as `typst eval` takes it: a string evaluated in code mode.
339fn evaluate_parsed(code: &str) -> (Engine, Outcome<Value>) {
340 let mut engine = Engine::new(World::new(PathBuf::from("/")));
341 let out = eval_string(&mut engine, code, EvalMode::Code, Scope::new(), Span::detached());
342 (engine, out)
343}
344
345// Values as `typst eval` serialises them
346
347fn num(f: f64) -> String {
348 // Typst's repr rounds unit values to two decimals and drops trailing zeros.
349 let r = (f * 100.0).round() / 100.0;
350 let s = fmt!("{:.2}", r);
351 let s = s.trim_end_matches('0').trim_end_matches('.').to_string();
352 if s == "-0" { "0".to_string() } else { s }
353}
354
355fn length_repr(l: &Length) -> String {
356 match (l.abs != 0.0, l.em != 0.0) {
357 (true, true) => fmt!("{}pt + {}em", num(l.abs), num(l.em)),
358 (false, true) => fmt!("{}em", num(l.em)),
359 _ => fmt!("{}pt", num(l.abs)),
360 }
361}
362
363fn json_str(s: &str) -> String {
364 let mut o = String::from("\"");
365 for c in s.chars() {
366 match c {
367 '"' => o.push_str("\\\""),
368 '\\' => o.push_str("\\\\"),
369 '\n' => o.push_str("\\n"),
370 '\r' => o.push_str("\\r"),
371 '\t' => o.push_str("\\t"),
372 c if (c as u32) < 0x20 => o.push_str(&fmt!("\\u{:04x}", c as u32)),
373 c => o.push(c),
374 }
375 }
376 o.push('"');
377 o
378}
379
380fn float(f: f64) -> String {
381 if f.fract() == 0.0 && f.abs() < 1e16 { fmt!("{:.1}", f) } else { fmt!("{}", f) }
382}
383
384fn short_repr(v: &Value) -> String {
385 match v {
386 Value::Int(i) => fmt!("{}", i),
387 Value::Str(s) => fmt!("\"{}\"", s),
388 other => to_json(other),
389 }
390}
391
392fn content_json(c: &Content) -> String {
393 let mut parts = Vec::new();
394 match c {
395 Content::Elem(e) => {
396 parts.push(fmt!("\"func\":{}", json_str(e.kind.name())));
397 let mut fields: Vec<_> = e.fields.iter().collect();
398 fields.sort_by_key(|(id, _)| *id);
399 for (id, v) in fields {
400 let name = e.kind.field_spec(*id).map(|s| s.name).unwrap_or("?");
401 // `context`'s closure is internal; Typst's serialisation leaves it out.
402 if name == "func" {
403 continue;
404 }
405 parts.push(fmt!("{}:{}", json_str(name), to_json(v)));
406 }
407 }
408 Content::Sequence(s) => {
409 parts.push("\"func\":\"sequence\"".to_string());
410 let kids: Vec<String> = s.children.iter().map(content_json).collect();
411 parts.push(fmt!("\"children\":[{}]", kids.join(",")));
412 }
413 Content::Styled(s) => {
414 parts.push("\"func\":\"styled\"".to_string());
415 parts.push(fmt!("\"child\":{}", content_json(&s.child)));
416 parts.push("\"styles\":\"styles(..)\"".to_string());
417 }
418 }
419 if let Some(l) = c.label() {
420 parts.push(fmt!("\"label\":\"<{}>\"", l.as_str()));
421 }
422 fmt!("{{{}}}", parts.join(","))
423}
424
425fn to_json(v: &Value) -> String {
426 match v {
427 Value::None => "null".to_string(),
428 Value::Auto => "\"auto\"".to_string(),
429 Value::Bool(b) => fmt!("{}", b),
430 Value::Int(i) => fmt!("{}", i),
431 Value::Float(f) => float(*f),
432 Value::Str(s) => json_str(s),
433 Value::Length(l) => json_str(&length_repr(l)),
434 Value::Ratio(r) => json_str(&fmt!("{}%", num(r.0 * 100.0))),
435 Value::Angle(a) => json_str(&fmt!("{}deg", num(a.0.to_degrees()))),
436 Value::Fraction(f) => json_str(&fmt!("{}fr", num(f.0))),
437 Value::Relative(r) => {
438 let mut s = fmt!("{}%", num(r.rel.0 * 100.0));
439 if r.abs.abs != 0.0 {
440 s.push_str(&fmt!(" + {}pt", num(r.abs.abs)));
441 }
442 if r.abs.em != 0.0 {
443 s.push_str(&fmt!(" + {}em", num(r.abs.em)));
444 }
445 json_str(&s)
446 }
447 Value::Label(l) => json_str(&fmt!("<{}>", l.as_str())),
448 Value::Array(a) => fmt!("[{}]", a.iter().map(to_json).collect::<Vec<_>>().join(",")),
449 Value::Dict(d) => fmt!("{{{}}}", d.iter().map(|(k, v)| fmt!("{}:{}", json_str(k), to_json(v)))
450 .collect::<Vec<_>>().join(",")),
451 Value::Content(c) => content_json(c),
452 Value::Func(f) => json_str(match f {
453 Func::Closure(c) if c.name().is_none() => "(..) => ..",
454 other => other.name().unwrap_or("?"),
455 }),
456 Value::Args(a) => {
457 let items: Vec<String> = a.items.iter().map(|i| match &i.name {
458 Some(n) => fmt!("{}: {}", n, short_repr(&i.value)),
459 None => short_repr(&i.value),
460 }).collect();
461 json_str(&fmt!("arguments({})", items.join(", ")))
462 }
463 other => json_str(&fmt!("<unserialised {}>", other.ty().name())),
464 }
465}
466
467// The oracle
468
469struct Case {
470 code: String,
471 tree: String,
472 dep: Option<String>, // another unit the case needs, from `%%[dep]`
473}
474
475// Is the unit a case depends on present yet? Probed through its own public surface, so a case is
476// enforced from the moment its dependency lands.
477fn available(deps: &str) -> bool {
478 deps.split(',').all(|dep| match dep.trim() {
479 "U3" => repr(&Value::Int(1)) == "1",
480 "U5" => ElemKind::Strong.field_id("body").is_some() && ElemKind::Heading.field_id("depth").is_some(),
481 "U6a" => ElemKind::SmartQuote.field_id("double").is_some(),
482 "U7" => ElemKind::MathFrac.field_id("num").is_some() && ElemKind::MathAttach.field_id("base").is_some(),
483 "U8" => ElemKind::Context.field_id("func").is_some(),
484 "symbol" => ElemKind::Symbol.field_id("text").is_some(),
485 _ => false,
486 })
487}
488
489// JSON, only as far as comparing `typst eval` output needs: an element's fields compare in any order
490// (the schema's order is the owning unit's), a dictionary's in insertion order (Typst's semantics).
491#[derive(Debug, PartialEq)]
492enum Json {
493 Null,
494 Bool(bool),
495 Num(String),
496 Str(String),
497 Arr(Vec<Json>),
498 Obj(Vec<(String, Json)>),
499}
500
501fn parse_json(s: &str) -> Outcome<Json> {
502 let cs: Vec<char> = s.chars().collect();
503 let mut i = 0;
504 let j = res!(json_value(&cs, &mut i));
505 Ok(canon(j))
506}
507
508fn json_ws(cs: &[char], i: &mut usize) {
509 while *i < cs.len() && cs[*i].is_whitespace() {
510 *i += 1;
511 }
512}
513
514fn json_string(cs: &[char], i: &mut usize) -> Outcome<String> {
515 *i += 1;
516 let mut out = String::new();
517 while *i < cs.len() && cs[*i] != '"' {
518 if cs[*i] == '\\' {
519 *i += 1;
520 match cs.get(*i) {
521 Some('n') => out.push('\n'),
522 Some('t') => out.push('\t'),
523 Some('r') => out.push('\r'),
524 Some('u') => {
525 let hex: String = cs[*i + 1..(*i + 5).min(cs.len())].iter().collect();
526 if let Some(c) = u32::from_str_radix(&hex, 16).ok().and_then(char::from_u32) {
527 out.push(c);
528 }
529 *i += 4;
530 }
531 Some(c) => out.push(*c),
532 None => (),
533 }
534 } else {
535 out.push(cs[*i]);
536 }
537 *i += 1;
538 }
539 *i += 1;
540 Ok(out)
541}
542
543fn json_value(cs: &[char], i: &mut usize) -> Outcome<Json> {
544 json_ws(cs, i);
545 let c = match cs.get(*i) {
546 Some(c) => *c,
547 None => return Err(err!("Truncated JSON"; Input, Invalid)),
548 };
549 match c {
550 '"' => Ok(Json::Str(res!(json_string(cs, i)))),
551 '[' => {
552 *i += 1;
553 let mut v = Vec::new();
554 loop {
555 json_ws(cs, i);
556 if cs.get(*i) == Some(&']') {
557 *i += 1;
558 break;
559 }
560 v.push(res!(json_value(cs, i)));
561 json_ws(cs, i);
562 if cs.get(*i) == Some(&',') {
563 *i += 1;
564 }
565 }
566 Ok(Json::Arr(v))
567 }
568 '{' => {
569 *i += 1;
570 let mut v = Vec::new();
571 loop {
572 json_ws(cs, i);
573 if cs.get(*i) == Some(&'}') {
574 *i += 1;
575 break;
576 }
577 let k = res!(json_string(cs, i));
578 json_ws(cs, i);
579 *i += 1;
580 let x = res!(json_value(cs, i));
581 v.push((k, x));
582 json_ws(cs, i);
583 if cs.get(*i) == Some(&',') {
584 *i += 1;
585 }
586 }
587 Ok(Json::Obj(v))
588 }
589 _ => {
590 let start = *i;
591 while *i < cs.len() && !matches!(cs[*i], ',' | ']' | '}') && !cs[*i].is_whitespace() {
592 *i += 1;
593 }
594 let t: String = cs[start..*i].iter().collect();
595 Ok(match t.as_str() {
596 "null" => Json::Null,
597 "true" => Json::Bool(true),
598 "false" => Json::Bool(false),
599 _ => Json::Num(t),
600 })
601 }
602 }
603}
604
605fn canon(j: Json) -> Json {
606 match j {
607 Json::Arr(v) => Json::Arr(v.into_iter().map(canon).collect()),
608 Json::Obj(v) => {
609 let elem = v.iter().any(|(k, _)| k == "func");
610 let mut v: Vec<(String, Json)> = v.into_iter().map(|(k, x)| (k, canon(x))).collect();
611 if elem {
612 v.sort_by(|a, b| a.0.cmp(&b.0));
613 }
614 Json::Obj(v)
615 }
616 other => other,
617 }
618}
619
620fn cases(file: &str) -> Outcome<Vec<Case>> {
621 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/eval/core").join(file);
622 let text = res!(std::fs::read_to_string(&path).map_err(|e| err!(
623 "Could not read {}: {}", path.display(), e; IO, File, Read)));
624 let mut out: Vec<Case> = Vec::new();
625 for line in text.lines() {
626 if line.starts_with(";;") {
627 continue;
628 }
629 if let Some(code) = line.strip_prefix("%% ") {
630 out.push(Case { code: code.to_string(), tree: String::new(), dep: None });
631 } else if let Some(tagged) = line.strip_prefix("%%[") {
632 let (dep, code) = match tagged.split_once("] ") {
633 Some(p) => p,
634 None => return Err(err!("Malformed case header {}", line; Input, Invalid)),
635 };
636 out.push(Case { code: code.to_string(), tree: String::new(), dep: Some(dep.to_string()) });
637 } else if let Some(c) = out.last_mut() {
638 c.tree.push_str(line);
639 c.tree.push('\n');
640 }
641 }
642 Ok(out)
643}
644
645fn skip() -> bool { std::env::var("EVAL_ORACLE_SKIP").map(|v| v == "1").unwrap_or(false) }
646
647// `typst eval`, memory-capped when systemd can scope it. Environment expansion is off, or systemd reads
648// the `$` of an equation as a variable.
649fn oracle(code: &str) -> Outcome<(bool, String)> {
650 let capped = Command::new("systemd-run")
651 .args(["--user", "--scope", "--quiet", "--expand-environment=no", "-p", "MemoryMax=3G",
652 "--slice=claude-rc.slice", "typst", "eval", "--", code])
653 .output();
654 let out = match capped {
655 Ok(o) if o.status.code() != Some(127) && !String::from_utf8_lossy(&o.stderr).contains("Failed to") => o,
656 _ => res!(Command::new("typst").args(["eval", "--", code]).output().map_err(|e| err!(
657 "The typst oracle could not run ({}); set EVAL_ORACLE_SKIP=1 to skip explicitly.", e; IO, Missing))),
658 };
659 if out.status.success() {
660 return Ok((true, String::from_utf8_lossy(&out.stdout).trim().to_string()));
661 }
662 let stderr = String::from_utf8_lossy(&out.stderr).to_string();
663 let first = stderr.lines().find_map(|l| l.strip_prefix("error: ")).unwrap_or(stderr.trim()).to_string();
664 Ok((false, first))
665}
666
667fn first_error(engine: &Engine) -> String {
668 engine.diags.iter().find(|d| d.is_error()).map(|d| d.message.clone()).unwrap_or_default()
669}
670
671#[test]
672fn values_match_the_typst_oracle() -> Outcome<()> {
673 if skip() {
674 println!("EVAL_ORACLE_SKIP=1: oracle comparison skipped");
675 return Ok(());
676 }
677 let all = res!(cases("values.txt"));
678 assert!(all.len() > 100, "the value corpus shrank to {}", all.len());
679 let mut failures = Vec::new();
680 let mut pending = Vec::new();
681 for c in &all {
682 if let Some(d) = &c.dep {
683 if !available(d) {
684 pending.push(fmt!("[{}] {}", d, c.code));
685 continue;
686 }
687 }
688 let (ok, expected) = res!(oracle(&c.code));
689 if !ok {
690 failures.push(fmt!("{}\n oracle failed: {}", c.code, expected));
691 continue;
692 }
693 let mut runs = vec![("parsed", evaluate_parsed(&c.code))];
694 if !c.tree.trim().is_empty() {
695 runs.insert(0, ("tree", res!(evaluate(&c.tree))));
696 }
697 for (how, (engine, got)) in runs {
698 match got {
699 Ok(v) => {
700 let got = to_json(&v);
701 let same = match (parse_json(&got), parse_json(&expected)) {
702 (Ok(a), Ok(b)) => a == b,
703 _ => got == expected,
704 };
705 if !same {
706 failures.push(fmt!("{} ({})\n typst: {}\n austenite: {}", c.code, how, expected, got));
707 }
708 }
709 Err(_) => failures.push(fmt!("{} ({})\n typst: {}\n austenite error: {}",
710 c.code, how, expected, first_error(&engine))),
711 }
712 }
713 }
714 println!("{} value cases pending on other units:\n{}", pending.len(), pending.join("\n"));
715 assert!(failures.is_empty(), "{} of {} value cases differ:\n{}", failures.len(), all.len(), failures.join("\n"));
716 Ok(())
717}
718
719#[test]
720fn errors_match_the_typst_oracle() -> Outcome<()> {
721 if skip() {
722 println!("EVAL_ORACLE_SKIP=1: oracle comparison skipped");
723 return Ok(());
724 }
725 let all = res!(cases("errors.txt"));
726 assert!(all.len() > 60, "the error corpus shrank to {}", all.len());
727 let mut failures = Vec::new();
728 let mut pending = Vec::new();
729 for c in &all {
730 if let Some(d) = &c.dep {
731 if !available(d) {
732 pending.push(fmt!("[{}] {}", d, c.code));
733 continue;
734 }
735 }
736 let (ok, expected) = res!(oracle(&c.code));
737 if ok {
738 failures.push(fmt!("{}\n oracle succeeded with {}; not an error case", c.code, expected));
739 continue;
740 }
741 let mut runs = vec![("parsed", evaluate_parsed(&c.code))];
742 if !c.tree.trim().is_empty() {
743 runs.insert(0, ("tree", res!(evaluate(&c.tree))));
744 }
745 for (how, (engine, got)) in runs {
746 match got {
747 Ok(v) => failures.push(fmt!("{} ({})\n typst: error: {}\n austenite: {}",
748 c.code, how, expected, to_json(&v))),
749 Err(_) => {
750 let msg = first_error(&engine);
751 if msg != expected {
752 failures.push(fmt!("{} ({})\n typst: error: {}\n austenite: error: {}",
753 c.code, how, expected, msg));
754 }
755 }
756 }
757 }
758 }
759 println!("{} error cases pending on other units:\n{}", pending.len(), pending.join("\n"));
760 assert!(failures.is_empty(), "{} of {} error cases differ:\n{}", failures.len(), all.len(), failures.join("\n"));
761 Ok(())
762}
763
764#[test]
765fn every_error_is_positioned_in_the_source() -> Outcome<()> {
766 for c in res!(cases("errors.txt")) {
767 if c.dep.as_deref().map(|d| !available(d)).unwrap_or(false) || c.tree.trim().is_empty() {
768 continue;
769 }
770 let (engine, got) = res!(evaluate(&c.tree));
771 assert!(got.is_err(), "{} evaluated without error", c.code);
772 let d = engine.diags.iter().find(|d| d.is_error());
773 assert!(d.map(|d| !d.span.is_detached()).unwrap_or(false), "{}: error without a position", c.code);
774 }
775 Ok(())
776}
777
778// Warnings
779
780/// One warning as both sides report it: message, 1-based line and column, hints.
781type Warning = (String, usize, usize, Vec<String>);
782
783fn warning_cases() -> Outcome<Vec<String>> {
784 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/eval/core/warnings.txt");
785 let text = res!(std::fs::read_to_string(&path).map_err(|e| err!(
786 "Could not read {}: {}", path.display(), e; IO, File, Read)));
787 let mut out: Vec<String> = Vec::new();
788 for line in text.lines() {
789 if line.starts_with(";;") {
790 continue;
791 }
792 if line == "%%" {
793 out.push(String::new());
794 } else if let Some(doc) = out.last_mut() {
795 doc.push_str(line);
796 doc.push('\n');
797 }
798 }
799 Ok(out)
800}
801
802fn work_dir() -> Outcome<PathBuf> {
803 let d = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join("eval_core_warnings");
804 res!(std::fs::create_dir_all(&d).map_err(|e| err!("Cannot create {}: {}", d.display(), e; IO, File, Write)));
805 Ok(d)
806}
807
808// `typst compile`'s warnings. Its location line gives a 0-based character column.
809fn oracle_warnings(doc: &str, n: usize) -> Outcome<Vec<Warning>> {
810 let dir = res!(work_dir());
811 let src = dir.join(fmt!("w{}.typ", n));
812 res!(std::fs::write(&src, doc).map_err(|e| err!("Cannot write {}: {}", src.display(), e; IO, File, Write)));
813 let pdf = dir.join(fmt!("w{}.pdf", n));
814 let out = res!(Command::new("systemd-run")
815 .args(["--user", "--scope", "--quiet", "-p", "MemoryMax=3G", "--slice=claude-rc.slice", "typst", "compile"])
816 .arg(&src).arg(&pdf).output()
817 .map_err(|e| err!("The typst oracle could not run: {}", e; IO, Missing)));
818 let stderr = String::from_utf8_lossy(&out.stderr).to_string();
819 if !out.status.success() {
820 return Err(err!("typst rejects warning case {}: {}", n, stderr; Input, Invalid));
821 }
822 let mut ws: Vec<Warning> = Vec::new();
823 let mut lines = stderr.lines().peekable();
824 while let Some(l) = lines.next() {
825 if let Some(msg) = l.strip_prefix("warning: ") {
826 let mut w = (msg.to_string(), 0, 0, Vec::new());
827 if let Some(loc) = lines.peek().and_then(|n| n.trim().strip_prefix("┌─ ")) {
828 let mut parts = loc.rsplitn(3, ':');
829 let col = parts.next().and_then(|c| c.parse::<usize>().ok()).unwrap_or(0);
830 let line = parts.next().and_then(|c| c.parse::<usize>().ok()).unwrap_or(0);
831 w.1 = line;
832 w.2 = col + 1;
833 }
834 ws.push(w);
835 } else if let Some(h) = l.trim().strip_prefix("= hint: ") {
836 if let Some(w) = ws.last_mut() {
837 w.3.push(h.to_string());
838 }
839 }
840 }
841 Ok(ws)
842}
843
844fn austenite_warnings(doc: &str) -> Outcome<Vec<Warning>> {
845 let mut world = World::new(PathBuf::from("/"));
846 let id = res!(world.add_source(PathBuf::from("/case.typ"), doc.to_string()));
847 let mut engine = Engine::new(world);
848 if let Err(e) = eval_source(&mut engine, id) {
849 return Err(err!("evaluation failed: {} ({:?})", first_error(&engine), e; Invalid));
850 }
851 let src = match engine.world.source(id) {
852 Some(s) => s,
853 None => return Err(err!("the case's source vanished"; Bug)),
854 };
855 Ok(engine.diags.iter().filter(|d| !d.is_error()).map(|d| {
856 let (l, c) = src.line_col(d.span.start);
857 (d.message.clone(), l, c, d.hints.clone())
858 }).collect())
859}
860
861#[test]
862fn warnings_match_the_typst_oracle() -> Outcome<()> {
863 if skip() {
864 println!("EVAL_ORACLE_SKIP=1: oracle comparison skipped");
865 return Ok(());
866 }
867 let docs = res!(warning_cases());
868 assert!(docs.len() >= 10, "the warning corpus shrank to {}", docs.len());
869 let mut warned = 0;
870 let mut failures = Vec::new();
871 for (n, doc) in docs.iter().enumerate() {
872 let want = res!(oracle_warnings(doc, n));
873 warned += want.len();
874 match austenite_warnings(doc) {
875 Ok(got) if got == want => (),
876 Ok(got) => failures.push(fmt!("{}\n typst: {:?}\n austenite: {:?}", doc, want, got)),
877 Err(e) => failures.push(fmt!("{}\n austenite: {}", doc, e)),
878 }
879 }
880 // The corpus must hold warnings to compare, or agreement means nothing.
881 assert!(warned >= 5, "only {} warnings in the corpus", warned);
882 assert!(failures.is_empty(), "{} of {} warning cases differ:\n{}", failures.len(), docs.len(), failures.join("\n"));
883 Ok(())
884}
885
886// Every diagnostic carries the kind of what it reports, set where it is raised, so a caller switches on
887// the kind and never on the wording. Each row is code, the kind of its first diagnostic, and whether that
888// diagnostic is an error.
889#[test]
890fn diagnostics_carry_the_kind_of_what_they_report() -> Outcome<()> {
891 use DiagnosticKind as K;
892 let dir = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join("eval_core_kinds");
893 res!(std::fs::create_dir_all(&dir).map_err(|e| err!("Cannot create {}: {}", dir.display(), e; IO, File, Write)));
894 // A file at a path needs a file to resolve it from, so each case is evaluated from a source in `dir`, as
895 // the string handed to `eval` is in Typst. 0xff is not UTF-8 in any position.
896 res!(std::fs::write(dir.join("bad.txt"), [255u8, 254, 65]).map_err(|e| err!(
897 "Cannot write bad.txt in {}: {}", dir.display(), e; IO, File, Write)));
898 let cases: [(&str, K, bool); 18] = [
899 ("missing", K::UnknownVariable, true),
900 ("(1 +", K::Syntax, true),
901 ("break", K::Syntax, true),
902 ("1 + \"a\"", K::Type, true),
903 ("int(\"x\")", K::Type, true),
904 ("{ let f() = f(); f() }", K::Limit, true),
905 // Not a module: a type error when plugins run, refused outright when the feature is off.
906 ("plugin(bytes(\"x\"))", if cfg!(feature = "plugin") { K::Type } else { K::Unsupported }, true),
907 ("json(bytes((255, 254, 65)))", K::Encoding, true),
908 ("read(\"/absent.txt\")", K::MissingFile, true),
909 ("read(\"/bad.txt\")", K::Encoding, true),
910 ("{ let f() = { [a]; return 1 }; f() }", K::Lint, false),
911 ("import heading as heading", K::Lint, false),
912 ("import list: item as item", K::Lint, false),
913 ("[<x>]", K::Lint, false),
914 ("[a <x> <y>]", K::Lint, false),
915 ("numbering(\"a\", 0)", K::Lint, false),
916 ("enum(([1], [a]))", K::Lint, false),
917 ("terms(([a], [b]))", K::Lint, false),
918 ];
919 let mut failures = Vec::new();
920 for (code, kind, error) in cases {
921 let mut engine = Engine::new(World::new(dir.clone()));
922 let id = res!(engine.world.add_source(dir.join("case.typ"), String::new()));
923 let _ = eval_string(&mut engine, code, EvalMode::Code, Scope::new(), Span::new(id, 0, 0));
924 match engine.diags.iter().find(|d| d.is_error() == error) {
925 Some(d) if d.kind == kind => (),
926 Some(d) => failures.push(fmt!("{}: kind {} ({}), wanted {}", code, d.kind, d.message, kind)),
927 None => failures.push(fmt!("{}: no {} raised", code, if error { "error" } else { "warning" })),
928 }
929 }
930 assert!(failures.is_empty(), "{} case(s) carry the wrong kind:\n{}", failures.len(), failures.join("\n"));
931 Ok(())
932}