Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/src/eval/lib/data.rs

72.2 KiB, 13 runs

created by r1870400018:63226, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// U3 owns this file. Data loading (paths resolved with `import::resolve_path`, files read with
2// `import::read_file`) and the readers and writers behind `read`, `json`, `csv`, `yaml`, `toml`, `xml` and
3// `cbor` (with `.encode` for JSON, YAML, TOML and CBOR). Every format is read here into Typst values directly: JSON keeps object
4// order and the integer/float distinction, TOML dates become `datetime`s, and a document a reader cannot
5// take is a diagnostic naming the line and column, never an empty value. XML goes through
6// `fe2o3_text::xml`. YAML is the block and flow subset real documents use (mappings, sequences, scalars
7// of the 1.2 core schema, quoted and block scalars, anchors and aliases); tags and complex keys are
8// refused with a diagnostic.
9
10use crate::diag::DiagnosticKind;
11use crate::eval::args::Args;
12use crate::eval::content::Content;
13use crate::eval::func::{
14 Func,
15 NativeFunc,
16};
17use crate::eval::import::{
18 read_file,
19 resolve_path,
20};
21use crate::eval::lib::foundations::{
22 finish,
23 mismatch,
24 need,
25 repr,
26 words,
27};
28use crate::eval::scope::Scope;
29use crate::eval::value::{
30 Datetime,
31 Dict,
32 Value,
33};
34use crate::eval::Engine;
35use crate::syntax::Span;
36
37use oxedyne_fe2o3_core::prelude::*;
38
39use std::sync::Arc;
40
41native_fns! {
42 pub enum DataFn {
43 Read => "read",
44 Json => "json",
45 Csv => "csv",
46 Yaml => "yaml",
47 Toml => "toml",
48 Xml => "xml",
49 Cbor => "cbor",
50 JsonEncode => "encode",
51 YamlEncode => "encode",
52 TomlEncode => "encode",
53 CborEncode => "encode",
54 }
55}
56
57pub fn define(scope: &mut Scope) {
58 for f in [DataFn::Read, DataFn::Json, DataFn::Csv, DataFn::Yaml, DataFn::Toml, DataFn::Xml, DataFn::Cbor] {
59 scope.define(f.name(), Value::Func(Func::Native(NativeFunc::Data(f))));
60 }
61}
62
63/// `json.encode` and its kin: the members a data function carries.
64pub fn func_scope(f: DataFn, name: &str) -> Option<Value> {
65 let g = match (f, name) {
66 (DataFn::Json, "encode") => DataFn::JsonEncode,
67 (DataFn::Yaml, "encode") => DataFn::YamlEncode,
68 (DataFn::Toml, "encode") => DataFn::TomlEncode,
69 (DataFn::Cbor, "encode") => DataFn::CborEncode,
70 _ => return None,
71 };
72 Some(Value::Func(Func::Native(NativeFunc::Data(g))))
73}
74
75// The bytes a data function reads: a path through the vfs, or bytes given inline.
76pub(crate) fn source(engine: &mut Engine, span: Span, v: Value) -> Outcome<Arc<Vec<u8>>> {
77 match v.symbol_as_str() {
78 Value::Str(p) => {
79 let path = res!(resolve_path(engine, &p, span.file, span));
80 Ok(Arc::new(res!(read_file(engine, &path, span))))
81 }
82 Value::Bytes(b) => Ok(b),
83 other => Err(mismatch(engine, span, "string or bytes", &other)),
84 }
85}
86
87fn utf8<'a>(engine: &mut Engine, span: Span, b: &'a [u8]) -> Outcome<&'a str> {
88 match std::str::from_utf8(b) {
89 Ok(s) => Ok(s.strip_prefix('\u{feff}').unwrap_or(s)),
90 Err(_) => Err(engine.error(DiagnosticKind::Encoding, span, "file is not valid utf-8")),
91 }
92}
93
94// A parse failure at a byte offset, as Typst words it: "failed to parse JSON (message at line L column C
95// at L:C)". The column counts the offending character, as serde's does, and at the end of input the
96// characters read.
97pub(crate) fn parse_error(engine: &mut Engine, span: Span, fmt_name: &str, text: &str, at: usize, msg: &str) -> Error<ErrTag> {
98 let (line, col) = line_col(text, at);
99 engine.error(DiagnosticKind::Type, span, fmt!("failed to parse {} ({} at line {} column {} at {}:{})",
100 fmt_name, msg, line, col, line, col.max(1)))
101}
102
103fn line_col(text: &str, at: usize) -> (usize, usize) {
104 let end = at >= text.len();
105 let at = at.min(text.len());
106 let before = &text[..at];
107 let line = before.matches('\n').count() + 1;
108 let col = before.rsplit('\n').next().map(|l| l.chars().count()).unwrap_or(0);
109 (line, if end { col } else { col + 1 })
110}
111
112pub fn call(f: DataFn, engine: &mut Engine, mut args: Args) -> Outcome<Value> {
113 let span = args.span;
114 // Typst places a path that cannot be resolved or a file that cannot be read at the argument naming it.
115 let at = args.items.iter().find(|a| a.name.is_none()).map(|a| a.span).unwrap_or(span);
116 match f {
117 DataFn::Read => {
118 let p = res!(need(engine, &mut args, "path"));
119 let enc = res!(args.named::<Value>("encoding"));
120 res!(finish(engine, args));
121 let path = match p.symbol_as_str() {
122 Value::Str(s) => s,
123 other => return Err(mismatch(engine, span, "string", &other)),
124 };
125 let b = res!(source(engine, at, Value::Str(path)));
126 match enc {
127 None | Some(Value::Str(_)) => {
128 if let Some(Value::Str(e)) = &enc {
129 if e.as_str() != "utf8" {
130 return Err(engine.error(DiagnosticKind::Type, span, "expected \"utf8\" or none"));
131 }
132 }
133 let s = res!(utf8(engine, span, &b));
134 Ok(Value::str(s))
135 }
136 Some(Value::None) => Ok(Value::Bytes(b)),
137 Some(other) => Err(mismatch(engine, span, "string or none", &other)),
138 }
139 }
140 DataFn::Json | DataFn::Yaml | DataFn::Toml | DataFn::Xml | DataFn::Cbor | DataFn::Csv => {
141 let src = res!(need(engine, &mut args, "source"));
142 let (delim, dict_rows) = if f == DataFn::Csv {
143 let d = match res!(args.named::<Value>("delimiter")) {
144 None => ',',
145 Some(Value::Str(s)) => {
146 let mut cs = s.chars();
147 match (cs.next(), cs.next()) {
148 (Some(c), None) if c.is_ascii() => c,
149 (Some(_), None) => return Err(engine.error(DiagnosticKind::Type, span, "delimiter must be an ASCII character")),
150 _ => return Err(engine.error(DiagnosticKind::Type, span, "delimiter must be a single character")),
151 }
152 }
153 Some(other) => return Err(mismatch(engine, span, "string", &other)),
154 };
155 let rows = match res!(args.named::<Value>("row-type")) {
156 None => false,
157 Some(Value::Type(t)) if t == crate::eval::value::Type::Array => false,
158 Some(Value::Type(t)) if t == crate::eval::value::Type::Dict => true,
159 Some(_) => return Err(engine.error(DiagnosticKind::Type, span, "expected `array` or `dictionary`")),
160 };
161 (d, rows)
162 } else {
163 (',', false)
164 };
165 res!(finish(engine, args));
166 let b = res!(source(engine, at, src));
167 match f {
168 DataFn::Cbor => {
169 let mut r = CborReader { b: &b, at: 0, depth: 0 };
170 match r.value() {
171 Ok(v) if r.at == b.len() => Ok(v),
172 Ok(_) => Err(engine.error(DiagnosticKind::Type, span, "failed to parse CBOR (trailing data)")),
173 Err(e) => Err(engine.error(DiagnosticKind::Type, span, fmt!("failed to parse CBOR ({})", words(&e)))),
174 }
175 }
176 _ => {
177 let text = res!(utf8(engine, span, &b)).to_string();
178 match f {
179 DataFn::Json => {
180 let mut p = JsonParser { s: text.as_bytes(), at: 0, depth: 0, fail: None };
181 match p.document() {
182 Ok(v) => Ok(v),
183 Err(_) => {
184 let (at, m) = p.fail.clone().unwrap_or((0, "invalid JSON".to_string()));
185 Err(parse_error(engine, span, "JSON", &text, at, &m))
186 }
187 }
188 }
189 DataFn::Toml => match toml_parse(&text) {
190 Ok(v) => Ok(v),
191 Err((at, m)) => Err(parse_error(engine, span, "TOML", &text, at, &m)),
192 },
193 DataFn::Yaml => match yaml_parse(&text) {
194 Ok(v) => Ok(v),
195 Err((at, m)) => Err(parse_error(engine, span, "YAML", &text, at, &m)),
196 },
197 DataFn::Xml => xml_value(engine, span, &text),
198 _ => csv_value(engine, span, &text, delim, dict_rows),
199 }
200 }
201 }
202 }
203 DataFn::JsonEncode | DataFn::YamlEncode | DataFn::TomlEncode | DataFn::CborEncode => {
204 let v = res!(need(engine, &mut args, "value"));
205 let pretty = if f == DataFn::CborEncode {
206 false
207 } else {
208 match res!(args.named::<Value>("pretty")) {
209 None => true,
210 Some(Value::Bool(b)) => b,
211 Some(other) => return Err(mismatch(engine, span, "boolean", &other)),
212 }
213 };
214 res!(finish(engine, args));
215 let data = to_data(&v);
216 match f {
217 DataFn::JsonEncode => {
218 let mut out = String::new();
219 json_write(&data, pretty, 0, &mut out);
220 Ok(Value::str(out))
221 }
222 DataFn::YamlEncode => {
223 let mut out = String::new();
224 yaml_write(&data, 0, &mut out, true);
225 Ok(Value::str(out))
226 }
227 DataFn::TomlEncode => match &data {
228 Data::Map(m) => {
229 let mut out = String::new();
230 toml_write_table(m, &[], pretty, &mut out);
231 Ok(Value::str(out))
232 }
233 _ => Err(mismatch(engine, span, "dictionary", &v)),
234 },
235 _ => {
236 let mut out = Vec::new();
237 cbor_write(&data, &mut out);
238 Ok(Value::Bytes(Arc::new(out)))
239 }
240 }
241 }
242 }
243}
244
245// Serialisable data: what a Typst value becomes on its way to a text or binary format.
246
247enum Data {
248 Null,
249 Bool(bool),
250 Int(i64),
251 Float(f64),
252 Str(String),
253 Bytes(Vec<u8>),
254 Seq(Vec<Data>),
255 Map(Vec<(String, Data)>),
256}
257
258fn to_data(v: &Value) -> Data {
259 match v {
260 Value::None => Data::Null,
261 Value::Bool(b) => Data::Bool(*b),
262 Value::Int(i) => Data::Int(*i),
263 Value::Float(f) => Data::Float(*f),
264 Value::Str(s) => Data::Str((**s).clone()),
265 Value::Bytes(b) => Data::Bytes((**b).clone()),
266 Value::Array(a) => Data::Seq(a.iter().map(to_data).collect()),
267 Value::Dict(d) => Data::Map(d.iter().map(|(k, v)| (k.to_string(), to_data(v))).collect()),
268 Value::Symbol(s) => Data::Str(crate::eval::lib::sym::text(s).to_string()),
269 Value::Content(c) => content_data(c),
270 other => Data::Str(repr(other)),
271 }
272}
273
274/// `v` as compact JSON text, for the values Typst writes as plain JSON: none, a boolean, an integer, a float, a
275/// string or symbol, and an array or dictionary of those. Any other value, content, a length, a colour, bytes,
276/// a function, is `None`: it has more than one way to be written, and a caller reading JSON must get Typst's or
277/// nothing.
278pub fn plain_json(v: &Value) -> Option<String> {
279 let data = match plain_data(v) {
280 Some(d) => d,
281 None => return None,
282 };
283 let mut out = String::new();
284 json_write(&data, false, 0, &mut out);
285 Some(out)
286}
287
288fn plain_data(v: &Value) -> Option<Data> {
289 match v {
290 Value::None => Some(Data::Null),
291 Value::Bool(b) => Some(Data::Bool(*b)),
292 Value::Int(i) => Some(Data::Int(*i)),
293 Value::Float(f) => Some(Data::Float(*f)),
294 Value::Str(s) => Some(Data::Str((**s).clone())),
295 Value::Symbol(s) => Some(Data::Str(crate::eval::lib::sym::text(s).to_string())),
296 Value::Array(a) => {
297 let mut out = Vec::with_capacity(a.len());
298 for x in a.iter() {
299 match plain_data(x) {
300 Some(d) => out.push(d),
301 None => return None,
302 }
303 }
304 Some(Data::Seq(out))
305 },
306 Value::Dict(d) => {
307 let mut out = Vec::with_capacity(d.len());
308 for (k, x) in d.iter() {
309 match plain_data(x) {
310 Some(dx) => out.push((k.to_string(), dx)),
311 None => return None,
312 }
313 }
314 Some(Data::Map(out))
315 },
316 _ => None,
317 }
318}
319
320// Content serialises as its fields under `func`, as Typst's does.
321fn content_data(c: &Content) -> Data {
322 match c {
323 Content::Sequence(s) => Data::Map(vec![
324 ("func".to_string(), Data::Str("sequence".to_string())),
325 ("children".to_string(), Data::Seq(s.children.iter().map(content_data).collect())),
326 ]),
327 Content::Styled(s) => content_data(&s.child),
328 Content::Elem(e) => {
329 let mut m = vec![("func".to_string(), Data::Str(e.kind.name().to_string()))];
330 let specs = e.kind.fields();
331 for (i, spec) in specs.iter().enumerate() {
332 if let Some((_, v)) = e.fields.iter().find(|(f, _)| f.0 as usize == i) {
333 m.push((spec.name.to_string(), to_data(v)));
334 }
335 }
336 if let Some(l) = &e.label {
337 m.push(("label".to_string(), Data::Str(fmt!("<{}>", l.as_str()))));
338 }
339 Data::Map(m)
340 }
341 }
342}
343
344/// A float as `ryu` (and so serde) writes it: shortest round-trip digits, scientific below 1e-5 and from
345/// 1e16.
346pub fn ryu_format(f: f64) -> String {
347 if f == 0.0 {
348 return if f.is_sign_negative() { "-0.0".to_string() } else { "0.0".to_string() };
349 }
350 let sci = fmt!("{:e}", f);
351 let (mant, exp) = match sci.split_once('e') {
352 Some(p) => p,
353 None => return sci,
354 };
355 let exp: i32 = exp.parse().unwrap_or(0);
356 let neg = mant.starts_with('-');
357 let digits: String = mant.chars().filter(|c| c.is_ascii_digit()).collect();
358 let len = digits.len() as i32;
359 let kk = exp + 1;
360 let mut out = String::new();
361 if neg {
362 out.push('-');
363 }
364 if len <= kk && kk <= 16 {
365 out.push_str(&digits);
366 out.push_str(&"0".repeat((kk - len) as usize));
367 out.push_str(".0");
368 } else if 0 < kk && kk <= 16 {
369 out.push_str(&digits[..kk as usize]);
370 out.push('.');
371 out.push_str(&digits[kk as usize..]);
372 } else if -5 < kk && kk <= 0 {
373 out.push_str("0.");
374 out.push_str(&"0".repeat((-kk) as usize));
375 out.push_str(&digits);
376 } else if len == 1 {
377 out.push_str(&digits);
378 out.push('e');
379 out.push_str(&(kk - 1).to_string());
380 } else {
381 out.push_str(&digits[..1]);
382 out.push('.');
383 out.push_str(&digits[1..]);
384 out.push('e');
385 out.push_str(&(kk - 1).to_string());
386 }
387 out
388}
389
390// JSON
391
392struct JsonParser<'a> {
393 s: &'a [u8],
394 at: usize,
395 depth: usize,
396 fail: Option<(usize, String)>, // where and why parsing stopped, for the diagnostic
397}
398
399
400impl<'a> JsonParser<'a> {
401 fn ws(&mut self) {
402 while self.at < self.s.len() && matches!(self.s[self.at], b' ' | b'\t' | b'\n' | b'\r') {
403 self.at += 1;
404 }
405 }
406
407 fn err<T>(&mut self, m: &str) -> Outcome<T> {
408 let at = self.at;
409 self.fail_at(at, m)
410 }
411
412 fn fail_at<T>(&mut self, at: usize, m: &str) -> Outcome<T> {
413 self.fail = Some((at, m.to_string()));
414 Err(err!("{}", m; Decode, Input))
415 }
416
417 fn document(&mut self) -> Outcome<Value> {
418 self.ws();
419 let v = res!(self.value());
420 self.ws();
421 if self.at < self.s.len() {
422 return self.err("trailing characters");
423 }
424 Ok(v)
425 }
426
427 fn value(&mut self) -> Outcome<Value> {
428 self.ws();
429 if self.depth > 128 {
430 return self.err("recursion limit exceeded");
431 }
432 match self.s.get(self.at) {
433 None => self.err("EOF while parsing a value"),
434 Some(b'{') => {
435 self.at += 1;
436 self.depth += 1;
437 let mut d = Dict::new();
438 self.ws();
439 if self.s.get(self.at) == Some(&b'}') {
440 self.at += 1;
441 self.depth -= 1;
442 return Ok(Value::dict(d));
443 }
444 loop {
445 self.ws();
446 if self.s.get(self.at) != Some(&b'"') {
447 return match self.s.get(self.at) {
448 Some(b'}') => self.err("trailing comma"),
449 None => self.err("EOF while parsing an object"),
450 Some(_) => self.err("key must be a string"),
451 };
452 }
453 let k = res!(self.string());
454 self.ws();
455 if self.s.get(self.at) != Some(&b':') {
456 return if self.at >= self.s.len() { self.err("EOF while parsing an object") } else { self.err("expected `:`") };
457 }
458 self.at += 1;
459 let v = res!(self.value());
460 d.insert(&k, v);
461 self.ws();
462 match self.s.get(self.at) {
463 Some(b',') => self.at += 1,
464 Some(b'}') => {
465 self.at += 1;
466 self.depth -= 1;
467 return Ok(Value::dict(d));
468 }
469 None => return self.err("EOF while parsing an object"),
470 _ => return self.err("expected `,` or `}`"),
471 }
472 }
473 }
474 Some(b'[') => {
475 self.at += 1;
476 self.depth += 1;
477 let mut a = Vec::new();
478 self.ws();
479 if self.s.get(self.at) == Some(&b']') {
480 self.at += 1;
481 self.depth -= 1;
482 return Ok(Value::array(a));
483 }
484 loop {
485 self.ws();
486 if self.s.get(self.at) == Some(&b']') {
487 return self.err("trailing comma");
488 }
489 match self.value() {
490 Ok(v) => a.push(v),
491 Err(e) => return Err(e),
492 }
493 self.ws();
494 match self.s.get(self.at) {
495 Some(b',') => self.at += 1,
496 Some(b']') => {
497 self.at += 1;
498 self.depth -= 1;
499 return Ok(Value::array(a));
500 }
501 None => return self.err("EOF while parsing a list"),
502 _ => return self.err("expected `,` or `]`"),
503 }
504 }
505 }
506 Some(b'"') => self.string().map(Value::str),
507 Some(b't') => self.word("true", Value::Bool(true)),
508 Some(b'f') => self.word("false", Value::Bool(false)),
509 Some(b'n') => self.word("null", Value::None),
510 Some(c) if *c == b'-' || c.is_ascii_digit() => self.number(),
511 Some(_) => self.err("expected value"),
512 }
513 }
514
515 fn word(&mut self, w: &str, v: Value) -> Outcome<Value> {
516 if self.s[self.at..].starts_with(w.as_bytes()) {
517 self.at += w.len();
518 Ok(v)
519 } else {
520 self.err("expected ident")
521 }
522 }
523
524 fn number(&mut self) -> Outcome<Value> {
525 let start = self.at;
526 if self.s.get(self.at) == Some(&b'-') {
527 self.at += 1;
528 }
529 let int_start = self.at;
530 while self.at < self.s.len() && self.s[self.at].is_ascii_digit() {
531 self.at += 1;
532 }
533 if self.at == int_start {
534 return self.err("invalid number");
535 }
536 if self.at - int_start > 1 && self.s[int_start] == b'0' {
537 return self.fail_at(int_start + 1, "invalid number");
538 }
539 let mut float = false;
540 if self.s.get(self.at) == Some(&b'.') {
541 float = true;
542 self.at += 1;
543 let f = self.at;
544 while self.at < self.s.len() && self.s[self.at].is_ascii_digit() {
545 self.at += 1;
546 }
547 if self.at == f {
548 return self.err("invalid number");
549 }
550 }
551 if matches!(self.s.get(self.at), Some(b'e') | Some(b'E')) {
552 float = true;
553 self.at += 1;
554 if matches!(self.s.get(self.at), Some(b'+') | Some(b'-')) {
555 self.at += 1;
556 }
557 let e = self.at;
558 while self.at < self.s.len() && self.s[self.at].is_ascii_digit() {
559 self.at += 1;
560 }
561 if self.at == e {
562 return self.err("invalid number");
563 }
564 }
565 let text = std::str::from_utf8(&self.s[start..self.at]).unwrap_or("0");
566 // A negative zero has no integer form; serde reads it as the float.
567 if !float && text != "-0" {
568 if let Ok(i) = text.parse::<i64>() {
569 return Ok(Value::Int(i));
570 }
571 }
572 match text.parse::<f64>() {
573 Ok(f) if f.is_finite() => Ok(if f == 0.0 && text.starts_with('-') { Value::Float(-0.0) } else { Value::Float(f) }),
574 _ => {
575 let at = self.at - 1;
576 self.fail_at(at, "number out of range")
577 }
578 }
579 }
580
581 fn hex4(&mut self) -> Outcome<u32> {
582 let h = self.s.get(self.at..self.at + 4).and_then(|b| std::str::from_utf8(b).ok())
583 .and_then(|t| u32::from_str_radix(t, 16).ok());
584 match h {
585 Some(v) => {
586 self.at += 4;
587 Ok(v)
588 }
589 None => self.err("invalid escape"),
590 }
591 }
592
593 fn string(&mut self) -> Outcome<String> {
594 self.at += 1;
595 let mut out = String::new();
596 loop {
597 let start = self.at;
598 while self.at < self.s.len() && self.s[self.at] != b'"' && self.s[self.at] != b'\\' && self.s[self.at] >= 0x20 {
599 self.at += 1;
600 }
601 out.push_str(std::str::from_utf8(&self.s[start..self.at]).unwrap_or(""));
602 match self.s.get(self.at) {
603 None => return self.err("EOF while parsing a string"),
604 Some(b'"') => {
605 self.at += 1;
606 return Ok(out);
607 }
608 Some(b'\\') => {
609 self.at += 1;
610 let c = match self.s.get(self.at) {
611 Some(c) => *c,
612 None => return self.err("EOF while parsing a string"),
613 };
614 self.at += 1;
615 match c {
616 b'"' => out.push('"'),
617 b'\\' => out.push('\\'),
618 b'/' => out.push('/'),
619 b'b' => out.push('\u{8}'),
620 b'f' => out.push('\u{c}'),
621 b'n' => out.push('\n'),
622 b'r' => out.push('\r'),
623 b't' => out.push('\t'),
624 b'u' => {
625 let hi = res!(self.hex4());
626 let cp = if (0xD800..0xDC00).contains(&hi) {
627 if !self.s[self.at..].starts_with(b"\\u") {
628 return self.err("lone leading surrogate in hex escape");
629 }
630 self.at += 2;
631 let lo = res!(self.hex4());
632 if !(0xDC00..0xE000).contains(&lo) {
633 return self.err("invalid unicode code point");
634 }
635 0x10000 + ((hi - 0xD800) << 10) + (lo - 0xDC00)
636 } else {
637 hi
638 };
639 match char::from_u32(cp) {
640 Some(ch) => out.push(ch),
641 None => return self.err("invalid unicode code point"),
642 }
643 }
644 _ => return self.err("invalid escape"),
645 }
646 }
647 Some(_) => return self.err("control character (\\u0000-\\u001F) found while parsing a string"),
648 }
649 }
650 }
651}
652
653fn json_str(s: &str, out: &mut String) {
654 out.push('"');
655 for c in s.chars() {
656 match c {
657 '"' => out.push_str("\\\""),
658 '\\' => out.push_str("\\\\"),
659 '\n' => out.push_str("\\n"),
660 '\r' => out.push_str("\\r"),
661 '\t' => out.push_str("\\t"),
662 '\u{8}' => out.push_str("\\b"),
663 '\u{c}' => out.push_str("\\f"),
664 c if (c as u32) < 0x20 => out.push_str(&fmt!("\\u{:04x}", c as u32)),
665 c => out.push(c),
666 }
667 }
668 out.push('"');
669}
670
671fn json_write(d: &Data, pretty: bool, indent: usize, out: &mut String) {
672 let nl = |out: &mut String, n: usize| {
673 if pretty {
674 out.push('\n');
675 out.push_str(&" ".repeat(n));
676 }
677 };
678 match d {
679 Data::Null => out.push_str("null"),
680 Data::Bool(b) => out.push_str(if *b { "true" } else { "false" }),
681 Data::Int(i) => out.push_str(&i.to_string()),
682 Data::Float(f) => if f.is_finite() { out.push_str(&ryu_format(*f)) } else { out.push_str("null") },
683 Data::Str(s) => json_str(s, out),
684 Data::Bytes(b) => json_write(&Data::Seq(b.iter().map(|x| Data::Int(*x as i64)).collect()), pretty, indent, out),
685 Data::Seq(a) => {
686 if a.is_empty() {
687 out.push_str("[]");
688 return;
689 }
690 out.push('[');
691 for (i, x) in a.iter().enumerate() {
692 if i > 0 {
693 out.push(',');
694 }
695 nl(out, indent + 1);
696 json_write(x, pretty, indent + 1, out);
697 }
698 nl(out, indent);
699 out.push(']');
700 }
701 Data::Map(m) => {
702 if m.is_empty() {
703 out.push_str("{}");
704 return;
705 }
706 out.push('{');
707 for (i, (k, x)) in m.iter().enumerate() {
708 if i > 0 {
709 out.push(',');
710 }
711 nl(out, indent + 1);
712 json_str(k, out);
713 out.push(':');
714 if pretty {
715 out.push(' ');
716 }
717 json_write(x, pretty, indent + 1, out);
718 }
719 nl(out, indent);
720 out.push('}');
721 }
722 }
723}
724
725// CSV
726
727fn csv_value(engine: &mut Engine, span: Span, text: &str, delim: char, dict_rows: bool) -> Outcome<Value> {
728 let mut rows: Vec<Vec<String>> = Vec::new();
729 let mut row: Vec<String> = Vec::new();
730 let mut field = String::new();
731 let mut quoted = false;
732 let mut in_quotes = false;
733 let mut chars = text.chars().peekable();
734 let mut line = 1usize;
735 let mut expected: Option<usize> = None;
736 let mut row_line = 1usize;
737 let finish_row = |row: &mut Vec<String>, rows: &mut Vec<Vec<String>>, expected: &mut Option<usize>, line: usize|
738 -> std::result::Result<(), String>
739 {
740 let r = std::mem::take(row);
741 match expected {
742 None => *expected = Some(r.len()),
743 Some(n) if *n != r.len() => return Err(fmt!(
744 "found {} instead of {} fields in line {} at {}:1", r.len(), n, line, line)),
745 _ => (),
746 }
747 rows.push(r);
748 Ok(())
749 };
750 let mut any = false;
751 while let Some(c) = chars.next() {
752 any = true;
753 if in_quotes {
754 if c == '"' {
755 if chars.peek() == Some(&'"') {
756 chars.next();
757 field.push('"');
758 } else {
759 in_quotes = false;
760 }
761 } else {
762 if c == '\n' {
763 line += 1;
764 }
765 field.push(c);
766 }
767 continue;
768 }
769 if c == '"' && field.is_empty() && !quoted {
770 in_quotes = true;
771 quoted = true;
772 } else if c == delim {
773 row.push(std::mem::take(&mut field));
774 quoted = false;
775 } else if c == '\n' || c == '\r' {
776 if c == '\r' && chars.peek() == Some(&'\n') {
777 chars.next();
778 }
779 row.push(std::mem::take(&mut field));
780 quoted = false;
781 if let Err(m) = finish_row(&mut row, &mut rows, &mut expected, row_line) {
782 return Err(engine.error(DiagnosticKind::Type, span, fmt!("failed to parse CSV ({})", m)));
783 }
784 line += 1;
785 row_line = line;
786 any = false;
787 } else {
788 field.push(c);
789 }
790 }
791 if any || !field.is_empty() || !row.is_empty() {
792 row.push(field);
793 if let Err(m) = finish_row(&mut row, &mut rows, &mut expected, row_line) {
794 return Err(engine.error(DiagnosticKind::Type, span, fmt!("failed to parse CSV ({})", m)));
795 }
796 }
797 if dict_rows {
798 let mut it = rows.into_iter();
799 let header = it.next().unwrap_or_default();
800 let out = it.map(|r| {
801 let mut d = Dict::new();
802 for (k, v) in header.iter().zip(r) {
803 d.insert(k, Value::str(v));
804 }
805 Value::dict(d)
806 }).collect();
807 Ok(Value::array(out))
808 } else {
809 Ok(Value::array(rows.into_iter().map(|r| Value::array(r.into_iter().map(Value::str).collect())).collect()))
810 }
811}
812
813// XML
814
815fn xml_value(engine: &mut Engine, span: Span, text: &str) -> Outcome<Value> {
816 use oxedyne_fe2o3_text::xml::{
817 Node,
818 Xml,
819 };
820 let doc = match Xml::parse(text) {
821 Ok(d) => d,
822 Err(e) => return Err(engine.error(DiagnosticKind::Type, span, fmt!("failed to parse XML ({})",
823 e.msgs().last().cloned().unwrap_or_default()))),
824 };
825 // Typst reads XML through roxmltree, whose tree this reproduces: adjacent text and CDATA merge
826 // into one string, a comment or a processing instruction is a node with an empty tag, and the XML
827 // declaration and the document type are not nodes at all.
828 fn empty_node() -> Value {
829 let mut d = Dict::new();
830 d.insert("namespace", Value::None);
831 d.insert("tag", Value::str(""));
832 d.insert("attrs", Value::dict(Dict::new()));
833 d.insert("children", Value::array(Vec::new()));
834 Value::dict(d)
835 }
836 fn is_decl(doc: &Xml, n: &Node) -> bool {
837 match n {
838 Node::Pi(s) => {
839 let raw = doc.raw(s);
840 raw.starts_with("<?xml") && raw[5..].starts_with(|c: char| c.is_whitespace() || c == '?')
841 }
842 Node::DocType(_) => true,
843 _ => false,
844 }
845 }
846 fn children(doc: &Xml, nodes: &[Node], top: bool) -> Vec<Value> {
847 let mut out = Vec::new();
848 let mut text: Option<String> = None;
849 for n in nodes {
850 let piece = match n {
851 Node::Text(s) => Some(doc.text(s).to_string()),
852 Node::CData(s) => {
853 let raw = doc.raw(s);
854 Some(raw.strip_prefix("<![CDATA[").and_then(|r| r.strip_suffix("]]>")).unwrap_or(raw).to_string())
855 }
856 _ => None,
857 };
858 match piece {
859 Some(p) => {
860 if !top {
861 text.get_or_insert_with(String::new).push_str(&p);
862 }
863 continue;
864 }
865 None => if let Some(t) = text.take() {
866 out.push(Value::str(t));
867 },
868 }
869 if is_decl(doc, n) {
870 continue;
871 }
872 out.push(match n {
873 Node::Elem(e) => elem(doc, e),
874 _ => empty_node(),
875 });
876 }
877 if let Some(t) = text.take() {
878 out.push(Value::str(t));
879 }
880 out
881 }
882 fn elem(doc: &Xml, e: &oxedyne_fe2o3_text::xml::Elem) -> Value {
883 let mut d = Dict::new();
884 d.insert("namespace", match e.name.ns.and_then(|i| doc.uris().get(i)) {
885 Some(u) => Value::str(u.as_str()),
886 None => Value::None,
887 });
888 d.insert("tag", Value::str(e.name.local()));
889 let mut attrs = Dict::new();
890 for a in &e.attrs {
891 if a.name.qname == "xmlns" || a.name.qname.starts_with("xmlns:") {
892 continue;
893 }
894 attrs.insert(a.name.local(), Value::str(oxedyne_fe2o3_text::xml::write::decode(&a.value)));
895 }
896 d.insert("attrs", Value::dict(attrs));
897 d.insert("children", Value::array(children(doc, &e.kids, false)));
898 Value::dict(d)
899 }
900 Ok(Value::array(children(&doc, &doc.nodes, true)))
901}
902
903// CBOR
904
905struct CborReader<'a> {
906 b: &'a [u8],
907 at: usize,
908 depth: usize,
909}
910
911impl<'a> CborReader<'a> {
912 fn byte(&mut self) -> Outcome<u8> {
913 match self.b.get(self.at) {
914 Some(x) => {
915 self.at += 1;
916 Ok(*x)
917 }
918 None => Err(err!("unexpected end of input"; Decode, Input)),
919 }
920 }
921
922 fn take(&mut self, n: usize) -> Outcome<&'a [u8]> {
923 match self.b.get(self.at..self.at.saturating_add(n)) {
924 Some(s) => {
925 self.at += n;
926 Ok(s)
927 }
928 None => Err(err!("unexpected end of input"; Decode, Input)),
929 }
930 }
931
932 fn arg(&mut self, info: u8) -> Outcome<Option<u64>> {
933 let n = match info {
934 0..=23 => return Ok(Some(info as u64)),
935 24 => 1,
936 25 => 2,
937 26 => 4,
938 27 => 8,
939 31 => return Ok(None),
940 _ => return Err(err!("invalid additional information"; Decode, Input)),
941 };
942 let s = res!(self.take(n));
943 Ok(Some(s.iter().fold(0u64, |a, x| (a << 8) | *x as u64)))
944 }
945
946 fn value(&mut self) -> Outcome<Value> {
947 if self.depth > 256 {
948 return Err(err!("recursion limit exceeded"; Decode, Input));
949 }
950 let ib = res!(self.byte());
951 let (major, info) = (ib >> 5, ib & 0x1f);
952 match major {
953 0 => {
954 let n = res!(res!(self.arg(info)).ok_or_else(|| err!("indefinite integer"; Decode, Input)));
955 i64::try_from(n).map(Value::Int).map_err(|_| err!("integer too large"; Decode, Input))
956 }
957 1 => {
958 let n = res!(res!(self.arg(info)).ok_or_else(|| err!("indefinite integer"; Decode, Input)));
959 i64::try_from(n).map(|x| Value::Int(-1 - x)).map_err(|_| err!("integer too large"; Decode, Input))
960 }
961 2 | 3 => {
962 let mut buf = Vec::new();
963 match res!(self.arg(info)) {
964 Some(n) => buf.extend_from_slice(res!(self.take(n as usize))),
965 None => loop {
966 if self.b.get(self.at) == Some(&0xff) {
967 self.at += 1;
968 break;
969 }
970 let ib2 = res!(self.byte());
971 if ib2 >> 5 != major {
972 return Err(err!("invalid chunk in indefinite string"; Decode, Input));
973 }
974 let n = res!(res!(self.arg(ib2 & 0x1f)).ok_or_else(|| err!("nested indefinite string"; Decode, Input)));
975 buf.extend_from_slice(res!(self.take(n as usize)));
976 },
977 }
978 if major == 2 {
979 Ok(Value::Bytes(Arc::new(buf)))
980 } else {
981 String::from_utf8(buf).map(Value::str).map_err(|_| err!("invalid utf-8 in text string"; Decode, Input))
982 }
983 }
984 4 => {
985 self.depth += 1;
986 let mut a = Vec::new();
987 match res!(self.arg(info)) {
988 Some(n) => for _ in 0..n {
989 a.push(res!(self.value()));
990 },
991 None => while self.b.get(self.at) != Some(&0xff) {
992 a.push(res!(self.value()));
993 },
994 }
995 if info == 31 {
996 self.at += 1;
997 }
998 self.depth -= 1;
999 Ok(Value::array(a))
1000 }
1001 5 => {
1002 self.depth += 1;
1003 let mut d = Dict::new();
1004 let n = res!(self.arg(info));
1005 let mut i = 0u64;
1006 loop {
1007 match n {
1008 Some(n) if i >= n => break,
1009 None if self.b.get(self.at) == Some(&0xff) => {
1010 self.at += 1;
1011 break;
1012 }
1013 _ => (),
1014 }
1015 let k = match res!(self.value()) {
1016 Value::Str(s) => (*s).clone(),
1017 Value::Int(i) => i.to_string(),
1018 other => repr(&other),
1019 };
1020 let v = res!(self.value());
1021 d.insert(&k, v);
1022 i += 1;
1023 }
1024 self.depth -= 1;
1025 Ok(Value::dict(d))
1026 }
1027 6 => {
1028 res!(self.arg(info));
1029 self.value()
1030 }
1031 _ => match info {
1032 20 => Ok(Value::Bool(false)),
1033 21 => Ok(Value::Bool(true)),
1034 22 | 23 => Ok(Value::None),
1035 25 => {
1036 let s = res!(self.take(2));
1037 Ok(Value::Float(f16_to_f64(((s[0] as u16) << 8) | s[1] as u16)))
1038 }
1039 26 => {
1040 let s = res!(self.take(4));
1041 Ok(Value::Float(f32::from_be_bytes([s[0], s[1], s[2], s[3]]) as f64))
1042 }
1043 27 => {
1044 let s = res!(self.take(8));
1045 let mut a = [0u8; 8];
1046 a.copy_from_slice(s);
1047 Ok(Value::Float(f64::from_be_bytes(a)))
1048 }
1049 _ => Err(err!("unsupported simple value"; Decode, Input)),
1050 },
1051 }
1052 }
1053}
1054
1055fn f16_to_f64(h: u16) -> f64 {
1056 let sign = if h & 0x8000 != 0 { -1.0 } else { 1.0 };
1057 let exp = ((h >> 10) & 0x1f) as i32;
1058 let frac = (h & 0x3ff) as f64;
1059 let v = match exp {
1060 0 => frac * 2f64.powi(-24),
1061 31 => if frac == 0.0 { f64::INFINITY } else { f64::NAN },
1062 e => (1.0 + frac / 1024.0) * 2f64.powi(e - 15),
1063 };
1064 sign * v
1065}
1066
1067// The half-precision bits of `f`, when it converts exactly.
1068fn f64_to_f16_exact(f: f64) -> Option<u16> {
1069 if f.is_nan() {
1070 return Some(0x7e00);
1071 }
1072 let sign: u16 = if f.is_sign_negative() { 0x8000 } else { 0 };
1073 let a = f.abs();
1074 if a == 0.0 {
1075 return Some(sign);
1076 }
1077 if a.is_infinite() {
1078 return Some(sign | 0x7c00);
1079 }
1080 for exp in 1..31i32 {
1081 let base = 2f64.powi(exp - 15);
1082 if a >= base && a < base * 2.0 {
1083 let frac = (a / base - 1.0) * 1024.0;
1084 return if frac.fract() == 0.0 { Some(sign | ((exp as u16) << 10) | frac as u16) } else { None };
1085 }
1086 }
1087 let frac = a / 2f64.powi(-24);
1088 if frac.fract() == 0.0 && frac < 1024.0 { Some(sign | frac as u16) } else { None }
1089}
1090
1091fn cbor_head(major: u8, n: u64, out: &mut Vec<u8>) {
1092 let m = major << 5;
1093 if n < 24 {
1094 out.push(m | n as u8);
1095 } else if n <= 0xff {
1096 out.push(m | 24);
1097 out.push(n as u8);
1098 } else if n <= 0xffff {
1099 out.push(m | 25);
1100 out.extend_from_slice(&(n as u16).to_be_bytes());
1101 } else if n <= 0xffff_ffff {
1102 out.push(m | 26);
1103 out.extend_from_slice(&(n as u32).to_be_bytes());
1104 } else {
1105 out.push(m | 27);
1106 out.extend_from_slice(&n.to_be_bytes());
1107 }
1108}
1109
1110fn cbor_write(d: &Data, out: &mut Vec<u8>) {
1111 match d {
1112 Data::Null => out.push(0xf6),
1113 Data::Bool(b) => out.push(if *b { 0xf5 } else { 0xf4 }),
1114 Data::Int(i) => if *i >= 0 { cbor_head(0, *i as u64, out) } else { cbor_head(1, (-1 - *i) as u64, out) },
1115 Data::Float(f) => {
1116 if let Some(h) = f64_to_f16_exact(*f) {
1117 out.push(0xf9);
1118 out.extend_from_slice(&h.to_be_bytes());
1119 } else if (*f as f32) as f64 == *f {
1120 out.push(0xfa);
1121 out.extend_from_slice(&(*f as f32).to_be_bytes());
1122 } else {
1123 out.push(0xfb);
1124 out.extend_from_slice(&f.to_be_bytes());
1125 }
1126 }
1127 Data::Str(s) => {
1128 cbor_head(3, s.len() as u64, out);
1129 out.extend_from_slice(s.as_bytes());
1130 }
1131 Data::Bytes(b) => {
1132 cbor_head(2, b.len() as u64, out);
1133 out.extend_from_slice(b);
1134 }
1135 Data::Seq(a) => {
1136 cbor_head(4, a.len() as u64, out);
1137 for x in a {
1138 cbor_write(x, out);
1139 }
1140 }
1141 Data::Map(m) => {
1142 cbor_head(5, m.len() as u64, out);
1143 for (k, x) in m {
1144 cbor_write(&Data::Str(k.clone()), out);
1145 cbor_write(x, out);
1146 }
1147 }
1148 }
1149}
1150
1151// TOML
1152
1153struct Toml<'a> {
1154 s: &'a str,
1155 b: &'a [u8],
1156 at: usize,
1157 fail: Option<(usize, String)>, // where and why parsing stopped, for the diagnostic
1158}
1159
1160// A table under construction: its entries, and which were defined how, for TOML's redefinition rules.
1161#[derive(Clone, Debug, Default)]
1162struct TTable {
1163 entries: Vec<(String, TVal)>,
1164 defined: bool, // by a `[header]` or as a value
1165 inline: bool, // an inline table, closed to further keys
1166 dotted: bool, // created implicitly by a dotted key
1167}
1168
1169#[derive(Clone, Debug)]
1170enum TVal {
1171 Val(Value),
1172 Table(TTable),
1173 Array(Vec<TVal>, bool), // the bool: an array of tables (`[[x]]`), still extendable
1174}
1175
1176impl TTable {
1177 fn get_mut(&mut self, k: &str) -> Option<&mut TVal> {
1178 self.entries.iter_mut().find(|(n, _)| n == k).map(|(_, v)| v)
1179 }
1180
1181 fn has(&self, k: &str) -> bool { self.entries.iter().any(|(n, _)| n == k) }
1182
1183 fn into_value(self) -> Value {
1184 let mut d = Dict::new();
1185 for (k, v) in self.entries {
1186 d.insert(&k, v.into_value());
1187 }
1188 Value::dict(d)
1189 }
1190}
1191
1192impl TVal {
1193 fn into_value(self) -> Value {
1194 match self {
1195 TVal::Val(v) => v,
1196 TVal::Table(t) => t.into_value(),
1197 TVal::Array(a, _) => Value::array(a.into_iter().map(TVal::into_value).collect()),
1198 }
1199 }
1200}
1201
1202pub(crate) fn toml_parse(s: &str) -> std::result::Result<Value, (usize, String)> {
1203 let mut p = Toml { s, b: s.as_bytes(), at: 0, fail: None };
1204 match toml_document(&mut p) {
1205 Ok(v) => Ok(v),
1206 Err(_) => Err(p.fail.unwrap_or((0, "invalid TOML".to_string()))),
1207 }
1208}
1209
1210fn toml_document(p: &mut Toml) -> Outcome<Value> {
1211 let mut root = TTable { defined: true, ..TTable::default() };
1212 let mut current: Vec<String> = Vec::new();
1213 loop {
1214 p.skip_ws_comments_newlines();
1215 if p.at >= p.b.len() {
1216 break;
1217 }
1218 if p.b[p.at] == b'[' {
1219 let aot = p.b.get(p.at + 1) == Some(&b'[');
1220 p.at += if aot { 2 } else { 1 };
1221 p.skip_ws();
1222 let key = res!(p.key());
1223 p.skip_ws();
1224 if aot {
1225 if !p.s[p.at..].starts_with("]]") {
1226 return p.err("invalid table header\nexpected `.`, `]]`");
1227 }
1228 p.at += 2;
1229 } else {
1230 if p.b.get(p.at) != Some(&b']') {
1231 return p.err("invalid table header\nexpected `.`, `]`");
1232 }
1233 p.at += 1;
1234 }
1235 res!(p.end_of_line());
1236 let at = p.at;
1237 if let Err(m) = toml_open_table(&mut root, &key, aot) {
1238 return p.fail_at(at, &m);
1239 }
1240 current = key;
1241 } else {
1242 let key = res!(p.key());
1243 p.skip_ws();
1244 if p.b.get(p.at) != Some(&b'=') {
1245 return p.err("expected `=`");
1246 }
1247 p.at += 1;
1248 p.skip_ws();
1249 let at = p.at;
1250 let v = res!(p.value());
1251 res!(p.end_of_line());
1252 let table = match toml_current(&mut root, &current) {
1253 Some(t) => t,
1254 None => return p.fail_at(at, "invalid table"),
1255 };
1256 if let Err(m) = toml_insert(table, &key, v) {
1257 return p.fail_at(at, &m);
1258 }
1259 }
1260 }
1261 Ok(root.into_value())
1262}
1263
1264// The table a path names, descending through the last element of arrays of tables.
1265fn toml_current<'t>(root: &'t mut TTable, path: &[String]) -> Option<&'t mut TTable> {
1266 let mut t = root;
1267 for k in path {
1268 t = match t.get_mut(k) {
1269 Some(TVal::Table(x)) => x,
1270 Some(TVal::Array(a, _)) => match a.last_mut() {
1271 Some(TVal::Table(x)) => x,
1272 _ => return None,
1273 },
1274 _ => return None,
1275 };
1276 }
1277 Some(t)
1278}
1279
1280fn toml_open_table(root: &mut TTable, key: &[String], aot: bool) -> std::result::Result<(), String> {
1281 let mut t = root;
1282 let (last, head) = match key.split_last() {
1283 Some(x) => x,
1284 None => return Err("empty table key".to_string()),
1285 };
1286 for k in head {
1287 if !t.has(k) {
1288 t.entries.push((k.clone(), TVal::Table(TTable::default())));
1289 }
1290 t = match t.get_mut(k) {
1291 Some(TVal::Table(x)) if !x.inline => x,
1292 Some(TVal::Array(a, true)) => match a.last_mut() {
1293 Some(TVal::Table(x)) => x,
1294 _ => return Err(fmt!("duplicate key `{}`", k)),
1295 },
1296 _ => return Err(fmt!("duplicate key `{}`", k)),
1297 };
1298 }
1299 if aot {
1300 match t.get_mut(last) {
1301 None => t.entries.push((last.clone(),
1302 TVal::Array(vec![TVal::Table(TTable { defined: true, ..TTable::default() })], true))),
1303 Some(TVal::Array(a, true)) => a.push(TVal::Table(TTable { defined: true, ..TTable::default() })),
1304 _ => return Err(fmt!("duplicate key `{}`", last)),
1305 }
1306 } else {
1307 match t.get_mut(last) {
1308 None => t.entries.push((last.clone(), TVal::Table(TTable { defined: true, ..TTable::default() }))),
1309 Some(TVal::Table(x)) if !x.defined && !x.inline && !x.dotted => x.defined = true,
1310 _ => return Err(fmt!("duplicate key `{}`", last)),
1311 }
1312 }
1313 Ok(())
1314}
1315
1316fn toml_insert(t: &mut TTable, key: &[String], v: TVal) -> std::result::Result<(), String> {
1317 let (last, head) = match key.split_last() {
1318 Some(x) => x,
1319 None => return Err("empty key".to_string()),
1320 };
1321 let mut t = t;
1322 for k in head {
1323 if !t.has(k) {
1324 t.entries.push((k.clone(), TVal::Table(TTable { dotted: true, ..TTable::default() })));
1325 }
1326 t = match t.get_mut(k) {
1327 Some(TVal::Table(x)) if !x.inline && (x.dotted || !x.defined) => x,
1328 _ => return Err(fmt!("duplicate key `{}`", k)),
1329 };
1330 }
1331 if t.has(last) {
1332 return Err(fmt!("duplicate key `{}`", last));
1333 }
1334 t.entries.push((last.clone(), v));
1335 Ok(())
1336}
1337
1338impl<'a> Toml<'a> {
1339 fn err<T>(&mut self, m: &str) -> Outcome<T> {
1340 let at = self.at;
1341 self.fail_at(at, m)
1342 }
1343
1344 fn fail_at<T>(&mut self, at: usize, m: &str) -> Outcome<T> {
1345 self.fail = Some((at, m.to_string()));
1346 Err(err!("{}", m; Decode, Input))
1347 }
1348
1349 fn skip_ws(&mut self) {
1350 while self.at < self.b.len() && matches!(self.b[self.at], b' ' | b'\t') {
1351 self.at += 1;
1352 }
1353 }
1354
1355 fn skip_comment(&mut self) {
1356 if self.b.get(self.at) == Some(&b'#') {
1357 while self.at < self.b.len() && self.b[self.at] != b'\n' {
1358 self.at += 1;
1359 }
1360 }
1361 }
1362
1363 fn skip_ws_comments_newlines(&mut self) {
1364 loop {
1365 self.skip_ws();
1366 self.skip_comment();
1367 match self.b.get(self.at) {
1368 Some(b'\n') => self.at += 1,
1369 Some(b'\r') if self.b.get(self.at + 1) == Some(&b'\n') => self.at += 2,
1370 _ => break,
1371 }
1372 }
1373 }
1374
1375 fn end_of_line(&mut self) -> Outcome<()> {
1376 self.skip_ws();
1377 self.skip_comment();
1378 match self.b.get(self.at) {
1379 None | Some(b'\n') => Ok(()),
1380 Some(b'\r') if self.b.get(self.at + 1) == Some(&b'\n') => Ok(()),
1381 _ => self.err("expected newline"),
1382 }
1383 }
1384
1385 fn key(&mut self) -> Outcome<Vec<String>> {
1386 let mut parts = Vec::new();
1387 loop {
1388 self.skip_ws();
1389 let part = match self.b.get(self.at) {
1390 Some(b'"') => res!(self.basic_string()),
1391 Some(b'\'') => res!(self.literal_string()),
1392 _ => {
1393 let start = self.at;
1394 while self.at < self.b.len() && (self.b[self.at].is_ascii_alphanumeric() || matches!(self.b[self.at], b'_' | b'-')) {
1395 self.at += 1;
1396 }
1397 if self.at == start {
1398 return self.err("invalid key");
1399 }
1400 self.s[start..self.at].to_string()
1401 }
1402 };
1403 parts.push(part);
1404 self.skip_ws();
1405 if self.b.get(self.at) == Some(&b'.') {
1406 self.at += 1;
1407 continue;
1408 }
1409 return Ok(parts);
1410 }
1411 }
1412
1413 fn value(&mut self) -> Outcome<TVal> {
1414 match self.b.get(self.at) {
1415 None => self.err("expected a value"),
1416 Some(b'"') => {
1417 if self.s[self.at..].starts_with("\"\"\"") {
1418 self.ml_basic_string().map(|s| TVal::Val(Value::str(s)))
1419 } else {
1420 self.basic_string().map(|s| TVal::Val(Value::str(s)))
1421 }
1422 }
1423 Some(b'\'') => {
1424 if self.s[self.at..].starts_with("'''") {
1425 self.ml_literal_string().map(|s| TVal::Val(Value::str(s)))
1426 } else {
1427 self.literal_string().map(|s| TVal::Val(Value::str(s)))
1428 }
1429 }
1430 Some(b'[') => {
1431 self.at += 1;
1432 let mut a = Vec::new();
1433 loop {
1434 self.skip_ws_comments_newlines();
1435 if self.b.get(self.at) == Some(&b']') {
1436 self.at += 1;
1437 break;
1438 }
1439 a.push(res!(self.value()));
1440 self.skip_ws_comments_newlines();
1441 match self.b.get(self.at) {
1442 Some(b',') => self.at += 1,
1443 Some(b']') => {
1444 self.at += 1;
1445 break;
1446 }
1447 _ => return self.err("expected `,` or `]`"),
1448 }
1449 }
1450 Ok(TVal::Array(a, false))
1451 }
1452 Some(b'{') => {
1453 self.at += 1;
1454 let mut t = TTable::default();
1455 self.skip_ws();
1456 if self.b.get(self.at) == Some(&b'}') {
1457 self.at += 1;
1458 } else {
1459 loop {
1460 self.skip_ws();
1461 let k = res!(self.key());
1462 self.skip_ws();
1463 if self.b.get(self.at) != Some(&b'=') {
1464 return self.err("expected `=`");
1465 }
1466 self.at += 1;
1467 self.skip_ws();
1468 let at = self.at;
1469 let v = res!(self.value());
1470 if let Err(m) = toml_insert(&mut t, &k, v) {
1471 return self.fail_at(at, &m);
1472 }
1473 self.skip_ws();
1474 match self.b.get(self.at) {
1475 Some(b',') => self.at += 1,
1476 Some(b'}') => {
1477 self.at += 1;
1478 break;
1479 }
1480 _ => return self.err("expected `,` or `}`"),
1481 }
1482 }
1483 }
1484 t.inline = true;
1485 t.defined = true;
1486 Ok(TVal::Table(t))
1487 }
1488 Some(b't') if self.s[self.at..].starts_with("true") => {
1489 self.at += 4;
1490 Ok(TVal::Val(Value::Bool(true)))
1491 }
1492 Some(b'f') if self.s[self.at..].starts_with("false") => {
1493 self.at += 5;
1494 Ok(TVal::Val(Value::Bool(false)))
1495 }
1496 Some(_) => self.scalar(),
1497 }
1498 }
1499
1500 // Numbers and dates: a bare run of characters, classified.
1501 fn scalar(&mut self) -> Outcome<TVal> {
1502 let start = self.at;
1503 while self.at < self.b.len() {
1504 let c = self.b[self.at];
1505 let date_space = c == b' ' && self.at - start == 10
1506 && self.b.get(self.at + 1).map(|d| d.is_ascii_digit()).unwrap_or(false)
1507 && self.s[start..self.at].bytes().filter(|x| *x == b'-').count() == 2;
1508 if c.is_ascii_alphanumeric() || matches!(c, b'+' | b'-' | b'_' | b'.' | b':') || date_space {
1509 self.at += 1;
1510 } else {
1511 break;
1512 }
1513 }
1514 let t = &self.s[start..self.at];
1515 if t.is_empty() {
1516 return self.err("expected a value");
1517 }
1518 if let Some(dt) = toml_datetime(t) {
1519 return Ok(TVal::Val(Value::Datetime(dt)));
1520 }
1521 match toml_number(t) {
1522 Some(v) => Ok(TVal::Val(v)),
1523 None => self.fail_at(start, "invalid number"),
1524 }
1525 }
1526
1527 fn escape(&mut self, out: &mut String) -> Outcome<()> {
1528 let c = match self.b.get(self.at) {
1529 Some(c) => *c,
1530 None => return self.err("unterminated string"),
1531 };
1532 self.at += 1;
1533 match c {
1534 b'b' => out.push('\u{8}'),
1535 b't' => out.push('\t'),
1536 b'n' => out.push('\n'),
1537 b'f' => out.push('\u{c}'),
1538 b'r' => out.push('\r'),
1539 b'e' => out.push('\u{1b}'),
1540 b'"' => out.push('"'),
1541 b'\\' => out.push('\\'),
1542 b'u' | b'U' => {
1543 let n = if c == b'u' { 4 } else { 8 };
1544 let cp = self.s.get(self.at..self.at + n).and_then(|h| u32::from_str_radix(h, 16).ok())
1545 .and_then(char::from_u32);
1546 match cp {
1547 Some(ch) => {
1548 out.push(ch);
1549 self.at += n;
1550 }
1551 None => return self.err("invalid unicode escape"),
1552 }
1553 }
1554 _ => return self.err("invalid escape"),
1555 }
1556 Ok(())
1557 }
1558
1559 fn basic_string(&mut self) -> Outcome<String> {
1560 self.at += 1;
1561 let mut out = String::new();
1562 loop {
1563 let rest = &self.s[self.at..];
1564 let ch = match rest.chars().next() {
1565 Some(c) => c,
1566 None => return self.err("unterminated string"),
1567 };
1568 match ch {
1569 '"' => {
1570 self.at += 1;
1571 return Ok(out);
1572 }
1573 '\\' => {
1574 self.at += 1;
1575 res!(self.escape(&mut out));
1576 }
1577 '\n' => return self.err("newline in string"),
1578 c => {
1579 out.push(c);
1580 self.at += c.len_utf8();
1581 }
1582 }
1583 }
1584 }
1585
1586 fn ml_basic_string(&mut self) -> Outcome<String> {
1587 self.at += 3;
1588 if self.s[self.at..].starts_with("\r\n") {
1589 self.at += 2;
1590 } else if self.s[self.at..].starts_with('\n') {
1591 self.at += 1;
1592 }
1593 let mut out = String::new();
1594 loop {
1595 if self.s[self.at..].starts_with("\"\"\"") {
1596 // Up to two quotes may close into the content: `""""` ends with one quote inside.
1597 let mut n = 3;
1598 while self.b.get(self.at + n) == Some(&b'"') && n < 5 {
1599 n += 1;
1600 }
1601 out.push_str(&"\"".repeat(n - 3));
1602 self.at += n;
1603 return Ok(out);
1604 }
1605 let ch = match self.s[self.at..].chars().next() {
1606 Some(c) => c,
1607 None => return self.err("unterminated string"),
1608 };
1609 if ch == '\\' {
1610 self.at += 1;
1611 // A line-ending backslash trims the newline and following whitespace.
1612 let save = self.at;
1613 let mut j = self.at;
1614 while j < self.b.len() && matches!(self.b[j], b' ' | b'\t') {
1615 j += 1;
1616 }
1617 if matches!(self.b.get(j), Some(b'\n') | Some(b'\r')) {
1618 while j < self.b.len() && matches!(self.b[j], b' ' | b'\t' | b'\n' | b'\r') {
1619 j += 1;
1620 }
1621 self.at = j;
1622 } else {
1623 self.at = save;
1624 res!(self.escape(&mut out));
1625 }
1626 } else {
1627 out.push(ch);
1628 self.at += ch.len_utf8();
1629 }
1630 }
1631 }
1632
1633 fn literal_string(&mut self) -> Outcome<String> {
1634 self.at += 1;
1635 match self.s[self.at..].find(['\'', '\n']) {
1636 Some(i) if self.b[self.at + i] == b'\'' => {
1637 let out = self.s[self.at..self.at + i].to_string();
1638 self.at += i + 1;
1639 Ok(out)
1640 }
1641 _ => self.err("unterminated string"),
1642 }
1643 }
1644
1645 fn ml_literal_string(&mut self) -> Outcome<String> {
1646 self.at += 3;
1647 if self.s[self.at..].starts_with("\r\n") {
1648 self.at += 2;
1649 } else if self.s[self.at..].starts_with('\n') {
1650 self.at += 1;
1651 }
1652 match self.s[self.at..].find("'''") {
1653 Some(i) => {
1654 let mut end = self.at + i;
1655 let mut n = 3;
1656 while self.b.get(end + n) == Some(&b'\'') && n < 5 {
1657 n += 1;
1658 }
1659 end += n - 3;
1660 let out = self.s[self.at..end].to_string();
1661 self.at = end + 3;
1662 Ok(out)
1663 }
1664 None => self.err("unterminated string"),
1665 }
1666 }
1667}
1668
1669fn toml_number(t: &str) -> Option<Value> {
1670 let (sign, body) = match t.as_bytes().first() {
1671 Some(b'+') => (1i64, &t[1..]),
1672 Some(b'-') => (-1i64, &t[1..]),
1673 _ => (1i64, t),
1674 };
1675 match body {
1676 "inf" => return Some(Value::Float(sign as f64 * f64::INFINITY)),
1677 "nan" => return Some(Value::Float(f64::NAN)),
1678 _ => (),
1679 }
1680 let underscores_ok = |s: &str| !s.starts_with('_') && !s.ends_with('_') && !s.contains("__");
1681 for (prefix, radix) in [("0x", 16), ("0o", 8), ("0b", 2)] {
1682 if let Some(d) = body.strip_prefix(prefix) {
1683 if sign != 1 || t.starts_with('+') || !underscores_ok(d) {
1684 return None;
1685 }
1686 return i64::from_str_radix(&d.replace('_', ""), radix).ok().map(Value::Int);
1687 }
1688 }
1689 if !underscores_ok(body) || body.is_empty() {
1690 return None;
1691 }
1692 let clean = body.replace('_', "");
1693 if clean.bytes().all(|b| b.is_ascii_digit()) {
1694 if clean.len() > 1 && clean.starts_with('0') {
1695 return None;
1696 }
1697 return clean.parse::<i64>().ok().map(|v| Value::Int(sign * v)).or_else(|| {
1698 if sign < 0 { clean.parse::<i128>().ok().filter(|v| -*v == i64::MIN as i128).map(|_| Value::Int(i64::MIN)) } else { None }
1699 });
1700 }
1701 let ok = clean.bytes().all(|b| b.is_ascii_digit() || matches!(b, b'.' | b'e' | b'E' | b'+' | b'-'))
1702 && !clean.starts_with('.') && !clean.ends_with('.') && !clean.contains(".e") && !clean.contains(".E");
1703 if !ok {
1704 return None;
1705 }
1706 clean.parse::<f64>().ok().map(|v| Value::Float(sign as f64 * v))
1707}
1708
1709// Fixed-width decimal digits, or `None`.
1710fn digits_n(s: Option<&str>, n: usize) -> Option<u32> {
1711 match s {
1712 Some(s) if s.len() == n && s.bytes().all(|b| b.is_ascii_digit()) => s.parse().ok(),
1713 _ => None,
1714 }
1715}
1716
1717// `hh:mm:ss`, a fraction and an offset dropped, as Typst keeps whole seconds.
1718fn toml_time(s: &str) -> Option<(u8, u8, u8)> {
1719 let s = s.split(['Z', 'z', '+']).next().unwrap_or("");
1720 let s = if s.len() > 8 && s.as_bytes()[8] == b'-' { &s[..8] } else { s };
1721 let s = s.split('.').next().unwrap_or("");
1722 let mut p = s.split(':');
1723 match (digits_n(p.next(), 2), digits_n(p.next(), 2), digits_n(p.next(), 2)) {
1724 (Some(h), Some(m), Some(sec)) if h <= 23 && m <= 59 && sec <= 60 => Some((h as u8, m as u8, sec.min(59) as u8)),
1725 _ => None,
1726 }
1727}
1728
1729fn toml_date(s: &str) -> Option<(i32, u8, u8)> {
1730 let mut p = s.split('-');
1731 match (digits_n(p.next(), 4), digits_n(p.next(), 2), digits_n(p.next(), 2), p.next()) {
1732 (Some(y), Some(m), Some(d), None) if (1..=12).contains(&m) && (1..=31).contains(&d) => Some((y as i32, m as u8, d as u8)),
1733 _ => None,
1734 }
1735}
1736
1737fn toml_datetime(t: &str) -> Option<Datetime> {
1738 if t.len() >= 10 && t.as_bytes().get(4) == Some(&b'-') {
1739 let (y, m, d) = match toml_date(&t[..10]) {
1740 Some(x) => x,
1741 None => return None,
1742 };
1743 let mut dt = Datetime { year: Some(y), month: Some(m), day: Some(d), ..Datetime::default() };
1744 if t.len() > 10 {
1745 if !matches!(t.as_bytes()[10], b'T' | b't' | b' ') {
1746 return None;
1747 }
1748 let (h, mi, s) = match toml_time(&t[11..]) {
1749 Some(x) => x,
1750 None => return None,
1751 };
1752 dt.hour = Some(h);
1753 dt.minute = Some(mi);
1754 dt.second = Some(s);
1755 }
1756 return Some(dt);
1757 }
1758 if t.len() >= 8 && t.as_bytes().get(2) == Some(&b':') {
1759 let (h, mi, s) = match toml_time(t) {
1760 Some(x) => x,
1761 None => return None,
1762 };
1763 return Some(Datetime { hour: Some(h), minute: Some(mi), second: Some(s), ..Datetime::default() });
1764 }
1765 None
1766}
1767
1768fn toml_key(k: &str) -> String {
1769 if !k.is_empty() && k.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-') {
1770 k.to_string()
1771 } else {
1772 toml_str(k)
1773 }
1774}
1775
1776// A TOML string: literal quotes when the text has a double quote and no single quote or control
1777// character, as the `toml` crate prefers; basic quotes otherwise.
1778fn toml_str(s: &str) -> String {
1779 if s.contains('"') && !s.contains('\'') && !s.chars().any(|c| c.is_control()) {
1780 return fmt!("'{}'", s);
1781 }
1782 let mut out = String::from("\"");
1783 for c in s.chars() {
1784 match c {
1785 '"' => out.push_str("\\\""),
1786 '\\' => out.push_str("\\\\"),
1787 '\n' => out.push_str("\\n"),
1788 '\r' => out.push_str("\\r"),
1789 '\t' => out.push_str("\\t"),
1790 '\u{8}' => out.push_str("\\b"),
1791 '\u{c}' => out.push_str("\\f"),
1792 c if c.is_control() => out.push_str(&fmt!("\\u{:04X}", c as u32)),
1793 c => out.push(c),
1794 }
1795 }
1796 out.push('"');
1797 out
1798}
1799
1800fn toml_inline(d: &Data, pretty: bool, out: &mut String) {
1801 match d {
1802 Data::Null => (),
1803 Data::Bool(b) => out.push_str(if *b { "true" } else { "false" }),
1804 Data::Int(i) => out.push_str(&i.to_string()),
1805 Data::Float(f) => out.push_str(&if f.is_nan() {
1806 "nan".to_string()
1807 } else if f.is_infinite() {
1808 if *f < 0.0 { "-inf".to_string() } else { "inf".to_string() }
1809 } else {
1810 ryu_format(*f)
1811 }),
1812 Data::Str(s) => out.push_str(&toml_str(s)),
1813 Data::Bytes(b) => toml_inline(&Data::Seq(b.iter().map(|x| Data::Int(*x as i64)).collect()), pretty, out),
1814 Data::Seq(a) => {
1815 if pretty && !a.is_empty() {
1816 out.push_str("[\n");
1817 for x in a {
1818 out.push_str(" ");
1819 toml_inline(x, false, out);
1820 out.push_str(",\n");
1821 }
1822 out.push(']');
1823 } else {
1824 out.push('[');
1825 for (i, x) in a.iter().enumerate() {
1826 if i > 0 {
1827 out.push_str(", ");
1828 }
1829 toml_inline(x, false, out);
1830 }
1831 out.push(']');
1832 }
1833 }
1834 Data::Map(m) => {
1835 out.push_str("{ ");
1836 for (i, (k, x)) in m.iter().filter(|(_, x)| !matches!(x, Data::Null)).enumerate() {
1837 if i > 0 {
1838 out.push_str(", ");
1839 }
1840 out.push_str(&toml_key(k));
1841 out.push_str(" = ");
1842 toml_inline(x, false, out);
1843 }
1844 out.push_str(" }");
1845 }
1846 }
1847}
1848
1849fn is_table_array(d: &Data) -> bool {
1850 matches!(d, Data::Seq(a) if !a.is_empty() && a.iter().all(|x| matches!(x, Data::Map(_))))
1851}
1852
1853fn toml_write_table(m: &[(String, Data)], path: &[String], pretty: bool, out: &mut String) {
1854 // Plain values first, then subtables and arrays of tables, as the `toml` crate orders them.
1855 for (k, v) in m {
1856 if matches!(v, Data::Null | Data::Map(_)) || is_table_array(v) {
1857 continue;
1858 }
1859 out.push_str(&toml_key(k));
1860 out.push_str(" = ");
1861 toml_inline(v, pretty, out);
1862 out.push('\n');
1863 }
1864 for (k, v) in m {
1865 let mut p = path.to_vec();
1866 p.push(k.clone());
1867 let header = p.iter().map(|x| toml_key(x)).collect::<Vec<_>>().join(".");
1868 match v {
1869 Data::Map(sub) => {
1870 let has_plain = sub.iter().any(|(_, x)| !matches!(x, Data::Null | Data::Map(_)) && !is_table_array(x));
1871 if has_plain || sub.is_empty() {
1872 if !out.is_empty() {
1873 out.push('\n');
1874 }
1875 out.push_str(&fmt!("[{}]\n", header));
1876 }
1877 toml_write_table(sub, &p, pretty, out);
1878 }
1879 Data::Seq(a) if is_table_array(v) => for x in a {
1880 if let Data::Map(sub) = x {
1881 if !out.is_empty() {
1882 out.push('\n');
1883 }
1884 out.push_str(&fmt!("[[{}]]\n", header));
1885 toml_write_table(sub, &p, pretty, out);
1886 }
1887 },
1888 _ => (),
1889 }
1890 }
1891}
1892
1893// YAML
1894
1895#[derive(Clone, Debug)]
1896struct YLine<'a> {
1897 indent: usize,
1898 text: &'a str, // without indentation; comments stripped outside quotes
1899 at: usize, // byte offset of the text in the source
1900}
1901
1902struct Yaml<'a> {
1903 lines: Vec<YLine<'a>>,
1904 i: usize,
1905 anchors: Vec<(String, Value)>,
1906 src: &'a str,
1907 fail: Option<(usize, String)>, // where and why parsing stopped, for the diagnostic
1908}
1909
1910fn strip_comment(t: &str) -> &str {
1911 let b = t.as_bytes();
1912 let (mut sq, mut dq) = (false, false);
1913 for i in 0..b.len() {
1914 match b[i] {
1915 b'\'' if !dq => sq = !sq,
1916 b'"' if !sq => dq = !dq,
1917 b'#' if !sq && !dq && (i == 0 || b[i - 1] == b' ' || b[i - 1] == b'\t') => return t[..i].trim_end(),
1918 _ => (),
1919 }
1920 }
1921 t.trim_end()
1922}
1923
1924pub(crate) fn yaml_parse(src: &str) -> std::result::Result<Value, (usize, String)> {
1925 let mut y = Yaml { lines: Vec::new(), i: 0, anchors: Vec::new(), src, fail: None };
1926 match yaml_document(&mut y) {
1927 Ok(v) => Ok(v),
1928 Err(_) => Err(y.fail.unwrap_or((0, "invalid YAML".to_string()))),
1929 }
1930}
1931
1932fn yaml_document(y: &mut Yaml) -> Outcome<Value> {
1933 let src = y.src;
1934 let mut lines = Vec::new();
1935 let mut off = 0;
1936 for raw in src.split('\n') {
1937 let line = raw.strip_suffix('\r').unwrap_or(raw);
1938 let indent = line.len() - line.trim_start_matches(' ').len();
1939 lines.push(YLine { indent, text: &line[indent..], at: off + indent });
1940 off += raw.len() + 1;
1941 }
1942 y.lines = lines;
1943 y.skip_blank();
1944 if let Some(l) = y.lines.get(y.i) {
1945 if l.text.starts_with("%") {
1946 let at = l.at;
1947 return y.fail_at(at, "directives are not supported");
1948 }
1949 if l.text == "---" || l.text.starts_with("--- ") {
1950 let rest = strip_comment(l.text[3..].trim_start());
1951 if rest.is_empty() {
1952 y.i += 1;
1953 } else {
1954 let at = l.at + (l.text.len() - l.text[3..].trim_start().len());
1955 let v = res!(y.inline_value(rest, at));
1956 return Ok(v);
1957 }
1958 }
1959 }
1960 y.skip_blank();
1961 if y.i >= y.lines.len() {
1962 return Ok(Value::None);
1963 }
1964 let ind = y.lines[y.i].indent;
1965 let v = res!(y.block(ind));
1966 y.skip_blank();
1967 if let Some(l) = y.lines.get(y.i) {
1968 if l.text != "..." && l.text != "---" {
1969 let at = l.at;
1970 return y.fail_at(at, "did not find expected end of document");
1971 }
1972 }
1973 Ok(v)
1974}
1975
1976impl<'a> Yaml<'a> {
1977 fn fail_at<T>(&mut self, at: usize, m: &str) -> Outcome<T> {
1978 self.fail = Some((at, m.to_string()));
1979 Err(err!("{}", m; Decode, Input))
1980 }
1981
1982 fn skip_blank(&mut self) {
1983 while let Some(l) = self.lines.get(self.i) {
1984 if strip_comment(l.text).is_empty() {
1985 self.i += 1;
1986 } else {
1987 break;
1988 }
1989 }
1990 }
1991
1992 // A block node whose lines are indented by `ind`.
1993 fn block(&mut self, ind: usize) -> Outcome<Value> {
1994 self.skip_blank();
1995 let l = match self.lines.get(self.i) {
1996 Some(l) => l.clone(),
1997 None => return Ok(Value::None),
1998 };
1999 let t = strip_comment(l.text);
2000 if t == "-" || t.starts_with("- ") {
2001 return self.sequence(ind);
2002 }
2003 if yaml_key_split(t).is_some() {
2004 return self.mapping(ind);
2005 }
2006 // A scalar spread over lines, or a flow collection.
2007 self.i += 1;
2008 let mut text = t.to_string();
2009 if t.starts_with('[') || t.starts_with('{') || t.starts_with('"') || t.starts_with('\'') {
2010 while !yaml_balanced(&text) && self.i < self.lines.len() {
2011 text.push(' ');
2012 text.push_str(strip_comment(self.lines[self.i].text).trim());
2013 self.i += 1;
2014 }
2015 } else {
2016 while let Some(n) = self.lines.get(self.i) {
2017 let nt = strip_comment(n.text);
2018 if nt.is_empty() || n.indent < ind || (n.indent == ind && (nt.starts_with("- ") || yaml_key_split(nt).is_some())) {
2019 break;
2020 }
2021 text.push(' ');
2022 text.push_str(nt.trim());
2023 self.i += 1;
2024 }
2025 }
2026 self.inline_value(&text, l.at)
2027 }
2028
2029 fn sequence(&mut self, ind: usize) -> Outcome<Value> {
2030 let mut out = Vec::new();
2031 loop {
2032 self.skip_blank();
2033 let l = match self.lines.get(self.i) {
2034 Some(l) if l.indent == ind => l.clone(),
2035 _ => break,
2036 };
2037 let t = strip_comment(l.text);
2038 if !(t == "-" || t.starts_with("- ")) {
2039 break;
2040 }
2041 let rest = t[1..].trim_start();
2042 let rest_at = l.at + (t.len() - rest.len());
2043 if rest.is_empty() {
2044 self.i += 1;
2045 self.skip_blank();
2046 match self.lines.get(self.i) {
2047 Some(n) if n.indent > ind => {
2048 let ni = n.indent;
2049 out.push(res!(self.block(ni)));
2050 }
2051 _ => out.push(Value::None),
2052 }
2053 } else {
2054 // The item's content starts on this line; treat it as a line indented to its column.
2055 let col = ind + (t.len() - rest.len());
2056 self.lines[self.i] = YLine { indent: col, text: rest, at: rest_at };
2057 out.push(res!(self.item(col, ind)));
2058 }
2059 }
2060 Ok(Value::array(out))
2061 }
2062
2063 // A node starting at the current line, which sits at column `col`.
2064 // A sequence item's content starting on the dash's line, at column `col`; a block scalar there is
2065 // indented relative to the sequence's own indentation `parent`, not to the item's column.
2066 fn item(&mut self, col: usize, parent: usize) -> Outcome<Value> {
2067 let l = match self.lines.get(self.i) {
2068 Some(l) => l.clone(),
2069 None => return Ok(Value::None),
2070 };
2071 let t = strip_comment(l.text);
2072 if let Some(v) = res!(self.block_scalar(t, parent)) {
2073 return Ok(v);
2074 }
2075 self.block(col)
2076 }
2077
2078 fn mapping(&mut self, ind: usize) -> Outcome<Value> {
2079 let mut d = Dict::new();
2080 loop {
2081 self.skip_blank();
2082 let l = match self.lines.get(self.i) {
2083 Some(l) if l.indent == ind => l.clone(),
2084 _ => break,
2085 };
2086 let t = strip_comment(l.text);
2087 let (k, rest) = match yaml_key_split(t) {
2088 Some(x) => x,
2089 None => break,
2090 };
2091 let key = match res!(self.inline_value(k, l.at)) {
2092 Value::Str(s) => (*s).clone(),
2093 Value::None => "null".to_string(),
2094 other => match crate::eval::lib::foundations::display(&other) {
2095 Some(s) => s.replace('\u{2212}', "-"),
2096 None => repr(&other),
2097 },
2098 };
2099 let rest = rest.trim_start();
2100 let rest_at = l.at + (t.len() - rest.len());
2101 let (anchor, rest) = yaml_anchor(rest);
2102 let v = if rest.is_empty() {
2103 self.i += 1;
2104 self.skip_blank();
2105 match self.lines.get(self.i) {
2106 Some(n) if n.indent > ind => {
2107 let ni = n.indent;
2108 res!(self.block(ni))
2109 }
2110 Some(n) if n.indent == ind && (strip_comment(n.text).starts_with("- ") || strip_comment(n.text) == "-") => {
2111 res!(self.sequence(ind))
2112 }
2113 _ => Value::None,
2114 }
2115 } else if let Some(v) = res!(self.block_scalar(rest, ind)) {
2116 v
2117 } else {
2118 self.i += 1;
2119 let mut text = rest.to_string();
2120 if rest.starts_with('[') || rest.starts_with('{') || rest.starts_with('"') || rest.starts_with('\'') {
2121 while !yaml_balanced(&text) && self.i < self.lines.len() {
2122 text.push(' ');
2123 text.push_str(strip_comment(self.lines[self.i].text).trim());
2124 self.i += 1;
2125 }
2126 } else {
2127 while let Some(n) = self.lines.get(self.i) {
2128 let nt = strip_comment(n.text);
2129 if nt.is_empty() || n.indent <= ind {
2130 break;
2131 }
2132 text.push(' ');
2133 text.push_str(nt.trim());
2134 self.i += 1;
2135 }
2136 }
2137 res!(self.inline_value(&text, rest_at))
2138 };
2139 if let Some(a) = anchor {
2140 self.anchors.push((a.to_string(), v.clone()));
2141 }
2142 d.insert(&key, v);
2143 }
2144 Ok(Value::dict(d))
2145 }
2146
2147 // `|` and `>` block scalars, with `-`/`+` chomping.
2148 fn block_scalar(&mut self, t: &str, ind: usize) -> Outcome<Option<Value>> {
2149 let head = t.trim();
2150 let (style, chomp) = match head {
2151 "|" => ('|', ' '),
2152 "|-" => ('|', '-'),
2153 "|+" => ('|', '+'),
2154 ">" => ('>', ' '),
2155 ">-" => ('>', '-'),
2156 ">+" => ('>', '+'),
2157 _ => return Ok(None),
2158 };
2159 self.i += 1;
2160 let mut body: Vec<&str> = Vec::new();
2161 let mut block_ind: Option<usize> = None;
2162 while let Some(n) = self.lines.get(self.i) {
2163 if n.text.is_empty() {
2164 body.push("");
2165 self.i += 1;
2166 continue;
2167 }
2168 let bi = *block_ind.get_or_insert(n.indent);
2169 if n.indent < bi || n.indent <= ind {
2170 break;
2171 }
2172 // Keep the extra indentation of a more indented line.
2173 let line_start = n.at - n.indent;
2174 let full = &self.src[line_start..line_start + n.indent + n.text.len()];
2175 body.push(&full[bi.min(full.len())..]);
2176 self.i += 1;
2177 }
2178 let mut trailing = 0;
2179 while body.last() == Some(&"") {
2180 body.pop();
2181 trailing += 1;
2182 }
2183 let mut text = if style == '|' {
2184 body.join("\n")
2185 } else {
2186 // Folding: a single break between text lines becomes a space, a run of blank lines keeps
2187 // one break per blank line, and more-indented lines keep their breaks.
2188 let mut s = String::new();
2189 let mut blanks = 0usize;
2190 let mut prev: Option<&str> = None;
2191 for line in body.iter() {
2192 if line.is_empty() {
2193 blanks += 1;
2194 continue;
2195 }
2196 if let Some(p) = prev {
2197 if blanks > 0 {
2198 s.push_str(&"\n".repeat(blanks));
2199 } else if line.starts_with(' ') || p.starts_with(' ') {
2200 s.push('\n');
2201 } else {
2202 s.push(' ');
2203 }
2204 } else {
2205 s.push_str(&"\n".repeat(blanks));
2206 }
2207 blanks = 0;
2208 s.push_str(line);
2209 prev = Some(line);
2210 }
2211 s
2212 };
2213 match chomp {
2214 '-' => (),
2215 '+' => {
2216 text.push('\n');
2217 text.push_str(&"\n".repeat(trailing));
2218 }
2219 _ => if !body.is_empty() {
2220 text.push('\n');
2221 },
2222 }
2223 Ok(Some(Value::str(text)))
2224 }
2225
2226 // A scalar, a flow collection, or an alias written on one line.
2227 fn inline_value(&mut self, t: &str, at: usize) -> Outcome<Value> {
2228 let mut p = YFlow { s: t, b: t.as_bytes(), i: 0, at, fail: None };
2229 let v = match p.value(self) {
2230 Ok(v) => v,
2231 Err(e) => {
2232 if self.fail.is_none() {
2233 self.fail = p.fail.take();
2234 }
2235 return Err(e);
2236 }
2237 };
2238 p.ws();
2239 if p.i < p.b.len() {
2240 return self.fail_at(at + p.i, "did not find expected end of flow value");
2241 }
2242 Ok(v)
2243 }
2244}
2245
2246fn yaml_anchor(rest: &str) -> (Option<&str>, &str) {
2247 if let Some(r) = rest.strip_prefix('&') {
2248 let end = r.find(' ').unwrap_or(r.len());
2249 return (Some(&r[..end]), r[end..].trim_start());
2250 }
2251 (None, rest)
2252}
2253
2254fn yaml_balanced(t: &str) -> bool {
2255 let (mut depth, mut sq, mut dq, mut esc) = (0i32, false, false, false);
2256 for c in t.chars() {
2257 if esc {
2258 esc = false;
2259 continue;
2260 }
2261 match c {
2262 '\\' if dq => esc = true,
2263 '"' if !sq => dq = !dq,
2264 '\'' if !dq => sq = !sq,
2265 '[' | '{' if !sq && !dq => depth += 1,
2266 ']' | '}' if !sq && !dq => depth -= 1,
2267 _ => (),
2268 }
2269 }
2270 depth <= 0 && !sq && !dq
2271}
2272
2273// `key: value` on a line, outside quotes and brackets.
2274fn yaml_key_split(t: &str) -> Option<(&str, &str)> {
2275 let b = t.as_bytes();
2276 let (mut sq, mut dq, mut depth) = (false, false, 0i32);
2277 if t.starts_with('[') || t.starts_with('{') || t.starts_with("- ") {
2278 return None;
2279 }
2280 for i in 0..b.len() {
2281 match b[i] {
2282 b'\'' if !dq => sq = !sq,
2283 b'"' if !sq => dq = !dq,
2284 b'[' | b'{' if !sq && !dq => depth += 1,
2285 b']' | b'}' if !sq && !dq => depth -= 1,
2286 b':' if !sq && !dq && depth == 0 && (i + 1 == b.len() || b[i + 1] == b' ' || b[i + 1] == b'\t') => {
2287 return Some((t[..i].trim_end(), &t[i + 1..]));
2288 }
2289 _ => (),
2290 }
2291 }
2292 None
2293}
2294
2295struct YFlow<'a> {
2296 s: &'a str,
2297 b: &'a [u8],
2298 i: usize,
2299 at: usize,
2300 fail: Option<(usize, String)>,
2301}
2302
2303impl<'a> YFlow<'a> {
2304 fn ws(&mut self) {
2305 while self.i < self.b.len() && matches!(self.b[self.i], b' ' | b'\t') {
2306 self.i += 1;
2307 }
2308 }
2309
2310 fn err<T>(&mut self, m: &str) -> Outcome<T> {
2311 self.fail = Some((self.at + self.i, m.to_string()));
2312 Err(err!("{}", m; Decode, Input))
2313 }
2314
2315 fn value(&mut self, y: &mut Yaml) -> Outcome<Value> {
2316 self.ws();
2317 let (anchor, _) = if self.b.get(self.i) == Some(&b'&') {
2318 let r = &self.s[self.i + 1..];
2319 let end = r.find([' ', ',', ']', '}']).unwrap_or(r.len());
2320 let a = r[..end].to_string();
2321 self.i += 1 + end;
2322 self.ws();
2323 (Some(a), ())
2324 } else {
2325 (None, ())
2326 };
2327 let v = match self.b.get(self.i) {
2328 None => Value::None,
2329 Some(b'*') => {
2330 let r = &self.s[self.i + 1..];
2331 let end = r.find([' ', ',', ']', '}']).unwrap_or(r.len());
2332 let name = &r[..end];
2333 self.i += 1 + end;
2334 match y.anchors.iter().rev().find(|(n, _)| n == name) {
2335 Some((_, v)) => v.clone(),
2336 None => return self.err("unknown anchor"),
2337 }
2338 }
2339 Some(b'!') => return self.err("tags are not supported"),
2340 Some(b'[') => {
2341 self.i += 1;
2342 let mut a = Vec::new();
2343 loop {
2344 self.ws();
2345 if self.b.get(self.i) == Some(&b']') {
2346 self.i += 1;
2347 break;
2348 }
2349 a.push(res!(self.value(y)));
2350 self.ws();
2351 match self.b.get(self.i) {
2352 Some(b',') => self.i += 1,
2353 Some(b']') => {
2354 self.i += 1;
2355 break;
2356 }
2357 _ => return self.err("did not find expected ',' or ']'"),
2358 }
2359 }
2360 Value::array(a)
2361 }
2362 Some(b'{') => {
2363 self.i += 1;
2364 let mut d = Dict::new();
2365 loop {
2366 self.ws();
2367 if self.b.get(self.i) == Some(&b'}') {
2368 self.i += 1;
2369 break;
2370 }
2371 let k = match res!(self.scalar(true)) {
2372 Value::Str(s) => (*s).clone(),
2373 other => crate::eval::lib::foundations::display(&other).unwrap_or_default(),
2374 };
2375 self.ws();
2376 let v = if self.b.get(self.i) == Some(&b':') {
2377 self.i += 1;
2378 res!(self.value(y))
2379 } else {
2380 Value::None
2381 };
2382 d.insert(&k, v);
2383 self.ws();
2384 match self.b.get(self.i) {
2385 Some(b',') => self.i += 1,
2386 Some(b'}') => {
2387 self.i += 1;
2388 break;
2389 }
2390 _ => return self.err("did not find expected ',' or '}'"),
2391 }
2392 }
2393 Value::dict(d)
2394 }
2395 Some(_) => res!(self.scalar(false)),
2396 };
2397 if let Some(a) = anchor {
2398 y.anchors.push((a, v.clone()));
2399 }
2400 Ok(v)
2401 }
2402
2403 // A quoted or plain scalar; in a flow collection it stops at `,`, `]`, `}` and (for keys) `:`.
2404 fn scalar(&mut self, key: bool) -> Outcome<Value> {
2405 match self.b.get(self.i) {
2406 Some(b'"') => {
2407 self.i += 1;
2408 let mut out = String::new();
2409 loop {
2410 let ch = match self.s[self.i..].chars().next() {
2411 Some(c) => c,
2412 None => return self.err("found unexpected end of stream"),
2413 };
2414 self.i += ch.len_utf8();
2415 match ch {
2416 '"' => break,
2417 '\\' => {
2418 let e = match self.s[self.i..].chars().next() {
2419 Some(c) => c,
2420 None => return self.err("found unexpected end of stream"),
2421 };
2422 self.i += e.len_utf8();
2423 match e {
2424 'n' => out.push('\n'),
2425 't' => out.push('\t'),
2426 'r' => out.push('\r'),
2427 '0' => out.push('\0'),
2428 '"' => out.push('"'),
2429 '\\' => out.push('\\'),
2430 '/' => out.push('/'),
2431 ' ' => out.push(' '),
2432 'e' => out.push('\u{1b}'),
2433 'x' | 'u' | 'U' => {
2434 let n = match e { 'x' => 2, 'u' => 4, _ => 8 };
2435 let c = self.s.get(self.i..self.i + n).and_then(|h| u32::from_str_radix(h, 16).ok())
2436 .and_then(char::from_u32);
2437 match c {
2438 Some(c) => {
2439 out.push(c);
2440 self.i += n;
2441 }
2442 None => return self.err("invalid escape"),
2443 }
2444 }
2445 _ => return self.err("found unknown escape character"),
2446 }
2447 }
2448 c => out.push(c),
2449 }
2450 }
2451 Ok(Value::str(out))
2452 }
2453 Some(b'\'') => {
2454 self.i += 1;
2455 let mut out = String::new();
2456 loop {
2457 let ch = match self.s[self.i..].chars().next() {
2458 Some(c) => c,
2459 None => return self.err("found unexpected end of stream"),
2460 };
2461 self.i += ch.len_utf8();
2462 if ch == '\'' {
2463 if self.b.get(self.i) == Some(&b'\'') {
2464 self.i += 1;
2465 out.push('\'');
2466 } else {
2467 break;
2468 }
2469 } else {
2470 out.push(ch);
2471 }
2472 }
2473 Ok(Value::str(out))
2474 }
2475 _ => {
2476 let start = self.i;
2477 let in_flow = self.s.len() != self.b.len() || true;
2478 while self.i < self.b.len() {
2479 let c = self.b[self.i];
2480 if in_flow && matches!(c, b',' | b']' | b'}') && self.flow_context() {
2481 break;
2482 }
2483 if key && c == b':' {
2484 break;
2485 }
2486 self.i += 1;
2487 }
2488 let t = self.s[start..self.i].trim();
2489 Ok(yaml_plain(t))
2490 }
2491 }
2492 }
2493
2494 // Whether the scalar being read sits inside a flow collection on this line.
2495 fn flow_context(&self) -> bool {
2496 let before = &self.s[..self.i];
2497 let opens = before.matches(['[', '{']).count();
2498 let closes = before.matches([']', '}']).count();
2499 opens > closes
2500 }
2501}
2502
2503// A plain scalar under the YAML 1.2 core schema.
2504fn yaml_plain(t: &str) -> Value {
2505 match t {
2506 "" | "~" | "null" | "Null" | "NULL" => return Value::None,
2507 "true" | "True" | "TRUE" => return Value::Bool(true),
2508 "false" | "False" | "FALSE" => return Value::Bool(false),
2509 ".inf" | ".Inf" | ".INF" | "+.inf" | "+.Inf" | "+.INF" => return Value::Float(f64::INFINITY),
2510 "-.inf" | "-.Inf" | "-.INF" => return Value::Float(f64::NEG_INFINITY),
2511 ".nan" | ".NaN" | ".NAN" => return Value::Float(f64::NAN),
2512 _ => (),
2513 }
2514 if let Some(h) = t.strip_prefix("0x") {
2515 if let Ok(i) = i64::from_str_radix(h, 16) {
2516 return Value::Int(i);
2517 }
2518 }
2519 if let Some(o) = t.strip_prefix("0o") {
2520 if let Ok(i) = i64::from_str_radix(o, 8) {
2521 return Value::Int(i);
2522 }
2523 }
2524 let body = t.strip_prefix(['+', '-']).unwrap_or(t);
2525 if !body.is_empty() && body.bytes().all(|b| b.is_ascii_digit()) {
2526 if let Ok(i) = t.parse::<i64>() {
2527 return Value::Int(i);
2528 }
2529 }
2530 let num = !body.is_empty()
2531 && body.bytes().all(|b| b.is_ascii_digit() || matches!(b, b'.' | b'e' | b'E' | b'+' | b'-'))
2532 && body.bytes().any(|b| b.is_ascii_digit());
2533 if num {
2534 if let Ok(f) = t.parse::<f64>() {
2535 return Value::Float(f);
2536 }
2537 }
2538 Value::str(t)
2539}
2540
2541// Whether serde_yaml would quote a string: when it would read back as another type, or starts or
2542// contains what YAML reads as syntax.
2543fn yaml_needs_quotes(s: &str) -> bool {
2544 if s.is_empty() || s == "-" {
2545 return true;
2546 }
2547 if !matches!(yaml_plain(s), Value::Str(_)) {
2548 return true;
2549 }
2550 let first = s.chars().next().unwrap_or(' ');
2551 if "@!%&*|>[{'\"#`,]}".contains(first) || first == ' ' || s.ends_with(' ') || s.starts_with("- ") {
2552 return true;
2553 }
2554 s.contains(": ") || s.contains(" #") || s.ends_with(':')
2555}
2556
2557fn yaml_scalar(s: &str) -> String {
2558 if s.chars().any(|c| c.is_control() && c != '\n') {
2559 let mut out = String::from("\"");
2560 for c in s.chars() {
2561 match c {
2562 '"' => out.push_str("\\\""),
2563 '\\' => out.push_str("\\\\"),
2564 '\n' => out.push_str("\\n"),
2565 '\t' => out.push_str("\\t"),
2566 c if c.is_control() => out.push_str(&fmt!("\\x{:02X}", c as u32)),
2567 c => out.push(c),
2568 }
2569 }
2570 out.push('"');
2571 return out;
2572 }
2573 if yaml_needs_quotes(s) {
2574 fmt!("'{}'", s.replace('\'', "''"))
2575 } else {
2576 s.to_string()
2577 }
2578}
2579
2580fn yaml_write(d: &Data, indent: usize, out: &mut String, top: bool) {
2581 let pad = " ".repeat(indent);
2582 match d {
2583 Data::Map(m) if !m.is_empty() => {
2584 for (i, (k, v)) in m.iter().enumerate() {
2585 if i > 0 || !top {
2586 out.push_str(&pad);
2587 }
2588 out.push_str(&yaml_scalar(k));
2589 out.push(':');
2590 yaml_child(v, indent, out);
2591 }
2592 }
2593 Data::Seq(a) if !a.is_empty() => {
2594 for (i, v) in a.iter().enumerate() {
2595 if i > 0 || !top {
2596 out.push_str(&pad);
2597 }
2598 out.push('-');
2599 match v {
2600 Data::Map(m) if !m.is_empty() => {
2601 out.push(' ');
2602 yaml_write(v, indent + 1, out, true);
2603 }
2604 Data::Seq(s) if !s.is_empty() => {
2605 out.push(' ');
2606 yaml_write(v, indent + 1, out, true);
2607 }
2608 _ => {
2609 out.push(' ');
2610 yaml_leaf(v, indent + 1, out);
2611 out.push('\n');
2612 }
2613 }
2614 }
2615 }
2616 other => {
2617 yaml_leaf(other, indent, out);
2618 out.push('\n');
2619 }
2620 }
2621}
2622
2623fn yaml_child(v: &Data, indent: usize, out: &mut String) {
2624 match v {
2625 Data::Map(m) if !m.is_empty() => {
2626 out.push('\n');
2627 yaml_write(v, indent + 1, out, false);
2628 }
2629 Data::Seq(a) if !a.is_empty() => {
2630 out.push('\n');
2631 yaml_write(v, indent, out, false);
2632 }
2633 _ => {
2634 out.push(' ');
2635 yaml_leaf(v, indent + 1, out);
2636 out.push('\n');
2637 }
2638 }
2639}
2640
2641fn yaml_leaf(v: &Data, indent: usize, out: &mut String) {
2642 match v {
2643 Data::Null => out.push_str("null"),
2644 Data::Bool(b) => out.push_str(if *b { "true" } else { "false" }),
2645 Data::Int(i) => out.push_str(&i.to_string()),
2646 Data::Float(f) => out.push_str(&if f.is_nan() {
2647 ".nan".to_string()
2648 } else if f.is_infinite() {
2649 if *f < 0.0 { "-.inf".to_string() } else { ".inf".to_string() }
2650 } else {
2651 ryu_format(*f)
2652 }),
2653 Data::Str(s) if s.contains('\n') && !s.chars().any(|c| c.is_control() && c != '\n') => {
2654 let chomp = if s.ends_with('\n') { "" } else { "-" };
2655 out.push('|');
2656 out.push_str(chomp);
2657 let pad = " ".repeat(indent);
2658 for line in s.trim_end_matches('\n').split('\n') {
2659 out.push('\n');
2660 if !line.is_empty() {
2661 out.push_str(&pad);
2662 out.push_str(line);
2663 }
2664 }
2665 }
2666 Data::Str(s) => out.push_str(&yaml_scalar(s)),
2667 Data::Bytes(b) => {
2668 let mut first = true;
2669 out.push('[');
2670 for x in b {
2671 if !first {
2672 out.push_str(", ");
2673 }
2674 first = false;
2675 out.push_str(&x.to_string());
2676 }
2677 out.push(']');
2678 }
2679 Data::Seq(_) => out.push_str("[]"),
2680 Data::Map(_) => out.push_str("{}"),
2681 }
2682}