Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/src/lang/lex.rs

76.2 KiB, 3 runs

created by r1870400018:62383, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! The one markup lexer every source scanner reads through, as a view over the concrete syntax tree of
2//! [`crate::syntax`]: the port of `typst-syntax` 0.15.1's lexer and parser. A text is parsed once, and every
3//! answer -- what a character is, where a line stands, what a group holds, where a conditional ends -- is read
4//! from that tree, so the scanners keep no second reading of what is a comment, a body or the top level.
5//! A `[` opens content only where code opened it (`#name[`, `#[`, a call's trailing argument, a reference's
6//! supplement); in prose it is text, balanced against a later `]` within its own markup scope. A `"` is a
7//! character in markup and a string only in code or maths. An automatic link is one token, so the `//` or
8//! `/*` in it opens no comment. Block comments nest. Raw text opens with one backtick or three or more and
9//! closes on a run as long, across lines. A `#!` opening the file is a comment to its line's end. A `$`
10//! whose maths never closes is read as a character, so the error stays at it and the markup after it is read
11//! as usual, where Typst's parser takes the rest of the source into the equation: the text is parsed again
12//! with each such `$` set aside, until none is left.
13
14use crate::syntax::FileId;
15use crate::syntax::SyntaxKind;
16use crate::syntax::SyntaxNode;
17use crate::syntax::parser;
18
19/// What a character is, as Typst reads it.
20#[derive(Clone, Copy, Debug, PartialEq, Eq)]
21pub(crate) enum Tok {
22 Text, // markup text, a bracket in prose included, and a `$` whose maths never closes
23 Code, // code: a name, a number, an operator, a space
24 Math, // an equation's own text
25 Str, // a string in code or maths, its quotes included
26 Comment, // a line or block comment, its delimiters included
27 Raw, // raw text, its backticks included
28 Link, // an automatic link
29 Escape, // an escape or a forced line break, its backslash included
30 Label, // `<name>`
31 Ref, // `@name`, and a supplement's opening `[`
32 Hash, // the `#` that embeds an expression
33 Open, // a delimiter code opens: `(`, `{`, a content block's `[`, an equation's `$`
34 Close, // the delimiter that closes one
35}
36
37/// Where a line stands, decided at its first token, as Typst's parser and realiser read it.
38#[derive(Clone, Copy, Debug, PartialEq, Eq)]
39pub(crate) enum Place {
40 Top, // the file's own markup, with no list item, heading, strong or emphasis open
41 Block, // a bare content block `#[`, or a run of them, opened where no scope was: joined into the file's markup
42 Contained, // in a list item, a heading, strong or emphasis of that markup: a container, which Typst sets apart
43 Content, // in content code opened: a call's argument, a reference's supplement, a conditional's or loop's body, a value
44}
45
46/// The keyword that heads a loop or a conditional.
47#[derive(Clone, Copy, Debug, PartialEq, Eq)]
48pub(crate) enum Kw {
49 If,
50 While,
51 For,
52}
53
54/// How the text of a node is read.
55#[derive(Clone, Copy, Debug, PartialEq, Eq)]
56enum Mode {
57 Markup,
58 Code,
59 Math,
60}
61
62const NONE: usize = usize::MAX; // no node
63
64/// One node of the tree, flattened: the leaves and the nodes above them, in source order.
65#[derive(Clone, Copy, Debug)]
66struct Node {
67 kind: SyntaxKind,
68 start: usize,
69 end: usize,
70 parent: usize, // `NONE` for the root
71 child: usize, // the first child; `NONE` for a leaf
72 next: usize, // the next sibling
73 prev: usize, // the previous sibling
74 mode: Mode, // of a leaf, how it is read; of an inner node, how its children are
75 bare: bool, // a content block opened directly after a `#`
76 leaf: usize, // the leaf's place in `leaves`; `NONE` for an inner node
77}
78
79/// A leaf's class, and what is open before it.
80#[derive(Clone, Copy, Debug)]
81struct Leaf {
82 node: usize,
83 tok: Tok,
84 groups: u32, // groups code opened and not yet closed, before this leaf
85 eqs: u32, // equations open before it
86 shut: bool, // false for a string, raw text or block comment the text ends inside
87}
88
89/// A text read once. A file's markup is read with [`Lexer::markup`], a line or fragment as Typst reads it
90/// with [`Lexer::markup_exact`], and the inside of a call's argument list with [`Lexer::code`].
91#[derive(Clone, Debug)]
92pub(crate) struct Lexer {
93 text: String,
94 nodes: Vec<Node>,
95 leaves: Vec<Leaf>,
96 lone: Vec<usize>, // sorted: the source bytes of each `$` whose maths never closes
97 groups: u32, // open at the end of the text
98 eqs: u32,
99}
100
101/// The tree under construction.
102struct Build {
103 nodes: Vec<Node>,
104 leaves: Vec<Leaf>,
105 pos: usize,
106 open: Vec<usize>, // the group nodes whose brackets are open, outermost first
107 eqs: u32,
108 last: Option<(SyntaxKind, usize)>, // the last leaf made: its kind and where it ended
109 dollars: Vec<usize>, // the byte of each `$` the parser found unclosed
110 deep: u32, // error text being read again
111}
112
113// Where the kinds of the syntax tree take a text apart.
114
115/// Is `kind` one whose brackets code opens and closes as groups?
116fn is_group(kind: SyntaxKind) -> bool {
117 matches!(kind,
118 SyntaxKind::CodeBlock | SyntaxKind::ContentBlock | SyntaxKind::Args | SyntaxKind::Parenthesized
119 | SyntaxKind::Array | SyntaxKind::Dict | SyntaxKind::Params | SyntaxKind::Destructuring)
120}
121
122/// Does `kind` open a markup scope of its own in the markup around it: an item, a heading, strong or emphasis?
123fn is_scope(kind: SyntaxKind) -> bool {
124 matches!(kind,
125 SyntaxKind::Strong | SyntaxKind::Emph | SyntaxKind::Heading | SyntaxKind::ListItem
126 | SyntaxKind::EnumItem | SyntaxKind::TermItem)
127}
128
129/// Is `kind` a statement an `#` embeds, which runs to its line's end?
130fn is_stmt(kind: SyntaxKind) -> bool {
131 matches!(kind,
132 SyntaxKind::LetBinding | SyntaxKind::SetRule | SyntaxKind::ShowRule | SyntaxKind::ModuleImport
133 | SyntaxKind::ModuleInclude | SyntaxKind::FuncReturn)
134}
135
136/// How the children of a node of `kind` are read, `mode` being how the node itself is.
137fn child_mode(kind: SyntaxKind, mode: Mode) -> Mode {
138 match kind {
139 SyntaxKind::Markup | SyntaxKind::Heading | SyntaxKind::Strong | SyntaxKind::Emph
140 | SyntaxKind::ListItem | SyntaxKind::EnumItem | SyntaxKind::TermItem | SyntaxKind::Ref
141 | SyntaxKind::Raw
142 => Mode::Markup,
143 SyntaxKind::Math | SyntaxKind::Equation | SyntaxKind::MathDelimited | SyntaxKind::MathCall
144 | SyntaxKind::MathArgs | SyntaxKind::MathAttach | SyntaxKind::MathFrac | SyntaxKind::MathRoot
145 => Mode::Math,
146 SyntaxKind::Code | SyntaxKind::CodeBlock | SyntaxKind::ContentBlock | SyntaxKind::Parenthesized
147 | SyntaxKind::Array | SyntaxKind::Dict | SyntaxKind::Keyed | SyntaxKind::Unary | SyntaxKind::Binary
148 | SyntaxKind::FieldAccess | SyntaxKind::FuncCall | SyntaxKind::Args | SyntaxKind::Closure
149 | SyntaxKind::Params | SyntaxKind::LetBinding | SyntaxKind::SetRule | SyntaxKind::ShowRule
150 | SyntaxKind::Contextual | SyntaxKind::Conditional | SyntaxKind::WhileLoop | SyntaxKind::ForLoop
151 | SyntaxKind::ModuleImport | SyntaxKind::ImportItems | SyntaxKind::ImportItemPath
152 | SyntaxKind::RenamedImportItem | SyntaxKind::ModuleInclude | SyntaxKind::LoopBreak
153 | SyntaxKind::LoopContinue | SyntaxKind::FuncReturn | SyntaxKind::Destructuring
154 | SyntaxKind::DestructAssignment
155 => Mode::Code,
156 _ => mode,
157 }
158}
159
160/// Does a block comment's text close, its nesting balanced?
161fn comment_closed(text: &str) -> bool {
162 let b = text.as_bytes();
163 let (mut i, mut depth) = (0usize, 0u32);
164 while i + 1 < b.len() {
165 match (b[i], b[i + 1]) {
166 (b'/', b'*') => { depth = depth.saturating_add(1); i += 2; },
167 (b'*', b'/') => { depth = depth.saturating_sub(1); i += 2; },
168 _ => i += 1,
169 }
170 }
171 depth == 0
172}
173
174impl Build {
175 fn new() -> Self {
176 Build {
177 nodes: Vec::new(),
178 leaves: Vec::new(),
179 pos: 0,
180 open: Vec::new(),
181 eqs: 0,
182 last: None,
183 dollars: Vec::new(),
184 deep: 0,
185 }
186 }
187
188 /// An inner node, opening at the current byte; its end is set when its children are done.
189 fn inner(&mut self, kind: SyntaxKind, parent: usize, mode: Mode) -> usize {
190 let bare = kind == SyntaxKind::ContentBlock && self.last == Some((SyntaxKind::Hash, self.pos));
191 self.nodes.push(Node { kind, start: self.pos, end: self.pos, parent, child: NONE, next: NONE, prev: NONE, mode, bare, leaf: NONE });
192 self.nodes.len() - 1
193 }
194
195 /// A leaf of `len` bytes, read as `tok`.
196 fn leaf(&mut self, kind: SyntaxKind, len: usize, tok: Tok, parent: usize, mode: Mode, shut: bool) -> usize {
197 let idx = self.nodes.len();
198 self.nodes.push(Node {
199 kind, start: self.pos, end: self.pos + len, parent, child: NONE, next: NONE, prev: NONE, mode,
200 bare: false, leaf: self.leaves.len(),
201 });
202 self.leaves.push(Leaf { node: idx, tok, groups: self.open.len() as u32, eqs: self.eqs, shut });
203 self.pos += len;
204 self.last = Some((kind, self.pos));
205 idx
206 }
207
208 /// Reads the children `kids` of the node `parent`, of `pkind`, whose children are read as `mode`.
209 fn kids(&mut self, kids: &[SyntaxNode], parent: usize, pkind: SyntaxKind, mode: Mode) {
210 let mut last = NONE;
211 let mut k1: Option<SyntaxKind> = None; // the kind of the sibling before
212 let mut k2: Option<SyntaxKind> = None; // and the one before that
213 let mut first = true;
214 for c in kids {
215 let kind = c.kind();
216 let idx = if c.is_inner() {
217 let cmode = child_mode(kind, mode);
218 let idx = self.inner(kind, parent, cmode);
219 self.kids(c.children(), idx, kind, cmode);
220 self.nodes[idx].end = self.pos;
221 idx
222 } else if c.len() == 0 {
223 continue; // a zero-width error: nothing of the text
224 } else if kind == SyntaxKind::Error {
225 self.error(c, parent, mode, k1)
226 } else {
227 self.token(c, parent, pkind, mode, first, k1, k2)
228 };
229 if last == NONE {
230 self.nodes[parent].child = idx;
231 } else {
232 self.nodes[last].next = idx;
233 self.nodes[idx].prev = last;
234 }
235 last = idx;
236 k2 = k1;
237 k1 = Some(kind);
238 first = false;
239 }
240 }
241
242 /// A leaf that is no error.
243 fn token(
244 &mut self,
245 c: &SyntaxNode,
246 parent: usize,
247 pkind: SyntaxKind,
248 mode: Mode,
249 first: bool,
250 k1: Option<SyntaxKind>,
251 k2: Option<SyntaxKind>,
252 )
253 -> usize
254 {
255 let kind = c.kind();
256 let len = c.len();
257 let gp = self.nodes[parent].parent;
258 let gkind = if gp == NONE { None } else { Some(self.nodes[gp].kind) };
259 let group = is_group(pkind);
260 // The atom an `#` embeds, which follows it directly.
261 let after_hash = k1 == Some(SyntaxKind::Hash) && !matches!(kind, SyntaxKind::Space | SyntaxKind::Parbreak);
262 let (mut opens, mut closes, mut eq, mut shut) = (false, false, 0i8, true);
263 let tok = match kind {
264 SyntaxKind::Shebang | SyntaxKind::LineComment => Tok::Comment,
265 SyntaxKind::BlockComment => { shut = comment_closed(c.text()); Tok::Comment },
266 SyntaxKind::Hash => Tok::Hash,
267 SyntaxKind::Str => Tok::Str,
268 SyntaxKind::Escape | SyntaxKind::Linebreak => Tok::Escape,
269 SyntaxKind::Link => Tok::Link,
270 SyntaxKind::Label => Tok::Label,
271 SyntaxKind::RefMarker => Tok::Ref,
272 SyntaxKind::Dollar if pkind == SyntaxKind::Equation => {
273 if first {
274 eq = 1;
275 Tok::Open
276 } else {
277 eq = -1;
278 Tok::Close
279 }
280 },
281 SyntaxKind::LeftParen | SyntaxKind::LeftBrace | SyntaxKind::LeftBracket if group => {
282 opens = true;
283 // A reference's supplement is a content block, its opening bracket part of the reference.
284 if kind == SyntaxKind::LeftBracket && pkind == SyntaxKind::ContentBlock && gkind == Some(SyntaxKind::Ref) {
285 Tok::Ref
286 } else {
287 Tok::Open
288 }
289 },
290 SyntaxKind::RightParen | SyntaxKind::RightBrace | SyntaxKind::RightBracket if group => {
291 closes = true;
292 Tok::Close
293 },
294 _ if pkind == SyntaxKind::Raw => Tok::Raw,
295 _ if after_hash => Tok::Code,
296 _ => match mode {
297 Mode::Code => Tok::Code,
298 Mode::Markup => if kind == SyntaxKind::Semicolon { Tok::Code } else { Tok::Text },
299 // A `;` straight after an embedded expression ends it, as it ends no maths argument.
300 Mode::Math => if kind == SyntaxKind::Semicolon && k2 == Some(SyntaxKind::Hash) { Tok::Code } else { Tok::Math },
301 },
302 };
303 let at = self.leaf(kind, len, tok, parent, mode, shut);
304 self.shift(opens, closes, eq, parent);
305 at
306 }
307
308 /// Moves the counts of what is open past a leaf that opens or closes a group or an equation.
309 fn shift(&mut self, opens: bool, closes: bool, eq: i8, parent: usize) {
310 if opens {
311 self.open.push(parent);
312 }
313 // A closer ends its own group and any inside it that never closed, as the parser leaves them.
314 if closes && self.open.contains(&parent) {
315 while let Some(g) = self.open.pop() {
316 if g == parent {
317 break;
318 }
319 }
320 }
321 if eq > 0 {
322 self.eqs = self.eqs.saturating_add(1);
323 } else if eq < 0 {
324 self.eqs = self.eqs.saturating_sub(1);
325 }
326 }
327
328 /// A leaf the parser could not read. Most are one token, by what the message says it is; one that is
329 /// the leftover of a construct is read again as code, so the strings and comments in it are still such.
330 fn error(&mut self, c: &SyntaxNode, parent: usize, mode: Mode, k1: Option<SyntaxKind>) -> usize {
331 let text = c.text();
332 let msg = c.error_info().map(|e| e.message.as_str()).unwrap_or("");
333 let len = c.len();
334 let (mut opens, mut eq, mut shut) = (false, 0i8, true);
335 let tok = if msg.starts_with("unclosed delimiter") {
336 match text {
337 "$" => {
338 eq = 1;
339 self.dollars.push(self.pos);
340 Some(Tok::Open)
341 },
342 "(" | "{" | "[" => { opens = true; Some(Tok::Open) },
343 _ => None,
344 }
345 } else if msg.starts_with("unclosed raw text") {
346 shut = false;
347 Some(Tok::Raw)
348 } else if msg.starts_with("unclosed string") {
349 shut = false;
350 Some(Tok::Str)
351 } else if msg.starts_with("unexpected end of block comment") {
352 Some(Tok::Text)
353 } else if msg.starts_with("unclosed label") {
354 Some(Tok::Label)
355 } else if msg.contains("Unicode") {
356 Some(Tok::Escape)
357 } else if msg.starts_with("automatic links") {
358 Some(Tok::Link)
359 } else {
360 None
361 };
362 if let Some(tok) = tok {
363 let at = self.leaf(SyntaxKind::Error, len, tok, parent, mode, shut);
364 self.shift(opens, false, eq, parent);
365 return at;
366 }
367 // What the parser left over in code may hold a string, a comment, a group: read it again as code.
368 if mode == Mode::Code && text.chars().nth(1).is_some() && self.deep < 3 {
369 let root = parser::parse_code(text, FileId::DETACHED);
370 let same = matches!(root.children(), [only] if only.kind() == SyntaxKind::Error && only.text() == text);
371 if !same {
372 let idx = self.inner(SyntaxKind::Error, parent, Mode::Code);
373 self.deep += 1;
374 self.kids(root.children(), idx, SyntaxKind::Code, Mode::Code);
375 self.deep -= 1;
376 self.nodes[idx].end = self.pos;
377 return idx;
378 }
379 }
380 let tok = match mode {
381 // What follows an `#` and is no expression is still code.
382 _ if k1 == Some(SyntaxKind::Hash) => Tok::Code,
383 Mode::Markup => Tok::Text,
384 Mode::Code => Tok::Code,
385 Mode::Math => Tok::Math,
386 };
387 self.leaf(SyntaxKind::Error, len, tok, parent, mode, true)
388 }
389}
390
391/// Is `c` a line break, as Typst counts one?
392pub(crate) fn is_newline(c: char) -> bool {
393 matches!(c, '\n' | '\x0B' | '\x0C' | '\r' | '\u{0085}' | '\u{2028}' | '\u{2029}')
394}
395
396/// Can `c` start a Typst name?
397pub(crate) fn is_id_start(c: char) -> bool {
398 c.is_alphabetic() || c == '_'
399}
400
401/// Can `c` continue a Typst name?
402pub(crate) fn is_id_continue(c: char) -> bool {
403 c.is_alphanumeric() || c == '_' || c == '-'
404}
405
406impl Lexer {
407 /// A file's markup, as Typst reads a `.typ` file, each `$` whose maths never closes read as a character.
408 pub(crate) fn markup(src: &str) -> Self {
409 Self::read(src, false, true)
410 }
411
412 /// A line or fragment of markup exactly as Typst reads it, an unclosed `$` opening an equation that
413 /// takes the rest.
414 #[cfg(test)]
415 pub(crate) fn markup_exact(src: &str) -> Self {
416 Self::read(src, false, false)
417 }
418
419 /// Code, as the inside of a call's argument list or a value reads.
420 pub(crate) fn code(src: &str) -> Self {
421 Self::read(src, true, false)
422 }
423
424 fn read(src: &str, code: bool, recover: bool) -> Self {
425 let mut lone: Vec<usize> = Vec::new();
426 let passes = if recover { src.matches('$').count() + 1 } else { 1 };
427 let mut done = None;
428 for _ in 0..passes {
429 let b = Self::pass(src, code, &lone);
430 // The innermost equation that never closes is the last, as each one that does not takes the rest.
431 match (recover, b.dollars.last()) {
432 (true, Some(&at)) => lone.insert(lone.partition_point(|&p| p < at), at),
433 _ => { done = Some(b); break; },
434 }
435 }
436 let b = match done {
437 Some(b) => b,
438 None => Self::pass(src, code, &lone),
439 };
440 Lexer { text: src.to_string(), nodes: b.nodes, leaves: b.leaves, lone, groups: b.open.len() as u32, eqs: b.eqs }
441 }
442
443 /// One reading, with each `$` of `lone` set aside as a comma, which no mode reads as more than text.
444 fn pass(src: &str, code: bool, lone: &[usize]) -> Build {
445 let mut bytes = src.as_bytes().to_vec();
446 for &at in lone {
447 bytes[at] = b',';
448 }
449 let text = String::from_utf8(bytes).unwrap_or_else(|_| src.to_string());
450 let (root, mode) = if code {
451 (parser::parse_code(&text, FileId::DETACHED), Mode::Code)
452 } else {
453 (parser::parse(&text, FileId::DETACHED), Mode::Markup)
454 };
455 let mut b = Build::new();
456 let idx = b.inner(root.kind(), NONE, mode);
457 b.kids(root.children(), idx, root.kind(), mode);
458 b.nodes[idx].end = b.pos;
459 b
460 }
461
462 /// The source bytes of each `$` whose maths never closes, in order. Typst refuses the file there, with the
463 /// rest of the source taken into the equation; the scan reads each of these as a character instead, so the
464 /// error stays at the `$` and the markup after it is read.
465 pub(crate) fn lone(&self) -> &[usize] {
466 &self.lone
467 }
468
469 /// The leaf holding byte `p`, or the last one starting before it.
470 fn leaf_at(&self, p: usize) -> Option<usize> {
471 self.leaves.partition_point(|l| self.nodes[l.node].start <= p).checked_sub(1)
472 }
473
474 /// What stands around byte `p`: the leaf it falls inside, when it is no leaf's first byte, and the nodes
475 /// that began before it and end after it, innermost first.
476 fn around(&self, p: usize) -> (Option<usize>, Vec<usize>) {
477 let Some(li) = self.leaf_at(p) else { return (None, Vec::new()); };
478 let n = self.nodes[self.leaves[li].node];
479 if p >= n.end {
480 return (None, Vec::new());
481 }
482 let inside = if n.start < p { Some(li) } else { None };
483 let mut out = Vec::new();
484 let mut a = n.parent;
485 while a != NONE {
486 let m = self.nodes[a];
487 if m.start < p && m.end > p {
488 out.push(a);
489 }
490 a = m.parent;
491 }
492 (inside, out)
493 }
494
495 /// Does the text stand in markup at the position `chain` and `inside` describe: neither in a string, a
496 /// comment, raw text, a code group, an equation, nor anywhere but the file's own markup or a content block?
497 fn in_markup(&self, inside: Option<usize>, chain: &[usize]) -> bool {
498 if let Some(li) = inside {
499 if matches!(self.leaves[li].tok, Tok::Str | Tok::Comment | Tok::Raw) {
500 return false;
501 }
502 }
503 let own = chain.iter().map(|&a| self.nodes[a].kind).find(|&k| k != SyntaxKind::Markup && !is_scope(k));
504 own.map_or(true, |k| k == SyntaxKind::ContentBlock)
505 }
506
507 /// The scopes and frames of `chain`, outermost first, as a line standing in them is placed.
508 fn place_of(&self, chain: &[usize]) -> Place {
509 let mut place = Place::Top;
510 for &a in chain.iter().rev() {
511 let n = &self.nodes[a];
512 match n.kind {
513 SyntaxKind::Markup => {},
514 k if is_scope(k) => return Place::Contained,
515 SyntaxKind::ContentBlock if n.bare => place = Place::Block,
516 _ => return Place::Content,
517 }
518 }
519 place
520 }
521
522 /// The first leaf of the line at byte `p` that is a token: past its spaces and any block comment that
523 /// closes on it. `None` when the line holds none (spaces, a comment or its break alone).
524 fn first_token(&self, p: usize) -> Option<usize> {
525 let line_end = self.text[p..].find(is_newline).map_or(self.text.len(), |k| p + k);
526 let mut li = self.leaf_at(p)?;
527 loop {
528 let n = self.nodes[self.leaves.get(li)?.node];
529 if n.start >= line_end {
530 return None;
531 }
532 match n.kind {
533 SyntaxKind::Space | SyntaxKind::Parbreak => {},
534 SyntaxKind::LineComment | SyntaxKind::Shebang => return None,
535 SyntaxKind::BlockComment => if n.end > line_end { return None; },
536 _ => return Some(n.start.max(p)),
537 }
538 li += 1;
539 }
540 }
541
542 /// The scopes still open where a line with no token stands, which ends none: those that held the last token
543 /// before it. A heading ended with its line, and strong or emphasis that closed before it is closed; one
544 /// that has not ended runs on through the trailing space the parser gives it.
545 fn open_chain(&self, p: usize) -> Vec<usize> {
546 let Some(mut li) = self.leaf_at(p) else { return Vec::new(); };
547 // The last token ending before the line.
548 loop {
549 let n = self.nodes[self.leaves[li].node];
550 let blank = matches!(n.kind, SyntaxKind::Space | SyntaxKind::Parbreak | SyntaxKind::LineComment
551 | SyntaxKind::BlockComment | SyntaxKind::Shebang);
552 if n.end <= p && !blank {
553 break;
554 }
555 match li.checked_sub(1) {
556 Some(k) => li = k,
557 None => return Vec::new(),
558 }
559 }
560 let mut out = Vec::new();
561 let mut a = self.nodes[self.leaves[li].node].parent;
562 while a != NONE {
563 let m = self.nodes[a];
564 let keep = match m.kind {
565 SyntaxKind::Heading => false,
566 SyntaxKind::Strong | SyntaxKind::Emph => m.end > p,
567 SyntaxKind::Markup | SyntaxKind::ListItem | SyntaxKind::EnumItem | SyntaxKind::TermItem => true,
568 _ => m.end > p,
569 };
570 if keep {
571 out.push(a);
572 } else if out.last().is_some_and(|&l| self.nodes[l].kind == SyntaxKind::Markup && self.nodes[l].parent == a) {
573 // The markup a scope that has ended held goes with it.
574 out.pop();
575 }
576 a = m.parent;
577 }
578 out
579 }
580
581 /// The chain a line at byte `p` is placed by, and whether it opens in markup at all: the strict ancestors of
582 /// its first token, or, for a line with none, the scopes left open before it.
583 fn line_chain(&self, p: usize) -> Option<Vec<usize>> {
584 let (inside, here) = self.around(p);
585 if !self.in_markup(inside, &here) {
586 return None;
587 }
588 Some(match self.first_token(p) {
589 Some(t) => self.around(t).1,
590 None => self.open_chain(p),
591 })
592 }
593
594 /// Where the line starting at byte `p` stands, or `None` when it opens outside markup: in a string, a
595 /// comment, raw text, an equation, a code group or an unfinished expression. The spaces that open a line
596 /// change no scope, so the place is decided at its first token, which ends the items it does not indent past.
597 pub(crate) fn place_at(&self, p: usize) -> Option<Place> {
598 self.line_chain(p).map(|chain| self.place_of(&chain))
599 }
600
601 /// Is the line starting at byte `p` standing directly in the innermost markup, whatever opened it, with no
602 /// list item, heading, strong or emphasis of that markup open around it?
603 pub(crate) fn bare_line_at(&self, p: usize) -> bool {
604 let Some(chain) = self.line_chain(p) else { return false; };
605 match chain.first() {
606 None => true,
607 // A line in a content block whose markup has not begun, or that opens with the block's own closer,
608 // stands in the block, bare.
609 Some(&a) if self.nodes[a].kind == SyntaxKind::ContentBlock => true,
610 Some(&a) => {
611 self.nodes[a].kind == SyntaxKind::Markup
612 && chain.get(1).map_or(true, |&b| self.nodes[b].kind == SyntaxKind::ContentBlock)
613 },
614 }
615 }
616
617 /// How many groups code opened are open at byte `p`: `(`, `{`, a content block. Prose brackets, strings,
618 /// equations, comments and raw text are not counted.
619 pub(crate) fn depth_at(&self, p: usize) -> usize {
620 match self.leaf_at(p) {
621 Some(li) if p < self.nodes[self.leaves[li].node].end => self.leaves[li].groups as usize,
622 _ => self.groups as usize,
623 }
624 }
625
626 /// The number of groups open at byte `p` when the text stands there in markup -- the file's own, or a
627 /// content block's -- and `None` when it stands in code, a string, an equation, a comment or raw text.
628 pub(crate) fn markup_level_at(&self, p: usize) -> Option<usize> {
629 let (inside, chain) = self.around(p);
630 self.in_markup(inside, &chain).then(|| self.depth_at(p))
631 }
632
633 /// How many frames are open at byte `p`: groups, equations, and the string, comment or raw text it falls
634 /// inside.
635 pub(crate) fn open_at(&self, p: usize) -> usize {
636 let end = |this: &Self| -> usize {
637 let shut = this.leaves.last().map_or(true, |l| l.shut);
638 this.groups as usize + this.eqs as usize + usize::from(!shut)
639 };
640 let Some(li) = self.leaf_at(p) else { return end(self); };
641 let l = self.leaves[li];
642 let n = self.nodes[l.node];
643 if p >= n.end {
644 return end(self);
645 }
646 let mut open = l.groups as usize + l.eqs as usize;
647 let in_raw = l.tok == Tok::Raw
648 && (n.start < p || (self.nodes[n.parent].kind == SyntaxKind::Raw && self.nodes[n.parent].start < p));
649 if (n.start < p && matches!(l.tok, Tok::Str | Tok::Comment)) || in_raw {
650 open += 1;
651 }
652 open
653 }
654
655 /// Is anything open at the end of the text?
656 #[cfg(test)]
657 pub(crate) fn is_open(&self) -> bool {
658 self.open_at(self.text.len()) > 0
659 }
660
661 /// What each byte of the text is: every byte of a character takes the character's.
662 pub(crate) fn byte_toks(&self) -> Vec<Tok> {
663 let mut out = vec![Tok::Text; self.text.len()];
664 for l in &self.leaves {
665 let n = &self.nodes[l.node];
666 for t in &mut out[n.start..n.end] {
667 *t = l.tok;
668 }
669 }
670 for &at in &self.lone {
671 out[at] = Tok::Text;
672 }
673 out
674 }
675
676 /// Each character of the text, with the byte it starts at.
677 pub(crate) fn tokens(&self) -> Vec<(usize, char, Tok)> {
678 let toks = self.byte_toks();
679 self.text.char_indices().map(|(at, c)| (at, c, toks[at])).collect()
680 }
681
682 /// The text of a leaf.
683 fn leaf_text(&self, li: usize) -> &str {
684 let n = &self.nodes[self.leaves[li].node];
685 &self.text[n.start..n.end]
686 }
687
688 /// The text with every character `blank` names turned into a space and every line break kept, so each byte
689 /// offset and line of the result is the text's own.
690 fn blanked(&self, blank: impl Fn(Tok) -> bool) -> String {
691 let mut out = String::with_capacity(self.text.len());
692 for (_, c, tok) in self.tokens() {
693 if blank(tok) && c != '\n' && c != '\r' {
694 for _ in 0..c.len_utf8() {
695 out.push(' ');
696 }
697 } else {
698 out.push(c);
699 }
700 }
701 out
702 }
703
704 /// The text with every character a comment or raw text holds blanked to spaces and every line break kept,
705 /// so each byte offset and line of the result is the text's own.
706 pub(crate) fn live_text(&self) -> String {
707 self.blanked(|t| matches!(t, Tok::Comment | Tok::Raw))
708 }
709
710 /// Each line of `src`, the text read, that opens in markup, with the byte it starts at and where it stands.
711 pub(crate) fn lines<'a>(&self, src: &'a str) -> Vec<(usize, &'a str, Place)> {
712 let mut out = Vec::new();
713 let mut offset = 0usize;
714 for raw in src.split_inclusive('\n') {
715 if let Some(place) = self.place_at(offset) {
716 out.push((offset, raw, place));
717 }
718 offset = offset.saturating_add(raw.len());
719 }
720 out
721 }
722}
723
724/// The source bytes of each `$` in `src` whose maths never closes, in order.
725#[cfg(test)]
726pub(crate) fn lone_dollars(src: &str) -> Vec<usize> {
727 if !src.contains('$') {
728 return Vec::new();
729 }
730 Lexer::markup(src).lone
731}
732
733/// Each character of `src` read as a file's markup, with the byte it starts at.
734pub(crate) fn tokens(src: &str) -> Vec<(usize, char, Tok)> {
735 Lexer::markup(src).tokens()
736}
737
738/// Which characters of `src` a comment or raw text holds, delimiters included, one entry per character.
739pub(crate) fn literal_chars(src: &str) -> Vec<bool> {
740 tokens(src).into_iter().map(|(_, _, t)| matches!(t, Tok::Comment | Tok::Raw)).collect()
741}
742
743/// What each byte of `src` is, read as a file's markup: every byte of a character takes the character's.
744#[cfg(test)]
745pub(crate) fn byte_toks(src: &str) -> Vec<Tok> {
746 Lexer::markup(src).byte_toks()
747}
748
749/// `src` with every character a comment holds blanked to spaces and every line break kept, so each byte
750/// offset and line of the result is the source's own.
751pub(crate) fn uncommented(src: &str) -> String {
752 Lexer::markup(src).blanked(|t| t == Tok::Comment)
753}
754
755/// `src` with every character a comment or raw text holds blanked to spaces and every line break kept, so
756/// each byte offset and line of the result is the source's own. A scan that finds a declaration, an
757/// `#include` or a field in this text finds none a comment holds or a raw block shows.
758pub(crate) fn live_text(src: &str) -> String {
759 Lexer::markup(src).live_text()
760}
761
762/// Each line of `src` that opens in markup, with the byte it starts at and where it stands.
763pub(crate) fn placed_lines(src: &str) -> Vec<(usize, &str, Place)> {
764 Lexer::markup(src).lines(src)
765}
766
767/// Where the line starting at byte `at` stands, among `lines` from [`placed_lines`].
768pub(crate) fn place_in(lines: &[(usize, &str, Place)], at: usize) -> Option<Place> {
769 lines.binary_search_by_key(&at, |&(start, _, _)| start).ok().map(|k| lines[k].2)
770}
771
772/// The lines of `src` that stand at its own level, each with the byte it starts at: a line opening inside
773/// a group, a content block, a string, an equation, a comment or raw text is none of them, and nor is a
774/// line inside a list item, a heading, strong or emphasis.
775pub(crate) fn top_level_lines(src: &str) -> Vec<(usize, &str)> {
776 placed_lines(src).into_iter().filter(|&(_, _, p)| p == Place::Top).map(|(at, line, _)| (at, line)).collect()
777}
778
779/// Is anything left open at the end of `text`, read as Typst reads a fragment of markup: a group, a string,
780/// an equation, a comment or raw text?
781#[cfg(test)]
782pub(crate) fn open_after(text: &str) -> bool {
783 Lexer::markup_exact(text).is_open()
784}
785
786// Code fragments. A group's end, a value's end and the like are asked of text that goes on past the answer,
787// so the text is read in growing prefixes: a prefix that holds the answer holds it as the whole would.
788
789/// `text` cut at the first char boundary at or after `n` bytes.
790fn prefix(text: &str, n: usize) -> &str {
791 let mut end = n.min(text.len());
792 while !text.is_char_boundary(end) {
793 end += 1;
794 }
795 &text[..end]
796}
797
798/// The first answer `ask` finds, reading `text` as code in prefixes of growing size; the whole, last, is
799/// asked with `full` set.
800fn in_prefixes<T>(text: &str, ask: impl Fn(&Lexer, bool) -> Option<T>) -> Option<T> {
801 let mut n = 512usize;
802 loop {
803 let head = prefix(text, n);
804 let full = head.len() >= text.len();
805 if let Some(r) = ask(&Lexer::code(head), full) {
806 return Some(r);
807 }
808 if full {
809 return None;
810 }
811 n = n.saturating_mul(4);
812 }
813}
814
815impl Lexer {
816 /// The byte just past the group the text opens with, a `(`, `[` or `{`; `None` when it never closes.
817 fn group_close(&self) -> Option<usize> {
818 let first = self.leaves.first()?;
819 if first.tok != Tok::Open || self.nodes[first.node].start != 0 {
820 return None;
821 }
822 let group = self.nodes[first.node].parent;
823 (1..self.leaves.len()).find(|&li| self.closes_group(li) && self.nodes[self.leaves[li].node].parent == group)
824 .map(|li| self.nodes[self.leaves[li].node].end)
825 }
826
827 /// Is the leaf `li` the `)`, `]` or `}` that closes a group?
828 fn closes_group(&self, li: usize) -> bool {
829 self.leaves[li].tok == Tok::Close
830 && matches!(self.nodes[self.leaves[li].node].kind, SyntaxKind::RightParen | SyntaxKind::RightBrace | SyntaxKind::RightBracket)
831 }
832
833 /// The byte of the first leaf that is a comma or a closer standing at the text's own level: in no group,
834 /// equation, string, comment or raw text.
835 fn comma(&self) -> Option<usize> {
836 (0..self.leaves.len()).find(|&li| {
837 let l = self.leaves[li];
838 l.groups == 0 && l.eqs == 0 && l.tok == Tok::Code && matches!(self.leaf_text(li), "," | ")" | "]" | "}")
839 }).map(|li| self.nodes[self.leaves[li].node].start)
840 }
841}
842
843/// The byte just past the group opening at byte `at` of `src` -- a `(`, `[` or `{` read as code opens one
844/// -- or `None` when it never closes.
845pub(crate) fn group_end(src: &str, at: usize) -> Option<usize> {
846 if !matches!(src[at..].chars().next(), Some('(') | Some('[') | Some('{')) {
847 return None;
848 }
849 in_prefixes(&src[at..], |lx, _| lx.group_close()).map(|end| at + end)
850}
851
852/// As [`group_end`] over a slice of chars, at the char `at` and in chars: the index just past the closer.
853pub(crate) fn group_end_chars(chars: &[char], at: usize) -> Option<usize> {
854 if !matches!(chars.get(at), Some('(') | Some('[') | Some('{')) {
855 return None;
856 }
857 let mut n = 128usize;
858 loop {
859 let upto = (at + n).min(chars.len());
860 let head: String = chars[at..upto].iter().collect();
861 if let Some(end) = Lexer::code(&head).group_close() {
862 return Some(at + head[..end].chars().count());
863 }
864 if upto >= chars.len() {
865 return None;
866 }
867 n = n.saturating_mul(4);
868 }
869}
870
871/// Where a field's value in an argument list or a dictionary ends, for `src` read as code from the value's
872/// start: the byte of the first comma at its own level, or of the closer of the group it stands in, or the
873/// length of `src` when there is neither.
874pub(crate) fn top_comma(src: &str) -> usize {
875 in_prefixes(src, |lx, full| lx.comma().or_else(|| full.then_some(src.len()))).unwrap_or(src.len())
876}
877
878/// The byte of the first colon at the own level of `src` read as code, or `None` when there is none.
879pub(crate) fn top_colon(src: &str) -> Option<usize> {
880 let lx = Lexer::code(src);
881 (0..lx.leaves.len()).find(|&li| {
882 let l = lx.leaves[li];
883 l.groups == 0 && l.eqs == 0 && l.tok == Tok::Code && lx.leaf_text(li) == ":"
884 }).map(|li| lx.nodes[lx.leaves[li].node].start)
885}
886
887/// Each group a `(` opens at the top level of `src` read as code, by the byte of its `(` and the byte just
888/// past its `)`. A group that never closes is not listed.
889pub(crate) fn top_parens(src: &str) -> Vec<(usize, usize)> {
890 let lx = Lexer::code(src);
891 let mut out = Vec::new();
892 let mut open: Option<usize> = None;
893 let mut group: Option<usize> = None; // the group node the open `(` belongs to
894 for li in 0..lx.leaves.len() {
895 let l = lx.leaves[li];
896 let n = lx.nodes[l.node];
897 if l.groups == 0 && l.eqs == 0 && l.tok == Tok::Open && lx.leaf_text(li) == "(" {
898 open = Some(n.start);
899 group = Some(n.parent);
900 }
901 if lx.closes_group(li) && Some(n.parent) == group {
902 if let Some(o) = open.take() {
903 out.push((o, n.end));
904 }
905 group = None;
906 }
907 }
908 out
909}
910
911/// One argument of a call's list, or one entry of an array or a dictionary, as Typst's parser reads it.
912#[derive(Clone, Debug, PartialEq, Eq)]
913pub(crate) struct Arg {
914 pub(crate) key: Option<String>, // a named argument's name; `None` when positional
915 pub(crate) value: String, // the value's text, its comments dropped, trimmed
916}
917
918/// What a leaf of an argument list's text leaves of what it read.
919#[derive(Clone, Copy, Debug, PartialEq, Eq)]
920pub(crate) enum Kept {
921 Text, // the characters as written
922 Space, // a comment in code or maths: one space, so the tokens either side of it stay apart
923 Nothing, // a comment in markup, where Typst sets nothing for it
924}
925
926impl Lexer {
927 /// What the leaf `li` leaves of an argument list: a comment is trivia, so a reader that keeps the text of
928 /// an argument keeps none of a comment's.
929 fn kept(&self, li: usize) -> Kept {
930 match self.leaves[li].tok {
931 Tok::Comment if self.nodes[self.leaves[li].node].mode == Mode::Markup => Kept::Nothing,
932 Tok::Comment => Kept::Space,
933 _ => Kept::Text,
934 }
935 }
936
937 /// Is the leaf `li` a comma at the list's own level?
938 fn parts(&self, li: usize) -> bool {
939 let l = self.leaves[li];
940 l.groups == 0 && l.eqs == 0 && l.tok == Tok::Code && self.leaf_text(li) == ","
941 }
942}
943
944/// How much of an argument's head has been read, to tell a name before its `:` from a value.
945#[derive(Clone, Debug, PartialEq, Eq)]
946enum Head {
947 Start, // nothing yet but spaces and comments
948 Name { name: String, spaced: bool }, // a name, and whether a space has followed it
949 Value, // anything else: a positional value, or the value after a `:`
950}
951
952/// The arguments of the list `inner`, the text inside a call's parentheses (or an array's or a
953/// dictionary's), read as code the way Typst's parser reads them. An argument ends at a comma only at the
954/// list's own level. A comment, at any depth, is trivia: it adds one space to the value in code and nothing
955/// in markup, where Typst sets nothing for it, so a field commented out is no field. Strings, raw text and
956/// groups are copied as written and close where the lexer closes them. An argument is named only when it
957/// opens with a name followed, past any spaces or comments, by a `:` at the list's own level; a spread, a
958/// content block and every other value are positional. An empty argument, as a trailing comma leaves, is
959/// not listed.
960pub(crate) fn args(inner: &str) -> Vec<Arg> {
961 let lx = Lexer::code(inner);
962 let mut out: Vec<Arg> = Vec::new();
963 let mut head = Head::Start;
964 let mut key = None;
965 let mut value = String::new();
966 let flush = |key: &mut Option<String>, value: &mut String, out: &mut Vec<Arg>| {
967 let v = value.trim();
968 if key.is_some() || !v.is_empty() {
969 out.push(Arg { key: key.take(), value: v.to_string() });
970 }
971 };
972 for li in 0..lx.leaves.len() {
973 if lx.parts(li) {
974 flush(&mut key, &mut value, &mut out);
975 head = Head::Start;
976 key = None;
977 value.clear();
978 continue;
979 }
980 match lx.kept(li) {
981 Kept::Space => { value.push(' '); continue; },
982 Kept::Nothing => continue,
983 Kept::Text => {},
984 }
985 let l = lx.leaves[li];
986 let piece = lx.leaf_text(li);
987 let top = l.groups == 0 && l.eqs == 0;
988 let blank = piece.chars().all(|c| c.is_whitespace());
989 if top && l.tok == Tok::Code {
990 let word = piece.chars().next().is_some_and(is_id_start) && piece.chars().all(is_id_continue);
991 let next = match std::mem::replace(&mut head, Head::Value) {
992 Head::Start if blank => Head::Start,
993 Head::Start if word => Head::Name { name: piece.to_string(), spaced: false },
994 Head::Name { name, .. } if blank => Head::Name { name, spaced: true },
995 Head::Name { mut name, spaced: false } if piece.chars().all(is_id_continue) => {
996 name.push_str(piece);
997 Head::Name { name, spaced: false }
998 },
999 Head::Name { name, .. } if piece == ":" => {
1000 // The name is the key, and the value starts after its colon.
1001 key = Some(name);
1002 value.clear();
1003 continue;
1004 },
1005 Head::Start | Head::Name { .. } | Head::Value => Head::Value,
1006 };
1007 head = next;
1008 } else if !blank {
1009 head = Head::Value;
1010 }
1011 value.push_str(piece);
1012 }
1013 flush(&mut key, &mut value, &mut out);
1014 out
1015}
1016
1017/// The arguments of the list `inner` as the text each keeps, comments as [`args`] reads them, split at the
1018/// commas of the list's own level. A last argument of nothing but space is not listed.
1019pub(crate) fn split_args(inner: &str) -> Vec<String> {
1020 let lx = Lexer::code(inner);
1021 let mut out: Vec<String> = Vec::new();
1022 let mut cur = String::new();
1023 for li in 0..lx.leaves.len() {
1024 if lx.parts(li) {
1025 out.push(std::mem::take(&mut cur));
1026 continue;
1027 }
1028 match lx.kept(li) {
1029 Kept::Text => cur.push_str(lx.leaf_text(li)),
1030 Kept::Space => cur.push(' '),
1031 Kept::Nothing => {},
1032 }
1033 }
1034 if !cur.trim().is_empty() {
1035 out.push(cur);
1036 }
1037 out
1038}
1039
1040/// The value of the argument named `key` in `list`, the last one when it is named twice.
1041pub(crate) fn named<'a>(list: &'a [Arg], key: &str) -> Option<&'a str> {
1042 list.iter().rev().find(|a| a.key.as_deref() == Some(key)).map(|a| a.value.as_str())
1043}
1044
1045/// A name `list` gives twice, which Typst refuses as a duplicate argument.
1046pub(crate) fn duplicate_key(list: &[Arg]) -> Option<&str> {
1047 for (k, a) in list.iter().enumerate() {
1048 if let Some(name) = a.key.as_deref() {
1049 if list[..k].iter().any(|b| b.key.as_deref() == Some(name)) {
1050 return Some(name);
1051 }
1052 }
1053 }
1054 None
1055}
1056
1057/// A conditional or a loop embedded in markup -- an `#if` with its `else` arms, a `#for` or a `#while` --
1058/// with its extent and each arm's condition and body.
1059#[derive(Clone, Debug, PartialEq, Eq)]
1060pub(crate) struct Flow {
1061 pub(crate) kw: Kw, // `If`, `For` or `While`
1062 pub(crate) start: usize, // the byte of its `#`
1063 pub(crate) end: usize, // the byte just past its last body, or the end of what it took
1064 pub(crate) arms: Vec<Arm>,
1065 pub(crate) whole: bool, // every arm closed, and no call or field follows the last
1066 pub(crate) coded: bool, // it stands in code, a code block or a function's body, whose names this scan cannot see
1067 pub(crate) math: bool, // it stands in an equation
1068}
1069
1070/// Which of a text's conditionals and loops [`flows_in`] lists.
1071#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1072pub(crate) enum Level {
1073 Own, // those standing in the text's own markup
1074 Deep, // those too in any content block or equation inside it: a cell's, a caption's, a note's, a call's argument
1075}
1076
1077/// One arm of a [`Flow`].
1078#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1079pub(crate) struct Arm {
1080 pub(crate) cond: Option<(usize, usize)>, // the condition's bytes, or a loop's head's; `None` for `else`
1081 pub(crate) body: (usize, usize), // the body's bytes, its delimiters included
1082 pub(crate) content: bool, // a `[...]` content block, not a `{...}` code block
1083}
1084
1085/// Every conditional and loop standing in `src`'s own markup, in order. One in another's body, or in any
1086/// other group, is that group's, and is not listed.
1087pub(crate) fn flows(src: &str) -> Vec<Flow> {
1088 flows_in(src, Level::Own)
1089}
1090
1091/// The conditionals and loops of `src` that `level` names, in order, each the outermost of its kind: one in
1092/// another's head or body is that statement's. At [`Level::Deep`] a flow in a content block or an equation is
1093/// listed wherever it stands, a call's argument list included, with its `coded` set where it stands in code
1094/// whose own bindings (a code block's `let`, a function's parameters) this scan does not read. A flow inside
1095/// a `#let`, `#set` or `#show` statement is that statement's, read when it is, and not listed.
1096pub(crate) fn flows_in(src: &str, level: Level) -> Vec<Flow> {
1097 Lexer::markup(src).flows(level)
1098}
1099
1100/// A `#let` or `#import` embedded in markup, and the stretch of the text its names are bound over.
1101#[derive(Clone, Debug, PartialEq, Eq)]
1102pub(crate) struct Binding {
1103 pub(crate) start: usize, // the byte of its `#`
1104 pub(crate) text: String, // the statement from its `#`, its comments dropped
1105 pub(crate) at: usize, // the byte just past it, where its names come into force
1106 pub(crate) until: Option<usize>, // the byte of the closer of the block it stands in; `None` at the text's own level
1107}
1108
1109/// Every `#let` and `#import` embedded in `src`'s markup, at any depth, in source order. Typst binds a
1110/// name in the innermost content block `[...]` or code block `{...}` the statement stands in, a
1111/// conditional's branch included, from the statement's end to the block's closer; a heading, a list item,
1112/// strong or emphasis opens no scope. A code block's own `let`, written with no `#`, is code rather than
1113/// markup and is not listed.
1114pub(crate) fn bindings(src: &str) -> Vec<Binding> {
1115 Lexer::markup(src).bindings()
1116}
1117
1118impl Lexer {
1119 fn kind(&self, a: usize) -> SyntaxKind {
1120 self.nodes[a].kind
1121 }
1122
1123 /// The expression an `#` at the leaf node `h` embeds, if one follows it directly.
1124 fn embedded(&self, h: usize) -> Option<usize> {
1125 let n = &self.nodes[h];
1126 (n.kind == SyntaxKind::Hash && n.next != NONE && self.nodes[n.next].start == n.end).then_some(n.next)
1127 }
1128
1129 /// The nodes beneath `a`, in order.
1130 fn kids(&self, a: usize) -> Vec<usize> {
1131 let mut out = Vec::new();
1132 let mut c = self.nodes[a].child;
1133 while c != NONE {
1134 out.push(c);
1135 c = self.nodes[c].next;
1136 }
1137 out
1138 }
1139
1140 /// The conditionals and loops `level` names.
1141 fn flows(&self, level: Level) -> Vec<Flow> {
1142 let mut out = Vec::new();
1143 self.find_flows(0, level, true, false, false, &mut out);
1144 out
1145 }
1146
1147 /// Walks the children of `a`: `own` while only markup and its scopes lie between the root and `a`, `stmt` inside
1148 /// an embedded statement, `coded` inside a code block or a closure.
1149 fn find_flows(&self, a: usize, level: Level, own: bool, stmt: bool, coded: bool, out: &mut Vec<Flow>) {
1150 let mode = self.nodes[a].mode;
1151 let mut skip = NONE; // the expression a flow took, which holds no other
1152 for c in self.kids(a) {
1153 if c == skip {
1154 continue;
1155 }
1156 let k = self.kind(c);
1157 if let Some(e) = self.embedded(c) {
1158 let listed = match level {
1159 Level::Own => own,
1160 Level::Deep => !stmt && matches!(mode, Mode::Markup | Mode::Math),
1161 };
1162 if listed {
1163 if let Some(flow) = self.flow(c, e, coded, mode == Mode::Math) {
1164 out.push(flow);
1165 skip = e;
1166 continue;
1167 }
1168 }
1169 }
1170 if self.nodes[c].child == NONE {
1171 continue;
1172 }
1173 // An embedded statement is the statement's, whatever stands inside it.
1174 let embedded_stmt = is_stmt(k) && self.nodes[c].prev != NONE && self.kind(self.nodes[c].prev) == SyntaxKind::Hash;
1175 let keeps = k == SyntaxKind::Markup || is_scope(k);
1176 self.find_flows(
1177 c, level, own && keeps, stmt || embedded_stmt,
1178 coded || matches!(k, SyntaxKind::CodeBlock | SyntaxKind::Closure), out,
1179 );
1180 }
1181 }
1182
1183 /// The flow the `#` at `h` opens over the expression `e`, if `e` is a conditional or a loop, or a call or a
1184 /// field taken from one.
1185 fn flow(&self, h: usize, e: usize, coded: bool, math: bool) -> Option<Flow> {
1186 let mut x = e;
1187 let mut whole = true;
1188 while matches!(self.kind(x), SyntaxKind::FuncCall | SyntaxKind::FieldAccess) {
1189 x = self.nodes[x].child;
1190 whole = false;
1191 if x == NONE {
1192 return None;
1193 }
1194 }
1195 let kw = match self.kind(x) {
1196 SyntaxKind::Conditional => Kw::If,
1197 SyntaxKind::WhileLoop => Kw::While,
1198 SyntaxKind::ForLoop => Kw::For,
1199 _ => return None,
1200 };
1201 let mut arms = Vec::new();
1202 let complete = self.arms(x, &mut arms);
1203 let whole = whole && complete && !arms.is_empty();
1204 let last = arms.last().map_or(0, |a| a.body.1);
1205 Some(Flow {
1206 kw,
1207 start: self.nodes[h].start,
1208 end: if whole { last } else { self.nodes[e].end.max(last) },
1209 arms,
1210 whole,
1211 coded,
1212 math,
1213 })
1214 }
1215
1216 /// Pushes the arms of the conditional or loop `x`: false when one is missing its body or leaves it open.
1217 fn arms(&self, x: usize, out: &mut Vec<Arm>) -> bool {
1218 let kids = self.kids(x);
1219 let Some(&kw) = kids.first() else { return false; };
1220 let from = self.nodes[kw].end;
1221 let block = |a: usize| matches!(self.kind(a), SyntaxKind::ContentBlock | SyntaxKind::CodeBlock);
1222 let at_else = kids.iter().position(|&a| self.kind(a) == SyntaxKind::Else);
1223 let head = &kids[1..at_else.unwrap_or(kids.len())];
1224 let Some(&body) = head.iter().rev().find(|&&a| block(a)) else { return false; };
1225 if !self.closed(body) {
1226 return false;
1227 }
1228 out.push(Arm {
1229 cond: Some((from, self.nodes[body].start)),
1230 body: (self.nodes[body].start, self.nodes[body].end),
1231 content: self.kind(body) == SyntaxKind::ContentBlock,
1232 });
1233 let Some(e) = at_else else { return true; };
1234 let tail = kids[e + 1..].iter().copied().find(|&a| !matches!(self.kind(a), SyntaxKind::Space | SyntaxKind::LineComment | SyntaxKind::BlockComment));
1235 match tail {
1236 Some(a) if self.kind(a) == SyntaxKind::Conditional => self.arms(a, out),
1237 Some(a) if block(a) && self.closed(a) => {
1238 out.push(Arm {
1239 cond: None,
1240 body: (self.nodes[a].start, self.nodes[a].end),
1241 content: self.kind(a) == SyntaxKind::ContentBlock,
1242 });
1243 true
1244 },
1245 _ => false,
1246 }
1247 }
1248
1249 /// Does the block `a` end with its closer?
1250 fn closed(&self, a: usize) -> bool {
1251 let mut c = self.nodes[a].child;
1252 let mut last = NONE;
1253 while c != NONE {
1254 last = c;
1255 c = self.nodes[c].next;
1256 }
1257 last != NONE && matches!(self.kind(last), SyntaxKind::RightBracket | SyntaxKind::RightBrace)
1258 }
1259
1260 /// The statements `bindings` lists.
1261 fn bindings(&self) -> Vec<Binding> {
1262 let mut out = Vec::new();
1263 self.find_bindings(0, &mut out);
1264 out
1265 }
1266
1267 fn find_bindings(&self, a: usize, out: &mut Vec<Binding>) {
1268 let mut skip = NONE; // the statement a binding took, which holds no other
1269 for c in self.kids(a) {
1270 if c == skip {
1271 continue;
1272 }
1273 if let Some(e) = self.embedded(c) {
1274 if matches!(self.kind(e), SyntaxKind::LetBinding | SyntaxKind::ModuleImport) {
1275 out.push(self.binding(c, e));
1276 skip = e;
1277 continue;
1278 }
1279 }
1280 if self.nodes[c].child != NONE {
1281 self.find_bindings(c, out);
1282 }
1283 }
1284 }
1285
1286 /// The binding the `#` at `h` makes of the statement `e`.
1287 fn binding(&self, h: usize, e: usize) -> Binding {
1288 let mut text = String::from("#");
1289 self.push_text(e, &mut text);
1290 // A `;` ends the statement, and is part of what it holds over, though not of its text.
1291 let mut at = self.nodes[e].end;
1292 let mut n = self.nodes[e].next;
1293 while n != NONE && matches!(self.kind(n), SyntaxKind::Space | SyntaxKind::LineComment | SyntaxKind::BlockComment) {
1294 n = self.nodes[n].next;
1295 }
1296 if n != NONE && self.kind(n) == SyntaxKind::Semicolon {
1297 at = self.nodes[n].end;
1298 }
1299 // The innermost block it stands in holds it to its closer.
1300 let mut until = None;
1301 let mut p = self.nodes[h].parent;
1302 while p != NONE {
1303 if matches!(self.kind(p), SyntaxKind::ContentBlock | SyntaxKind::CodeBlock) {
1304 let close = self.kids(p).last().copied().filter(|&k| matches!(self.kind(k), SyntaxKind::RightBracket | SyntaxKind::RightBrace));
1305 until = Some(close.map_or(self.text.len(), |k| self.nodes[k].start));
1306 break;
1307 }
1308 p = self.nodes[p].parent;
1309 }
1310 Binding { start: self.nodes[h].start, text, at, until }
1311 }
1312
1313 /// Appends the text beneath `a`, its comments dropped.
1314 fn push_text(&self, a: usize, out: &mut String) {
1315 let n = &self.nodes[a];
1316 if n.leaf != NONE {
1317 if self.leaves[n.leaf].tok != Tok::Comment {
1318 out.push_str(&self.text[n.start..n.end]);
1319 }
1320 return;
1321 }
1322 for c in self.kids(a) {
1323 self.push_text(c, out);
1324 }
1325 }
1326}
1327
1328#[cfg(test)]
1329mod tests {
1330 use super::*;
1331 use oxedyne_fe2o3_core::prelude::*;
1332
1333 /// The top-level lines of `src`, by their trimmed text.
1334 fn tops(src: &str) -> Vec<String> {
1335 top_level_lines(src).into_iter().map(|(_, l)| l.trim().to_string()).collect()
1336 }
1337
1338 #[test]
1339 fn a_bracket_in_prose_is_text_and_hides_no_line() {
1340 let src = "The interval [0, 1) is half-open.\n#include \"ch1.typ\"\n= Intervals [0, 1)\n#set document(title: \"T\")\n";
1341 assert_eq!(tops(src), vec![
1342 "The interval [0, 1) is half-open.", "#include \"ch1.typ\"", "= Intervals [0, 1)",
1343 "#set document(title: \"T\")",
1344 ]);
1345 }
1346
1347 #[test]
1348 fn a_bracket_code_opens_is_content() {
1349 let src = "#box[\n#include \"a.typ\"\n]\n#link(\"x\")[\nB\n]\n#[\nC\n]\n@fig[\nD\n]\nE\n";
1350 assert_eq!(tops(src), vec!["#box[", "#link(\"x\")[", "#[", "@fig[", "E"]);
1351 }
1352
1353 #[test]
1354 fn a_scope_forgets_its_own_open_brackets() {
1355 // A heading's, an item's and a strong span's unbalanced `[` does not keep the block open.
1356 let src = "#box[\n= Intervals [0, 1)\n- item [a\n*[b*\n]\nAfter\n";
1357 assert_eq!(tops(src), vec!["#box[", "After"]);
1358 // In the block's own markup, a prose `[` does balance the next `]`.
1359 let src = "#box[\nprose [a\n]\nStill in\n]\nAfter\n";
1360 assert_eq!(tops(src), vec!["#box[", "After"]);
1361 }
1362
1363 #[test]
1364 fn a_quote_in_markup_is_a_character() {
1365 let src = "He said \"use /* here\" today.\n#set document(title: \"After Quote\")\nMore */ tail words.\n";
1366 assert_eq!(tops(src), vec!["He said \"use /* here\" today."]);
1367 let live = live_text(src);
1368 assert!(!live.contains("#set"), "the comment the quote does not hide holds the declaration");
1369 }
1370
1371 #[test]
1372 fn a_link_is_one_token() {
1373 let src = "See https://example.com/x/*y here.\n#include \"ch1.typ\"\nAnd http://a.io//b too.\n";
1374 assert_eq!(live_text(src), src);
1375 assert_eq!(tops(src).len(), 3);
1376 // Only `http://` and `https://` make a link; another scheme's `//` opens a comment.
1377 assert_eq!(live_text("ftp://x.io y\n"), "ftp: \n");
1378 }
1379
1380 #[test]
1381 fn a_link_drops_the_punctuation_that_ends_its_sentence() {
1382 let src = "https://x.io/a(b). /* c */";
1383 let link = "https://x.io/a(b)".len();
1384 let toks = byte_toks(src);
1385 assert!(toks[..link].iter().all(|&t| t == Tok::Link));
1386 assert_eq!(toks[link], Tok::Text);
1387 let src = "https://x.io. /* c */ d\n";
1388 assert_eq!(live_text(src), "https://x.io. d\n");
1389 }
1390
1391 #[test]
1392 fn block_comments_nest() {
1393 let src = "/* outer /* inner */\n#set document(title: \"Commented Out\")\n*/\nAfter\n";
1394 assert_eq!(tops(src), vec!["/* outer /* inner */", "After"]);
1395 assert_eq!(live_text("/* a /* b */ (c */ d"), fmt_spaces(19, "d"));
1396 }
1397
1398 fn fmt_spaces(n: usize, tail: &str) -> String {
1399 let mut s = " ".repeat(n);
1400 s.push_str(tail);
1401 s
1402 }
1403
1404 #[test]
1405 fn a_stray_closer_opens_no_comment() {
1406 // Typst reads `*/` outside a comment as one error token, so the `/*` inside `*/*` is not an opener.
1407 assert_eq!(live_text("a */* b\nc\n"), "a */* b\nc\n");
1408 }
1409
1410 #[test]
1411 fn raw_text_runs_across_lines_until_a_run_as_long() {
1412 let src = "`one\n#set text(size: 9pt)\n` two\n````\n```\n#set x()\n```\n````\nEnd\n";
1413 assert_eq!(tops(src), vec!["`one", "````", "End"]);
1414 // Two backticks are empty raw text, and open nothing.
1415 assert_eq!(tops("``\n#include \"a.typ\"\n"), vec!["``", "#include \"a.typ\""]);
1416 }
1417
1418 #[test]
1419 fn an_escape_takes_the_character_after_it() {
1420 // The `*` after the escaped slash opens a strong the `x*` closes, so the line after it stands at the top.
1421 assert_eq!(tops("\\[ \\/* x*\n#include \"a.typ\"\n"), vec!["\\[ \\/* x*", "#include \"a.typ\""]);
1422 assert_eq!(tops("#box[a \\] b\n]\nc\n"), vec!["#box[a \\] b", "c"]);
1423 }
1424
1425 #[test]
1426 fn a_string_holds_comment_markers_in_code_and_maths() {
1427 let src = "#let x = \"a // b /* c\"\n$ \"d /* e\" $\nAfter\n";
1428 assert_eq!(live_text(src), src);
1429 assert_eq!(tops(src).len(), 3);
1430 }
1431
1432 #[test]
1433 fn lone_dollars_leaves_earlier_pairs() {
1434 // Typst 0.15.1 refuses each of these files with "unclosed delimiter" at exactly the `$` named.
1435 assert_eq!(lone_dollars("a $x$ b $ c"), [8]);
1436 // A `]` is maths inside an equation, so the second `$` closes the first; the group never closes.
1437 assert_eq!(lone_dollars("#emph[a $b] c $d$"), [16]);
1438 // An equation opened in a group inside another: neither closes, the inner found first.
1439 assert_eq!(lone_dollars("$ a #[ b $ c"), [0, 9]);
1440 assert!(lone_dollars("a $x$ b $ y $\n").is_empty());
1441 let toks: Vec<Tok> = tokens("a $x$ b $ c").into_iter().map(|(_, _, t)| t).collect();
1442 assert_eq!(toks[2..5], [Tok::Open, Tok::Math, Tok::Close]);
1443 assert_eq!(toks[8..], [Tok::Text; 3]);
1444 }
1445
1446 #[test]
1447 fn a_dollar_that_never_closes_hides_no_line() {
1448 let src = "Alpha $x + y.\n\n= Heading Later\n#include \"a.typ\"\n#if a [B]\n";
1449 assert_eq!(tops(src), vec!["Alpha $x + y.", "", "= Heading Later", "#include \"a.typ\"", "#if a [B]"]);
1450 assert_eq!(flows_of(src).len(), 1);
1451 // A backtick after it opens raw text, as after any character of prose, so the `/*` in it opens nothing.
1452 assert_eq!(live_text("A $ `b /* c` d\n"), "A $".to_string() + &fmt_spaces(10, "d\n"));
1453 }
1454
1455 #[test]
1456 fn a_statement_ends_at_its_line_and_an_expression_after_its_operand() {
1457 // The `[` after a closed call and a space is prose, and so is the `"` after it.
1458 let src = "#link(\"x\")[a] then [b and \"c\n#include \"e.typ\"\n";
1459 assert_eq!(tops(src), vec!["#link(\"x\")[a] then [b and \"c", "#include \"e.typ\""]);
1460 // A statement's group runs across lines; the statement ends at the line after it closes.
1461 let src = "#let x = (\n a: 1,\n)\n#include \"e.typ\"\n";
1462 assert_eq!(tops(src), vec!["#let x = (", "#include \"e.typ\""]);
1463 }
1464
1465 #[test]
1466 fn a_shebang_is_a_comment_to_its_line_end() {
1467 let src = "#!/usr/bin/env typst /*\n= Root\n#include \"ch1.typ\"\n";
1468 assert_eq!(tops(src), vec!["#!/usr/bin/env typst /*", "= Root", "#include \"ch1.typ\""]);
1469 let first = src.find('\n').unwrap_or(src.len());
1470 assert!(tokens(src).iter().filter(|&&(at, _, _)| at < first).all(|&(_, _, t)| t == Tok::Comment));
1471 assert_eq!(live_text(src), fmt_spaces(first, &src[first..]));
1472 }
1473
1474 #[test]
1475 fn a_shebang_only_opens_the_file() {
1476 // Past the file's first character, `#!` is an embedded expression Typst refuses, and the `/*` after it
1477 // opens a comment.
1478 let src = "x\n#!y /*\n= Root\n";
1479 assert_eq!(tops(src), vec!["x", "#!y /*"]);
1480 assert_eq!(tokens(src)[2].2, Tok::Hash);
1481 }
1482
1483 #[test]
1484 fn a_line_in_open_strong_or_emphasis_is_not_top_level() {
1485 for m in ["*", "_"] {
1486 let src = fmt!("{m}bold [\n#set document(title: \"T\")\nstill bold{m}\nAfter\n");
1487 assert_eq!(tops(&src), vec![fmt!("{m}bold ["), "After".to_string()], "{}", m);
1488 }
1489 }
1490
1491 #[test]
1492 fn an_unclosed_star_holds_to_the_paragraph_break() {
1493 let src = "2 * 3 = 6\n#set document(title: \"S\")\n\n#set text(size: 9pt)\n";
1494 assert_eq!(tops(src), vec!["2 * 3 = 6", "#set text(size: 9pt)"]);
1495 }
1496
1497 #[test]
1498 fn an_underscore_between_cjk_letters_opens_emphasis() {
1499 assert_eq!(tops("漢_字\n#set document(title: \"C\")\n"), vec!["漢_字"]);
1500 // Between Latin letters it is within a word, and opens nothing.
1501 assert_eq!(tops("a_b\n#set document(title: \"C\")\n"), vec!["a_b", "#set document(title: \"C\")"]);
1502 }
1503
1504 #[test]
1505 fn an_indented_line_stays_in_its_item() {
1506 let src = "- item\n #set heading(numbering: \"1.\")\n= Next\n";
1507 assert_eq!(tops(src), vec!["- item", "= Next"]);
1508 }
1509
1510 #[test]
1511 fn a_line_that_does_not_indent_ends_the_item() {
1512 let src = "- item\n#set document(title: \"T\")\n";
1513 assert_eq!(tops(src), vec!["- item", "#set document(title: \"T\")"]);
1514 }
1515
1516 #[test]
1517 fn a_comment_ahead_of_the_first_token_does_not_hide_where_it_stands() {
1518 // The `#` stands at column 4, which does not indent past an item at column 6: the comment ahead of it
1519 // is trivia, and the token is what ends the item.
1520 let src = " - item\n/**/#set document(title: \"T\")\n";
1521 assert_eq!(tops(src), vec!["- item", "/**/#set document(title: \"T\")"]);
1522 // A comment alone on a line holds no token, so it ends no item and stands in it.
1523 assert_eq!(tops("- item\n// c\n #set text(size: 9pt)\n"), vec!["- item"]);
1524 assert_eq!(tops("- item\n/* c\n d */ #set text(size: 9pt)\n"), vec!["- item"]);
1525 }
1526
1527 /// Each placed line of `src` by its trimmed text.
1528 fn places(src: &str) -> Vec<(String, Place)> {
1529 placed_lines(src).into_iter().map(|(_, l, p)| (l.trim().to_string(), p)).collect()
1530 }
1531
1532 #[test]
1533 fn placed_lines_tells_a_bare_block_from_a_call() {
1534 let src = "#[\n#set document(title: \"B\")\n]\n#box[\n#set document(title: \"X\")\n]\n";
1535 let got = places(src);
1536 assert_eq!(got[1], ("#set document(title: \"B\")".to_string(), Place::Block));
1537 assert_eq!(got[4], ("#set document(title: \"X\")".to_string(), Place::Content));
1538 // A bare block in a bare block is joined too; one opened in a list item is held by the item.
1539 assert_eq!(places("#[\n#[\nX\n]\n]\n")[2], ("X".to_string(), Place::Block));
1540 assert_eq!(places("- a #[\nX\n]\n")[1], ("X".to_string(), Place::Contained));
1541 // Content a conditional, `context` or a reference opens is not joined as written.
1542 for src in ["#if a [\nX\n]\n", "#context [\nX\n]\n", "#context[\nX\n]\n", "@r[\nX\n]\n", "#f(a, [\nX\n])\n"] {
1543 assert_eq!(places(src)[1], ("X".to_string(), Place::Content), "{:?}", src);
1544 }
1545 }
1546
1547 #[test]
1548 fn a_conditional_takes_else_only_on_its_own_line() {
1549 let src = "#if a [\nX\n] else [\nY\n]\nAfter\n";
1550 assert_eq!(tops(src), vec!["#if a [", "After"]);
1551 // On the next line, `else [` is prose, and its `[` a prose bracket.
1552 let src = "#if a [X]\nelse [Y\nAfter\n";
1553 assert_eq!(tops(src), vec!["#if a [X]", "else [Y", "After"]);
1554 }
1555
1556 /// The byte ranges of `src`'s flows, as `(kw, whole, text, arms)` with each arm as its condition and body
1557 /// text.
1558 fn flows_of(src: &str) -> Vec<(Kw, bool, &str, Vec<(Option<&str>, &str, bool)>)> {
1559 flows(src).into_iter().map(|f| {
1560 let arms = f.arms.iter()
1561 .map(|a| (a.cond.map(|(x, y)| src[x..y].trim()), &src[a.body.0..a.body.1], a.content))
1562 .collect();
1563 (f.kw, f.whole, &src[f.start..f.end], arms)
1564 }).collect()
1565 }
1566
1567 #[test]
1568 fn a_conditional_is_read_whole_with_its_arms() {
1569 let src = "Text #if a == \"x\" [A] else [B] more.\n";
1570 assert_eq!(flows_of(src), vec![(Kw::If, true, "#if a == \"x\" [A] else [B]",
1571 vec![(Some("a == \"x\""), "[A]", true), (None, "[B]", true)])]);
1572 let src = "#if a [\nA\n] else if b [\nB\n] else {\n[C]\n}\nAfter\n";
1573 assert_eq!(flows_of(src), vec![(Kw::If, true, "#if a [\nA\n] else if b [\nB\n] else {\n[C]\n}",
1574 vec![(Some("a"), "[\nA\n]", true), (Some("b"), "[\nB\n]", true), (None, "{\n[C]\n}", false)])]);
1575 // On the next line, `else` is prose, and the conditional ends with its first body.
1576 let src = "#if a [A]\nelse [B]\n";
1577 assert_eq!(flows_of(src), vec![(Kw::If, true, "#if a [A]", vec![(Some("a"), "[A]", true)])]);
1578 }
1579
1580 #[test]
1581 fn a_loop_is_read_whole() {
1582 let src = "#for x in xs [\nX #x\n]\n#while n < 3 { n += 1 }\n";
1583 assert_eq!(flows_of(src), vec![
1584 (Kw::For, true, "#for x in xs [\nX #x\n]", vec![(Some("x in xs"), "[\nX #x\n]", true)]),
1585 (Kw::While, true, "#while n < 3 { n += 1 }", vec![(Some("n < 3"), "{ n += 1 }", false)]),
1586 ]);
1587 }
1588
1589 #[test]
1590 fn only_a_flow_at_the_scans_own_level_is_listed() {
1591 // One in another's body, in a content block, in maths or after `context` is not the scan's own.
1592 let src = "#if a [\n#if b [x]\n]\n#box[#if c [y]]\n$#if d [z]$\n#context if e [w]\n";
1593 assert_eq!(flows_of(src).len(), 1);
1594 assert_eq!(flows_of(src)[0].2, "#if a [\n#if b [x]\n]");
1595 // Two flows back to back are two.
1596 assert_eq!(flows_of("#if a [x]#if b [y]\n").len(), 2);
1597 }
1598
1599 #[test]
1600 fn a_flow_that_does_not_close_as_one_statement_is_not_whole() {
1601 // A call on its value, a head with no body, and a body that never closes.
1602 assert_eq!(flows_of("#if a [x](y) tail\n")[0].2, "#if a [x](y)");
1603 assert!(!flows_of("#if a [x](y) tail\n")[0].1);
1604 assert!(!flows_of("#if a\n[x]\n")[0].1);
1605 assert_eq!(flows_of("#if a [x\ny\n")[0].2, "#if a [x\ny\n");
1606 assert!(!flows_of("#if a [x\ny\n")[0].1);
1607 }
1608
1609 /// The statements `bindings` finds, as `(text, the source before it comes into force, the closer's source
1610 /// from where it stops)`.
1611 fn binds_of(src: &str) -> Vec<(String, String, Option<String>)> {
1612 bindings(src).into_iter()
1613 .map(|b| (b.text.clone(), src[..b.at].to_string(), b.until.map(|u| src[u..].to_string())))
1614 .collect()
1615 }
1616
1617 #[test]
1618 fn a_binding_is_listed_from_its_end_to_its_blocks_closer() {
1619 // At the text's own level it holds to the end; a `;` ends it there, and is part of what it holds over.
1620 let src = "#let a = \"x\"\nText\n#let b = true; more\n";
1621 assert_eq!(binds_of(src), vec![
1622 ("#let a = \"x\"".to_string(), "#let a = \"x\"".to_string(), None),
1623 ("#let b = true".to_string(), "#let a = \"x\"\nText\n#let b = true;".to_string(), None),
1624 ]);
1625 // In a callout's body, or a conditional's branch, it stops at the closer; text after the block is
1626 // outside it, and a heading, strong or emphasis opens no scope of its own.
1627 let src = "#box[\n#let a = 1\nIN\n]\n#if x [\n#let b = 2\n]\n= H #let c = 3;\nEND\n";
1628 let got = binds_of(src);
1629 assert_eq!(got.len(), 3);
1630 assert_eq!(got[0], ("#let a = 1".to_string(), "#box[\n#let a = 1".to_string(), Some("]\n#if x [\n#let b = 2\n]\n= H #let c = 3;\nEND\n".to_string())));
1631 assert_eq!(got[1].0, "#let b = 2");
1632 assert_eq!(got[1].2.as_deref(), Some("]\n= H #let c = 3;\nEND\n"));
1633 assert_eq!(got[2].0, "#let c = 3");
1634 assert_eq!(got[2].2, None);
1635 }
1636
1637 #[test]
1638 fn a_binding_a_comment_a_raw_or_a_code_block_holds_is_not_listed_as_markup() {
1639 // Comments inside a statement are dropped from its text; one standing alone, a raw span, and a `let`
1640 // of a code block (no `#`) are not statements of the markup.
1641 let src = "#let a = /* c */ \"x\" // t\n// #let b = 1\n`#let c = 1`\n#{ let d = 1 }\n#include \"f.typ\"\n#import \"g.typ\": *\n";
1642 let got: Vec<String> = binds_of(src).into_iter().map(|(t, _, _)| t.split_whitespace().collect::<Vec<_>>().join(" ")).collect();
1643 assert_eq!(got, ["#let a = \"x\"", "#import \"g.typ\": *"]);
1644 }
1645
1646 #[test]
1647 fn a_value_ends_at_its_comma_or_its_groups_closer() {
1648 assert_eq!(top_comma(" \"a, b\", c"), " \"a, b\"".len());
1649 assert_eq!(top_comma(" [x, y) z], c"), " [x, y) z]".len());
1650 assert_eq!(top_comma(" none)\n#let x = \"b\", c"), " none".len());
1651 assert_eq!(top_comma(" /* , */ 1"), " /* , */ 1".len());
1652 }
1653
1654 /// Each argument of `inner` as its key and value.
1655 fn pairs(inner: &str) -> Vec<(Option<String>, String)> {
1656 args(inner).into_iter().map(|a| (a.key, a.value)).collect()
1657 }
1658
1659 fn keys(inner: &str) -> Vec<String> {
1660 args(inner).into_iter().filter_map(|a| a.key).collect()
1661 }
1662
1663 #[test]
1664 fn an_argument_list_reads_comments_as_trivia() {
1665 assert_eq!(pairs("title: \"T\" /* c */"), vec![(Some("title".to_string()), "\"T\"".to_string())]);
1666 // In code a comment parts the tokens beside it; in a content block it leaves nothing, as Typst sets
1667 // nothing for it.
1668 assert_eq!(pairs("a: 1/* c */+ 2, [x/* c */y // d\n]"), vec![
1669 (Some("a".to_string()), "1 + 2".to_string()), (None, "[xy \n]".to_string()),
1670 ]);
1671 // A comment between a name and its colon leaves the name a key.
1672 assert_eq!(keys("title /* c */ : \"T\""), vec!["title"]);
1673 }
1674
1675 #[test]
1676 fn a_commented_out_named_argument_is_no_argument() {
1677 let list = args("\n // author: \"Old\",\n title: \"T\",\n");
1678 assert_eq!(list.len(), 1);
1679 assert_eq!(list[0].key.as_deref(), Some("title"));
1680 assert_eq!(named(&list, "author"), None);
1681 }
1682
1683 #[test]
1684 fn a_comment_between_arguments_parts_nothing() {
1685 assert_eq!(keys("a: 1, /* b: 2, */ c: 3"), vec!["a", "c"]);
1686 assert_eq!(keys("title: \"T\" // , author: \"X\"\n"), vec!["title"]);
1687 assert_eq!(pairs("(\"a\", /* \"b\", */ \"c\")")[0].1, "(\"a\", \"c\")");
1688 assert_eq!(pairs("\"a\", /* \"b\", */ \"c\""), vec![(None, "\"a\"".to_string()), (None, "\"c\"".to_string())]);
1689 }
1690
1691 #[test]
1692 fn a_key_inside_a_string_or_group_is_no_key() {
1693 let list = args("title: \"T, author: Q\" /* , author: \"X\" */, keywords: (\"k\",)");
1694 assert_eq!(keys("title: \"T, author: Q\" /* , author: \"X\" */, keywords: (\"k\",)"), vec!["title", "keywords"]);
1695 assert_eq!(named(&list, "title"), Some("\"T, author: Q\""));
1696 assert_eq!(named(&list, "keywords"), Some("(\"k\",)"));
1697 assert_eq!(keys("header: [page: 1]"), vec!["header"]);
1698 assert_eq!(named(&args("header: [page: 1]"), "page"), None);
1699 // A string keeps its comment markers, and raw text its commas.
1700 assert_eq!(pairs("author: \"A /* not */ B\""), vec![(Some("author".to_string()), "\"A /* not */ B\"".to_string())]);
1701 assert_eq!(keys("title: [T `raw, author: \"X\"` z]"), vec!["title"]);
1702 // A spread, a positional value and a name without its colon are positional.
1703 assert_eq!(pairs("..xs, 2pt, body"), vec![
1704 (None, "..xs".to_string()), (None, "2pt".to_string()), (None, "body".to_string()),
1705 ]);
1706 assert_eq!(keys("a b: 1, (c: 1)"), Vec::<String>::new());
1707 }
1708
1709 /// The text a reader of `src` as an argument list keeps, as [`split_args`] does.
1710 fn kept(src: &str) -> String {
1711 let lx = Lexer::code(src);
1712 let mut out = String::new();
1713 for li in 0..lx.leaves.len() {
1714 match lx.kept(li) {
1715 Kept::Text => out.push_str(lx.leaf_text(li)),
1716 Kept::Space => out.push(' '),
1717 Kept::Nothing => {},
1718 }
1719 }
1720 out
1721 }
1722
1723 #[test]
1724 fn a_step_over_an_argument_list_keeps_no_comment_text() {
1725 // One space for a comment in code, however it nests, and none in a content block, as in Typst.
1726 assert_eq!(kept("a /* x, y */, b"), "a , b");
1727 assert_eq!(kept("a/* b /* c */ d */e"), "a e");
1728 assert_eq!(kept("a // c, d\n, b"), "a \n, b");
1729 assert_eq!(kept("[p // q\n r /* s */]"), "[p \n r ]");
1730 // A string and raw text keep their comment markers.
1731 assert_eq!(kept("\"/* x */\", `// y`"), "\"/* x */\", `// y`");
1732 }
1733
1734 #[test]
1735 fn a_name_given_twice_is_a_duplicate() {
1736 let list = args("title: \"A\", title: \"B\"");
1737 assert_eq!(duplicate_key(&list), Some("title"));
1738 assert_eq!(named(&list, "title"), Some("\"B\""));
1739 assert_eq!(duplicate_key(&args("title: \"A\" /* , title: \"B\" */")), None);
1740 // A name is read whole, so a longer one holding it is another name.
1741 let list = args("heading-font: \"A\", first-line-indent: 1em");
1742 assert_eq!(named(&list, "font"), None);
1743 assert_eq!(named(&list, "indent"), None);
1744 assert_eq!(named(&list, "first-line-indent"), Some("1em"));
1745 }
1746
1747 #[test]
1748 fn a_top_level_paren_group_is_read_as_code() {
1749 let src = "// (a\n(b, \")\"), /* ( */ [(], (c (d)), (e";
1750 let groups: Vec<&str> = top_parens(src).into_iter().map(|(a, b)| &src[a..b]).collect();
1751 assert_eq!(groups, ["(b, \")\")", "(c (d))"]);
1752 }
1753
1754 #[test]
1755 fn a_group_end_is_read_as_code() {
1756 let src = "(a, \"b)\", [c (d], /* ) */ e) tail";
1757 assert_eq!(group_end(src, 0).map(|e| &src[e..]), Some(" tail"));
1758 assert_eq!(group_end("(a", 0), None);
1759 }
1760
1761 /// The texts of the flows `level` lists in `src`, with whether each stands in code and in an equation.
1762 fn deep(src: &str, level: Level) -> Vec<(String, bool, bool)> {
1763 flows_in(src, level).into_iter().map(|f| (src[f.start..f.end].to_string(), f.coded, f.math)).collect()
1764 }
1765
1766 #[test]
1767 fn a_flow_in_a_content_block_is_listed_at_the_deep_level_only() {
1768 let src = "#table(columns: 2, [c1], [#if a [X] else [Y]])\nNote#footnote[F #if b [Z]].\n#if c [W]\n";
1769 assert_eq!(deep(src, Level::Own), vec![("#if c [W]".to_string(), false, false)]);
1770 assert_eq!(deep(src, Level::Deep), vec![
1771 ("#if a [X] else [Y]".to_string(), false, false),
1772 ("#if b [Z]".to_string(), false, false),
1773 ("#if c [W]".to_string(), false, false),
1774 ]);
1775 }
1776
1777 #[test]
1778 fn a_flow_in_an_equation_is_listed_in_one_flagged_math() {
1779 let src = "P $x #if a [MX] else [MY]$ q $y$ #if b [Z]\n";
1780 assert_eq!(deep(src, Level::Own), vec![("#if b [Z]".to_string(), false, false)]);
1781 assert_eq!(deep(src, Level::Deep), vec![
1782 ("#if a [MX] else [MY]".to_string(), false, true),
1783 ("#if b [Z]".to_string(), false, false),
1784 ]);
1785 }
1786
1787 #[test]
1788 fn a_flow_in_code_is_flagged_and_one_in_a_statement_is_not_listed() {
1789 let src = "#{ let a = 1; [#if a [X]] }\n#f(r => [#if r [Y]])\n#let blk = [#if a [Z]]\n#set text(fill: if a [W])\n";
1790 assert_eq!(deep(src, Level::Deep), vec![
1791 ("#if a [X]".to_string(), true, false),
1792 ("#if r [Y]".to_string(), true, false),
1793 ]);
1794 }
1795
1796 #[test]
1797 fn a_flow_inside_a_flow_is_that_flows_and_a_loop_is_listed_whole() {
1798 let src = "#table([#for x in (1, 2) [#if x [A]]])\n";
1799 let got = flows_in(src, Level::Deep);
1800 assert_eq!(got.len(), 1);
1801 assert_eq!(got[0].kw, Kw::For);
1802 assert_eq!(&src[got[0].start..got[0].end], "#for x in (1, 2) [#if x [A]]");
1803 }
1804
1805 #[test]
1806 fn a_let_in_a_text_is_listed_with_the_byte_of_its_hash() {
1807 let src = "A [#let a = 1\nB]";
1808 let b = bindings(src);
1809 assert_eq!(b.len(), 1);
1810 assert_eq!(&src[b[0].start..b[0].at], "#let a = 1");
1811 }
1812
1813 // What the syntax tree settles that the state machine it replaces read otherwise. Each source below was
1814 // compiled by typst 0.15.1 as a probe, and the diagnostics it gave are named where they decide the reading.
1815
1816 #[test]
1817 fn a_space_after_a_statement_is_markup_beside_it() {
1818 // The spaces before a trailing comment are the markup's, outside the statement; typst compiles this
1819 // without a diagnostic.
1820 let src = "#let a = 230 // note\n#let b = 1; // c\n";
1821 let toks = byte_toks(src);
1822 let first = src.find(" //").unwrap_or(0);
1823 assert!(toks[first..first + 3].iter().all(|&t| t == Tok::Text));
1824 assert_eq!(toks[first + 3], Tok::Comment);
1825 let second = src.find("; //").map_or(0, |k| k + 1);
1826 assert_eq!(toks[second - 1], Tok::Code);
1827 assert_eq!(toks[second], Tok::Text);
1828 // The statement's own text ends before them, and what it binds holds from its end.
1829 let b = bindings(src);
1830 assert_eq!(b[0].text, "#let a = 230");
1831 assert_eq!(&src[..b[0].at], "#let a = 230");
1832 assert_eq!(&src[..b[1].at], "#let a = 230 // note\n#let b = 1;");
1833 }
1834
1835 #[test]
1836 fn a_statement_ends_where_typst_ends_it() {
1837 // typst: `expected semicolon or line break` after the `1` and nothing about `*b*`, so what follows is
1838 // markup, a strong span included.
1839 let src = "#let x = 1 a *b*\n";
1840 let toks = byte_toks(src);
1841 let a = src.find(" a").unwrap_or(0);
1842 assert!(toks[a..a + 2].iter().all(|&t| t == Tok::Text));
1843 assert_eq!(bindings(src)[0].text, "#let x = 1");
1844 // typst: `expected colon` after `heading`, then the same.
1845 let src = "#show heading it => it\n";
1846 let toks = byte_toks(src);
1847 let it = src.find(" it").unwrap_or(0);
1848 assert!(toks[it..src.len() - 1].iter().all(|&t| t == Tok::Text));
1849 }
1850
1851 #[test]
1852 fn an_unclosed_paren_ends_at_a_terminator() {
1853 // typst: `unclosed delimiter` at the `(`, `unexpected closing bracket` at the `]`, and the duplicate
1854 // argument of the call after it, so the `]` and what follows are markup.
1855 let src = "#let x = (1, 2\n\n*unclosed strong\n\ntext ] more\n\n#f(a: 1, a: 2)\n";
1856 let toks = byte_toks(src);
1857 let bracket = src.find(" ] more").unwrap_or(0);
1858 assert!(toks[bracket..bracket + 7].iter().all(|&t| t == Tok::Text));
1859 let call = src.find("#f(").unwrap_or(0);
1860 assert_eq!((toks[call], toks[call + 1], toks[call + 2]), (Tok::Hash, Tok::Code, Tok::Open));
1861 // The group never closed, and so is still counted open.
1862 assert_eq!(Lexer::markup(src).depth_at(call), 1);
1863 }
1864
1865 #[test]
1866 fn an_unclosed_paren_ends_with_the_code_block_round_it() {
1867 // typst: `unclosed delimiter` at the `(` alone, so the `}` closes the block.
1868 let src = "#{ import \"x\": (a, b }\n";
1869 assert_eq!(group_end(src, 1), Some(src.len() - 1));
1870 assert_eq!(group_end(src, src.find('(').unwrap_or(0)), None);
1871 assert_eq!(byte_toks(src)[src.len() - 1], Tok::Text);
1872 // The block ended what was open inside it, so the line after it is not in a group.
1873 let src = "#{ import \"x\": (a, b }\nnext\n";
1874 let lx = Lexer::markup(src);
1875 let next = src.find("next").unwrap_or(0);
1876 assert_eq!((lx.depth_at(next), lx.open_at(next)), (0, 0));
1877 }
1878
1879 #[test]
1880 fn an_error_token_after_a_hash_is_code() {
1881 // typst: `invalid number suffix`, `expected a hexadecimal number` and `unexpected minus`, each reported
1882 // at the `#` and each part of the embedded expression.
1883 for (src, upto) in [("#12p\n", 4), ("#0x\n", 3), ("#-1\n", 2)] {
1884 assert!(byte_toks(src)[1..upto].iter().all(|&t| t == Tok::Code), "{:?}", src);
1885 }
1886 assert_eq!(byte_toks("#-1\n")[2], Tok::Text);
1887 }
1888
1889 #[test]
1890 fn what_the_parser_left_over_in_code_is_read_again() {
1891 // typst: `expected named or keyed pair`, and nothing about what follows. The expression it refuses
1892 // is one error node holding the text of its tokens, whose string stays a string.
1893 let src = "#let d = (a: 1, \"x\" + \"y // z\")\n#set x()\n";
1894 let toks = byte_toks(src);
1895 let slashes = src.find("//").unwrap_or(0);
1896 assert_eq!(toks[slashes], Tok::Str);
1897 assert_eq!(tops(src).len(), 2);
1898 }
1899
1900 #[test]
1901 fn a_line_with_no_token_stands_where_the_last_token_left_it() {
1902 // The blank line is inside the strong span, which a paragraph break has not yet ended; the line after it
1903 // is not.
1904 let src = "2 * 3\n#set x()\n\n#set y()\n";
1905 let lx = Lexer::markup(src);
1906 let blank = src.find("\n\n").map_or(0, |k| k + 1);
1907 assert_eq!(lx.place_at(blank), Some(Place::Contained));
1908 assert_eq!(lx.place_at(blank + 1), Some(Place::Top));
1909 // A heading ended with its line, and an item has not ended until a token says so.
1910 let src = "= H\n\ntext\n";
1911 assert_eq!(Lexer::markup(src).place_at(4), Some(Place::Top));
1912 assert!(Lexer::markup(src).bare_line_at(4));
1913 let src = "- item\n\n more\n";
1914 assert_eq!(Lexer::markup(src).place_at(7), Some(Place::Contained));
1915 }
1916
1917 #[test]
1918 fn a_block_closer_and_a_blank_line_in_an_empty_block_are_bare() {
1919 let src = "#box[\n\n]\n";
1920 let lx = Lexer::markup(src);
1921 assert!(lx.bare_line_at(6), "a blank line in a block that holds nothing yet");
1922 assert!(lx.bare_line_at(7), "the closer stands in the block's markup");
1923 assert_eq!(lx.place_at(7), Some(Place::Content));
1924 // With a list item open in the block, the closer at its own margin ends it.
1925 let src = "#box[\n- a\n]\n";
1926 assert!(Lexer::markup(src).bare_line_at(10));
1927 assert!(!Lexer::markup(src).bare_line_at(6) || Lexer::markup(src).place_at(6) == Some(Place::Content));
1928 // A line in strong inside the block is not bare.
1929 let src = "#box[\n*a\nb*\n]\n";
1930 assert!(!Lexer::markup(src).bare_line_at(8));
1931 }
1932
1933 #[test]
1934 fn what_is_open_at_a_line_start_counts_groups_equations_and_literals() {
1935 let src = "#let x = (\n a,\n)\nb\n";
1936 let lx = Lexer::markup(src);
1937 assert_eq!((lx.open_at(0), lx.open_at(11), lx.open_at(src.len() - 2)), (0, 1, 0));
1938 assert_eq!(lx.depth_at(11), 1);
1939 assert_eq!(lx.markup_level_at(11), None, "inside a code group");
1940 assert_eq!(lx.markup_level_at(src.len() - 2), Some(0));
1941 for (src, line) in [("$ a\n b $\n", 4), ("#\"a\nb\"\n", 4), ("`a\nb`\n", 3), ("/* a\nb */\n", 5)] {
1942 let lx = Lexer::markup(src);
1943 assert_eq!(lx.open_at(line), 1, "{:?}", src);
1944 assert_eq!(lx.open_at(src.len()), 0, "{:?}", src);
1945 assert_eq!(lx.place_at(line), None, "{:?}", src);
1946 }
1947 // What is open at the end of a text that stops inside a construct.
1948 for src in ["#f(a,\n", "$ x\n", "#[a\n", "`raw\n", "/* c\n", "#\"s\n"] {
1949 assert!(open_after(src), "{:?}", src);
1950 }
1951 for src in ["#f(a)\n", "$ x $\n", "#[a]\n", "`raw`\n", "/* c */\n", "#\"s\"\n"] {
1952 assert!(!open_after(src), "{:?}", src);
1953 }
1954 }
1955
1956 #[test]
1957 fn a_group_is_found_in_the_prefix_that_holds_it() {
1958 // A group far longer than the first prefix, nested and holding strings, comments and brackets that
1959 // close nothing, ends where the whole text says.
1960 let mut inner = String::new();
1961 for k in 0..400 {
1962 inner.push_str(&fmt!("s{}: \"a ] b ) c\", item{}: [text ) with \\] {} /* ) */ (n{})], ", k, k, k, k));
1963 }
1964 let src = fmt!("({}) tail ( not part", inner);
1965 let end = src.find(") tail").map(|k| k + 1);
1966 assert_eq!(group_end(&src, 0), end);
1967 let chars: Vec<char> = src.chars().collect();
1968 assert_eq!(group_end_chars(&chars, 0), end);
1969 assert_eq!(group_end_chars(&chars, chars.len() - 9), None);
1970 assert_eq!(group_end("(a, b", 0), None);
1971 assert_eq!(top_comma(&fmt!("{} , more", &src[..end.unwrap_or(0)])), end.unwrap_or(0) + 1);
1972 // An index in chars is not an index in bytes.
1973 let src = "(é, \"ü ) €\", [ñ])x";
1974 let chars: Vec<char> = src.chars().collect();
1975 assert_eq!((group_end_chars(&chars, 0), group_end(src, 0)), (Some(17), Some(22)));
1976 }
1977
1978 #[test]
1979 fn the_first_colon_at_the_own_level_names_an_argument() {
1980 assert_eq!(top_colon("key: value"), Some(3));
1981 assert_eq!(top_colon("(a: 1) b: 2"), Some(8));
1982 assert_eq!(top_colon("[a: 1], \"b: 2\", $c: 3$"), None);
1983 assert_eq!(split_args("a, [b, c], \"d, e\" /* , */, f"), vec!["a", " [b, c]", " \"d, e\" ", " f"]);
1984 }
1985}