oxedyne/fe2o3/fe2o3_austenite/src/lang/lex.rs
76.2 KiB, 3 runs
created by r1870400018:62383, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! The one markup lexer every source scanner reads through, as a view over the concrete syntax tree of |
| 2 | //! [`crate::syntax`]: the port of `typst-syntax` 0.15.1's lexer and parser. A text is parsed once, and every |
| 3 | //! answer -- what a character is, where a line stands, what a group holds, where a conditional ends -- is read |
| 4 | //! from that tree, so the scanners keep no second reading of what is a comment, a body or the top level. |
| 5 | //! A `[` opens content only where code opened it (`#name[`, `#[`, a call's trailing argument, a reference's |
| 6 | //! supplement); in prose it is text, balanced against a later `]` within its own markup scope. A `"` is a |
| 7 | //! character in markup and a string only in code or maths. An automatic link is one token, so the `//` or |
| 8 | //! `/*` in it opens no comment. Block comments nest. Raw text opens with one backtick or three or more and |
| 9 | //! closes on a run as long, across lines. A `#!` opening the file is a comment to its line's end. A `$` |
| 10 | //! whose maths never closes is read as a character, so the error stays at it and the markup after it is read |
| 11 | //! as usual, where Typst's parser takes the rest of the source into the equation: the text is parsed again |
| 12 | //! with each such `$` set aside, until none is left. |
| 13 | |
| 14 | use crate::syntax::FileId; |
| 15 | use crate::syntax::SyntaxKind; |
| 16 | use crate::syntax::SyntaxNode; |
| 17 | use crate::syntax::parser; |
| 18 | |
| 19 | /// What a character is, as Typst reads it. |
| 20 | #[derive(Clone, Copy, Debug, PartialEq, Eq)] |
| 21 | pub(crate) enum Tok { |
| 22 | Text, // markup text, a bracket in prose included, and a `$` whose maths never closes |
| 23 | Code, // code: a name, a number, an operator, a space |
| 24 | Math, // an equation's own text |
| 25 | Str, // a string in code or maths, its quotes included |
| 26 | Comment, // a line or block comment, its delimiters included |
| 27 | Raw, // raw text, its backticks included |
| 28 | Link, // an automatic link |
| 29 | Escape, // an escape or a forced line break, its backslash included |
| 30 | Label, // `<name>` |
| 31 | Ref, // `@name`, and a supplement's opening `[` |
| 32 | Hash, // the `#` that embeds an expression |
| 33 | Open, // a delimiter code opens: `(`, `{`, a content block's `[`, an equation's `$` |
| 34 | Close, // the delimiter that closes one |
| 35 | } |
| 36 | |
| 37 | /// Where a line stands, decided at its first token, as Typst's parser and realiser read it. |
| 38 | #[derive(Clone, Copy, Debug, PartialEq, Eq)] |
| 39 | pub(crate) enum Place { |
| 40 | Top, // the file's own markup, with no list item, heading, strong or emphasis open |
| 41 | Block, // a bare content block `#[`, or a run of them, opened where no scope was: joined into the file's markup |
| 42 | Contained, // in a list item, a heading, strong or emphasis of that markup: a container, which Typst sets apart |
| 43 | Content, // in content code opened: a call's argument, a reference's supplement, a conditional's or loop's body, a value |
| 44 | } |
| 45 | |
| 46 | /// The keyword that heads a loop or a conditional. |
| 47 | #[derive(Clone, Copy, Debug, PartialEq, Eq)] |
| 48 | pub(crate) enum Kw { |
| 49 | If, |
| 50 | While, |
| 51 | For, |
| 52 | } |
| 53 | |
| 54 | /// How the text of a node is read. |
| 55 | #[derive(Clone, Copy, Debug, PartialEq, Eq)] |
| 56 | enum Mode { |
| 57 | Markup, |
| 58 | Code, |
| 59 | Math, |
| 60 | } |
| 61 | |
| 62 | const NONE: usize = usize::MAX; // no node |
| 63 | |
| 64 | /// One node of the tree, flattened: the leaves and the nodes above them, in source order. |
| 65 | #[derive(Clone, Copy, Debug)] |
| 66 | struct Node { |
| 67 | kind: SyntaxKind, |
| 68 | start: usize, |
| 69 | end: usize, |
| 70 | parent: usize, // `NONE` for the root |
| 71 | child: usize, // the first child; `NONE` for a leaf |
| 72 | next: usize, // the next sibling |
| 73 | prev: usize, // the previous sibling |
| 74 | mode: Mode, // of a leaf, how it is read; of an inner node, how its children are |
| 75 | bare: bool, // a content block opened directly after a `#` |
| 76 | leaf: usize, // the leaf's place in `leaves`; `NONE` for an inner node |
| 77 | } |
| 78 | |
| 79 | /// A leaf's class, and what is open before it. |
| 80 | #[derive(Clone, Copy, Debug)] |
| 81 | struct Leaf { |
| 82 | node: usize, |
| 83 | tok: Tok, |
| 84 | groups: u32, // groups code opened and not yet closed, before this leaf |
| 85 | eqs: u32, // equations open before it |
| 86 | shut: bool, // false for a string, raw text or block comment the text ends inside |
| 87 | } |
| 88 | |
| 89 | /// A text read once. A file's markup is read with [`Lexer::markup`], a line or fragment as Typst reads it |
| 90 | /// with [`Lexer::markup_exact`], and the inside of a call's argument list with [`Lexer::code`]. |
| 91 | #[derive(Clone, Debug)] |
| 92 | pub(crate) struct Lexer { |
| 93 | text: String, |
| 94 | nodes: Vec<Node>, |
| 95 | leaves: Vec<Leaf>, |
| 96 | lone: Vec<usize>, // sorted: the source bytes of each `$` whose maths never closes |
| 97 | groups: u32, // open at the end of the text |
| 98 | eqs: u32, |
| 99 | } |
| 100 | |
| 101 | /// The tree under construction. |
| 102 | struct Build { |
| 103 | nodes: Vec<Node>, |
| 104 | leaves: Vec<Leaf>, |
| 105 | pos: usize, |
| 106 | open: Vec<usize>, // the group nodes whose brackets are open, outermost first |
| 107 | eqs: u32, |
| 108 | last: Option<(SyntaxKind, usize)>, // the last leaf made: its kind and where it ended |
| 109 | dollars: Vec<usize>, // the byte of each `$` the parser found unclosed |
| 110 | deep: u32, // error text being read again |
| 111 | } |
| 112 | |
| 113 | // Where the kinds of the syntax tree take a text apart. |
| 114 | |
| 115 | /// Is `kind` one whose brackets code opens and closes as groups? |
| 116 | fn is_group(kind: SyntaxKind) -> bool { |
| 117 | matches!(kind, |
| 118 | SyntaxKind::CodeBlock | SyntaxKind::ContentBlock | SyntaxKind::Args | SyntaxKind::Parenthesized |
| 119 | | SyntaxKind::Array | SyntaxKind::Dict | SyntaxKind::Params | SyntaxKind::Destructuring) |
| 120 | } |
| 121 | |
| 122 | /// Does `kind` open a markup scope of its own in the markup around it: an item, a heading, strong or emphasis? |
| 123 | fn is_scope(kind: SyntaxKind) -> bool { |
| 124 | matches!(kind, |
| 125 | SyntaxKind::Strong | SyntaxKind::Emph | SyntaxKind::Heading | SyntaxKind::ListItem |
| 126 | | SyntaxKind::EnumItem | SyntaxKind::TermItem) |
| 127 | } |
| 128 | |
| 129 | /// Is `kind` a statement an `#` embeds, which runs to its line's end? |
| 130 | fn is_stmt(kind: SyntaxKind) -> bool { |
| 131 | matches!(kind, |
| 132 | SyntaxKind::LetBinding | SyntaxKind::SetRule | SyntaxKind::ShowRule | SyntaxKind::ModuleImport |
| 133 | | SyntaxKind::ModuleInclude | SyntaxKind::FuncReturn) |
| 134 | } |
| 135 | |
| 136 | /// How the children of a node of `kind` are read, `mode` being how the node itself is. |
| 137 | fn child_mode(kind: SyntaxKind, mode: Mode) -> Mode { |
| 138 | match kind { |
| 139 | SyntaxKind::Markup | SyntaxKind::Heading | SyntaxKind::Strong | SyntaxKind::Emph |
| 140 | | SyntaxKind::ListItem | SyntaxKind::EnumItem | SyntaxKind::TermItem | SyntaxKind::Ref |
| 141 | | SyntaxKind::Raw |
| 142 | => Mode::Markup, |
| 143 | SyntaxKind::Math | SyntaxKind::Equation | SyntaxKind::MathDelimited | SyntaxKind::MathCall |
| 144 | | SyntaxKind::MathArgs | SyntaxKind::MathAttach | SyntaxKind::MathFrac | SyntaxKind::MathRoot |
| 145 | => Mode::Math, |
| 146 | SyntaxKind::Code | SyntaxKind::CodeBlock | SyntaxKind::ContentBlock | SyntaxKind::Parenthesized |
| 147 | | SyntaxKind::Array | SyntaxKind::Dict | SyntaxKind::Keyed | SyntaxKind::Unary | SyntaxKind::Binary |
| 148 | | SyntaxKind::FieldAccess | SyntaxKind::FuncCall | SyntaxKind::Args | SyntaxKind::Closure |
| 149 | | SyntaxKind::Params | SyntaxKind::LetBinding | SyntaxKind::SetRule | SyntaxKind::ShowRule |
| 150 | | SyntaxKind::Contextual | SyntaxKind::Conditional | SyntaxKind::WhileLoop | SyntaxKind::ForLoop |
| 151 | | SyntaxKind::ModuleImport | SyntaxKind::ImportItems | SyntaxKind::ImportItemPath |
| 152 | | SyntaxKind::RenamedImportItem | SyntaxKind::ModuleInclude | SyntaxKind::LoopBreak |
| 153 | | SyntaxKind::LoopContinue | SyntaxKind::FuncReturn | SyntaxKind::Destructuring |
| 154 | | SyntaxKind::DestructAssignment |
| 155 | => Mode::Code, |
| 156 | _ => mode, |
| 157 | } |
| 158 | } |
| 159 | |
| 160 | /// Does a block comment's text close, its nesting balanced? |
| 161 | fn comment_closed(text: &str) -> bool { |
| 162 | let b = text.as_bytes(); |
| 163 | let (mut i, mut depth) = (0usize, 0u32); |
| 164 | while i + 1 < b.len() { |
| 165 | match (b[i], b[i + 1]) { |
| 166 | (b'/', b'*') => { depth = depth.saturating_add(1); i += 2; }, |
| 167 | (b'*', b'/') => { depth = depth.saturating_sub(1); i += 2; }, |
| 168 | _ => i += 1, |
| 169 | } |
| 170 | } |
| 171 | depth == 0 |
| 172 | } |
| 173 | |
| 174 | impl Build { |
| 175 | fn new() -> Self { |
| 176 | Build { |
| 177 | nodes: Vec::new(), |
| 178 | leaves: Vec::new(), |
| 179 | pos: 0, |
| 180 | open: Vec::new(), |
| 181 | eqs: 0, |
| 182 | last: None, |
| 183 | dollars: Vec::new(), |
| 184 | deep: 0, |
| 185 | } |
| 186 | } |
| 187 | |
| 188 | /// An inner node, opening at the current byte; its end is set when its children are done. |
| 189 | fn inner(&mut self, kind: SyntaxKind, parent: usize, mode: Mode) -> usize { |
| 190 | let bare = kind == SyntaxKind::ContentBlock && self.last == Some((SyntaxKind::Hash, self.pos)); |
| 191 | self.nodes.push(Node { kind, start: self.pos, end: self.pos, parent, child: NONE, next: NONE, prev: NONE, mode, bare, leaf: NONE }); |
| 192 | self.nodes.len() - 1 |
| 193 | } |
| 194 | |
| 195 | /// A leaf of `len` bytes, read as `tok`. |
| 196 | fn leaf(&mut self, kind: SyntaxKind, len: usize, tok: Tok, parent: usize, mode: Mode, shut: bool) -> usize { |
| 197 | let idx = self.nodes.len(); |
| 198 | self.nodes.push(Node { |
| 199 | kind, start: self.pos, end: self.pos + len, parent, child: NONE, next: NONE, prev: NONE, mode, |
| 200 | bare: false, leaf: self.leaves.len(), |
| 201 | }); |
| 202 | self.leaves.push(Leaf { node: idx, tok, groups: self.open.len() as u32, eqs: self.eqs, shut }); |
| 203 | self.pos += len; |
| 204 | self.last = Some((kind, self.pos)); |
| 205 | idx |
| 206 | } |
| 207 | |
| 208 | /// Reads the children `kids` of the node `parent`, of `pkind`, whose children are read as `mode`. |
| 209 | fn kids(&mut self, kids: &[SyntaxNode], parent: usize, pkind: SyntaxKind, mode: Mode) { |
| 210 | let mut last = NONE; |
| 211 | let mut k1: Option<SyntaxKind> = None; // the kind of the sibling before |
| 212 | let mut k2: Option<SyntaxKind> = None; // and the one before that |
| 213 | let mut first = true; |
| 214 | for c in kids { |
| 215 | let kind = c.kind(); |
| 216 | let idx = if c.is_inner() { |
| 217 | let cmode = child_mode(kind, mode); |
| 218 | let idx = self.inner(kind, parent, cmode); |
| 219 | self.kids(c.children(), idx, kind, cmode); |
| 220 | self.nodes[idx].end = self.pos; |
| 221 | idx |
| 222 | } else if c.len() == 0 { |
| 223 | continue; // a zero-width error: nothing of the text |
| 224 | } else if kind == SyntaxKind::Error { |
| 225 | self.error(c, parent, mode, k1) |
| 226 | } else { |
| 227 | self.token(c, parent, pkind, mode, first, k1, k2) |
| 228 | }; |
| 229 | if last == NONE { |
| 230 | self.nodes[parent].child = idx; |
| 231 | } else { |
| 232 | self.nodes[last].next = idx; |
| 233 | self.nodes[idx].prev = last; |
| 234 | } |
| 235 | last = idx; |
| 236 | k2 = k1; |
| 237 | k1 = Some(kind); |
| 238 | first = false; |
| 239 | } |
| 240 | } |
| 241 | |
| 242 | /// A leaf that is no error. |
| 243 | fn token( |
| 244 | &mut self, |
| 245 | c: &SyntaxNode, |
| 246 | parent: usize, |
| 247 | pkind: SyntaxKind, |
| 248 | mode: Mode, |
| 249 | first: bool, |
| 250 | k1: Option<SyntaxKind>, |
| 251 | k2: Option<SyntaxKind>, |
| 252 | ) |
| 253 | -> usize |
| 254 | { |
| 255 | let kind = c.kind(); |
| 256 | let len = c.len(); |
| 257 | let gp = self.nodes[parent].parent; |
| 258 | let gkind = if gp == NONE { None } else { Some(self.nodes[gp].kind) }; |
| 259 | let group = is_group(pkind); |
| 260 | // The atom an `#` embeds, which follows it directly. |
| 261 | let after_hash = k1 == Some(SyntaxKind::Hash) && !matches!(kind, SyntaxKind::Space | SyntaxKind::Parbreak); |
| 262 | let (mut opens, mut closes, mut eq, mut shut) = (false, false, 0i8, true); |
| 263 | let tok = match kind { |
| 264 | SyntaxKind::Shebang | SyntaxKind::LineComment => Tok::Comment, |
| 265 | SyntaxKind::BlockComment => { shut = comment_closed(c.text()); Tok::Comment }, |
| 266 | SyntaxKind::Hash => Tok::Hash, |
| 267 | SyntaxKind::Str => Tok::Str, |
| 268 | SyntaxKind::Escape | SyntaxKind::Linebreak => Tok::Escape, |
| 269 | SyntaxKind::Link => Tok::Link, |
| 270 | SyntaxKind::Label => Tok::Label, |
| 271 | SyntaxKind::RefMarker => Tok::Ref, |
| 272 | SyntaxKind::Dollar if pkind == SyntaxKind::Equation => { |
| 273 | if first { |
| 274 | eq = 1; |
| 275 | Tok::Open |
| 276 | } else { |
| 277 | eq = -1; |
| 278 | Tok::Close |
| 279 | } |
| 280 | }, |
| 281 | SyntaxKind::LeftParen | SyntaxKind::LeftBrace | SyntaxKind::LeftBracket if group => { |
| 282 | opens = true; |
| 283 | // A reference's supplement is a content block, its opening bracket part of the reference. |
| 284 | if kind == SyntaxKind::LeftBracket && pkind == SyntaxKind::ContentBlock && gkind == Some(SyntaxKind::Ref) { |
| 285 | Tok::Ref |
| 286 | } else { |
| 287 | Tok::Open |
| 288 | } |
| 289 | }, |
| 290 | SyntaxKind::RightParen | SyntaxKind::RightBrace | SyntaxKind::RightBracket if group => { |
| 291 | closes = true; |
| 292 | Tok::Close |
| 293 | }, |
| 294 | _ if pkind == SyntaxKind::Raw => Tok::Raw, |
| 295 | _ if after_hash => Tok::Code, |
| 296 | _ => match mode { |
| 297 | Mode::Code => Tok::Code, |
| 298 | Mode::Markup => if kind == SyntaxKind::Semicolon { Tok::Code } else { Tok::Text }, |
| 299 | // A `;` straight after an embedded expression ends it, as it ends no maths argument. |
| 300 | Mode::Math => if kind == SyntaxKind::Semicolon && k2 == Some(SyntaxKind::Hash) { Tok::Code } else { Tok::Math }, |
| 301 | }, |
| 302 | }; |
| 303 | let at = self.leaf(kind, len, tok, parent, mode, shut); |
| 304 | self.shift(opens, closes, eq, parent); |
| 305 | at |
| 306 | } |
| 307 | |
| 308 | /// Moves the counts of what is open past a leaf that opens or closes a group or an equation. |
| 309 | fn shift(&mut self, opens: bool, closes: bool, eq: i8, parent: usize) { |
| 310 | if opens { |
| 311 | self.open.push(parent); |
| 312 | } |
| 313 | // A closer ends its own group and any inside it that never closed, as the parser leaves them. |
| 314 | if closes && self.open.contains(&parent) { |
| 315 | while let Some(g) = self.open.pop() { |
| 316 | if g == parent { |
| 317 | break; |
| 318 | } |
| 319 | } |
| 320 | } |
| 321 | if eq > 0 { |
| 322 | self.eqs = self.eqs.saturating_add(1); |
| 323 | } else if eq < 0 { |
| 324 | self.eqs = self.eqs.saturating_sub(1); |
| 325 | } |
| 326 | } |
| 327 | |
| 328 | /// A leaf the parser could not read. Most are one token, by what the message says it is; one that is |
| 329 | /// the leftover of a construct is read again as code, so the strings and comments in it are still such. |
| 330 | fn error(&mut self, c: &SyntaxNode, parent: usize, mode: Mode, k1: Option<SyntaxKind>) -> usize { |
| 331 | let text = c.text(); |
| 332 | let msg = c.error_info().map(|e| e.message.as_str()).unwrap_or(""); |
| 333 | let len = c.len(); |
| 334 | let (mut opens, mut eq, mut shut) = (false, 0i8, true); |
| 335 | let tok = if msg.starts_with("unclosed delimiter") { |
| 336 | match text { |
| 337 | "$" => { |
| 338 | eq = 1; |
| 339 | self.dollars.push(self.pos); |
| 340 | Some(Tok::Open) |
| 341 | }, |
| 342 | "(" | "{" | "[" => { opens = true; Some(Tok::Open) }, |
| 343 | _ => None, |
| 344 | } |
| 345 | } else if msg.starts_with("unclosed raw text") { |
| 346 | shut = false; |
| 347 | Some(Tok::Raw) |
| 348 | } else if msg.starts_with("unclosed string") { |
| 349 | shut = false; |
| 350 | Some(Tok::Str) |
| 351 | } else if msg.starts_with("unexpected end of block comment") { |
| 352 | Some(Tok::Text) |
| 353 | } else if msg.starts_with("unclosed label") { |
| 354 | Some(Tok::Label) |
| 355 | } else if msg.contains("Unicode") { |
| 356 | Some(Tok::Escape) |
| 357 | } else if msg.starts_with("automatic links") { |
| 358 | Some(Tok::Link) |
| 359 | } else { |
| 360 | None |
| 361 | }; |
| 362 | if let Some(tok) = tok { |
| 363 | let at = self.leaf(SyntaxKind::Error, len, tok, parent, mode, shut); |
| 364 | self.shift(opens, false, eq, parent); |
| 365 | return at; |
| 366 | } |
| 367 | // What the parser left over in code may hold a string, a comment, a group: read it again as code. |
| 368 | if mode == Mode::Code && text.chars().nth(1).is_some() && self.deep < 3 { |
| 369 | let root = parser::parse_code(text, FileId::DETACHED); |
| 370 | let same = matches!(root.children(), [only] if only.kind() == SyntaxKind::Error && only.text() == text); |
| 371 | if !same { |
| 372 | let idx = self.inner(SyntaxKind::Error, parent, Mode::Code); |
| 373 | self.deep += 1; |
| 374 | self.kids(root.children(), idx, SyntaxKind::Code, Mode::Code); |
| 375 | self.deep -= 1; |
| 376 | self.nodes[idx].end = self.pos; |
| 377 | return idx; |
| 378 | } |
| 379 | } |
| 380 | let tok = match mode { |
| 381 | // What follows an `#` and is no expression is still code. |
| 382 | _ if k1 == Some(SyntaxKind::Hash) => Tok::Code, |
| 383 | Mode::Markup => Tok::Text, |
| 384 | Mode::Code => Tok::Code, |
| 385 | Mode::Math => Tok::Math, |
| 386 | }; |
| 387 | self.leaf(SyntaxKind::Error, len, tok, parent, mode, true) |
| 388 | } |
| 389 | } |
| 390 | |
| 391 | /// Is `c` a line break, as Typst counts one? |
| 392 | pub(crate) fn is_newline(c: char) -> bool { |
| 393 | matches!(c, '\n' | '\x0B' | '\x0C' | '\r' | '\u{0085}' | '\u{2028}' | '\u{2029}') |
| 394 | } |
| 395 | |
| 396 | /// Can `c` start a Typst name? |
| 397 | pub(crate) fn is_id_start(c: char) -> bool { |
| 398 | c.is_alphabetic() || c == '_' |
| 399 | } |
| 400 | |
| 401 | /// Can `c` continue a Typst name? |
| 402 | pub(crate) fn is_id_continue(c: char) -> bool { |
| 403 | c.is_alphanumeric() || c == '_' || c == '-' |
| 404 | } |
| 405 | |
| 406 | impl Lexer { |
| 407 | /// A file's markup, as Typst reads a `.typ` file, each `$` whose maths never closes read as a character. |
| 408 | pub(crate) fn markup(src: &str) -> Self { |
| 409 | Self::read(src, false, true) |
| 410 | } |
| 411 | |
| 412 | /// A line or fragment of markup exactly as Typst reads it, an unclosed `$` opening an equation that |
| 413 | /// takes the rest. |
| 414 | #[cfg(test)] |
| 415 | pub(crate) fn markup_exact(src: &str) -> Self { |
| 416 | Self::read(src, false, false) |
| 417 | } |
| 418 | |
| 419 | /// Code, as the inside of a call's argument list or a value reads. |
| 420 | pub(crate) fn code(src: &str) -> Self { |
| 421 | Self::read(src, true, false) |
| 422 | } |
| 423 | |
| 424 | fn read(src: &str, code: bool, recover: bool) -> Self { |
| 425 | let mut lone: Vec<usize> = Vec::new(); |
| 426 | let passes = if recover { src.matches('$').count() + 1 } else { 1 }; |
| 427 | let mut done = None; |
| 428 | for _ in 0..passes { |
| 429 | let b = Self::pass(src, code, &lone); |
| 430 | // The innermost equation that never closes is the last, as each one that does not takes the rest. |
| 431 | match (recover, b.dollars.last()) { |
| 432 | (true, Some(&at)) => lone.insert(lone.partition_point(|&p| p < at), at), |
| 433 | _ => { done = Some(b); break; }, |
| 434 | } |
| 435 | } |
| 436 | let b = match done { |
| 437 | Some(b) => b, |
| 438 | None => Self::pass(src, code, &lone), |
| 439 | }; |
| 440 | Lexer { text: src.to_string(), nodes: b.nodes, leaves: b.leaves, lone, groups: b.open.len() as u32, eqs: b.eqs } |
| 441 | } |
| 442 | |
| 443 | /// One reading, with each `$` of `lone` set aside as a comma, which no mode reads as more than text. |
| 444 | fn pass(src: &str, code: bool, lone: &[usize]) -> Build { |
| 445 | let mut bytes = src.as_bytes().to_vec(); |
| 446 | for &at in lone { |
| 447 | bytes[at] = b','; |
| 448 | } |
| 449 | let text = String::from_utf8(bytes).unwrap_or_else(|_| src.to_string()); |
| 450 | let (root, mode) = if code { |
| 451 | (parser::parse_code(&text, FileId::DETACHED), Mode::Code) |
| 452 | } else { |
| 453 | (parser::parse(&text, FileId::DETACHED), Mode::Markup) |
| 454 | }; |
| 455 | let mut b = Build::new(); |
| 456 | let idx = b.inner(root.kind(), NONE, mode); |
| 457 | b.kids(root.children(), idx, root.kind(), mode); |
| 458 | b.nodes[idx].end = b.pos; |
| 459 | b |
| 460 | } |
| 461 | |
| 462 | /// The source bytes of each `$` whose maths never closes, in order. Typst refuses the file there, with the |
| 463 | /// rest of the source taken into the equation; the scan reads each of these as a character instead, so the |
| 464 | /// error stays at the `$` and the markup after it is read. |
| 465 | pub(crate) fn lone(&self) -> &[usize] { |
| 466 | &self.lone |
| 467 | } |
| 468 | |
| 469 | /// The leaf holding byte `p`, or the last one starting before it. |
| 470 | fn leaf_at(&self, p: usize) -> Option<usize> { |
| 471 | self.leaves.partition_point(|l| self.nodes[l.node].start <= p).checked_sub(1) |
| 472 | } |
| 473 | |
| 474 | /// What stands around byte `p`: the leaf it falls inside, when it is no leaf's first byte, and the nodes |
| 475 | /// that began before it and end after it, innermost first. |
| 476 | fn around(&self, p: usize) -> (Option<usize>, Vec<usize>) { |
| 477 | let Some(li) = self.leaf_at(p) else { return (None, Vec::new()); }; |
| 478 | let n = self.nodes[self.leaves[li].node]; |
| 479 | if p >= n.end { |
| 480 | return (None, Vec::new()); |
| 481 | } |
| 482 | let inside = if n.start < p { Some(li) } else { None }; |
| 483 | let mut out = Vec::new(); |
| 484 | let mut a = n.parent; |
| 485 | while a != NONE { |
| 486 | let m = self.nodes[a]; |
| 487 | if m.start < p && m.end > p { |
| 488 | out.push(a); |
| 489 | } |
| 490 | a = m.parent; |
| 491 | } |
| 492 | (inside, out) |
| 493 | } |
| 494 | |
| 495 | /// Does the text stand in markup at the position `chain` and `inside` describe: neither in a string, a |
| 496 | /// comment, raw text, a code group, an equation, nor anywhere but the file's own markup or a content block? |
| 497 | fn in_markup(&self, inside: Option<usize>, chain: &[usize]) -> bool { |
| 498 | if let Some(li) = inside { |
| 499 | if matches!(self.leaves[li].tok, Tok::Str | Tok::Comment | Tok::Raw) { |
| 500 | return false; |
| 501 | } |
| 502 | } |
| 503 | let own = chain.iter().map(|&a| self.nodes[a].kind).find(|&k| k != SyntaxKind::Markup && !is_scope(k)); |
| 504 | own.map_or(true, |k| k == SyntaxKind::ContentBlock) |
| 505 | } |
| 506 | |
| 507 | /// The scopes and frames of `chain`, outermost first, as a line standing in them is placed. |
| 508 | fn place_of(&self, chain: &[usize]) -> Place { |
| 509 | let mut place = Place::Top; |
| 510 | for &a in chain.iter().rev() { |
| 511 | let n = &self.nodes[a]; |
| 512 | match n.kind { |
| 513 | SyntaxKind::Markup => {}, |
| 514 | k if is_scope(k) => return Place::Contained, |
| 515 | SyntaxKind::ContentBlock if n.bare => place = Place::Block, |
| 516 | _ => return Place::Content, |
| 517 | } |
| 518 | } |
| 519 | place |
| 520 | } |
| 521 | |
| 522 | /// The first leaf of the line at byte `p` that is a token: past its spaces and any block comment that |
| 523 | /// closes on it. `None` when the line holds none (spaces, a comment or its break alone). |
| 524 | fn first_token(&self, p: usize) -> Option<usize> { |
| 525 | let line_end = self.text[p..].find(is_newline).map_or(self.text.len(), |k| p + k); |
| 526 | let mut li = self.leaf_at(p)?; |
| 527 | loop { |
| 528 | let n = self.nodes[self.leaves.get(li)?.node]; |
| 529 | if n.start >= line_end { |
| 530 | return None; |
| 531 | } |
| 532 | match n.kind { |
| 533 | SyntaxKind::Space | SyntaxKind::Parbreak => {}, |
| 534 | SyntaxKind::LineComment | SyntaxKind::Shebang => return None, |
| 535 | SyntaxKind::BlockComment => if n.end > line_end { return None; }, |
| 536 | _ => return Some(n.start.max(p)), |
| 537 | } |
| 538 | li += 1; |
| 539 | } |
| 540 | } |
| 541 | |
| 542 | /// The scopes still open where a line with no token stands, which ends none: those that held the last token |
| 543 | /// before it. A heading ended with its line, and strong or emphasis that closed before it is closed; one |
| 544 | /// that has not ended runs on through the trailing space the parser gives it. |
| 545 | fn open_chain(&self, p: usize) -> Vec<usize> { |
| 546 | let Some(mut li) = self.leaf_at(p) else { return Vec::new(); }; |
| 547 | // The last token ending before the line. |
| 548 | loop { |
| 549 | let n = self.nodes[self.leaves[li].node]; |
| 550 | let blank = matches!(n.kind, SyntaxKind::Space | SyntaxKind::Parbreak | SyntaxKind::LineComment |
| 551 | | SyntaxKind::BlockComment | SyntaxKind::Shebang); |
| 552 | if n.end <= p && !blank { |
| 553 | break; |
| 554 | } |
| 555 | match li.checked_sub(1) { |
| 556 | Some(k) => li = k, |
| 557 | None => return Vec::new(), |
| 558 | } |
| 559 | } |
| 560 | let mut out = Vec::new(); |
| 561 | let mut a = self.nodes[self.leaves[li].node].parent; |
| 562 | while a != NONE { |
| 563 | let m = self.nodes[a]; |
| 564 | let keep = match m.kind { |
| 565 | SyntaxKind::Heading => false, |
| 566 | SyntaxKind::Strong | SyntaxKind::Emph => m.end > p, |
| 567 | SyntaxKind::Markup | SyntaxKind::ListItem | SyntaxKind::EnumItem | SyntaxKind::TermItem => true, |
| 568 | _ => m.end > p, |
| 569 | }; |
| 570 | if keep { |
| 571 | out.push(a); |
| 572 | } else if out.last().is_some_and(|&l| self.nodes[l].kind == SyntaxKind::Markup && self.nodes[l].parent == a) { |
| 573 | // The markup a scope that has ended held goes with it. |
| 574 | out.pop(); |
| 575 | } |
| 576 | a = m.parent; |
| 577 | } |
| 578 | out |
| 579 | } |
| 580 | |
| 581 | /// The chain a line at byte `p` is placed by, and whether it opens in markup at all: the strict ancestors of |
| 582 | /// its first token, or, for a line with none, the scopes left open before it. |
| 583 | fn line_chain(&self, p: usize) -> Option<Vec<usize>> { |
| 584 | let (inside, here) = self.around(p); |
| 585 | if !self.in_markup(inside, &here) { |
| 586 | return None; |
| 587 | } |
| 588 | Some(match self.first_token(p) { |
| 589 | Some(t) => self.around(t).1, |
| 590 | None => self.open_chain(p), |
| 591 | }) |
| 592 | } |
| 593 | |
| 594 | /// Where the line starting at byte `p` stands, or `None` when it opens outside markup: in a string, a |
| 595 | /// comment, raw text, an equation, a code group or an unfinished expression. The spaces that open a line |
| 596 | /// change no scope, so the place is decided at its first token, which ends the items it does not indent past. |
| 597 | pub(crate) fn place_at(&self, p: usize) -> Option<Place> { |
| 598 | self.line_chain(p).map(|chain| self.place_of(&chain)) |
| 599 | } |
| 600 | |
| 601 | /// Is the line starting at byte `p` standing directly in the innermost markup, whatever opened it, with no |
| 602 | /// list item, heading, strong or emphasis of that markup open around it? |
| 603 | pub(crate) fn bare_line_at(&self, p: usize) -> bool { |
| 604 | let Some(chain) = self.line_chain(p) else { return false; }; |
| 605 | match chain.first() { |
| 606 | None => true, |
| 607 | // A line in a content block whose markup has not begun, or that opens with the block's own closer, |
| 608 | // stands in the block, bare. |
| 609 | Some(&a) if self.nodes[a].kind == SyntaxKind::ContentBlock => true, |
| 610 | Some(&a) => { |
| 611 | self.nodes[a].kind == SyntaxKind::Markup |
| 612 | && chain.get(1).map_or(true, |&b| self.nodes[b].kind == SyntaxKind::ContentBlock) |
| 613 | }, |
| 614 | } |
| 615 | } |
| 616 | |
| 617 | /// How many groups code opened are open at byte `p`: `(`, `{`, a content block. Prose brackets, strings, |
| 618 | /// equations, comments and raw text are not counted. |
| 619 | pub(crate) fn depth_at(&self, p: usize) -> usize { |
| 620 | match self.leaf_at(p) { |
| 621 | Some(li) if p < self.nodes[self.leaves[li].node].end => self.leaves[li].groups as usize, |
| 622 | _ => self.groups as usize, |
| 623 | } |
| 624 | } |
| 625 | |
| 626 | /// The number of groups open at byte `p` when the text stands there in markup -- the file's own, or a |
| 627 | /// content block's -- and `None` when it stands in code, a string, an equation, a comment or raw text. |
| 628 | pub(crate) fn markup_level_at(&self, p: usize) -> Option<usize> { |
| 629 | let (inside, chain) = self.around(p); |
| 630 | self.in_markup(inside, &chain).then(|| self.depth_at(p)) |
| 631 | } |
| 632 | |
| 633 | /// How many frames are open at byte `p`: groups, equations, and the string, comment or raw text it falls |
| 634 | /// inside. |
| 635 | pub(crate) fn open_at(&self, p: usize) -> usize { |
| 636 | let end = |this: &Self| -> usize { |
| 637 | let shut = this.leaves.last().map_or(true, |l| l.shut); |
| 638 | this.groups as usize + this.eqs as usize + usize::from(!shut) |
| 639 | }; |
| 640 | let Some(li) = self.leaf_at(p) else { return end(self); }; |
| 641 | let l = self.leaves[li]; |
| 642 | let n = self.nodes[l.node]; |
| 643 | if p >= n.end { |
| 644 | return end(self); |
| 645 | } |
| 646 | let mut open = l.groups as usize + l.eqs as usize; |
| 647 | let in_raw = l.tok == Tok::Raw |
| 648 | && (n.start < p || (self.nodes[n.parent].kind == SyntaxKind::Raw && self.nodes[n.parent].start < p)); |
| 649 | if (n.start < p && matches!(l.tok, Tok::Str | Tok::Comment)) || in_raw { |
| 650 | open += 1; |
| 651 | } |
| 652 | open |
| 653 | } |
| 654 | |
| 655 | /// Is anything open at the end of the text? |
| 656 | #[cfg(test)] |
| 657 | pub(crate) fn is_open(&self) -> bool { |
| 658 | self.open_at(self.text.len()) > 0 |
| 659 | } |
| 660 | |
| 661 | /// What each byte of the text is: every byte of a character takes the character's. |
| 662 | pub(crate) fn byte_toks(&self) -> Vec<Tok> { |
| 663 | let mut out = vec![Tok::Text; self.text.len()]; |
| 664 | for l in &self.leaves { |
| 665 | let n = &self.nodes[l.node]; |
| 666 | for t in &mut out[n.start..n.end] { |
| 667 | *t = l.tok; |
| 668 | } |
| 669 | } |
| 670 | for &at in &self.lone { |
| 671 | out[at] = Tok::Text; |
| 672 | } |
| 673 | out |
| 674 | } |
| 675 | |
| 676 | /// Each character of the text, with the byte it starts at. |
| 677 | pub(crate) fn tokens(&self) -> Vec<(usize, char, Tok)> { |
| 678 | let toks = self.byte_toks(); |
| 679 | self.text.char_indices().map(|(at, c)| (at, c, toks[at])).collect() |
| 680 | } |
| 681 | |
| 682 | /// The text of a leaf. |
| 683 | fn leaf_text(&self, li: usize) -> &str { |
| 684 | let n = &self.nodes[self.leaves[li].node]; |
| 685 | &self.text[n.start..n.end] |
| 686 | } |
| 687 | |
| 688 | /// The text with every character `blank` names turned into a space and every line break kept, so each byte |
| 689 | /// offset and line of the result is the text's own. |
| 690 | fn blanked(&self, blank: impl Fn(Tok) -> bool) -> String { |
| 691 | let mut out = String::with_capacity(self.text.len()); |
| 692 | for (_, c, tok) in self.tokens() { |
| 693 | if blank(tok) && c != '\n' && c != '\r' { |
| 694 | for _ in 0..c.len_utf8() { |
| 695 | out.push(' '); |
| 696 | } |
| 697 | } else { |
| 698 | out.push(c); |
| 699 | } |
| 700 | } |
| 701 | out |
| 702 | } |
| 703 | |
| 704 | /// The text with every character a comment or raw text holds blanked to spaces and every line break kept, |
| 705 | /// so each byte offset and line of the result is the text's own. |
| 706 | pub(crate) fn live_text(&self) -> String { |
| 707 | self.blanked(|t| matches!(t, Tok::Comment | Tok::Raw)) |
| 708 | } |
| 709 | |
| 710 | /// Each line of `src`, the text read, that opens in markup, with the byte it starts at and where it stands. |
| 711 | pub(crate) fn lines<'a>(&self, src: &'a str) -> Vec<(usize, &'a str, Place)> { |
| 712 | let mut out = Vec::new(); |
| 713 | let mut offset = 0usize; |
| 714 | for raw in src.split_inclusive('\n') { |
| 715 | if let Some(place) = self.place_at(offset) { |
| 716 | out.push((offset, raw, place)); |
| 717 | } |
| 718 | offset = offset.saturating_add(raw.len()); |
| 719 | } |
| 720 | out |
| 721 | } |
| 722 | } |
| 723 | |
| 724 | /// The source bytes of each `$` in `src` whose maths never closes, in order. |
| 725 | #[cfg(test)] |
| 726 | pub(crate) fn lone_dollars(src: &str) -> Vec<usize> { |
| 727 | if !src.contains('$') { |
| 728 | return Vec::new(); |
| 729 | } |
| 730 | Lexer::markup(src).lone |
| 731 | } |
| 732 | |
| 733 | /// Each character of `src` read as a file's markup, with the byte it starts at. |
| 734 | pub(crate) fn tokens(src: &str) -> Vec<(usize, char, Tok)> { |
| 735 | Lexer::markup(src).tokens() |
| 736 | } |
| 737 | |
| 738 | /// Which characters of `src` a comment or raw text holds, delimiters included, one entry per character. |
| 739 | pub(crate) fn literal_chars(src: &str) -> Vec<bool> { |
| 740 | tokens(src).into_iter().map(|(_, _, t)| matches!(t, Tok::Comment | Tok::Raw)).collect() |
| 741 | } |
| 742 | |
| 743 | /// What each byte of `src` is, read as a file's markup: every byte of a character takes the character's. |
| 744 | #[cfg(test)] |
| 745 | pub(crate) fn byte_toks(src: &str) -> Vec<Tok> { |
| 746 | Lexer::markup(src).byte_toks() |
| 747 | } |
| 748 | |
| 749 | /// `src` with every character a comment holds blanked to spaces and every line break kept, so each byte |
| 750 | /// offset and line of the result is the source's own. |
| 751 | pub(crate) fn uncommented(src: &str) -> String { |
| 752 | Lexer::markup(src).blanked(|t| t == Tok::Comment) |
| 753 | } |
| 754 | |
| 755 | /// `src` with every character a comment or raw text holds blanked to spaces and every line break kept, so |
| 756 | /// each byte offset and line of the result is the source's own. A scan that finds a declaration, an |
| 757 | /// `#include` or a field in this text finds none a comment holds or a raw block shows. |
| 758 | pub(crate) fn live_text(src: &str) -> String { |
| 759 | Lexer::markup(src).live_text() |
| 760 | } |
| 761 | |
| 762 | /// Each line of `src` that opens in markup, with the byte it starts at and where it stands. |
| 763 | pub(crate) fn placed_lines(src: &str) -> Vec<(usize, &str, Place)> { |
| 764 | Lexer::markup(src).lines(src) |
| 765 | } |
| 766 | |
| 767 | /// Where the line starting at byte `at` stands, among `lines` from [`placed_lines`]. |
| 768 | pub(crate) fn place_in(lines: &[(usize, &str, Place)], at: usize) -> Option<Place> { |
| 769 | lines.binary_search_by_key(&at, |&(start, _, _)| start).ok().map(|k| lines[k].2) |
| 770 | } |
| 771 | |
| 772 | /// The lines of `src` that stand at its own level, each with the byte it starts at: a line opening inside |
| 773 | /// a group, a content block, a string, an equation, a comment or raw text is none of them, and nor is a |
| 774 | /// line inside a list item, a heading, strong or emphasis. |
| 775 | pub(crate) fn top_level_lines(src: &str) -> Vec<(usize, &str)> { |
| 776 | placed_lines(src).into_iter().filter(|&(_, _, p)| p == Place::Top).map(|(at, line, _)| (at, line)).collect() |
| 777 | } |
| 778 | |
| 779 | /// Is anything left open at the end of `text`, read as Typst reads a fragment of markup: a group, a string, |
| 780 | /// an equation, a comment or raw text? |
| 781 | #[cfg(test)] |
| 782 | pub(crate) fn open_after(text: &str) -> bool { |
| 783 | Lexer::markup_exact(text).is_open() |
| 784 | } |
| 785 | |
| 786 | // Code fragments. A group's end, a value's end and the like are asked of text that goes on past the answer, |
| 787 | // so the text is read in growing prefixes: a prefix that holds the answer holds it as the whole would. |
| 788 | |
| 789 | /// `text` cut at the first char boundary at or after `n` bytes. |
| 790 | fn prefix(text: &str, n: usize) -> &str { |
| 791 | let mut end = n.min(text.len()); |
| 792 | while !text.is_char_boundary(end) { |
| 793 | end += 1; |
| 794 | } |
| 795 | &text[..end] |
| 796 | } |
| 797 | |
| 798 | /// The first answer `ask` finds, reading `text` as code in prefixes of growing size; the whole, last, is |
| 799 | /// asked with `full` set. |
| 800 | fn in_prefixes<T>(text: &str, ask: impl Fn(&Lexer, bool) -> Option<T>) -> Option<T> { |
| 801 | let mut n = 512usize; |
| 802 | loop { |
| 803 | let head = prefix(text, n); |
| 804 | let full = head.len() >= text.len(); |
| 805 | if let Some(r) = ask(&Lexer::code(head), full) { |
| 806 | return Some(r); |
| 807 | } |
| 808 | if full { |
| 809 | return None; |
| 810 | } |
| 811 | n = n.saturating_mul(4); |
| 812 | } |
| 813 | } |
| 814 | |
| 815 | impl Lexer { |
| 816 | /// The byte just past the group the text opens with, a `(`, `[` or `{`; `None` when it never closes. |
| 817 | fn group_close(&self) -> Option<usize> { |
| 818 | let first = self.leaves.first()?; |
| 819 | if first.tok != Tok::Open || self.nodes[first.node].start != 0 { |
| 820 | return None; |
| 821 | } |
| 822 | let group = self.nodes[first.node].parent; |
| 823 | (1..self.leaves.len()).find(|&li| self.closes_group(li) && self.nodes[self.leaves[li].node].parent == group) |
| 824 | .map(|li| self.nodes[self.leaves[li].node].end) |
| 825 | } |
| 826 | |
| 827 | /// Is the leaf `li` the `)`, `]` or `}` that closes a group? |
| 828 | fn closes_group(&self, li: usize) -> bool { |
| 829 | self.leaves[li].tok == Tok::Close |
| 830 | && matches!(self.nodes[self.leaves[li].node].kind, SyntaxKind::RightParen | SyntaxKind::RightBrace | SyntaxKind::RightBracket) |
| 831 | } |
| 832 | |
| 833 | /// The byte of the first leaf that is a comma or a closer standing at the text's own level: in no group, |
| 834 | /// equation, string, comment or raw text. |
| 835 | fn comma(&self) -> Option<usize> { |
| 836 | (0..self.leaves.len()).find(|&li| { |
| 837 | let l = self.leaves[li]; |
| 838 | l.groups == 0 && l.eqs == 0 && l.tok == Tok::Code && matches!(self.leaf_text(li), "," | ")" | "]" | "}") |
| 839 | }).map(|li| self.nodes[self.leaves[li].node].start) |
| 840 | } |
| 841 | } |
| 842 | |
| 843 | /// The byte just past the group opening at byte `at` of `src` -- a `(`, `[` or `{` read as code opens one |
| 844 | /// -- or `None` when it never closes. |
| 845 | pub(crate) fn group_end(src: &str, at: usize) -> Option<usize> { |
| 846 | if !matches!(src[at..].chars().next(), Some('(') | Some('[') | Some('{')) { |
| 847 | return None; |
| 848 | } |
| 849 | in_prefixes(&src[at..], |lx, _| lx.group_close()).map(|end| at + end) |
| 850 | } |
| 851 | |
| 852 | /// As [`group_end`] over a slice of chars, at the char `at` and in chars: the index just past the closer. |
| 853 | pub(crate) fn group_end_chars(chars: &[char], at: usize) -> Option<usize> { |
| 854 | if !matches!(chars.get(at), Some('(') | Some('[') | Some('{')) { |
| 855 | return None; |
| 856 | } |
| 857 | let mut n = 128usize; |
| 858 | loop { |
| 859 | let upto = (at + n).min(chars.len()); |
| 860 | let head: String = chars[at..upto].iter().collect(); |
| 861 | if let Some(end) = Lexer::code(&head).group_close() { |
| 862 | return Some(at + head[..end].chars().count()); |
| 863 | } |
| 864 | if upto >= chars.len() { |
| 865 | return None; |
| 866 | } |
| 867 | n = n.saturating_mul(4); |
| 868 | } |
| 869 | } |
| 870 | |
| 871 | /// Where a field's value in an argument list or a dictionary ends, for `src` read as code from the value's |
| 872 | /// start: the byte of the first comma at its own level, or of the closer of the group it stands in, or the |
| 873 | /// length of `src` when there is neither. |
| 874 | pub(crate) fn top_comma(src: &str) -> usize { |
| 875 | in_prefixes(src, |lx, full| lx.comma().or_else(|| full.then_some(src.len()))).unwrap_or(src.len()) |
| 876 | } |
| 877 | |
| 878 | /// The byte of the first colon at the own level of `src` read as code, or `None` when there is none. |
| 879 | pub(crate) fn top_colon(src: &str) -> Option<usize> { |
| 880 | let lx = Lexer::code(src); |
| 881 | (0..lx.leaves.len()).find(|&li| { |
| 882 | let l = lx.leaves[li]; |
| 883 | l.groups == 0 && l.eqs == 0 && l.tok == Tok::Code && lx.leaf_text(li) == ":" |
| 884 | }).map(|li| lx.nodes[lx.leaves[li].node].start) |
| 885 | } |
| 886 | |
| 887 | /// Each group a `(` opens at the top level of `src` read as code, by the byte of its `(` and the byte just |
| 888 | /// past its `)`. A group that never closes is not listed. |
| 889 | pub(crate) fn top_parens(src: &str) -> Vec<(usize, usize)> { |
| 890 | let lx = Lexer::code(src); |
| 891 | let mut out = Vec::new(); |
| 892 | let mut open: Option<usize> = None; |
| 893 | let mut group: Option<usize> = None; // the group node the open `(` belongs to |
| 894 | for li in 0..lx.leaves.len() { |
| 895 | let l = lx.leaves[li]; |
| 896 | let n = lx.nodes[l.node]; |
| 897 | if l.groups == 0 && l.eqs == 0 && l.tok == Tok::Open && lx.leaf_text(li) == "(" { |
| 898 | open = Some(n.start); |
| 899 | group = Some(n.parent); |
| 900 | } |
| 901 | if lx.closes_group(li) && Some(n.parent) == group { |
| 902 | if let Some(o) = open.take() { |
| 903 | out.push((o, n.end)); |
| 904 | } |
| 905 | group = None; |
| 906 | } |
| 907 | } |
| 908 | out |
| 909 | } |
| 910 | |
| 911 | /// One argument of a call's list, or one entry of an array or a dictionary, as Typst's parser reads it. |
| 912 | #[derive(Clone, Debug, PartialEq, Eq)] |
| 913 | pub(crate) struct Arg { |
| 914 | pub(crate) key: Option<String>, // a named argument's name; `None` when positional |
| 915 | pub(crate) value: String, // the value's text, its comments dropped, trimmed |
| 916 | } |
| 917 | |
| 918 | /// What a leaf of an argument list's text leaves of what it read. |
| 919 | #[derive(Clone, Copy, Debug, PartialEq, Eq)] |
| 920 | pub(crate) enum Kept { |
| 921 | Text, // the characters as written |
| 922 | Space, // a comment in code or maths: one space, so the tokens either side of it stay apart |
| 923 | Nothing, // a comment in markup, where Typst sets nothing for it |
| 924 | } |
| 925 | |
| 926 | impl Lexer { |
| 927 | /// What the leaf `li` leaves of an argument list: a comment is trivia, so a reader that keeps the text of |
| 928 | /// an argument keeps none of a comment's. |
| 929 | fn kept(&self, li: usize) -> Kept { |
| 930 | match self.leaves[li].tok { |
| 931 | Tok::Comment if self.nodes[self.leaves[li].node].mode == Mode::Markup => Kept::Nothing, |
| 932 | Tok::Comment => Kept::Space, |
| 933 | _ => Kept::Text, |
| 934 | } |
| 935 | } |
| 936 | |
| 937 | /// Is the leaf `li` a comma at the list's own level? |
| 938 | fn parts(&self, li: usize) -> bool { |
| 939 | let l = self.leaves[li]; |
| 940 | l.groups == 0 && l.eqs == 0 && l.tok == Tok::Code && self.leaf_text(li) == "," |
| 941 | } |
| 942 | } |
| 943 | |
| 944 | /// How much of an argument's head has been read, to tell a name before its `:` from a value. |
| 945 | #[derive(Clone, Debug, PartialEq, Eq)] |
| 946 | enum Head { |
| 947 | Start, // nothing yet but spaces and comments |
| 948 | Name { name: String, spaced: bool }, // a name, and whether a space has followed it |
| 949 | Value, // anything else: a positional value, or the value after a `:` |
| 950 | } |
| 951 | |
| 952 | /// The arguments of the list `inner`, the text inside a call's parentheses (or an array's or a |
| 953 | /// dictionary's), read as code the way Typst's parser reads them. An argument ends at a comma only at the |
| 954 | /// list's own level. A comment, at any depth, is trivia: it adds one space to the value in code and nothing |
| 955 | /// in markup, where Typst sets nothing for it, so a field commented out is no field. Strings, raw text and |
| 956 | /// groups are copied as written and close where the lexer closes them. An argument is named only when it |
| 957 | /// opens with a name followed, past any spaces or comments, by a `:` at the list's own level; a spread, a |
| 958 | /// content block and every other value are positional. An empty argument, as a trailing comma leaves, is |
| 959 | /// not listed. |
| 960 | pub(crate) fn args(inner: &str) -> Vec<Arg> { |
| 961 | let lx = Lexer::code(inner); |
| 962 | let mut out: Vec<Arg> = Vec::new(); |
| 963 | let mut head = Head::Start; |
| 964 | let mut key = None; |
| 965 | let mut value = String::new(); |
| 966 | let flush = |key: &mut Option<String>, value: &mut String, out: &mut Vec<Arg>| { |
| 967 | let v = value.trim(); |
| 968 | if key.is_some() || !v.is_empty() { |
| 969 | out.push(Arg { key: key.take(), value: v.to_string() }); |
| 970 | } |
| 971 | }; |
| 972 | for li in 0..lx.leaves.len() { |
| 973 | if lx.parts(li) { |
| 974 | flush(&mut key, &mut value, &mut out); |
| 975 | head = Head::Start; |
| 976 | key = None; |
| 977 | value.clear(); |
| 978 | continue; |
| 979 | } |
| 980 | match lx.kept(li) { |
| 981 | Kept::Space => { value.push(' '); continue; }, |
| 982 | Kept::Nothing => continue, |
| 983 | Kept::Text => {}, |
| 984 | } |
| 985 | let l = lx.leaves[li]; |
| 986 | let piece = lx.leaf_text(li); |
| 987 | let top = l.groups == 0 && l.eqs == 0; |
| 988 | let blank = piece.chars().all(|c| c.is_whitespace()); |
| 989 | if top && l.tok == Tok::Code { |
| 990 | let word = piece.chars().next().is_some_and(is_id_start) && piece.chars().all(is_id_continue); |
| 991 | let next = match std::mem::replace(&mut head, Head::Value) { |
| 992 | Head::Start if blank => Head::Start, |
| 993 | Head::Start if word => Head::Name { name: piece.to_string(), spaced: false }, |
| 994 | Head::Name { name, .. } if blank => Head::Name { name, spaced: true }, |
| 995 | Head::Name { mut name, spaced: false } if piece.chars().all(is_id_continue) => { |
| 996 | name.push_str(piece); |
| 997 | Head::Name { name, spaced: false } |
| 998 | }, |
| 999 | Head::Name { name, .. } if piece == ":" => { |
| 1000 | // The name is the key, and the value starts after its colon. |
| 1001 | key = Some(name); |
| 1002 | value.clear(); |
| 1003 | continue; |
| 1004 | }, |
| 1005 | Head::Start | Head::Name { .. } | Head::Value => Head::Value, |
| 1006 | }; |
| 1007 | head = next; |
| 1008 | } else if !blank { |
| 1009 | head = Head::Value; |
| 1010 | } |
| 1011 | value.push_str(piece); |
| 1012 | } |
| 1013 | flush(&mut key, &mut value, &mut out); |
| 1014 | out |
| 1015 | } |
| 1016 | |
| 1017 | /// The arguments of the list `inner` as the text each keeps, comments as [`args`] reads them, split at the |
| 1018 | /// commas of the list's own level. A last argument of nothing but space is not listed. |
| 1019 | pub(crate) fn split_args(inner: &str) -> Vec<String> { |
| 1020 | let lx = Lexer::code(inner); |
| 1021 | let mut out: Vec<String> = Vec::new(); |
| 1022 | let mut cur = String::new(); |
| 1023 | for li in 0..lx.leaves.len() { |
| 1024 | if lx.parts(li) { |
| 1025 | out.push(std::mem::take(&mut cur)); |
| 1026 | continue; |
| 1027 | } |
| 1028 | match lx.kept(li) { |
| 1029 | Kept::Text => cur.push_str(lx.leaf_text(li)), |
| 1030 | Kept::Space => cur.push(' '), |
| 1031 | Kept::Nothing => {}, |
| 1032 | } |
| 1033 | } |
| 1034 | if !cur.trim().is_empty() { |
| 1035 | out.push(cur); |
| 1036 | } |
| 1037 | out |
| 1038 | } |
| 1039 | |
| 1040 | /// The value of the argument named `key` in `list`, the last one when it is named twice. |
| 1041 | pub(crate) fn named<'a>(list: &'a [Arg], key: &str) -> Option<&'a str> { |
| 1042 | list.iter().rev().find(|a| a.key.as_deref() == Some(key)).map(|a| a.value.as_str()) |
| 1043 | } |
| 1044 | |
| 1045 | /// A name `list` gives twice, which Typst refuses as a duplicate argument. |
| 1046 | pub(crate) fn duplicate_key(list: &[Arg]) -> Option<&str> { |
| 1047 | for (k, a) in list.iter().enumerate() { |
| 1048 | if let Some(name) = a.key.as_deref() { |
| 1049 | if list[..k].iter().any(|b| b.key.as_deref() == Some(name)) { |
| 1050 | return Some(name); |
| 1051 | } |
| 1052 | } |
| 1053 | } |
| 1054 | None |
| 1055 | } |
| 1056 | |
| 1057 | /// A conditional or a loop embedded in markup -- an `#if` with its `else` arms, a `#for` or a `#while` -- |
| 1058 | /// with its extent and each arm's condition and body. |
| 1059 | #[derive(Clone, Debug, PartialEq, Eq)] |
| 1060 | pub(crate) struct Flow { |
| 1061 | pub(crate) kw: Kw, // `If`, `For` or `While` |
| 1062 | pub(crate) start: usize, // the byte of its `#` |
| 1063 | pub(crate) end: usize, // the byte just past its last body, or the end of what it took |
| 1064 | pub(crate) arms: Vec<Arm>, |
| 1065 | pub(crate) whole: bool, // every arm closed, and no call or field follows the last |
| 1066 | pub(crate) coded: bool, // it stands in code, a code block or a function's body, whose names this scan cannot see |
| 1067 | pub(crate) math: bool, // it stands in an equation |
| 1068 | } |
| 1069 | |
| 1070 | /// Which of a text's conditionals and loops [`flows_in`] lists. |
| 1071 | #[derive(Clone, Copy, Debug, PartialEq, Eq)] |
| 1072 | pub(crate) enum Level { |
| 1073 | Own, // those standing in the text's own markup |
| 1074 | Deep, // those too in any content block or equation inside it: a cell's, a caption's, a note's, a call's argument |
| 1075 | } |
| 1076 | |
| 1077 | /// One arm of a [`Flow`]. |
| 1078 | #[derive(Clone, Copy, Debug, PartialEq, Eq)] |
| 1079 | pub(crate) struct Arm { |
| 1080 | pub(crate) cond: Option<(usize, usize)>, // the condition's bytes, or a loop's head's; `None` for `else` |
| 1081 | pub(crate) body: (usize, usize), // the body's bytes, its delimiters included |
| 1082 | pub(crate) content: bool, // a `[...]` content block, not a `{...}` code block |
| 1083 | } |
| 1084 | |
| 1085 | /// Every conditional and loop standing in `src`'s own markup, in order. One in another's body, or in any |
| 1086 | /// other group, is that group's, and is not listed. |
| 1087 | pub(crate) fn flows(src: &str) -> Vec<Flow> { |
| 1088 | flows_in(src, Level::Own) |
| 1089 | } |
| 1090 | |
| 1091 | /// The conditionals and loops of `src` that `level` names, in order, each the outermost of its kind: one in |
| 1092 | /// another's head or body is that statement's. At [`Level::Deep`] a flow in a content block or an equation is |
| 1093 | /// listed wherever it stands, a call's argument list included, with its `coded` set where it stands in code |
| 1094 | /// whose own bindings (a code block's `let`, a function's parameters) this scan does not read. A flow inside |
| 1095 | /// a `#let`, `#set` or `#show` statement is that statement's, read when it is, and not listed. |
| 1096 | pub(crate) fn flows_in(src: &str, level: Level) -> Vec<Flow> { |
| 1097 | Lexer::markup(src).flows(level) |
| 1098 | } |
| 1099 | |
| 1100 | /// A `#let` or `#import` embedded in markup, and the stretch of the text its names are bound over. |
| 1101 | #[derive(Clone, Debug, PartialEq, Eq)] |
| 1102 | pub(crate) struct Binding { |
| 1103 | pub(crate) start: usize, // the byte of its `#` |
| 1104 | pub(crate) text: String, // the statement from its `#`, its comments dropped |
| 1105 | pub(crate) at: usize, // the byte just past it, where its names come into force |
| 1106 | pub(crate) until: Option<usize>, // the byte of the closer of the block it stands in; `None` at the text's own level |
| 1107 | } |
| 1108 | |
| 1109 | /// Every `#let` and `#import` embedded in `src`'s markup, at any depth, in source order. Typst binds a |
| 1110 | /// name in the innermost content block `[...]` or code block `{...}` the statement stands in, a |
| 1111 | /// conditional's branch included, from the statement's end to the block's closer; a heading, a list item, |
| 1112 | /// strong or emphasis opens no scope. A code block's own `let`, written with no `#`, is code rather than |
| 1113 | /// markup and is not listed. |
| 1114 | pub(crate) fn bindings(src: &str) -> Vec<Binding> { |
| 1115 | Lexer::markup(src).bindings() |
| 1116 | } |
| 1117 | |
| 1118 | impl Lexer { |
| 1119 | fn kind(&self, a: usize) -> SyntaxKind { |
| 1120 | self.nodes[a].kind |
| 1121 | } |
| 1122 | |
| 1123 | /// The expression an `#` at the leaf node `h` embeds, if one follows it directly. |
| 1124 | fn embedded(&self, h: usize) -> Option<usize> { |
| 1125 | let n = &self.nodes[h]; |
| 1126 | (n.kind == SyntaxKind::Hash && n.next != NONE && self.nodes[n.next].start == n.end).then_some(n.next) |
| 1127 | } |
| 1128 | |
| 1129 | /// The nodes beneath `a`, in order. |
| 1130 | fn kids(&self, a: usize) -> Vec<usize> { |
| 1131 | let mut out = Vec::new(); |
| 1132 | let mut c = self.nodes[a].child; |
| 1133 | while c != NONE { |
| 1134 | out.push(c); |
| 1135 | c = self.nodes[c].next; |
| 1136 | } |
| 1137 | out |
| 1138 | } |
| 1139 | |
| 1140 | /// The conditionals and loops `level` names. |
| 1141 | fn flows(&self, level: Level) -> Vec<Flow> { |
| 1142 | let mut out = Vec::new(); |
| 1143 | self.find_flows(0, level, true, false, false, &mut out); |
| 1144 | out |
| 1145 | } |
| 1146 | |
| 1147 | /// Walks the children of `a`: `own` while only markup and its scopes lie between the root and `a`, `stmt` inside |
| 1148 | /// an embedded statement, `coded` inside a code block or a closure. |
| 1149 | fn find_flows(&self, a: usize, level: Level, own: bool, stmt: bool, coded: bool, out: &mut Vec<Flow>) { |
| 1150 | let mode = self.nodes[a].mode; |
| 1151 | let mut skip = NONE; // the expression a flow took, which holds no other |
| 1152 | for c in self.kids(a) { |
| 1153 | if c == skip { |
| 1154 | continue; |
| 1155 | } |
| 1156 | let k = self.kind(c); |
| 1157 | if let Some(e) = self.embedded(c) { |
| 1158 | let listed = match level { |
| 1159 | Level::Own => own, |
| 1160 | Level::Deep => !stmt && matches!(mode, Mode::Markup | Mode::Math), |
| 1161 | }; |
| 1162 | if listed { |
| 1163 | if let Some(flow) = self.flow(c, e, coded, mode == Mode::Math) { |
| 1164 | out.push(flow); |
| 1165 | skip = e; |
| 1166 | continue; |
| 1167 | } |
| 1168 | } |
| 1169 | } |
| 1170 | if self.nodes[c].child == NONE { |
| 1171 | continue; |
| 1172 | } |
| 1173 | // An embedded statement is the statement's, whatever stands inside it. |
| 1174 | let embedded_stmt = is_stmt(k) && self.nodes[c].prev != NONE && self.kind(self.nodes[c].prev) == SyntaxKind::Hash; |
| 1175 | let keeps = k == SyntaxKind::Markup || is_scope(k); |
| 1176 | self.find_flows( |
| 1177 | c, level, own && keeps, stmt || embedded_stmt, |
| 1178 | coded || matches!(k, SyntaxKind::CodeBlock | SyntaxKind::Closure), out, |
| 1179 | ); |
| 1180 | } |
| 1181 | } |
| 1182 | |
| 1183 | /// The flow the `#` at `h` opens over the expression `e`, if `e` is a conditional or a loop, or a call or a |
| 1184 | /// field taken from one. |
| 1185 | fn flow(&self, h: usize, e: usize, coded: bool, math: bool) -> Option<Flow> { |
| 1186 | let mut x = e; |
| 1187 | let mut whole = true; |
| 1188 | while matches!(self.kind(x), SyntaxKind::FuncCall | SyntaxKind::FieldAccess) { |
| 1189 | x = self.nodes[x].child; |
| 1190 | whole = false; |
| 1191 | if x == NONE { |
| 1192 | return None; |
| 1193 | } |
| 1194 | } |
| 1195 | let kw = match self.kind(x) { |
| 1196 | SyntaxKind::Conditional => Kw::If, |
| 1197 | SyntaxKind::WhileLoop => Kw::While, |
| 1198 | SyntaxKind::ForLoop => Kw::For, |
| 1199 | _ => return None, |
| 1200 | }; |
| 1201 | let mut arms = Vec::new(); |
| 1202 | let complete = self.arms(x, &mut arms); |
| 1203 | let whole = whole && complete && !arms.is_empty(); |
| 1204 | let last = arms.last().map_or(0, |a| a.body.1); |
| 1205 | Some(Flow { |
| 1206 | kw, |
| 1207 | start: self.nodes[h].start, |
| 1208 | end: if whole { last } else { self.nodes[e].end.max(last) }, |
| 1209 | arms, |
| 1210 | whole, |
| 1211 | coded, |
| 1212 | math, |
| 1213 | }) |
| 1214 | } |
| 1215 | |
| 1216 | /// Pushes the arms of the conditional or loop `x`: false when one is missing its body or leaves it open. |
| 1217 | fn arms(&self, x: usize, out: &mut Vec<Arm>) -> bool { |
| 1218 | let kids = self.kids(x); |
| 1219 | let Some(&kw) = kids.first() else { return false; }; |
| 1220 | let from = self.nodes[kw].end; |
| 1221 | let block = |a: usize| matches!(self.kind(a), SyntaxKind::ContentBlock | SyntaxKind::CodeBlock); |
| 1222 | let at_else = kids.iter().position(|&a| self.kind(a) == SyntaxKind::Else); |
| 1223 | let head = &kids[1..at_else.unwrap_or(kids.len())]; |
| 1224 | let Some(&body) = head.iter().rev().find(|&&a| block(a)) else { return false; }; |
| 1225 | if !self.closed(body) { |
| 1226 | return false; |
| 1227 | } |
| 1228 | out.push(Arm { |
| 1229 | cond: Some((from, self.nodes[body].start)), |
| 1230 | body: (self.nodes[body].start, self.nodes[body].end), |
| 1231 | content: self.kind(body) == SyntaxKind::ContentBlock, |
| 1232 | }); |
| 1233 | let Some(e) = at_else else { return true; }; |
| 1234 | let tail = kids[e + 1..].iter().copied().find(|&a| !matches!(self.kind(a), SyntaxKind::Space | SyntaxKind::LineComment | SyntaxKind::BlockComment)); |
| 1235 | match tail { |
| 1236 | Some(a) if self.kind(a) == SyntaxKind::Conditional => self.arms(a, out), |
| 1237 | Some(a) if block(a) && self.closed(a) => { |
| 1238 | out.push(Arm { |
| 1239 | cond: None, |
| 1240 | body: (self.nodes[a].start, self.nodes[a].end), |
| 1241 | content: self.kind(a) == SyntaxKind::ContentBlock, |
| 1242 | }); |
| 1243 | true |
| 1244 | }, |
| 1245 | _ => false, |
| 1246 | } |
| 1247 | } |
| 1248 | |
| 1249 | /// Does the block `a` end with its closer? |
| 1250 | fn closed(&self, a: usize) -> bool { |
| 1251 | let mut c = self.nodes[a].child; |
| 1252 | let mut last = NONE; |
| 1253 | while c != NONE { |
| 1254 | last = c; |
| 1255 | c = self.nodes[c].next; |
| 1256 | } |
| 1257 | last != NONE && matches!(self.kind(last), SyntaxKind::RightBracket | SyntaxKind::RightBrace) |
| 1258 | } |
| 1259 | |
| 1260 | /// The statements `bindings` lists. |
| 1261 | fn bindings(&self) -> Vec<Binding> { |
| 1262 | let mut out = Vec::new(); |
| 1263 | self.find_bindings(0, &mut out); |
| 1264 | out |
| 1265 | } |
| 1266 | |
| 1267 | fn find_bindings(&self, a: usize, out: &mut Vec<Binding>) { |
| 1268 | let mut skip = NONE; // the statement a binding took, which holds no other |
| 1269 | for c in self.kids(a) { |
| 1270 | if c == skip { |
| 1271 | continue; |
| 1272 | } |
| 1273 | if let Some(e) = self.embedded(c) { |
| 1274 | if matches!(self.kind(e), SyntaxKind::LetBinding | SyntaxKind::ModuleImport) { |
| 1275 | out.push(self.binding(c, e)); |
| 1276 | skip = e; |
| 1277 | continue; |
| 1278 | } |
| 1279 | } |
| 1280 | if self.nodes[c].child != NONE { |
| 1281 | self.find_bindings(c, out); |
| 1282 | } |
| 1283 | } |
| 1284 | } |
| 1285 | |
| 1286 | /// The binding the `#` at `h` makes of the statement `e`. |
| 1287 | fn binding(&self, h: usize, e: usize) -> Binding { |
| 1288 | let mut text = String::from("#"); |
| 1289 | self.push_text(e, &mut text); |
| 1290 | // A `;` ends the statement, and is part of what it holds over, though not of its text. |
| 1291 | let mut at = self.nodes[e].end; |
| 1292 | let mut n = self.nodes[e].next; |
| 1293 | while n != NONE && matches!(self.kind(n), SyntaxKind::Space | SyntaxKind::LineComment | SyntaxKind::BlockComment) { |
| 1294 | n = self.nodes[n].next; |
| 1295 | } |
| 1296 | if n != NONE && self.kind(n) == SyntaxKind::Semicolon { |
| 1297 | at = self.nodes[n].end; |
| 1298 | } |
| 1299 | // The innermost block it stands in holds it to its closer. |
| 1300 | let mut until = None; |
| 1301 | let mut p = self.nodes[h].parent; |
| 1302 | while p != NONE { |
| 1303 | if matches!(self.kind(p), SyntaxKind::ContentBlock | SyntaxKind::CodeBlock) { |
| 1304 | let close = self.kids(p).last().copied().filter(|&k| matches!(self.kind(k), SyntaxKind::RightBracket | SyntaxKind::RightBrace)); |
| 1305 | until = Some(close.map_or(self.text.len(), |k| self.nodes[k].start)); |
| 1306 | break; |
| 1307 | } |
| 1308 | p = self.nodes[p].parent; |
| 1309 | } |
| 1310 | Binding { start: self.nodes[h].start, text, at, until } |
| 1311 | } |
| 1312 | |
| 1313 | /// Appends the text beneath `a`, its comments dropped. |
| 1314 | fn push_text(&self, a: usize, out: &mut String) { |
| 1315 | let n = &self.nodes[a]; |
| 1316 | if n.leaf != NONE { |
| 1317 | if self.leaves[n.leaf].tok != Tok::Comment { |
| 1318 | out.push_str(&self.text[n.start..n.end]); |
| 1319 | } |
| 1320 | return; |
| 1321 | } |
| 1322 | for c in self.kids(a) { |
| 1323 | self.push_text(c, out); |
| 1324 | } |
| 1325 | } |
| 1326 | } |
| 1327 | |
| 1328 | #[cfg(test)] |
| 1329 | mod tests { |
| 1330 | use super::*; |
| 1331 | use oxedyne_fe2o3_core::prelude::*; |
| 1332 | |
| 1333 | /// The top-level lines of `src`, by their trimmed text. |
| 1334 | fn tops(src: &str) -> Vec<String> { |
| 1335 | top_level_lines(src).into_iter().map(|(_, l)| l.trim().to_string()).collect() |
| 1336 | } |
| 1337 | |
| 1338 | #[test] |
| 1339 | fn a_bracket_in_prose_is_text_and_hides_no_line() { |
| 1340 | let src = "The interval [0, 1) is half-open.\n#include \"ch1.typ\"\n= Intervals [0, 1)\n#set document(title: \"T\")\n"; |
| 1341 | assert_eq!(tops(src), vec![ |
| 1342 | "The interval [0, 1) is half-open.", "#include \"ch1.typ\"", "= Intervals [0, 1)", |
| 1343 | "#set document(title: \"T\")", |
| 1344 | ]); |
| 1345 | } |
| 1346 | |
| 1347 | #[test] |
| 1348 | fn a_bracket_code_opens_is_content() { |
| 1349 | let src = "#box[\n#include \"a.typ\"\n]\n#link(\"x\")[\nB\n]\n#[\nC\n]\n@fig[\nD\n]\nE\n"; |
| 1350 | assert_eq!(tops(src), vec!["#box[", "#link(\"x\")[", "#[", "@fig[", "E"]); |
| 1351 | } |
| 1352 | |
| 1353 | #[test] |
| 1354 | fn a_scope_forgets_its_own_open_brackets() { |
| 1355 | // A heading's, an item's and a strong span's unbalanced `[` does not keep the block open. |
| 1356 | let src = "#box[\n= Intervals [0, 1)\n- item [a\n*[b*\n]\nAfter\n"; |
| 1357 | assert_eq!(tops(src), vec!["#box[", "After"]); |
| 1358 | // In the block's own markup, a prose `[` does balance the next `]`. |
| 1359 | let src = "#box[\nprose [a\n]\nStill in\n]\nAfter\n"; |
| 1360 | assert_eq!(tops(src), vec!["#box[", "After"]); |
| 1361 | } |
| 1362 | |
| 1363 | #[test] |
| 1364 | fn a_quote_in_markup_is_a_character() { |
| 1365 | let src = "He said \"use /* here\" today.\n#set document(title: \"After Quote\")\nMore */ tail words.\n"; |
| 1366 | assert_eq!(tops(src), vec!["He said \"use /* here\" today."]); |
| 1367 | let live = live_text(src); |
| 1368 | assert!(!live.contains("#set"), "the comment the quote does not hide holds the declaration"); |
| 1369 | } |
| 1370 | |
| 1371 | #[test] |
| 1372 | fn a_link_is_one_token() { |
| 1373 | let src = "See https://example.com/x/*y here.\n#include \"ch1.typ\"\nAnd http://a.io//b too.\n"; |
| 1374 | assert_eq!(live_text(src), src); |
| 1375 | assert_eq!(tops(src).len(), 3); |
| 1376 | // Only `http://` and `https://` make a link; another scheme's `//` opens a comment. |
| 1377 | assert_eq!(live_text("ftp://x.io y\n"), "ftp: \n"); |
| 1378 | } |
| 1379 | |
| 1380 | #[test] |
| 1381 | fn a_link_drops_the_punctuation_that_ends_its_sentence() { |
| 1382 | let src = "https://x.io/a(b). /* c */"; |
| 1383 | let link = "https://x.io/a(b)".len(); |
| 1384 | let toks = byte_toks(src); |
| 1385 | assert!(toks[..link].iter().all(|&t| t == Tok::Link)); |
| 1386 | assert_eq!(toks[link], Tok::Text); |
| 1387 | let src = "https://x.io. /* c */ d\n"; |
| 1388 | assert_eq!(live_text(src), "https://x.io. d\n"); |
| 1389 | } |
| 1390 | |
| 1391 | #[test] |
| 1392 | fn block_comments_nest() { |
| 1393 | let src = "/* outer /* inner */\n#set document(title: \"Commented Out\")\n*/\nAfter\n"; |
| 1394 | assert_eq!(tops(src), vec!["/* outer /* inner */", "After"]); |
| 1395 | assert_eq!(live_text("/* a /* b */ (c */ d"), fmt_spaces(19, "d")); |
| 1396 | } |
| 1397 | |
| 1398 | fn fmt_spaces(n: usize, tail: &str) -> String { |
| 1399 | let mut s = " ".repeat(n); |
| 1400 | s.push_str(tail); |
| 1401 | s |
| 1402 | } |
| 1403 | |
| 1404 | #[test] |
| 1405 | fn a_stray_closer_opens_no_comment() { |
| 1406 | // Typst reads `*/` outside a comment as one error token, so the `/*` inside `*/*` is not an opener. |
| 1407 | assert_eq!(live_text("a */* b\nc\n"), "a */* b\nc\n"); |
| 1408 | } |
| 1409 | |
| 1410 | #[test] |
| 1411 | fn raw_text_runs_across_lines_until_a_run_as_long() { |
| 1412 | let src = "`one\n#set text(size: 9pt)\n` two\n````\n```\n#set x()\n```\n````\nEnd\n"; |
| 1413 | assert_eq!(tops(src), vec!["`one", "````", "End"]); |
| 1414 | // Two backticks are empty raw text, and open nothing. |
| 1415 | assert_eq!(tops("``\n#include \"a.typ\"\n"), vec!["``", "#include \"a.typ\""]); |
| 1416 | } |
| 1417 | |
| 1418 | #[test] |
| 1419 | fn an_escape_takes_the_character_after_it() { |
| 1420 | // The `*` after the escaped slash opens a strong the `x*` closes, so the line after it stands at the top. |
| 1421 | assert_eq!(tops("\\[ \\/* x*\n#include \"a.typ\"\n"), vec!["\\[ \\/* x*", "#include \"a.typ\""]); |
| 1422 | assert_eq!(tops("#box[a \\] b\n]\nc\n"), vec!["#box[a \\] b", "c"]); |
| 1423 | } |
| 1424 | |
| 1425 | #[test] |
| 1426 | fn a_string_holds_comment_markers_in_code_and_maths() { |
| 1427 | let src = "#let x = \"a // b /* c\"\n$ \"d /* e\" $\nAfter\n"; |
| 1428 | assert_eq!(live_text(src), src); |
| 1429 | assert_eq!(tops(src).len(), 3); |
| 1430 | } |
| 1431 | |
| 1432 | #[test] |
| 1433 | fn lone_dollars_leaves_earlier_pairs() { |
| 1434 | // Typst 0.15.1 refuses each of these files with "unclosed delimiter" at exactly the `$` named. |
| 1435 | assert_eq!(lone_dollars("a $x$ b $ c"), [8]); |
| 1436 | // A `]` is maths inside an equation, so the second `$` closes the first; the group never closes. |
| 1437 | assert_eq!(lone_dollars("#emph[a $b] c $d$"), [16]); |
| 1438 | // An equation opened in a group inside another: neither closes, the inner found first. |
| 1439 | assert_eq!(lone_dollars("$ a #[ b $ c"), [0, 9]); |
| 1440 | assert!(lone_dollars("a $x$ b $ y $\n").is_empty()); |
| 1441 | let toks: Vec<Tok> = tokens("a $x$ b $ c").into_iter().map(|(_, _, t)| t).collect(); |
| 1442 | assert_eq!(toks[2..5], [Tok::Open, Tok::Math, Tok::Close]); |
| 1443 | assert_eq!(toks[8..], [Tok::Text; 3]); |
| 1444 | } |
| 1445 | |
| 1446 | #[test] |
| 1447 | fn a_dollar_that_never_closes_hides_no_line() { |
| 1448 | let src = "Alpha $x + y.\n\n= Heading Later\n#include \"a.typ\"\n#if a [B]\n"; |
| 1449 | assert_eq!(tops(src), vec!["Alpha $x + y.", "", "= Heading Later", "#include \"a.typ\"", "#if a [B]"]); |
| 1450 | assert_eq!(flows_of(src).len(), 1); |
| 1451 | // A backtick after it opens raw text, as after any character of prose, so the `/*` in it opens nothing. |
| 1452 | assert_eq!(live_text("A $ `b /* c` d\n"), "A $".to_string() + &fmt_spaces(10, "d\n")); |
| 1453 | } |
| 1454 | |
| 1455 | #[test] |
| 1456 | fn a_statement_ends_at_its_line_and_an_expression_after_its_operand() { |
| 1457 | // The `[` after a closed call and a space is prose, and so is the `"` after it. |
| 1458 | let src = "#link(\"x\")[a] then [b and \"c\n#include \"e.typ\"\n"; |
| 1459 | assert_eq!(tops(src), vec!["#link(\"x\")[a] then [b and \"c", "#include \"e.typ\""]); |
| 1460 | // A statement's group runs across lines; the statement ends at the line after it closes. |
| 1461 | let src = "#let x = (\n a: 1,\n)\n#include \"e.typ\"\n"; |
| 1462 | assert_eq!(tops(src), vec!["#let x = (", "#include \"e.typ\""]); |
| 1463 | } |
| 1464 | |
| 1465 | #[test] |
| 1466 | fn a_shebang_is_a_comment_to_its_line_end() { |
| 1467 | let src = "#!/usr/bin/env typst /*\n= Root\n#include \"ch1.typ\"\n"; |
| 1468 | assert_eq!(tops(src), vec!["#!/usr/bin/env typst /*", "= Root", "#include \"ch1.typ\""]); |
| 1469 | let first = src.find('\n').unwrap_or(src.len()); |
| 1470 | assert!(tokens(src).iter().filter(|&&(at, _, _)| at < first).all(|&(_, _, t)| t == Tok::Comment)); |
| 1471 | assert_eq!(live_text(src), fmt_spaces(first, &src[first..])); |
| 1472 | } |
| 1473 | |
| 1474 | #[test] |
| 1475 | fn a_shebang_only_opens_the_file() { |
| 1476 | // Past the file's first character, `#!` is an embedded expression Typst refuses, and the `/*` after it |
| 1477 | // opens a comment. |
| 1478 | let src = "x\n#!y /*\n= Root\n"; |
| 1479 | assert_eq!(tops(src), vec!["x", "#!y /*"]); |
| 1480 | assert_eq!(tokens(src)[2].2, Tok::Hash); |
| 1481 | } |
| 1482 | |
| 1483 | #[test] |
| 1484 | fn a_line_in_open_strong_or_emphasis_is_not_top_level() { |
| 1485 | for m in ["*", "_"] { |
| 1486 | let src = fmt!("{m}bold [\n#set document(title: \"T\")\nstill bold{m}\nAfter\n"); |
| 1487 | assert_eq!(tops(&src), vec![fmt!("{m}bold ["), "After".to_string()], "{}", m); |
| 1488 | } |
| 1489 | } |
| 1490 | |
| 1491 | #[test] |
| 1492 | fn an_unclosed_star_holds_to_the_paragraph_break() { |
| 1493 | let src = "2 * 3 = 6\n#set document(title: \"S\")\n\n#set text(size: 9pt)\n"; |
| 1494 | assert_eq!(tops(src), vec!["2 * 3 = 6", "#set text(size: 9pt)"]); |
| 1495 | } |
| 1496 | |
| 1497 | #[test] |
| 1498 | fn an_underscore_between_cjk_letters_opens_emphasis() { |
| 1499 | assert_eq!(tops("漢_字\n#set document(title: \"C\")\n"), vec!["漢_字"]); |
| 1500 | // Between Latin letters it is within a word, and opens nothing. |
| 1501 | assert_eq!(tops("a_b\n#set document(title: \"C\")\n"), vec!["a_b", "#set document(title: \"C\")"]); |
| 1502 | } |
| 1503 | |
| 1504 | #[test] |
| 1505 | fn an_indented_line_stays_in_its_item() { |
| 1506 | let src = "- item\n #set heading(numbering: \"1.\")\n= Next\n"; |
| 1507 | assert_eq!(tops(src), vec!["- item", "= Next"]); |
| 1508 | } |
| 1509 | |
| 1510 | #[test] |
| 1511 | fn a_line_that_does_not_indent_ends_the_item() { |
| 1512 | let src = "- item\n#set document(title: \"T\")\n"; |
| 1513 | assert_eq!(tops(src), vec!["- item", "#set document(title: \"T\")"]); |
| 1514 | } |
| 1515 | |
| 1516 | #[test] |
| 1517 | fn a_comment_ahead_of_the_first_token_does_not_hide_where_it_stands() { |
| 1518 | // The `#` stands at column 4, which does not indent past an item at column 6: the comment ahead of it |
| 1519 | // is trivia, and the token is what ends the item. |
| 1520 | let src = " - item\n/**/#set document(title: \"T\")\n"; |
| 1521 | assert_eq!(tops(src), vec!["- item", "/**/#set document(title: \"T\")"]); |
| 1522 | // A comment alone on a line holds no token, so it ends no item and stands in it. |
| 1523 | assert_eq!(tops("- item\n// c\n #set text(size: 9pt)\n"), vec!["- item"]); |
| 1524 | assert_eq!(tops("- item\n/* c\n d */ #set text(size: 9pt)\n"), vec!["- item"]); |
| 1525 | } |
| 1526 | |
| 1527 | /// Each placed line of `src` by its trimmed text. |
| 1528 | fn places(src: &str) -> Vec<(String, Place)> { |
| 1529 | placed_lines(src).into_iter().map(|(_, l, p)| (l.trim().to_string(), p)).collect() |
| 1530 | } |
| 1531 | |
| 1532 | #[test] |
| 1533 | fn placed_lines_tells_a_bare_block_from_a_call() { |
| 1534 | let src = "#[\n#set document(title: \"B\")\n]\n#box[\n#set document(title: \"X\")\n]\n"; |
| 1535 | let got = places(src); |
| 1536 | assert_eq!(got[1], ("#set document(title: \"B\")".to_string(), Place::Block)); |
| 1537 | assert_eq!(got[4], ("#set document(title: \"X\")".to_string(), Place::Content)); |
| 1538 | // A bare block in a bare block is joined too; one opened in a list item is held by the item. |
| 1539 | assert_eq!(places("#[\n#[\nX\n]\n]\n")[2], ("X".to_string(), Place::Block)); |
| 1540 | assert_eq!(places("- a #[\nX\n]\n")[1], ("X".to_string(), Place::Contained)); |
| 1541 | // Content a conditional, `context` or a reference opens is not joined as written. |
| 1542 | for src in ["#if a [\nX\n]\n", "#context [\nX\n]\n", "#context[\nX\n]\n", "@r[\nX\n]\n", "#f(a, [\nX\n])\n"] { |
| 1543 | assert_eq!(places(src)[1], ("X".to_string(), Place::Content), "{:?}", src); |
| 1544 | } |
| 1545 | } |
| 1546 | |
| 1547 | #[test] |
| 1548 | fn a_conditional_takes_else_only_on_its_own_line() { |
| 1549 | let src = "#if a [\nX\n] else [\nY\n]\nAfter\n"; |
| 1550 | assert_eq!(tops(src), vec!["#if a [", "After"]); |
| 1551 | // On the next line, `else [` is prose, and its `[` a prose bracket. |
| 1552 | let src = "#if a [X]\nelse [Y\nAfter\n"; |
| 1553 | assert_eq!(tops(src), vec!["#if a [X]", "else [Y", "After"]); |
| 1554 | } |
| 1555 | |
| 1556 | /// The byte ranges of `src`'s flows, as `(kw, whole, text, arms)` with each arm as its condition and body |
| 1557 | /// text. |
| 1558 | fn flows_of(src: &str) -> Vec<(Kw, bool, &str, Vec<(Option<&str>, &str, bool)>)> { |
| 1559 | flows(src).into_iter().map(|f| { |
| 1560 | let arms = f.arms.iter() |
| 1561 | .map(|a| (a.cond.map(|(x, y)| src[x..y].trim()), &src[a.body.0..a.body.1], a.content)) |
| 1562 | .collect(); |
| 1563 | (f.kw, f.whole, &src[f.start..f.end], arms) |
| 1564 | }).collect() |
| 1565 | } |
| 1566 | |
| 1567 | #[test] |
| 1568 | fn a_conditional_is_read_whole_with_its_arms() { |
| 1569 | let src = "Text #if a == \"x\" [A] else [B] more.\n"; |
| 1570 | assert_eq!(flows_of(src), vec![(Kw::If, true, "#if a == \"x\" [A] else [B]", |
| 1571 | vec![(Some("a == \"x\""), "[A]", true), (None, "[B]", true)])]); |
| 1572 | let src = "#if a [\nA\n] else if b [\nB\n] else {\n[C]\n}\nAfter\n"; |
| 1573 | assert_eq!(flows_of(src), vec![(Kw::If, true, "#if a [\nA\n] else if b [\nB\n] else {\n[C]\n}", |
| 1574 | vec![(Some("a"), "[\nA\n]", true), (Some("b"), "[\nB\n]", true), (None, "{\n[C]\n}", false)])]); |
| 1575 | // On the next line, `else` is prose, and the conditional ends with its first body. |
| 1576 | let src = "#if a [A]\nelse [B]\n"; |
| 1577 | assert_eq!(flows_of(src), vec![(Kw::If, true, "#if a [A]", vec![(Some("a"), "[A]", true)])]); |
| 1578 | } |
| 1579 | |
| 1580 | #[test] |
| 1581 | fn a_loop_is_read_whole() { |
| 1582 | let src = "#for x in xs [\nX #x\n]\n#while n < 3 { n += 1 }\n"; |
| 1583 | assert_eq!(flows_of(src), vec![ |
| 1584 | (Kw::For, true, "#for x in xs [\nX #x\n]", vec![(Some("x in xs"), "[\nX #x\n]", true)]), |
| 1585 | (Kw::While, true, "#while n < 3 { n += 1 }", vec![(Some("n < 3"), "{ n += 1 }", false)]), |
| 1586 | ]); |
| 1587 | } |
| 1588 | |
| 1589 | #[test] |
| 1590 | fn only_a_flow_at_the_scans_own_level_is_listed() { |
| 1591 | // One in another's body, in a content block, in maths or after `context` is not the scan's own. |
| 1592 | let src = "#if a [\n#if b [x]\n]\n#box[#if c [y]]\n$#if d [z]$\n#context if e [w]\n"; |
| 1593 | assert_eq!(flows_of(src).len(), 1); |
| 1594 | assert_eq!(flows_of(src)[0].2, "#if a [\n#if b [x]\n]"); |
| 1595 | // Two flows back to back are two. |
| 1596 | assert_eq!(flows_of("#if a [x]#if b [y]\n").len(), 2); |
| 1597 | } |
| 1598 | |
| 1599 | #[test] |
| 1600 | fn a_flow_that_does_not_close_as_one_statement_is_not_whole() { |
| 1601 | // A call on its value, a head with no body, and a body that never closes. |
| 1602 | assert_eq!(flows_of("#if a [x](y) tail\n")[0].2, "#if a [x](y)"); |
| 1603 | assert!(!flows_of("#if a [x](y) tail\n")[0].1); |
| 1604 | assert!(!flows_of("#if a\n[x]\n")[0].1); |
| 1605 | assert_eq!(flows_of("#if a [x\ny\n")[0].2, "#if a [x\ny\n"); |
| 1606 | assert!(!flows_of("#if a [x\ny\n")[0].1); |
| 1607 | } |
| 1608 | |
| 1609 | /// The statements `bindings` finds, as `(text, the source before it comes into force, the closer's source |
| 1610 | /// from where it stops)`. |
| 1611 | fn binds_of(src: &str) -> Vec<(String, String, Option<String>)> { |
| 1612 | bindings(src).into_iter() |
| 1613 | .map(|b| (b.text.clone(), src[..b.at].to_string(), b.until.map(|u| src[u..].to_string()))) |
| 1614 | .collect() |
| 1615 | } |
| 1616 | |
| 1617 | #[test] |
| 1618 | fn a_binding_is_listed_from_its_end_to_its_blocks_closer() { |
| 1619 | // At the text's own level it holds to the end; a `;` ends it there, and is part of what it holds over. |
| 1620 | let src = "#let a = \"x\"\nText\n#let b = true; more\n"; |
| 1621 | assert_eq!(binds_of(src), vec![ |
| 1622 | ("#let a = \"x\"".to_string(), "#let a = \"x\"".to_string(), None), |
| 1623 | ("#let b = true".to_string(), "#let a = \"x\"\nText\n#let b = true;".to_string(), None), |
| 1624 | ]); |
| 1625 | // In a callout's body, or a conditional's branch, it stops at the closer; text after the block is |
| 1626 | // outside it, and a heading, strong or emphasis opens no scope of its own. |
| 1627 | let src = "#box[\n#let a = 1\nIN\n]\n#if x [\n#let b = 2\n]\n= H #let c = 3;\nEND\n"; |
| 1628 | let got = binds_of(src); |
| 1629 | assert_eq!(got.len(), 3); |
| 1630 | assert_eq!(got[0], ("#let a = 1".to_string(), "#box[\n#let a = 1".to_string(), Some("]\n#if x [\n#let b = 2\n]\n= H #let c = 3;\nEND\n".to_string()))); |
| 1631 | assert_eq!(got[1].0, "#let b = 2"); |
| 1632 | assert_eq!(got[1].2.as_deref(), Some("]\n= H #let c = 3;\nEND\n")); |
| 1633 | assert_eq!(got[2].0, "#let c = 3"); |
| 1634 | assert_eq!(got[2].2, None); |
| 1635 | } |
| 1636 | |
| 1637 | #[test] |
| 1638 | fn a_binding_a_comment_a_raw_or_a_code_block_holds_is_not_listed_as_markup() { |
| 1639 | // Comments inside a statement are dropped from its text; one standing alone, a raw span, and a `let` |
| 1640 | // of a code block (no `#`) are not statements of the markup. |
| 1641 | let src = "#let a = /* c */ \"x\" // t\n// #let b = 1\n`#let c = 1`\n#{ let d = 1 }\n#include \"f.typ\"\n#import \"g.typ\": *\n"; |
| 1642 | let got: Vec<String> = binds_of(src).into_iter().map(|(t, _, _)| t.split_whitespace().collect::<Vec<_>>().join(" ")).collect(); |
| 1643 | assert_eq!(got, ["#let a = \"x\"", "#import \"g.typ\": *"]); |
| 1644 | } |
| 1645 | |
| 1646 | #[test] |
| 1647 | fn a_value_ends_at_its_comma_or_its_groups_closer() { |
| 1648 | assert_eq!(top_comma(" \"a, b\", c"), " \"a, b\"".len()); |
| 1649 | assert_eq!(top_comma(" [x, y) z], c"), " [x, y) z]".len()); |
| 1650 | assert_eq!(top_comma(" none)\n#let x = \"b\", c"), " none".len()); |
| 1651 | assert_eq!(top_comma(" /* , */ 1"), " /* , */ 1".len()); |
| 1652 | } |
| 1653 | |
| 1654 | /// Each argument of `inner` as its key and value. |
| 1655 | fn pairs(inner: &str) -> Vec<(Option<String>, String)> { |
| 1656 | args(inner).into_iter().map(|a| (a.key, a.value)).collect() |
| 1657 | } |
| 1658 | |
| 1659 | fn keys(inner: &str) -> Vec<String> { |
| 1660 | args(inner).into_iter().filter_map(|a| a.key).collect() |
| 1661 | } |
| 1662 | |
| 1663 | #[test] |
| 1664 | fn an_argument_list_reads_comments_as_trivia() { |
| 1665 | assert_eq!(pairs("title: \"T\" /* c */"), vec![(Some("title".to_string()), "\"T\"".to_string())]); |
| 1666 | // In code a comment parts the tokens beside it; in a content block it leaves nothing, as Typst sets |
| 1667 | // nothing for it. |
| 1668 | assert_eq!(pairs("a: 1/* c */+ 2, [x/* c */y // d\n]"), vec![ |
| 1669 | (Some("a".to_string()), "1 + 2".to_string()), (None, "[xy \n]".to_string()), |
| 1670 | ]); |
| 1671 | // A comment between a name and its colon leaves the name a key. |
| 1672 | assert_eq!(keys("title /* c */ : \"T\""), vec!["title"]); |
| 1673 | } |
| 1674 | |
| 1675 | #[test] |
| 1676 | fn a_commented_out_named_argument_is_no_argument() { |
| 1677 | let list = args("\n // author: \"Old\",\n title: \"T\",\n"); |
| 1678 | assert_eq!(list.len(), 1); |
| 1679 | assert_eq!(list[0].key.as_deref(), Some("title")); |
| 1680 | assert_eq!(named(&list, "author"), None); |
| 1681 | } |
| 1682 | |
| 1683 | #[test] |
| 1684 | fn a_comment_between_arguments_parts_nothing() { |
| 1685 | assert_eq!(keys("a: 1, /* b: 2, */ c: 3"), vec!["a", "c"]); |
| 1686 | assert_eq!(keys("title: \"T\" // , author: \"X\"\n"), vec!["title"]); |
| 1687 | assert_eq!(pairs("(\"a\", /* \"b\", */ \"c\")")[0].1, "(\"a\", \"c\")"); |
| 1688 | assert_eq!(pairs("\"a\", /* \"b\", */ \"c\""), vec![(None, "\"a\"".to_string()), (None, "\"c\"".to_string())]); |
| 1689 | } |
| 1690 | |
| 1691 | #[test] |
| 1692 | fn a_key_inside_a_string_or_group_is_no_key() { |
| 1693 | let list = args("title: \"T, author: Q\" /* , author: \"X\" */, keywords: (\"k\",)"); |
| 1694 | assert_eq!(keys("title: \"T, author: Q\" /* , author: \"X\" */, keywords: (\"k\",)"), vec!["title", "keywords"]); |
| 1695 | assert_eq!(named(&list, "title"), Some("\"T, author: Q\"")); |
| 1696 | assert_eq!(named(&list, "keywords"), Some("(\"k\",)")); |
| 1697 | assert_eq!(keys("header: [page: 1]"), vec!["header"]); |
| 1698 | assert_eq!(named(&args("header: [page: 1]"), "page"), None); |
| 1699 | // A string keeps its comment markers, and raw text its commas. |
| 1700 | assert_eq!(pairs("author: \"A /* not */ B\""), vec![(Some("author".to_string()), "\"A /* not */ B\"".to_string())]); |
| 1701 | assert_eq!(keys("title: [T `raw, author: \"X\"` z]"), vec!["title"]); |
| 1702 | // A spread, a positional value and a name without its colon are positional. |
| 1703 | assert_eq!(pairs("..xs, 2pt, body"), vec![ |
| 1704 | (None, "..xs".to_string()), (None, "2pt".to_string()), (None, "body".to_string()), |
| 1705 | ]); |
| 1706 | assert_eq!(keys("a b: 1, (c: 1)"), Vec::<String>::new()); |
| 1707 | } |
| 1708 | |
| 1709 | /// The text a reader of `src` as an argument list keeps, as [`split_args`] does. |
| 1710 | fn kept(src: &str) -> String { |
| 1711 | let lx = Lexer::code(src); |
| 1712 | let mut out = String::new(); |
| 1713 | for li in 0..lx.leaves.len() { |
| 1714 | match lx.kept(li) { |
| 1715 | Kept::Text => out.push_str(lx.leaf_text(li)), |
| 1716 | Kept::Space => out.push(' '), |
| 1717 | Kept::Nothing => {}, |
| 1718 | } |
| 1719 | } |
| 1720 | out |
| 1721 | } |
| 1722 | |
| 1723 | #[test] |
| 1724 | fn a_step_over_an_argument_list_keeps_no_comment_text() { |
| 1725 | // One space for a comment in code, however it nests, and none in a content block, as in Typst. |
| 1726 | assert_eq!(kept("a /* x, y */, b"), "a , b"); |
| 1727 | assert_eq!(kept("a/* b /* c */ d */e"), "a e"); |
| 1728 | assert_eq!(kept("a // c, d\n, b"), "a \n, b"); |
| 1729 | assert_eq!(kept("[p // q\n r /* s */]"), "[p \n r ]"); |
| 1730 | // A string and raw text keep their comment markers. |
| 1731 | assert_eq!(kept("\"/* x */\", `// y`"), "\"/* x */\", `// y`"); |
| 1732 | } |
| 1733 | |
| 1734 | #[test] |
| 1735 | fn a_name_given_twice_is_a_duplicate() { |
| 1736 | let list = args("title: \"A\", title: \"B\""); |
| 1737 | assert_eq!(duplicate_key(&list), Some("title")); |
| 1738 | assert_eq!(named(&list, "title"), Some("\"B\"")); |
| 1739 | assert_eq!(duplicate_key(&args("title: \"A\" /* , title: \"B\" */")), None); |
| 1740 | // A name is read whole, so a longer one holding it is another name. |
| 1741 | let list = args("heading-font: \"A\", first-line-indent: 1em"); |
| 1742 | assert_eq!(named(&list, "font"), None); |
| 1743 | assert_eq!(named(&list, "indent"), None); |
| 1744 | assert_eq!(named(&list, "first-line-indent"), Some("1em")); |
| 1745 | } |
| 1746 | |
| 1747 | #[test] |
| 1748 | fn a_top_level_paren_group_is_read_as_code() { |
| 1749 | let src = "// (a\n(b, \")\"), /* ( */ [(], (c (d)), (e"; |
| 1750 | let groups: Vec<&str> = top_parens(src).into_iter().map(|(a, b)| &src[a..b]).collect(); |
| 1751 | assert_eq!(groups, ["(b, \")\")", "(c (d))"]); |
| 1752 | } |
| 1753 | |
| 1754 | #[test] |
| 1755 | fn a_group_end_is_read_as_code() { |
| 1756 | let src = "(a, \"b)\", [c (d], /* ) */ e) tail"; |
| 1757 | assert_eq!(group_end(src, 0).map(|e| &src[e..]), Some(" tail")); |
| 1758 | assert_eq!(group_end("(a", 0), None); |
| 1759 | } |
| 1760 | |
| 1761 | /// The texts of the flows `level` lists in `src`, with whether each stands in code and in an equation. |
| 1762 | fn deep(src: &str, level: Level) -> Vec<(String, bool, bool)> { |
| 1763 | flows_in(src, level).into_iter().map(|f| (src[f.start..f.end].to_string(), f.coded, f.math)).collect() |
| 1764 | } |
| 1765 | |
| 1766 | #[test] |
| 1767 | fn a_flow_in_a_content_block_is_listed_at_the_deep_level_only() { |
| 1768 | let src = "#table(columns: 2, [c1], [#if a [X] else [Y]])\nNote#footnote[F #if b [Z]].\n#if c [W]\n"; |
| 1769 | assert_eq!(deep(src, Level::Own), vec![("#if c [W]".to_string(), false, false)]); |
| 1770 | assert_eq!(deep(src, Level::Deep), vec![ |
| 1771 | ("#if a [X] else [Y]".to_string(), false, false), |
| 1772 | ("#if b [Z]".to_string(), false, false), |
| 1773 | ("#if c [W]".to_string(), false, false), |
| 1774 | ]); |
| 1775 | } |
| 1776 | |
| 1777 | #[test] |
| 1778 | fn a_flow_in_an_equation_is_listed_in_one_flagged_math() { |
| 1779 | let src = "P $x #if a [MX] else [MY]$ q $y$ #if b [Z]\n"; |
| 1780 | assert_eq!(deep(src, Level::Own), vec![("#if b [Z]".to_string(), false, false)]); |
| 1781 | assert_eq!(deep(src, Level::Deep), vec![ |
| 1782 | ("#if a [MX] else [MY]".to_string(), false, true), |
| 1783 | ("#if b [Z]".to_string(), false, false), |
| 1784 | ]); |
| 1785 | } |
| 1786 | |
| 1787 | #[test] |
| 1788 | fn a_flow_in_code_is_flagged_and_one_in_a_statement_is_not_listed() { |
| 1789 | let src = "#{ let a = 1; [#if a [X]] }\n#f(r => [#if r [Y]])\n#let blk = [#if a [Z]]\n#set text(fill: if a [W])\n"; |
| 1790 | assert_eq!(deep(src, Level::Deep), vec![ |
| 1791 | ("#if a [X]".to_string(), true, false), |
| 1792 | ("#if r [Y]".to_string(), true, false), |
| 1793 | ]); |
| 1794 | } |
| 1795 | |
| 1796 | #[test] |
| 1797 | fn a_flow_inside_a_flow_is_that_flows_and_a_loop_is_listed_whole() { |
| 1798 | let src = "#table([#for x in (1, 2) [#if x [A]]])\n"; |
| 1799 | let got = flows_in(src, Level::Deep); |
| 1800 | assert_eq!(got.len(), 1); |
| 1801 | assert_eq!(got[0].kw, Kw::For); |
| 1802 | assert_eq!(&src[got[0].start..got[0].end], "#for x in (1, 2) [#if x [A]]"); |
| 1803 | } |
| 1804 | |
| 1805 | #[test] |
| 1806 | fn a_let_in_a_text_is_listed_with_the_byte_of_its_hash() { |
| 1807 | let src = "A [#let a = 1\nB]"; |
| 1808 | let b = bindings(src); |
| 1809 | assert_eq!(b.len(), 1); |
| 1810 | assert_eq!(&src[b[0].start..b[0].at], "#let a = 1"); |
| 1811 | } |
| 1812 | |
| 1813 | // What the syntax tree settles that the state machine it replaces read otherwise. Each source below was |
| 1814 | // compiled by typst 0.15.1 as a probe, and the diagnostics it gave are named where they decide the reading. |
| 1815 | |
| 1816 | #[test] |
| 1817 | fn a_space_after_a_statement_is_markup_beside_it() { |
| 1818 | // The spaces before a trailing comment are the markup's, outside the statement; typst compiles this |
| 1819 | // without a diagnostic. |
| 1820 | let src = "#let a = 230 // note\n#let b = 1; // c\n"; |
| 1821 | let toks = byte_toks(src); |
| 1822 | let first = src.find(" //").unwrap_or(0); |
| 1823 | assert!(toks[first..first + 3].iter().all(|&t| t == Tok::Text)); |
| 1824 | assert_eq!(toks[first + 3], Tok::Comment); |
| 1825 | let second = src.find("; //").map_or(0, |k| k + 1); |
| 1826 | assert_eq!(toks[second - 1], Tok::Code); |
| 1827 | assert_eq!(toks[second], Tok::Text); |
| 1828 | // The statement's own text ends before them, and what it binds holds from its end. |
| 1829 | let b = bindings(src); |
| 1830 | assert_eq!(b[0].text, "#let a = 230"); |
| 1831 | assert_eq!(&src[..b[0].at], "#let a = 230"); |
| 1832 | assert_eq!(&src[..b[1].at], "#let a = 230 // note\n#let b = 1;"); |
| 1833 | } |
| 1834 | |
| 1835 | #[test] |
| 1836 | fn a_statement_ends_where_typst_ends_it() { |
| 1837 | // typst: `expected semicolon or line break` after the `1` and nothing about `*b*`, so what follows is |
| 1838 | // markup, a strong span included. |
| 1839 | let src = "#let x = 1 a *b*\n"; |
| 1840 | let toks = byte_toks(src); |
| 1841 | let a = src.find(" a").unwrap_or(0); |
| 1842 | assert!(toks[a..a + 2].iter().all(|&t| t == Tok::Text)); |
| 1843 | assert_eq!(bindings(src)[0].text, "#let x = 1"); |
| 1844 | // typst: `expected colon` after `heading`, then the same. |
| 1845 | let src = "#show heading it => it\n"; |
| 1846 | let toks = byte_toks(src); |
| 1847 | let it = src.find(" it").unwrap_or(0); |
| 1848 | assert!(toks[it..src.len() - 1].iter().all(|&t| t == Tok::Text)); |
| 1849 | } |
| 1850 | |
| 1851 | #[test] |
| 1852 | fn an_unclosed_paren_ends_at_a_terminator() { |
| 1853 | // typst: `unclosed delimiter` at the `(`, `unexpected closing bracket` at the `]`, and the duplicate |
| 1854 | // argument of the call after it, so the `]` and what follows are markup. |
| 1855 | let src = "#let x = (1, 2\n\n*unclosed strong\n\ntext ] more\n\n#f(a: 1, a: 2)\n"; |
| 1856 | let toks = byte_toks(src); |
| 1857 | let bracket = src.find(" ] more").unwrap_or(0); |
| 1858 | assert!(toks[bracket..bracket + 7].iter().all(|&t| t == Tok::Text)); |
| 1859 | let call = src.find("#f(").unwrap_or(0); |
| 1860 | assert_eq!((toks[call], toks[call + 1], toks[call + 2]), (Tok::Hash, Tok::Code, Tok::Open)); |
| 1861 | // The group never closed, and so is still counted open. |
| 1862 | assert_eq!(Lexer::markup(src).depth_at(call), 1); |
| 1863 | } |
| 1864 | |
| 1865 | #[test] |
| 1866 | fn an_unclosed_paren_ends_with_the_code_block_round_it() { |
| 1867 | // typst: `unclosed delimiter` at the `(` alone, so the `}` closes the block. |
| 1868 | let src = "#{ import \"x\": (a, b }\n"; |
| 1869 | assert_eq!(group_end(src, 1), Some(src.len() - 1)); |
| 1870 | assert_eq!(group_end(src, src.find('(').unwrap_or(0)), None); |
| 1871 | assert_eq!(byte_toks(src)[src.len() - 1], Tok::Text); |
| 1872 | // The block ended what was open inside it, so the line after it is not in a group. |
| 1873 | let src = "#{ import \"x\": (a, b }\nnext\n"; |
| 1874 | let lx = Lexer::markup(src); |
| 1875 | let next = src.find("next").unwrap_or(0); |
| 1876 | assert_eq!((lx.depth_at(next), lx.open_at(next)), (0, 0)); |
| 1877 | } |
| 1878 | |
| 1879 | #[test] |
| 1880 | fn an_error_token_after_a_hash_is_code() { |
| 1881 | // typst: `invalid number suffix`, `expected a hexadecimal number` and `unexpected minus`, each reported |
| 1882 | // at the `#` and each part of the embedded expression. |
| 1883 | for (src, upto) in [("#12p\n", 4), ("#0x\n", 3), ("#-1\n", 2)] { |
| 1884 | assert!(byte_toks(src)[1..upto].iter().all(|&t| t == Tok::Code), "{:?}", src); |
| 1885 | } |
| 1886 | assert_eq!(byte_toks("#-1\n")[2], Tok::Text); |
| 1887 | } |
| 1888 | |
| 1889 | #[test] |
| 1890 | fn what_the_parser_left_over_in_code_is_read_again() { |
| 1891 | // typst: `expected named or keyed pair`, and nothing about what follows. The expression it refuses |
| 1892 | // is one error node holding the text of its tokens, whose string stays a string. |
| 1893 | let src = "#let d = (a: 1, \"x\" + \"y // z\")\n#set x()\n"; |
| 1894 | let toks = byte_toks(src); |
| 1895 | let slashes = src.find("//").unwrap_or(0); |
| 1896 | assert_eq!(toks[slashes], Tok::Str); |
| 1897 | assert_eq!(tops(src).len(), 2); |
| 1898 | } |
| 1899 | |
| 1900 | #[test] |
| 1901 | fn a_line_with_no_token_stands_where_the_last_token_left_it() { |
| 1902 | // The blank line is inside the strong span, which a paragraph break has not yet ended; the line after it |
| 1903 | // is not. |
| 1904 | let src = "2 * 3\n#set x()\n\n#set y()\n"; |
| 1905 | let lx = Lexer::markup(src); |
| 1906 | let blank = src.find("\n\n").map_or(0, |k| k + 1); |
| 1907 | assert_eq!(lx.place_at(blank), Some(Place::Contained)); |
| 1908 | assert_eq!(lx.place_at(blank + 1), Some(Place::Top)); |
| 1909 | // A heading ended with its line, and an item has not ended until a token says so. |
| 1910 | let src = "= H\n\ntext\n"; |
| 1911 | assert_eq!(Lexer::markup(src).place_at(4), Some(Place::Top)); |
| 1912 | assert!(Lexer::markup(src).bare_line_at(4)); |
| 1913 | let src = "- item\n\n more\n"; |
| 1914 | assert_eq!(Lexer::markup(src).place_at(7), Some(Place::Contained)); |
| 1915 | } |
| 1916 | |
| 1917 | #[test] |
| 1918 | fn a_block_closer_and_a_blank_line_in_an_empty_block_are_bare() { |
| 1919 | let src = "#box[\n\n]\n"; |
| 1920 | let lx = Lexer::markup(src); |
| 1921 | assert!(lx.bare_line_at(6), "a blank line in a block that holds nothing yet"); |
| 1922 | assert!(lx.bare_line_at(7), "the closer stands in the block's markup"); |
| 1923 | assert_eq!(lx.place_at(7), Some(Place::Content)); |
| 1924 | // With a list item open in the block, the closer at its own margin ends it. |
| 1925 | let src = "#box[\n- a\n]\n"; |
| 1926 | assert!(Lexer::markup(src).bare_line_at(10)); |
| 1927 | assert!(!Lexer::markup(src).bare_line_at(6) || Lexer::markup(src).place_at(6) == Some(Place::Content)); |
| 1928 | // A line in strong inside the block is not bare. |
| 1929 | let src = "#box[\n*a\nb*\n]\n"; |
| 1930 | assert!(!Lexer::markup(src).bare_line_at(8)); |
| 1931 | } |
| 1932 | |
| 1933 | #[test] |
| 1934 | fn what_is_open_at_a_line_start_counts_groups_equations_and_literals() { |
| 1935 | let src = "#let x = (\n a,\n)\nb\n"; |
| 1936 | let lx = Lexer::markup(src); |
| 1937 | assert_eq!((lx.open_at(0), lx.open_at(11), lx.open_at(src.len() - 2)), (0, 1, 0)); |
| 1938 | assert_eq!(lx.depth_at(11), 1); |
| 1939 | assert_eq!(lx.markup_level_at(11), None, "inside a code group"); |
| 1940 | assert_eq!(lx.markup_level_at(src.len() - 2), Some(0)); |
| 1941 | for (src, line) in [("$ a\n b $\n", 4), ("#\"a\nb\"\n", 4), ("`a\nb`\n", 3), ("/* a\nb */\n", 5)] { |
| 1942 | let lx = Lexer::markup(src); |
| 1943 | assert_eq!(lx.open_at(line), 1, "{:?}", src); |
| 1944 | assert_eq!(lx.open_at(src.len()), 0, "{:?}", src); |
| 1945 | assert_eq!(lx.place_at(line), None, "{:?}", src); |
| 1946 | } |
| 1947 | // What is open at the end of a text that stops inside a construct. |
| 1948 | for src in ["#f(a,\n", "$ x\n", "#[a\n", "`raw\n", "/* c\n", "#\"s\n"] { |
| 1949 | assert!(open_after(src), "{:?}", src); |
| 1950 | } |
| 1951 | for src in ["#f(a)\n", "$ x $\n", "#[a]\n", "`raw`\n", "/* c */\n", "#\"s\"\n"] { |
| 1952 | assert!(!open_after(src), "{:?}", src); |
| 1953 | } |
| 1954 | } |
| 1955 | |
| 1956 | #[test] |
| 1957 | fn a_group_is_found_in_the_prefix_that_holds_it() { |
| 1958 | // A group far longer than the first prefix, nested and holding strings, comments and brackets that |
| 1959 | // close nothing, ends where the whole text says. |
| 1960 | let mut inner = String::new(); |
| 1961 | for k in 0..400 { |
| 1962 | inner.push_str(&fmt!("s{}: \"a ] b ) c\", item{}: [text ) with \\] {} /* ) */ (n{})], ", k, k, k, k)); |
| 1963 | } |
| 1964 | let src = fmt!("({}) tail ( not part", inner); |
| 1965 | let end = src.find(") tail").map(|k| k + 1); |
| 1966 | assert_eq!(group_end(&src, 0), end); |
| 1967 | let chars: Vec<char> = src.chars().collect(); |
| 1968 | assert_eq!(group_end_chars(&chars, 0), end); |
| 1969 | assert_eq!(group_end_chars(&chars, chars.len() - 9), None); |
| 1970 | assert_eq!(group_end("(a, b", 0), None); |
| 1971 | assert_eq!(top_comma(&fmt!("{} , more", &src[..end.unwrap_or(0)])), end.unwrap_or(0) + 1); |
| 1972 | // An index in chars is not an index in bytes. |
| 1973 | let src = "(é, \"ü ) €\", [ñ])x"; |
| 1974 | let chars: Vec<char> = src.chars().collect(); |
| 1975 | assert_eq!((group_end_chars(&chars, 0), group_end(src, 0)), (Some(17), Some(22))); |
| 1976 | } |
| 1977 | |
| 1978 | #[test] |
| 1979 | fn the_first_colon_at_the_own_level_names_an_argument() { |
| 1980 | assert_eq!(top_colon("key: value"), Some(3)); |
| 1981 | assert_eq!(top_colon("(a: 1) b: 2"), Some(8)); |
| 1982 | assert_eq!(top_colon("[a: 1], \"b: 2\", $c: 3$"), None); |
| 1983 | assert_eq!(split_args("a, [b, c], \"d, e\" /* , */, f"), vec!["a", " [b, c]", " \"d, e\" ", " f"]); |
| 1984 | } |
| 1985 | } |