oxedyne/fe2o3/fe2o3_austenite/src/syntax/lexer.rs
37.8 KiB, 1 run
created by r1870400018:63564, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | // SPDX-License-Identifier: Apache-2.0 |
| 2 | // Portions Copyright (c) The Typst Project Developers, from typst-syntax 0.15.1's `src/lexer.rs`; see |
| 3 | // fe2o3_austenite/licences/typst-LICENSE.txt for the full Apache-2.0 text. |
| 4 | // Modified for Austenite. |
| 5 | //! Ported from typst-syntax 0.15.1 (`src/lexer.rs`, Apache-2.0, (c) the Typst authors), token for token: |
| 6 | //! the same modes, the same kinds, the same error messages and hints, so a document lexes as Typst |
| 7 | //! lexes it. The Unicode predicates it needs (XID identifiers, the CJK scripts, the maths delimiter |
| 8 | //! classes, grapheme clusters) come from `fe2o3_text::unicode` rather than crates. |
| 9 | |
| 10 | use crate::syntax::{ |
| 11 | SyntaxKind, |
| 12 | SyntaxNode, |
| 13 | Span, |
| 14 | }; |
| 15 | |
| 16 | use oxedyne_fe2o3_text::unicode::lookup::Partitioned; |
| 17 | use oxedyne_fe2o3_text::unicode::math::{ |
| 18 | self, |
| 19 | MathClass, |
| 20 | }; |
| 21 | use oxedyne_fe2o3_text::unicode::prop::Script; |
| 22 | use oxedyne_fe2o3_text::unicode::property; |
| 23 | use oxedyne_fe2o3_text::unicode::segment; |
| 24 | |
| 25 | /// The three lexical modes Typst switches between. |
| 26 | #[derive(Clone, Copy, Debug, PartialEq, Eq)] |
| 27 | pub enum LexMode { |
| 28 | Markup, |
| 29 | Math, |
| 30 | Code, |
| 31 | } |
| 32 | |
| 33 | // ┌───────────────────────────────────────────────────────────────────────────────────────────┐ |
| 34 | // │ Scanner │ |
| 35 | // └───────────────────────────────────────────────────────────────────────────────────────────┘ |
| 36 | |
| 37 | /// A cursor over a string, after `unscanny`. Every position is a byte offset on a char boundary. |
| 38 | #[derive(Clone, Copy, Debug)] |
| 39 | pub struct Scanner<'s> { |
| 40 | text: &'s str, |
| 41 | cursor: usize, |
| 42 | } |
| 43 | |
| 44 | /// Something a [`Scanner`] can match at its cursor: a char, a string, a set of chars or a predicate. |
| 45 | pub trait Pattern { |
| 46 | /// The byte length matched at the start of `s`, if any. |
| 47 | fn matches(&mut self, s: &str) -> Option<usize>; |
| 48 | } |
| 49 | |
| 50 | impl Pattern for char { |
| 51 | fn matches(&mut self, s: &str) -> Option<usize> { |
| 52 | if s.starts_with(*self) { Some(self.len_utf8()) } else { None } |
| 53 | } |
| 54 | } |
| 55 | |
| 56 | impl Pattern for &str { |
| 57 | fn matches(&mut self, s: &str) -> Option<usize> { |
| 58 | if s.starts_with(*self) { Some(self.len()) } else { None } |
| 59 | } |
| 60 | } |
| 61 | |
| 62 | impl<const N: usize> Pattern for [char; N] { |
| 63 | fn matches(&mut self, s: &str) -> Option<usize> { |
| 64 | match s.chars().next() { |
| 65 | Some(c) if self.contains(&c) => Some(c.len_utf8()), |
| 66 | _ => None, |
| 67 | } |
| 68 | } |
| 69 | } |
| 70 | |
| 71 | impl<F: FnMut(char) -> bool> Pattern for F { |
| 72 | fn matches(&mut self, s: &str) -> Option<usize> { |
| 73 | match s.chars().next() { |
| 74 | Some(c) if self(c) => Some(c.len_utf8()), |
| 75 | _ => None, |
| 76 | } |
| 77 | } |
| 78 | } |
| 79 | |
| 80 | impl<'s> Scanner<'s> { |
| 81 | pub fn new(text: &'s str) -> Self { Self { text, cursor: 0 } } |
| 82 | |
| 83 | pub fn string(&self) -> &'s str { self.text } |
| 84 | pub fn cursor(&self) -> usize { self.cursor } |
| 85 | pub fn done(&self) -> bool { self.cursor >= self.text.len() } |
| 86 | |
| 87 | /// Moves to `index`, snapped back onto a char boundary and clamped to the text. |
| 88 | pub fn jump(&mut self, index: usize) { |
| 89 | let mut i = index.min(self.text.len()); |
| 90 | while !self.text.is_char_boundary(i) { |
| 91 | i -= 1; |
| 92 | } |
| 93 | self.cursor = i; |
| 94 | } |
| 95 | |
| 96 | pub fn advance(&mut self, by: usize) { self.jump(self.cursor + by); } |
| 97 | |
| 98 | pub fn before(&self) -> &'s str { self.get(0, self.cursor) } |
| 99 | pub fn after(&self) -> &'s str { self.get(self.cursor, self.text.len()) } |
| 100 | pub fn from(&self, start: usize) -> &'s str { self.get(start, self.cursor) } |
| 101 | |
| 102 | /// The text between two offsets, empty when they do not bound a slice. |
| 103 | pub fn get(&self, a: usize, b: usize) -> &'s str { |
| 104 | match self.text.get(a.min(b)..b) { |
| 105 | Some(s) => s, |
| 106 | None => "", |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | pub fn peek(&self) -> Option<char> { self.after().chars().next() } |
| 111 | |
| 112 | /// The char `n` places from the cursor: `0` is the next, `-1` the previous. |
| 113 | pub fn scout(&self, n: isize) -> Option<char> { |
| 114 | if n >= 0 { |
| 115 | self.after().chars().nth(n as usize) |
| 116 | } else { |
| 117 | self.before().chars().rev().nth((-n - 1) as usize) |
| 118 | } |
| 119 | } |
| 120 | |
| 121 | pub fn eat(&mut self) -> Option<char> { |
| 122 | let c = self.peek(); |
| 123 | if let Some(c) = c { |
| 124 | self.cursor += c.len_utf8(); |
| 125 | } |
| 126 | c |
| 127 | } |
| 128 | |
| 129 | /// Steps back over the previous char. |
| 130 | pub fn uneat(&mut self) { |
| 131 | if let Some(c) = self.before().chars().next_back() { |
| 132 | self.cursor -= c.len_utf8(); |
| 133 | } |
| 134 | } |
| 135 | |
| 136 | pub fn at<P: Pattern>(&self, mut pat: P) -> bool { pat.matches(self.after()).is_some() } |
| 137 | |
| 138 | pub fn eat_if<P: Pattern>(&mut self, mut pat: P) -> bool { |
| 139 | match pat.matches(self.after()) { |
| 140 | Some(n) => { self.cursor += n; true }, |
| 141 | None => false, |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | pub fn eat_while<P: Pattern>(&mut self, mut pat: P) -> &'s str { |
| 146 | let start = self.cursor; |
| 147 | while let Some(n) = pat.matches(self.after()) { |
| 148 | if n == 0 { |
| 149 | break; |
| 150 | } |
| 151 | self.cursor += n; |
| 152 | } |
| 153 | self.from(start) |
| 154 | } |
| 155 | |
| 156 | pub fn eat_until<P: Pattern>(&mut self, mut pat: P) -> &'s str { |
| 157 | let start = self.cursor; |
| 158 | while !self.done() && pat.matches(self.after()).is_none() { |
| 159 | self.eat(); |
| 160 | } |
| 161 | self.from(start) |
| 162 | } |
| 163 | |
| 164 | /// Eats one newline, a CR LF pair counting as one. |
| 165 | pub fn eat_newline(&mut self) -> bool { |
| 166 | let ate = self.eat_if(is_newline); |
| 167 | if ate && self.before().ends_with('\r') { |
| 168 | self.eat_if('\n'); |
| 169 | } |
| 170 | ate |
| 171 | } |
| 172 | } |
| 173 | |
| 174 | // ┌───────────────────────────────────────────────────────────────────────────────────────────┐ |
| 175 | // │ Lexer │ |
| 176 | // └───────────────────────────────────────────────────────────────────────────────────────────┘ |
| 177 | |
| 178 | /// Turns source text into tokens, one call to [`Lexer::next`] at a time. Each token comes back as a |
| 179 | /// detached node; the parser assigns spans once the tree is whole. |
| 180 | #[derive(Clone, Debug)] |
| 181 | pub struct Lexer<'s> { |
| 182 | s: Scanner<'s>, |
| 183 | mode: LexMode, |
| 184 | newline: bool, // the last token held a newline |
| 185 | error: Option<(String, Vec<String>)>, // the current token's error and hints |
| 186 | } |
| 187 | |
| 188 | impl<'s> Lexer<'s> { |
| 189 | pub fn new(text: &'s str, mode: LexMode) -> Self { |
| 190 | Self { s: Scanner::new(text), mode, newline: false, error: None } |
| 191 | } |
| 192 | |
| 193 | pub fn mode(&self) -> LexMode { self.mode } |
| 194 | pub fn set_mode(&mut self, mode: LexMode) { self.mode = mode; } |
| 195 | pub fn cursor(&self) -> usize { self.s.cursor() } |
| 196 | pub fn jump(&mut self, index: usize) { self.s.jump(index); } |
| 197 | |
| 198 | /// Did the last token contain a newline? |
| 199 | pub fn newline(&self) -> bool { self.newline } |
| 200 | |
| 201 | /// The number of chars between the last newline before `index` and `index`. |
| 202 | pub fn column(&self, index: usize) -> usize { |
| 203 | let mut s = self.s; |
| 204 | s.jump(index); |
| 205 | s.before().chars().rev().take_while(|c| !is_newline(*c)).count() |
| 206 | } |
| 207 | |
| 208 | fn error(&mut self, message: String) -> SyntaxKind { |
| 209 | self.error = Some((message, Vec::new())); |
| 210 | SyntaxKind::Error |
| 211 | } |
| 212 | |
| 213 | fn hint(&mut self, hint: &str) { |
| 214 | if let Some((_, hints)) = &mut self.error { |
| 215 | hints.push(hint.to_string()); |
| 216 | } |
| 217 | } |
| 218 | |
| 219 | /// The next token, as its kind and its node. `End` at the end of the text. |
| 220 | pub fn next(&mut self) -> (SyntaxKind, SyntaxNode) { |
| 221 | self.error = None; |
| 222 | let start = self.s.cursor(); |
| 223 | self.newline = false; |
| 224 | let kind = match self.s.eat() { |
| 225 | Some(c) if is_space(c, self.mode) => self.whitespace(start, c), |
| 226 | Some('#') if start == 0 && self.s.eat_if('!') => self.shebang(), |
| 227 | Some('/') if self.s.eat_if('/') => self.line_comment(), |
| 228 | Some('/') if self.s.eat_if('*') => self.block_comment(), |
| 229 | Some('*') if self.s.eat_if('/') => { |
| 230 | let k = self.error("unexpected end of block comment".to_string()); |
| 231 | self.hint("consider escaping the `*` with a backslash or opening the block comment with `/*`"); |
| 232 | k |
| 233 | }, |
| 234 | Some('`') if self.mode != LexMode::Math => return self.raw(), |
| 235 | Some(c) => match self.mode { |
| 236 | LexMode::Markup => self.markup(start, c), |
| 237 | LexMode::Math => match self.math(start, c) { |
| 238 | (kind, None) => kind, |
| 239 | (kind, Some(node)) => return (kind, node), |
| 240 | }, |
| 241 | LexMode::Code => self.code(start, c), |
| 242 | }, |
| 243 | None => SyntaxKind::End, |
| 244 | }; |
| 245 | |
| 246 | let text = self.s.from(start); |
| 247 | let node = match self.error.take() { |
| 248 | Some((message, hints)) => { |
| 249 | let mut n = SyntaxNode::error(text, &message, Span::detached()); |
| 250 | for h in &hints { |
| 251 | n.hint(h); |
| 252 | } |
| 253 | n |
| 254 | }, |
| 255 | None => SyntaxNode::leaf(kind, text, Span::detached()), |
| 256 | }; |
| 257 | (kind, node) |
| 258 | } |
| 259 | |
| 260 | fn whitespace(&mut self, start: usize, c: char) -> SyntaxKind { |
| 261 | let mode = self.mode; |
| 262 | let more = self.s.eat_while(|c| is_space(c, mode)); |
| 263 | let newlines = match c { |
| 264 | ' ' if more.is_empty() => 0, |
| 265 | _ => count_newlines(self.s.from(start)), |
| 266 | }; |
| 267 | self.newline = newlines > 0; |
| 268 | if self.mode == LexMode::Markup && newlines >= 2 { SyntaxKind::Parbreak } else { SyntaxKind::Space } |
| 269 | } |
| 270 | |
| 271 | fn shebang(&mut self) -> SyntaxKind { |
| 272 | self.s.eat_until(is_newline); |
| 273 | SyntaxKind::Shebang |
| 274 | } |
| 275 | |
| 276 | fn line_comment(&mut self) -> SyntaxKind { |
| 277 | self.s.eat_until(is_newline); |
| 278 | SyntaxKind::LineComment |
| 279 | } |
| 280 | |
| 281 | fn block_comment(&mut self) -> SyntaxKind { |
| 282 | let mut state = '_'; |
| 283 | let mut depth = 1; |
| 284 | // The first `*/` that closes the outermost `/*`. |
| 285 | while let Some(c) = self.s.eat() { |
| 286 | state = match (state, c) { |
| 287 | ('*', '/') => { |
| 288 | depth -= 1; |
| 289 | if depth == 0 { |
| 290 | break; |
| 291 | } |
| 292 | '_' |
| 293 | }, |
| 294 | ('/', '*') => { |
| 295 | depth += 1; |
| 296 | '_' |
| 297 | }, |
| 298 | _ => c, |
| 299 | } |
| 300 | } |
| 301 | SyntaxKind::BlockComment |
| 302 | } |
| 303 | } |
| 304 | |
| 305 | // Raw. |
| 306 | impl Lexer<'_> { |
| 307 | /// A whole raw element, delimiters and all, lexed in one go. |
| 308 | fn raw(&mut self) -> (SyntaxKind, SyntaxNode) { |
| 309 | let start = self.s.cursor() - 1; |
| 310 | let mut backticks = 1; |
| 311 | while self.s.eat_if('`') { |
| 312 | backticks += 1; |
| 313 | } |
| 314 | |
| 315 | // Two backticks are an empty inline raw. |
| 316 | if backticks == 2 { |
| 317 | let nodes = vec![ |
| 318 | SyntaxNode::leaf(SyntaxKind::RawDelim, "`", Span::detached()), |
| 319 | SyntaxNode::leaf(SyntaxKind::RawDelim, "`", Span::detached()), |
| 320 | ]; |
| 321 | return (SyntaxKind::Raw, SyntaxNode::inner(SyntaxKind::Raw, nodes)); |
| 322 | } |
| 323 | |
| 324 | let mut found = 0; |
| 325 | while found < backticks { |
| 326 | match self.s.eat() { |
| 327 | Some('`') => found += 1, |
| 328 | Some(_) => found = 0, |
| 329 | None => { |
| 330 | let node = SyntaxNode::error(self.s.from(start), "unclosed raw text", Span::detached()); |
| 331 | return (SyntaxKind::Error, node); |
| 332 | }, |
| 333 | } |
| 334 | } |
| 335 | let end = self.s.cursor(); |
| 336 | |
| 337 | let mut inner = Scanner::new(self.s.get(start + backticks, end - backticks)); |
| 338 | let inner_len = inner.string().len(); |
| 339 | |
| 340 | let delim = SyntaxNode::leaf(SyntaxKind::RawDelim, self.s.get(end - backticks, end), Span::detached()); |
| 341 | let mut nodes = vec![delim.clone()]; |
| 342 | |
| 343 | let mut tag = None; |
| 344 | let mut future_len = None; |
| 345 | if backticks >= 3 { |
| 346 | let (t, f) = raw_lang_tag(&mut inner); |
| 347 | tag = t; |
| 348 | future_len = f; |
| 349 | if let Some(t) = tag { |
| 350 | nodes.push(SyntaxNode::leaf(SyntaxKind::RawLang, t, Span::detached())); |
| 351 | } |
| 352 | blocky_raw(&mut inner, &mut nodes); |
| 353 | } else { |
| 354 | inline_raw(&mut inner, &mut nodes); |
| 355 | } |
| 356 | nodes.push(delim); |
| 357 | |
| 358 | let mut raw = SyntaxNode::inner(SyntaxKind::Raw, nodes); |
| 359 | add_raw_warnings(&mut raw, backticks, future_len, tag, inner_len); |
| 360 | (SyntaxKind::Raw, raw) |
| 361 | } |
| 362 | } |
| 363 | |
| 364 | /// The language tag, if any, and the length the tag will have in the next version of Typst when that |
| 365 | /// differs (all text up to whitespace or a backtick). |
| 366 | fn raw_lang_tag<'a>(s: &mut Scanner<'a>) -> (Option<&'a str>, Option<usize>) { |
| 367 | let start = s.cursor(); |
| 368 | let future = s.eat_until(|c: char| c.is_whitespace() || c == '`'); |
| 369 | if future.is_empty() { |
| 370 | return (None, None); |
| 371 | } |
| 372 | s.jump(start); |
| 373 | let tag = if s.eat_if(is_id_start) { |
| 374 | s.eat_while(is_id_continue); |
| 375 | Some(s.from(start)) |
| 376 | } else { |
| 377 | None |
| 378 | }; |
| 379 | let differs = match tag { |
| 380 | Some(t) => t.len() != future.len(), |
| 381 | None => true, |
| 382 | }; |
| 383 | (tag, if differs { Some(future.len()) } else { None }) |
| 384 | } |
| 385 | |
| 386 | // A blocky raw trims the rest of the opening line when it is blank (else one space), dedents the inner |
| 387 | // lines by their common leading whitespace (the closing line counts, the opening line does not), and |
| 388 | // drops a blank last line; a last line ending in a backtick loses one trailing space. |
| 389 | fn blocky_raw(s: &mut Scanner, nodes: &mut Vec<SyntaxNode>) { |
| 390 | let mut lines = split_newlines(s.after()); |
| 391 | |
| 392 | let mut dedent = usize::MAX; |
| 393 | for line in lines.iter().skip(1) { |
| 394 | if !line.chars().all(char::is_whitespace) { |
| 395 | dedent = dedent.min(line.chars().take_while(|c| c.is_whitespace()).count()); |
| 396 | } |
| 397 | } |
| 398 | if let Some(last) = lines.last() { |
| 399 | dedent = dedent.min(last.chars().take_while(|c| c.is_whitespace()).count()); |
| 400 | } |
| 401 | if dedent == usize::MAX { |
| 402 | dedent = 0; |
| 403 | } |
| 404 | |
| 405 | let last_blank = match lines.last() { |
| 406 | Some(last) => last.chars().all(char::is_whitespace), |
| 407 | None => false, |
| 408 | }; |
| 409 | if last_blank { |
| 410 | lines.pop(); |
| 411 | } else if let Some(last) = lines.last_mut() { |
| 412 | if last.trim_end().ends_with('`') { |
| 413 | if let Some(stripped) = last.strip_suffix(' ') { |
| 414 | *last = stripped; |
| 415 | } |
| 416 | } |
| 417 | } |
| 418 | |
| 419 | let mut prev = s.cursor(); |
| 420 | let mut push = |kind: SyntaxKind, s: &Scanner, prev: &mut usize| { |
| 421 | nodes.push(SyntaxNode::leaf(kind, s.from(*prev), Span::detached())); |
| 422 | *prev = s.cursor(); |
| 423 | }; |
| 424 | |
| 425 | let mut lines = lines.into_iter(); |
| 426 | if let Some(first) = lines.next() { |
| 427 | if first.chars().all(char::is_whitespace) { |
| 428 | // Folded into the next `RawTrimmed`. |
| 429 | s.advance(first.len()); |
| 430 | } else { |
| 431 | let line_end = s.cursor() + first.len(); |
| 432 | if s.eat_if(' ') { |
| 433 | push(SyntaxKind::RawTrimmed, s, &mut prev); |
| 434 | } |
| 435 | s.jump(line_end); |
| 436 | push(SyntaxKind::Text, s, &mut prev); |
| 437 | } |
| 438 | } |
| 439 | |
| 440 | for line in lines { |
| 441 | let offset: usize = line.chars().take(dedent).map(char::len_utf8).sum(); |
| 442 | s.eat_newline(); |
| 443 | s.advance(offset); |
| 444 | push(SyntaxKind::RawTrimmed, s, &mut prev); |
| 445 | s.advance(line.len() - offset); |
| 446 | push(SyntaxKind::Text, s, &mut prev); |
| 447 | } |
| 448 | |
| 449 | if !s.done() { |
| 450 | nodes.push(SyntaxNode::leaf(SyntaxKind::RawTrimmed, s.get(prev, s.string().len()), Span::detached())); |
| 451 | } |
| 452 | } |
| 453 | |
| 454 | // Inline raw keeps all whitespace; each newline becomes a `RawTrimmed` between `Text` lines. |
| 455 | fn inline_raw(s: &mut Scanner, nodes: &mut Vec<SyntaxNode>) { |
| 456 | let mut prev = s.cursor(); |
| 457 | while !s.done() { |
| 458 | if s.at(is_newline) { |
| 459 | nodes.push(SyntaxNode::leaf(SyntaxKind::Text, s.from(prev), Span::detached())); |
| 460 | prev = s.cursor(); |
| 461 | s.eat_newline(); |
| 462 | nodes.push(SyntaxNode::leaf(SyntaxKind::RawTrimmed, s.from(prev), Span::detached())); |
| 463 | prev = s.cursor(); |
| 464 | continue; |
| 465 | } |
| 466 | s.eat(); |
| 467 | } |
| 468 | nodes.push(SyntaxNode::leaf(SyntaxKind::Text, s.from(prev), Span::detached())); |
| 469 | } |
| 470 | |
| 471 | // Typst 0.15 warns where the next version will read the language tag differently, and on a tag with no |
| 472 | // text after it. |
| 473 | fn add_raw_warnings( |
| 474 | raw: &mut SyntaxNode, |
| 475 | backticks: usize, |
| 476 | future_len: Option<usize>, |
| 477 | tag: Option<&str>, |
| 478 | inner_len: usize, |
| 479 | ) { |
| 480 | if let Some(future_len) = future_len { |
| 481 | let (a, b) = (backticks, backticks + future_len); |
| 482 | match tag { |
| 483 | Some(tag) => { |
| 484 | raw.warn_at(a, b, "no whitespace between language tag and raw text"); |
| 485 | raw.hint(&format!("currently, Typst is treating `{}` as the language tag", tag)); |
| 486 | raw.hint("in the next version of Typst, this will change and we will treat all text \ |
| 487 | until the first whitespace as the language tag"); |
| 488 | let tag_end = backticks + tag.len(); |
| 489 | raw.hint_at(a, tag_end, &format!("if the current behavior is correct, please add a space after `{}`", tag)); |
| 490 | raw.hint_at(a, tag_end, "otherwise, add a space or newline after the initial backticks"); |
| 491 | }, |
| 492 | None => { |
| 493 | raw.warn_at(a, b, "no whitespace before raw text"); |
| 494 | raw.hint("in the next version of Typst, this text will be treated as the language tag \ |
| 495 | for this element"); |
| 496 | raw.hint("to avoid this, add a space after the initial backticks"); |
| 497 | }, |
| 498 | } |
| 499 | } else if let Some(tag) = tag { |
| 500 | if inner_len == tag.len() { |
| 501 | raw.warn("empty raw text"); |
| 502 | raw.hint(&format!("Typst is treating `{}` as the language tag", tag)); |
| 503 | raw.hint_at(backticks, backticks + tag.len(), "to treat this as text, add a space after the initial backticks"); |
| 504 | } |
| 505 | } |
| 506 | } |
| 507 | |
| 508 | // Markup. |
| 509 | impl Lexer<'_> { |
| 510 | fn markup(&mut self, start: usize, c: char) -> SyntaxKind { |
| 511 | match c { |
| 512 | '\\' => self.backslash(), |
| 513 | 'h' if self.s.eat_if("ttp://") => self.link(), |
| 514 | 'h' if self.s.eat_if("ttps://") => self.link(), |
| 515 | '<' if self.s.at(is_id_continue) => self.label(), |
| 516 | '@' if self.s.at(is_id_continue) => self.ref_marker(), |
| 517 | |
| 518 | '.' if self.s.eat_if("..") => SyntaxKind::Shorthand, |
| 519 | '-' if self.s.eat_if("--") => SyntaxKind::Shorthand, |
| 520 | '-' if self.s.eat_if('-') => SyntaxKind::Shorthand, |
| 521 | '-' if self.s.eat_if('?') => SyntaxKind::Shorthand, |
| 522 | '-' if self.s.at(char::is_numeric) => SyntaxKind::Shorthand, |
| 523 | '*' if !self.in_word() => SyntaxKind::Star, |
| 524 | '_' if !self.in_word() => SyntaxKind::Underscore, |
| 525 | |
| 526 | '#' => SyntaxKind::Hash, |
| 527 | '[' => SyntaxKind::LeftBracket, |
| 528 | ']' => SyntaxKind::RightBracket, |
| 529 | '\'' => SyntaxKind::SmartQuote, |
| 530 | '"' => SyntaxKind::SmartQuote, |
| 531 | '$' => SyntaxKind::Dollar, |
| 532 | '~' => SyntaxKind::Shorthand, |
| 533 | ':' => SyntaxKind::Colon, |
| 534 | '=' => { |
| 535 | self.s.eat_while('='); |
| 536 | if self.space_or_end() { SyntaxKind::HeadingMarker } else { self.text() } |
| 537 | }, |
| 538 | '-' if self.space_or_end() => SyntaxKind::ListMarker, |
| 539 | '+' if self.space_or_end() => SyntaxKind::EnumMarker, |
| 540 | '/' if self.space_or_end() => SyntaxKind::TermMarker, |
| 541 | '0'..='9' => self.numbering(start), |
| 542 | |
| 543 | _ => self.text(), |
| 544 | } |
| 545 | } |
| 546 | |
| 547 | fn backslash(&mut self) -> SyntaxKind { |
| 548 | if self.s.eat_if("u{") { |
| 549 | let hex = self.s.eat_while(|c: char| c.is_ascii_alphanumeric()); |
| 550 | if !self.s.eat_if('}') { |
| 551 | return self.error("unclosed Unicode escape sequence".to_string()); |
| 552 | } |
| 553 | let valid = match u32::from_str_radix(hex, 16) { |
| 554 | Ok(n) => char::from_u32(n).is_some(), |
| 555 | Err(_) => false, |
| 556 | }; |
| 557 | if !valid { |
| 558 | return self.error(format!("invalid Unicode codepoint: {}", hex)); |
| 559 | } |
| 560 | return SyntaxKind::Escape; |
| 561 | } |
| 562 | if self.s.done() || self.s.at(char::is_whitespace) { |
| 563 | SyntaxKind::Linebreak |
| 564 | } else { |
| 565 | self.s.eat(); |
| 566 | SyntaxKind::Escape |
| 567 | } |
| 568 | } |
| 569 | |
| 570 | fn link(&mut self) -> SyntaxKind { |
| 571 | let (link, balanced) = link_prefix(self.s.after()); |
| 572 | self.s.advance(link.len()); |
| 573 | if !balanced { |
| 574 | return self.error("automatic links cannot contain unbalanced brackets, use the `link` function \ |
| 575 | instead".to_string()); |
| 576 | } |
| 577 | SyntaxKind::Link |
| 578 | } |
| 579 | |
| 580 | fn numbering(&mut self, start: usize) -> SyntaxKind { |
| 581 | self.s.eat_while(|c: char| c.is_ascii_digit()); |
| 582 | let read = self.s.from(start); |
| 583 | if self.s.eat_if('.') && self.space_or_end() && read.parse::<u64>().is_ok() { |
| 584 | return SyntaxKind::EnumMarker; |
| 585 | } |
| 586 | self.text() |
| 587 | } |
| 588 | |
| 589 | fn ref_marker(&mut self) -> SyntaxKind { |
| 590 | self.s.eat_while(is_valid_in_label_literal); |
| 591 | // Trailing dots and colons are likely prose. |
| 592 | while matches!(self.s.scout(-1), Some('.' | ':')) { |
| 593 | self.s.uneat(); |
| 594 | } |
| 595 | SyntaxKind::RefMarker |
| 596 | } |
| 597 | |
| 598 | fn label(&mut self) -> SyntaxKind { |
| 599 | let label = self.s.eat_while(is_valid_in_label_literal); |
| 600 | if label.is_empty() { |
| 601 | return self.error("label cannot be empty".to_string()); |
| 602 | } |
| 603 | if !self.s.eat_if('>') { |
| 604 | return self.error("unclosed label".to_string()); |
| 605 | } |
| 606 | SyntaxKind::Label |
| 607 | } |
| 608 | |
| 609 | fn text(&mut self) -> SyntaxKind { |
| 610 | loop { |
| 611 | self.s.eat_until(|c: char| { |
| 612 | if c.is_ascii() { TEXT_STOP[c as usize] } else { c.is_whitespace() } |
| 613 | }); |
| 614 | // Carry on in the same text node where the stop char would become text anyway. |
| 615 | let mut s = self.s; |
| 616 | let go_on = match s.eat() { |
| 617 | Some(' ') => s.at(char::is_alphanumeric), |
| 618 | Some('/') => !s.at(['/', '*']), |
| 619 | Some('-') => !s.at(['-', '?']), |
| 620 | Some('.') => !s.at(".."), |
| 621 | Some('h') => !s.at("ttp://") && !s.at("ttps://"), |
| 622 | Some('@') => !s.at(is_valid_in_label_literal), |
| 623 | _ => false, |
| 624 | }; |
| 625 | if !go_on { |
| 626 | break; |
| 627 | } |
| 628 | self.s = s; |
| 629 | } |
| 630 | SyntaxKind::Text |
| 631 | } |
| 632 | |
| 633 | /// Is the char just eaten flanked by word characters (outside the CJK scripts)? |
| 634 | fn in_word(&self) -> bool { |
| 635 | let wordy = |c: Option<char>| match c { |
| 636 | Some(c) => c.is_alphanumeric() && !is_cjk(c), |
| 637 | None => false, |
| 638 | }; |
| 639 | wordy(self.s.scout(-2)) && wordy(self.s.peek()) |
| 640 | } |
| 641 | |
| 642 | fn space_or_end(&self) -> bool { |
| 643 | self.s.done() || self.s.at(char::is_whitespace) || self.s.at("//") || self.s.at("/*") |
| 644 | } |
| 645 | } |
| 646 | |
| 647 | // The ASCII chars at which a markup text token stops. |
| 648 | const TEXT_STOP: [bool; 128] = { |
| 649 | let mut t = [false; 128]; |
| 650 | let stops = b" \t\n\x0b\x0c\r\\/[]~-.'\"*_:h`$<>@#"; |
| 651 | let mut i = 0; |
| 652 | while i < stops.len() { |
| 653 | t[stops[i] as usize] = true; |
| 654 | i += 1; |
| 655 | } |
| 656 | t |
| 657 | }; |
| 658 | |
| 659 | // Maths. |
| 660 | impl Lexer<'_> { |
| 661 | fn math(&mut self, start: usize, c: char) -> (SyntaxKind, Option<SyntaxNode>) { |
| 662 | let kind = match c { |
| 663 | '\\' => self.backslash(), |
| 664 | '"' => self.string(), |
| 665 | |
| 666 | '-' if self.s.eat_if(">>") => SyntaxKind::MathShorthand, |
| 667 | '-' if self.s.eat_if('>') => SyntaxKind::MathShorthand, |
| 668 | '-' if self.s.eat_if("->") => SyntaxKind::MathShorthand, |
| 669 | ':' if self.s.eat_if('=') => SyntaxKind::MathShorthand, |
| 670 | ':' if self.s.eat_if(":=") => SyntaxKind::MathShorthand, |
| 671 | '!' if self.s.eat_if('=') => SyntaxKind::MathShorthand, |
| 672 | '.' if self.s.eat_if("..") => SyntaxKind::MathShorthand, |
| 673 | '<' if self.s.eat_if("==>") => SyntaxKind::MathShorthand, |
| 674 | '<' if self.s.eat_if("-->") => SyntaxKind::MathShorthand, |
| 675 | '<' if self.s.eat_if("--") => SyntaxKind::MathShorthand, |
| 676 | '<' if self.s.eat_if("-<") => SyntaxKind::MathShorthand, |
| 677 | '<' if self.s.eat_if("->") => SyntaxKind::MathShorthand, |
| 678 | '<' if self.s.eat_if("<-") => SyntaxKind::MathShorthand, |
| 679 | '<' if self.s.eat_if("<<") => SyntaxKind::MathShorthand, |
| 680 | '<' if self.s.eat_if("=>") => SyntaxKind::MathShorthand, |
| 681 | '<' if self.s.eat_if("==") => SyntaxKind::MathShorthand, |
| 682 | '<' if self.s.eat_if("~~") => SyntaxKind::MathShorthand, |
| 683 | '<' if self.s.eat_if('=') => SyntaxKind::MathShorthand, |
| 684 | '<' if self.s.eat_if('<') => SyntaxKind::MathShorthand, |
| 685 | '<' if self.s.eat_if('-') => SyntaxKind::MathShorthand, |
| 686 | '<' if self.s.eat_if('~') => SyntaxKind::MathShorthand, |
| 687 | '>' if self.s.eat_if("->") => SyntaxKind::MathShorthand, |
| 688 | '>' if self.s.eat_if(">>") => SyntaxKind::MathShorthand, |
| 689 | '=' if self.s.eat_if("=>") => SyntaxKind::MathShorthand, |
| 690 | '=' if self.s.eat_if('>') => SyntaxKind::MathShorthand, |
| 691 | '=' if self.s.eat_if(':') => SyntaxKind::MathShorthand, |
| 692 | '>' if self.s.eat_if('=') => SyntaxKind::MathShorthand, |
| 693 | '>' if self.s.eat_if('>') => SyntaxKind::MathShorthand, |
| 694 | '|' if self.s.eat_if("->") => SyntaxKind::MathShorthand, |
| 695 | '|' if self.s.eat_if("=>") => SyntaxKind::MathShorthand, |
| 696 | '|' if self.s.eat_if('|') => SyntaxKind::MathShorthand, |
| 697 | '~' if self.s.eat_if("~>") => SyntaxKind::MathShorthand, |
| 698 | '~' if self.s.eat_if('>') => SyntaxKind::MathShorthand, |
| 699 | '*' | '-' | '~' => SyntaxKind::MathShorthand, |
| 700 | |
| 701 | '.' => SyntaxKind::Dot, |
| 702 | ',' => SyntaxKind::Comma, |
| 703 | ';' => SyntaxKind::Semicolon, |
| 704 | |
| 705 | '#' => SyntaxKind::Hash, |
| 706 | '_' => SyntaxKind::Underscore, |
| 707 | '$' => SyntaxKind::Dollar, |
| 708 | '/' => SyntaxKind::Slash, |
| 709 | '^' => SyntaxKind::Hat, |
| 710 | '&' => SyntaxKind::MathAlignPoint, |
| 711 | '√' | '∛' | '∜' => SyntaxKind::Root, |
| 712 | '!' => SyntaxKind::Bang, |
| 713 | |
| 714 | '\'' => { |
| 715 | self.s.eat_while('\''); |
| 716 | SyntaxKind::MathPrimes |
| 717 | }, |
| 718 | |
| 719 | // Delimiters lex as braces and parens; the parser turns them back into text. |
| 720 | '(' => SyntaxKind::LeftParen, |
| 721 | ')' => SyntaxKind::RightParen, |
| 722 | '[' if self.s.eat_if('|') => SyntaxKind::LeftBrace, |
| 723 | '|' if self.s.eat_if(']') => SyntaxKind::RightBrace, |
| 724 | c if is_math_opening(c) => SyntaxKind::LeftBrace, |
| 725 | c if is_math_closing(c) => SyntaxKind::RightBrace, |
| 726 | |
| 727 | c if is_math_id_start(c) && self.s.at(is_math_id_continue) => { |
| 728 | self.s.eat_while(is_math_id_continue); |
| 729 | let text = self.s.from(start); |
| 730 | if grapheme_len(text) == text.len() { |
| 731 | // One grapheme, a letter and its marks. |
| 732 | SyntaxKind::MathText |
| 733 | } else { |
| 734 | let (kind, node) = self.math_ident_or_field(start); |
| 735 | return (kind, Some(node)); |
| 736 | } |
| 737 | }, |
| 738 | |
| 739 | _ => self.math_text(start, c), |
| 740 | }; |
| 741 | (kind, None) |
| 742 | } |
| 743 | |
| 744 | /// A `MathIdent`, or a whole `MathFieldAccess` chain when dots and identifiers follow. |
| 745 | fn math_ident_or_field(&mut self, start: usize) -> (SyntaxKind, SyntaxNode) { |
| 746 | let mut kind = SyntaxKind::MathIdent; |
| 747 | let mut node = SyntaxNode::leaf(kind, self.s.from(start), Span::detached()); |
| 748 | while let Some(ident) = self.maybe_dot_ident() { |
| 749 | kind = SyntaxKind::MathFieldAccess; |
| 750 | node = SyntaxNode::inner(kind, vec![ |
| 751 | node, |
| 752 | SyntaxNode::leaf(SyntaxKind::Dot, ".", Span::detached()), |
| 753 | SyntaxNode::leaf(SyntaxKind::MathIdent, ident, Span::detached()), |
| 754 | ]); |
| 755 | } |
| 756 | (kind, node) |
| 757 | } |
| 758 | |
| 759 | fn maybe_dot_ident(&mut self) -> Option<&str> { |
| 760 | let next_starts = match self.s.scout(1) { |
| 761 | Some(c) => is_math_id_start(c), |
| 762 | None => false, |
| 763 | }; |
| 764 | if next_starts && self.s.eat_if('.') { |
| 765 | let start = self.s.cursor(); |
| 766 | self.s.eat(); |
| 767 | self.s.eat_while(is_math_id_continue); |
| 768 | Some(self.s.from(start)) |
| 769 | } else { |
| 770 | None |
| 771 | } |
| 772 | } |
| 773 | |
| 774 | fn math_text(&mut self, start: usize, c: char) -> SyntaxKind { |
| 775 | if c.is_numeric() { |
| 776 | // A number keeps its digits and one decimal part together. |
| 777 | self.s.eat_while(char::is_numeric); |
| 778 | let mut s = self.s; |
| 779 | if s.eat_if('.') && !s.eat_while(char::is_numeric).is_empty() { |
| 780 | self.s = s; |
| 781 | } |
| 782 | } else { |
| 783 | let rest = self.s.get(start, self.s.string().len()); |
| 784 | self.s.jump(start + grapheme_len(rest)); |
| 785 | } |
| 786 | SyntaxKind::MathText |
| 787 | } |
| 788 | |
| 789 | /// A named argument's name, `name:` but not `:=` or `::=`, at `start`; the cursor is restored when |
| 790 | /// there is none. |
| 791 | pub fn maybe_math_named_arg(&mut self, start: usize) -> Option<SyntaxNode> { |
| 792 | let cursor = self.s.cursor(); |
| 793 | self.s.jump(start); |
| 794 | if self.s.eat_if(is_id_start) { |
| 795 | self.s.eat_while(is_id_continue); |
| 796 | if self.s.at(':') && !self.s.at(":=") && !self.s.at("::=") { |
| 797 | let text = self.s.from(start); |
| 798 | let node = if text != "_" { |
| 799 | SyntaxNode::leaf(SyntaxKind::Ident, text, Span::detached()) |
| 800 | } else { |
| 801 | SyntaxNode::error(text, "expected identifier, found underscore", Span::detached()) |
| 802 | }; |
| 803 | return Some(node); |
| 804 | } |
| 805 | } |
| 806 | self.s.jump(cursor); |
| 807 | None |
| 808 | } |
| 809 | |
| 810 | /// A spread's `..` at `start`, unless it spreads nothing or is the `...` shorthand. |
| 811 | pub fn maybe_math_spread_arg(&mut self, start: usize) -> Option<SyntaxNode> { |
| 812 | let cursor = self.s.cursor(); |
| 813 | self.s.jump(start); |
| 814 | if self.s.eat_if("..") && !self.space_or_end() && !self.s.at(['.', ',', ';', ')', '$']) { |
| 815 | return Some(SyntaxNode::leaf(SyntaxKind::Dots, self.s.from(start), Span::detached())); |
| 816 | } |
| 817 | self.s.jump(cursor); |
| 818 | None |
| 819 | } |
| 820 | } |
| 821 | |
| 822 | // Code. |
| 823 | impl Lexer<'_> { |
| 824 | fn code(&mut self, start: usize, c: char) -> SyntaxKind { |
| 825 | match c { |
| 826 | '<' if self.s.at(is_id_continue) => self.label(), |
| 827 | '0'..='9' => self.number(start, c), |
| 828 | '.' if self.s.at(|c: char| c.is_ascii_digit()) => self.number(start, c), |
| 829 | '"' => self.string(), |
| 830 | |
| 831 | '=' if self.s.eat_if('=') => SyntaxKind::EqEq, |
| 832 | '!' if self.s.eat_if('=') => SyntaxKind::ExclEq, |
| 833 | '<' if self.s.eat_if('=') => SyntaxKind::LtEq, |
| 834 | '>' if self.s.eat_if('=') => SyntaxKind::GtEq, |
| 835 | '+' if self.s.eat_if('=') => SyntaxKind::PlusEq, |
| 836 | '-' | '\u{2212}' if self.s.eat_if('=') => SyntaxKind::HyphEq, |
| 837 | '*' if self.s.eat_if('=') => SyntaxKind::StarEq, |
| 838 | '/' if self.s.eat_if('=') => SyntaxKind::SlashEq, |
| 839 | '.' if self.s.eat_if('.') => SyntaxKind::Dots, |
| 840 | '=' if self.s.eat_if('>') => SyntaxKind::Arrow, |
| 841 | |
| 842 | '{' => SyntaxKind::LeftBrace, |
| 843 | '}' => SyntaxKind::RightBrace, |
| 844 | '[' => SyntaxKind::LeftBracket, |
| 845 | ']' => SyntaxKind::RightBracket, |
| 846 | '(' => SyntaxKind::LeftParen, |
| 847 | ')' => SyntaxKind::RightParen, |
| 848 | '$' => SyntaxKind::Dollar, |
| 849 | ',' => SyntaxKind::Comma, |
| 850 | ';' => SyntaxKind::Semicolon, |
| 851 | ':' => SyntaxKind::Colon, |
| 852 | '.' => SyntaxKind::Dot, |
| 853 | '+' => SyntaxKind::Plus, |
| 854 | '-' | '\u{2212}' => SyntaxKind::Minus, |
| 855 | '*' => SyntaxKind::Star, |
| 856 | '/' => SyntaxKind::Slash, |
| 857 | '=' => SyntaxKind::Eq, |
| 858 | '<' => SyntaxKind::Lt, |
| 859 | '>' => SyntaxKind::Gt, |
| 860 | |
| 861 | c if is_id_start(c) => self.ident(start), |
| 862 | |
| 863 | c => self.invalid_char_in_code(c), |
| 864 | } |
| 865 | } |
| 866 | |
| 867 | // Typst's hints for the operators people bring from other languages. |
| 868 | fn invalid_char_in_code(&mut self, c: char) -> SyntaxKind { |
| 869 | let invalid_char = format!("the character `{}` is not valid in code", c); |
| 870 | match c { |
| 871 | _ if self.s.scout(-2) == Some('#') => { |
| 872 | self.error(invalid_char); |
| 873 | self.hint("the preceding hash is causing this to parse in code mode"); |
| 874 | self.hint("try escaping the preceding hash: `\\#`"); |
| 875 | }, |
| 876 | '#' => { |
| 877 | self.error(invalid_char); |
| 878 | self.hint("you are already in code mode"); |
| 879 | self.hint("try removing the `#`"); |
| 880 | }, |
| 881 | '&' if self.s.eat_if('&') => { |
| 882 | self.error("`&&` is not valid in code".to_string()); |
| 883 | self.hint("in Typst, `and` is used for logical AND"); |
| 884 | }, |
| 885 | '|' if self.s.eat_if('|') => { |
| 886 | self.error("`||` is not valid in code".to_string()); |
| 887 | self.hint("in Typst, `or` is used for logical OR"); |
| 888 | }, |
| 889 | '!' => { |
| 890 | self.error(invalid_char); |
| 891 | self.hint("in Typst, `not` is used for negation"); |
| 892 | self.hint("or did you mean to write `!=` for not-equal?"); |
| 893 | }, |
| 894 | '~' if self.s.eat_if('=') => { |
| 895 | self.error("`~=` is not valid in code".to_string()); |
| 896 | self.hint("in Typst, `!=` is used for not-equal"); |
| 897 | }, |
| 898 | _ => { |
| 899 | self.error(invalid_char); |
| 900 | }, |
| 901 | } |
| 902 | SyntaxKind::Error |
| 903 | } |
| 904 | |
| 905 | fn ident(&mut self, start: usize) -> SyntaxKind { |
| 906 | self.s.eat_while(is_id_continue); |
| 907 | let ident = self.s.from(start); |
| 908 | let prev = self.s.get(0, start); |
| 909 | // After a dot or an at sign the word is a field or a reference, never a keyword. |
| 910 | if !prev.ends_with(['.', '@']) || prev.ends_with("..") { |
| 911 | if let Some(k) = keyword(ident) { |
| 912 | return k; |
| 913 | } |
| 914 | } |
| 915 | if ident == "_" { SyntaxKind::Underscore } else { SyntaxKind::Ident } |
| 916 | } |
| 917 | |
| 918 | /// An integer, a float or a numeric with a unit; only base-ten numbers may carry a unit. |
| 919 | fn number(&mut self, start: usize, first: char) -> SyntaxKind { |
| 920 | let base = match first { |
| 921 | '0' if self.s.eat_if('b') => 2, |
| 922 | '0' if self.s.eat_if('o') => 8, |
| 923 | '0' if self.s.eat_if('x') => 16, |
| 924 | _ => 10, |
| 925 | }; |
| 926 | |
| 927 | if base == 16 { |
| 928 | self.s.eat_while(|c: char| c.is_ascii_alphanumeric()); |
| 929 | } else { |
| 930 | self.s.eat_while(|c: char| c.is_ascii_digit()); |
| 931 | } |
| 932 | |
| 933 | let mut is_float = false; |
| 934 | if base == 10 { |
| 935 | // Not a spread (`1..`) and not a method call (`1.abs`). |
| 936 | if first == '.' { |
| 937 | is_float = true; |
| 938 | } else { |
| 939 | let id_follows = match self.s.scout(1) { |
| 940 | Some(c) => is_id_start(c), |
| 941 | None => false, |
| 942 | }; |
| 943 | if !self.s.at("..") && !id_follows && self.s.eat_if('.') { |
| 944 | is_float = true; |
| 945 | self.s.eat_while(|c: char| c.is_ascii_digit()); |
| 946 | } |
| 947 | } |
| 948 | if !self.s.at("em") && self.s.eat_if(['e', 'E']) { |
| 949 | is_float = true; |
| 950 | self.s.eat_if(['+', '-']); |
| 951 | self.s.eat_while(|c: char| c.is_ascii_digit()); |
| 952 | } |
| 953 | } |
| 954 | |
| 955 | let number = self.s.from(start); |
| 956 | let suffix = self.s.eat_while(|c: char| c.is_ascii_alphanumeric() || c == '%'); |
| 957 | |
| 958 | // A base-ten integer too large for i64 reads as a float. |
| 959 | if base == 10 && !is_float && number.parse::<i64>().is_err() && number.parse::<f64>().is_ok() { |
| 960 | let overflow = number.chars().all(|c| c.is_ascii_digit()) && !number.is_empty(); |
| 961 | if overflow { |
| 962 | is_float = true; |
| 963 | } |
| 964 | } |
| 965 | |
| 966 | let mut suffix_res: Result<bool, String> = match suffix { |
| 967 | "" => Ok(false), |
| 968 | "pt" | "mm" | "cm" | "in" | "deg" | "rad" | "em" | "fr" | "%" => Ok(true), |
| 969 | _ => |
| 970 | Err(format!("invalid number suffix: `{}`", suffix)), |
| 971 | }; |
| 972 | |
| 973 | let number_res: Result<(), String> = if is_float && number.parse::<f64>().is_err() { |
| 974 | Err(format!("invalid floating point number: `{}`", number)) |
| 975 | } else if base == 10 { |
| 976 | Ok(()) |
| 977 | } else { |
| 978 | let name = match base { |
| 979 | 2 => "binary", |
| 980 | 8 => "octal", |
| 981 | _ => "hexadecimal", |
| 982 | }; |
| 983 | let digits = number.get(2..).unwrap_or(""); |
| 984 | match i64::from_str_radix(digits, base) { |
| 985 | Ok(_) if suffix.is_empty() => Ok(()), |
| 986 | Ok(value) => { |
| 987 | if suffix_res.is_ok() { |
| 988 | suffix_res = Err(format!("try using a decimal number: `{}{}`", value, suffix)); |
| 989 | } |
| 990 | Err(format!("{} numbers cannot have a suffix", name)) |
| 991 | }, |
| 992 | Err(_) if digits.is_empty() => |
| 993 | Err(format!("expected a{} {} number", if base == 8 { "n" } else { "" }, name)), |
| 994 | Err(_) => Err(format!("invalid {} number: `{}`", name, number)), |
| 995 | } |
| 996 | }; |
| 997 | |
| 998 | match (number_res, suffix_res) { |
| 999 | (Ok(()), Ok(false)) if is_float => SyntaxKind::Float, |
| 1000 | (Ok(()), Ok(false)) => SyntaxKind::Int, |
| 1001 | (Ok(()), Ok(true)) => SyntaxKind::Numeric, |
| 1002 | (Err(n), Err(s)) => { |
| 1003 | let k = self.error(n); |
| 1004 | self.hint(&s); |
| 1005 | k |
| 1006 | }, |
| 1007 | (Ok(()), Err(m)) => self.error(m), |
| 1008 | (Err(m), Ok(_)) => self.error(m), |
| 1009 | } |
| 1010 | } |
| 1011 | |
| 1012 | fn string(&mut self) -> SyntaxKind { |
| 1013 | let mut escaped = false; |
| 1014 | self.s.eat_until(|c: char| { |
| 1015 | let stop = c == '"' && !escaped; |
| 1016 | escaped = c == '\\' && !escaped; |
| 1017 | stop |
| 1018 | }); |
| 1019 | if !self.s.eat_if('"') { |
| 1020 | return self.error("unclosed string".to_string()); |
| 1021 | } |
| 1022 | SyntaxKind::Str |
| 1023 | } |
| 1024 | } |
| 1025 | |
| 1026 | fn keyword(ident: &str) -> Option<SyntaxKind> { |
| 1027 | Some(match ident { |
| 1028 | "none" => SyntaxKind::None, |
| 1029 | "auto" => SyntaxKind::Auto, |
| 1030 | "true" => SyntaxKind::Bool, |
| 1031 | "false" => SyntaxKind::Bool, |
| 1032 | "not" => SyntaxKind::Not, |
| 1033 | "and" => SyntaxKind::And, |
| 1034 | "or" => SyntaxKind::Or, |
| 1035 | "let" => SyntaxKind::Let, |
| 1036 | "set" => SyntaxKind::Set, |
| 1037 | "show" => SyntaxKind::Show, |
| 1038 | "context" => SyntaxKind::Context, |
| 1039 | "if" => SyntaxKind::If, |
| 1040 | "else" => SyntaxKind::Else, |
| 1041 | "for" => SyntaxKind::For, |
| 1042 | "in" => SyntaxKind::In, |
| 1043 | "while" => SyntaxKind::While, |
| 1044 | "break" => SyntaxKind::Break, |
| 1045 | "continue" => SyntaxKind::Continue, |
| 1046 | "return" => SyntaxKind::Return, |
| 1047 | "import" => SyntaxKind::Import, |
| 1048 | "include" => SyntaxKind::Include, |
| 1049 | "as" => SyntaxKind::As, |
| 1050 | _ => return None, |
| 1051 | }) |
| 1052 | } |
| 1053 | |
| 1054 | // ┌───────────────────────────────────────────────────────────────────────────────────────────┐ |
| 1055 | // │ Character classes │ |
| 1056 | // └───────────────────────────────────────────────────────────────────────────────────────────┘ |
| 1057 | |
| 1058 | fn is_space(c: char, mode: LexMode) -> bool { |
| 1059 | match mode { |
| 1060 | LexMode::Markup => matches!(c, ' ' | '\t') || is_newline(c), |
| 1061 | _ => c.is_whitespace(), |
| 1062 | } |
| 1063 | } |
| 1064 | |
| 1065 | /// Is the char one Typst reads as a newline (LF, VT, FF, CR, NEL, LS, PS)? |
| 1066 | pub fn is_newline(c: char) -> bool { |
| 1067 | matches!(c, '\n' | '\x0B' | '\x0C' | '\r' | '\u{0085}' | '\u{2028}' | '\u{2029}') |
| 1068 | } |
| 1069 | |
| 1070 | pub fn is_xid_start(c: char) -> bool { property::is_xid_start(c) } |
| 1071 | |
| 1072 | pub fn is_xid_continue(c: char) -> bool { property::is_xid_continue(c) } |
| 1073 | |
| 1074 | /// Can the char start an identifier? XID_Start plus the underscore. |
| 1075 | pub fn is_id_start(c: char) -> bool { is_xid_start(c) || c == '_' } |
| 1076 | |
| 1077 | /// Can the char continue an identifier? XID_Continue plus the underscore and the hyphen. |
| 1078 | pub fn is_id_continue(c: char) -> bool { is_xid_continue(c) || c == '_' || c == '-' } |
| 1079 | |
| 1080 | /// Is the string a valid Typst identifier? |
| 1081 | pub fn is_ident(s: &str) -> bool { |
| 1082 | let mut chars = s.chars(); |
| 1083 | match chars.next() { |
| 1084 | Some(c) => is_id_start(c) && chars.all(is_id_continue), |
| 1085 | None => false, |
| 1086 | } |
| 1087 | } |
| 1088 | |
| 1089 | fn is_math_id_start(c: char) -> bool { is_xid_start(c) } |
| 1090 | |
| 1091 | fn is_math_id_continue(c: char) -> bool { is_xid_continue(c) && c != '_' } |
| 1092 | |
| 1093 | fn is_valid_in_label_literal(c: char) -> bool { is_id_continue(c) || matches!(c, ':' | '.') } |
| 1094 | |
| 1095 | /// Is the string usable as the name inside a `<label>` literal? |
| 1096 | pub fn is_valid_label_literal_id(id: &str) -> bool { |
| 1097 | !id.is_empty() && id.chars().all(is_valid_in_label_literal) |
| 1098 | } |
| 1099 | |
| 1100 | /// Is the char in the Han, Hiragana, Katakana or Hangul script, where `*` and `_` never sit inside a word? |
| 1101 | fn is_cjk(c: char) -> bool { |
| 1102 | !c.is_ascii() && matches!(Script::of(c), Script::Han | Script::Hiragana | Script::Katakana | Script::Hangul) |
| 1103 | } |
| 1104 | |
| 1105 | pub fn is_math_opening(c: char) -> bool { math::class(c) == Some(MathClass::Opening) } |
| 1106 | |
| 1107 | pub fn is_math_closing(c: char) -> bool { math::class(c) == Some(MathClass::Closing) } |
| 1108 | |
| 1109 | /// Does the char have the Unicode maths class Alphabetic? Typst's own class overrides touch no letter. |
| 1110 | pub fn is_math_alphabetic_class(c: char) -> bool { math::class(c) == Some(MathClass::Alphabetic) } |
| 1111 | |
| 1112 | /// The byte length of the first extended grapheme cluster of `s`, reading only that cluster. |
| 1113 | pub fn grapheme_len(s: &str) -> usize { segment::grapheme_len_at(s, 0) } |
| 1114 | |
| 1115 | /// The prefix of `text` that an automatic link takes, and whether its brackets balance. Trailing |
| 1116 | /// punctuation likely to be prose is left out. |
| 1117 | pub fn link_prefix(text: &str) -> (&str, bool) { |
| 1118 | let mut s = Scanner::new(text); |
| 1119 | let mut brackets: Vec<u8> = Vec::new(); |
| 1120 | s.eat_while(|c: char| match c { |
| 1121 | '0'..='9' | 'a'..='z' | 'A'..='Z' |
| 1122 | | '!' | '#' | '$' | '%' | '&' | '*' | '+' |
| 1123 | | ',' | '-' | '.' | '/' | ':' | ';' | '=' |
| 1124 | | '?' | '@' | '_' | '~' | '\'' => true, |
| 1125 | '[' => { brackets.push(b'['); true }, |
| 1126 | '(' => { brackets.push(b'('); true }, |
| 1127 | ']' => brackets.pop() == Some(b'['), |
| 1128 | ')' => brackets.pop() == Some(b'('), |
| 1129 | _ => false, |
| 1130 | }); |
| 1131 | while matches!(s.scout(-1), Some('!' | ',' | '.' | ':' | ';' | '?' | '\'')) { |
| 1132 | s.uneat(); |
| 1133 | } |
| 1134 | (s.before(), brackets.is_empty()) |
| 1135 | } |
| 1136 | |
| 1137 | /// Splits at newlines, a CR LF pair counting as one; the newlines are dropped. |
| 1138 | pub fn split_newlines(text: &str) -> Vec<&str> { |
| 1139 | let mut s = Scanner::new(text); |
| 1140 | let mut lines = Vec::new(); |
| 1141 | let mut start = 0; |
| 1142 | let mut end = 0; |
| 1143 | while let Some(c) = s.eat() { |
| 1144 | if is_newline(c) { |
| 1145 | if c == '\r' { |
| 1146 | s.eat_if('\n'); |
| 1147 | } |
| 1148 | lines.push(s.get(start, end)); |
| 1149 | start = s.cursor(); |
| 1150 | } |
| 1151 | end = s.cursor(); |
| 1152 | } |
| 1153 | lines.push(s.get(start, text.len())); |
| 1154 | lines |
| 1155 | } |
| 1156 | |
| 1157 | fn count_newlines(text: &str) -> usize { |
| 1158 | let mut n = 0; |
| 1159 | let mut s = Scanner::new(text); |
| 1160 | while let Some(c) = s.eat() { |
| 1161 | if is_newline(c) { |
| 1162 | if c == '\r' { |
| 1163 | s.eat_if('\n'); |
| 1164 | } |
| 1165 | n += 1; |
| 1166 | } |
| 1167 | } |
| 1168 | n |
| 1169 | } |