Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/src/syntax/lexer.rs

37.8 KiB, 1 run

created by r1870400018:63564, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// SPDX-License-Identifier: Apache-2.0
2// Portions Copyright (c) The Typst Project Developers, from typst-syntax 0.15.1's `src/lexer.rs`; see
3// fe2o3_austenite/licences/typst-LICENSE.txt for the full Apache-2.0 text.
4// Modified for Austenite.
5//! Ported from typst-syntax 0.15.1 (`src/lexer.rs`, Apache-2.0, (c) the Typst authors), token for token:
6//! the same modes, the same kinds, the same error messages and hints, so a document lexes as Typst
7//! lexes it. The Unicode predicates it needs (XID identifiers, the CJK scripts, the maths delimiter
8//! classes, grapheme clusters) come from `fe2o3_text::unicode` rather than crates.
9
10use crate::syntax::{
11 SyntaxKind,
12 SyntaxNode,
13 Span,
14};
15
16use oxedyne_fe2o3_text::unicode::lookup::Partitioned;
17use oxedyne_fe2o3_text::unicode::math::{
18 self,
19 MathClass,
20};
21use oxedyne_fe2o3_text::unicode::prop::Script;
22use oxedyne_fe2o3_text::unicode::property;
23use oxedyne_fe2o3_text::unicode::segment;
24
25/// The three lexical modes Typst switches between.
26#[derive(Clone, Copy, Debug, PartialEq, Eq)]
27pub enum LexMode {
28 Markup,
29 Math,
30 Code,
31}
32
33// ┌───────────────────────────────────────────────────────────────────────────────────────────┐
34// │ Scanner │
35// └───────────────────────────────────────────────────────────────────────────────────────────┘
36
37/// A cursor over a string, after `unscanny`. Every position is a byte offset on a char boundary.
38#[derive(Clone, Copy, Debug)]
39pub struct Scanner<'s> {
40 text: &'s str,
41 cursor: usize,
42}
43
44/// Something a [`Scanner`] can match at its cursor: a char, a string, a set of chars or a predicate.
45pub trait Pattern {
46 /// The byte length matched at the start of `s`, if any.
47 fn matches(&mut self, s: &str) -> Option<usize>;
48}
49
50impl Pattern for char {
51 fn matches(&mut self, s: &str) -> Option<usize> {
52 if s.starts_with(*self) { Some(self.len_utf8()) } else { None }
53 }
54}
55
56impl Pattern for &str {
57 fn matches(&mut self, s: &str) -> Option<usize> {
58 if s.starts_with(*self) { Some(self.len()) } else { None }
59 }
60}
61
62impl<const N: usize> Pattern for [char; N] {
63 fn matches(&mut self, s: &str) -> Option<usize> {
64 match s.chars().next() {
65 Some(c) if self.contains(&c) => Some(c.len_utf8()),
66 _ => None,
67 }
68 }
69}
70
71impl<F: FnMut(char) -> bool> Pattern for F {
72 fn matches(&mut self, s: &str) -> Option<usize> {
73 match s.chars().next() {
74 Some(c) if self(c) => Some(c.len_utf8()),
75 _ => None,
76 }
77 }
78}
79
80impl<'s> Scanner<'s> {
81 pub fn new(text: &'s str) -> Self { Self { text, cursor: 0 } }
82
83 pub fn string(&self) -> &'s str { self.text }
84 pub fn cursor(&self) -> usize { self.cursor }
85 pub fn done(&self) -> bool { self.cursor >= self.text.len() }
86
87 /// Moves to `index`, snapped back onto a char boundary and clamped to the text.
88 pub fn jump(&mut self, index: usize) {
89 let mut i = index.min(self.text.len());
90 while !self.text.is_char_boundary(i) {
91 i -= 1;
92 }
93 self.cursor = i;
94 }
95
96 pub fn advance(&mut self, by: usize) { self.jump(self.cursor + by); }
97
98 pub fn before(&self) -> &'s str { self.get(0, self.cursor) }
99 pub fn after(&self) -> &'s str { self.get(self.cursor, self.text.len()) }
100 pub fn from(&self, start: usize) -> &'s str { self.get(start, self.cursor) }
101
102 /// The text between two offsets, empty when they do not bound a slice.
103 pub fn get(&self, a: usize, b: usize) -> &'s str {
104 match self.text.get(a.min(b)..b) {
105 Some(s) => s,
106 None => "",
107 }
108 }
109
110 pub fn peek(&self) -> Option<char> { self.after().chars().next() }
111
112 /// The char `n` places from the cursor: `0` is the next, `-1` the previous.
113 pub fn scout(&self, n: isize) -> Option<char> {
114 if n >= 0 {
115 self.after().chars().nth(n as usize)
116 } else {
117 self.before().chars().rev().nth((-n - 1) as usize)
118 }
119 }
120
121 pub fn eat(&mut self) -> Option<char> {
122 let c = self.peek();
123 if let Some(c) = c {
124 self.cursor += c.len_utf8();
125 }
126 c
127 }
128
129 /// Steps back over the previous char.
130 pub fn uneat(&mut self) {
131 if let Some(c) = self.before().chars().next_back() {
132 self.cursor -= c.len_utf8();
133 }
134 }
135
136 pub fn at<P: Pattern>(&self, mut pat: P) -> bool { pat.matches(self.after()).is_some() }
137
138 pub fn eat_if<P: Pattern>(&mut self, mut pat: P) -> bool {
139 match pat.matches(self.after()) {
140 Some(n) => { self.cursor += n; true },
141 None => false,
142 }
143 }
144
145 pub fn eat_while<P: Pattern>(&mut self, mut pat: P) -> &'s str {
146 let start = self.cursor;
147 while let Some(n) = pat.matches(self.after()) {
148 if n == 0 {
149 break;
150 }
151 self.cursor += n;
152 }
153 self.from(start)
154 }
155
156 pub fn eat_until<P: Pattern>(&mut self, mut pat: P) -> &'s str {
157 let start = self.cursor;
158 while !self.done() && pat.matches(self.after()).is_none() {
159 self.eat();
160 }
161 self.from(start)
162 }
163
164 /// Eats one newline, a CR LF pair counting as one.
165 pub fn eat_newline(&mut self) -> bool {
166 let ate = self.eat_if(is_newline);
167 if ate && self.before().ends_with('\r') {
168 self.eat_if('\n');
169 }
170 ate
171 }
172}
173
174// ┌───────────────────────────────────────────────────────────────────────────────────────────┐
175// │ Lexer │
176// └───────────────────────────────────────────────────────────────────────────────────────────┘
177
178/// Turns source text into tokens, one call to [`Lexer::next`] at a time. Each token comes back as a
179/// detached node; the parser assigns spans once the tree is whole.
180#[derive(Clone, Debug)]
181pub struct Lexer<'s> {
182 s: Scanner<'s>,
183 mode: LexMode,
184 newline: bool, // the last token held a newline
185 error: Option<(String, Vec<String>)>, // the current token's error and hints
186}
187
188impl<'s> Lexer<'s> {
189 pub fn new(text: &'s str, mode: LexMode) -> Self {
190 Self { s: Scanner::new(text), mode, newline: false, error: None }
191 }
192
193 pub fn mode(&self) -> LexMode { self.mode }
194 pub fn set_mode(&mut self, mode: LexMode) { self.mode = mode; }
195 pub fn cursor(&self) -> usize { self.s.cursor() }
196 pub fn jump(&mut self, index: usize) { self.s.jump(index); }
197
198 /// Did the last token contain a newline?
199 pub fn newline(&self) -> bool { self.newline }
200
201 /// The number of chars between the last newline before `index` and `index`.
202 pub fn column(&self, index: usize) -> usize {
203 let mut s = self.s;
204 s.jump(index);
205 s.before().chars().rev().take_while(|c| !is_newline(*c)).count()
206 }
207
208 fn error(&mut self, message: String) -> SyntaxKind {
209 self.error = Some((message, Vec::new()));
210 SyntaxKind::Error
211 }
212
213 fn hint(&mut self, hint: &str) {
214 if let Some((_, hints)) = &mut self.error {
215 hints.push(hint.to_string());
216 }
217 }
218
219 /// The next token, as its kind and its node. `End` at the end of the text.
220 pub fn next(&mut self) -> (SyntaxKind, SyntaxNode) {
221 self.error = None;
222 let start = self.s.cursor();
223 self.newline = false;
224 let kind = match self.s.eat() {
225 Some(c) if is_space(c, self.mode) => self.whitespace(start, c),
226 Some('#') if start == 0 && self.s.eat_if('!') => self.shebang(),
227 Some('/') if self.s.eat_if('/') => self.line_comment(),
228 Some('/') if self.s.eat_if('*') => self.block_comment(),
229 Some('*') if self.s.eat_if('/') => {
230 let k = self.error("unexpected end of block comment".to_string());
231 self.hint("consider escaping the `*` with a backslash or opening the block comment with `/*`");
232 k
233 },
234 Some('`') if self.mode != LexMode::Math => return self.raw(),
235 Some(c) => match self.mode {
236 LexMode::Markup => self.markup(start, c),
237 LexMode::Math => match self.math(start, c) {
238 (kind, None) => kind,
239 (kind, Some(node)) => return (kind, node),
240 },
241 LexMode::Code => self.code(start, c),
242 },
243 None => SyntaxKind::End,
244 };
245
246 let text = self.s.from(start);
247 let node = match self.error.take() {
248 Some((message, hints)) => {
249 let mut n = SyntaxNode::error(text, &message, Span::detached());
250 for h in &hints {
251 n.hint(h);
252 }
253 n
254 },
255 None => SyntaxNode::leaf(kind, text, Span::detached()),
256 };
257 (kind, node)
258 }
259
260 fn whitespace(&mut self, start: usize, c: char) -> SyntaxKind {
261 let mode = self.mode;
262 let more = self.s.eat_while(|c| is_space(c, mode));
263 let newlines = match c {
264 ' ' if more.is_empty() => 0,
265 _ => count_newlines(self.s.from(start)),
266 };
267 self.newline = newlines > 0;
268 if self.mode == LexMode::Markup && newlines >= 2 { SyntaxKind::Parbreak } else { SyntaxKind::Space }
269 }
270
271 fn shebang(&mut self) -> SyntaxKind {
272 self.s.eat_until(is_newline);
273 SyntaxKind::Shebang
274 }
275
276 fn line_comment(&mut self) -> SyntaxKind {
277 self.s.eat_until(is_newline);
278 SyntaxKind::LineComment
279 }
280
281 fn block_comment(&mut self) -> SyntaxKind {
282 let mut state = '_';
283 let mut depth = 1;
284 // The first `*/` that closes the outermost `/*`.
285 while let Some(c) = self.s.eat() {
286 state = match (state, c) {
287 ('*', '/') => {
288 depth -= 1;
289 if depth == 0 {
290 break;
291 }
292 '_'
293 },
294 ('/', '*') => {
295 depth += 1;
296 '_'
297 },
298 _ => c,
299 }
300 }
301 SyntaxKind::BlockComment
302 }
303}
304
305// Raw.
306impl Lexer<'_> {
307 /// A whole raw element, delimiters and all, lexed in one go.
308 fn raw(&mut self) -> (SyntaxKind, SyntaxNode) {
309 let start = self.s.cursor() - 1;
310 let mut backticks = 1;
311 while self.s.eat_if('`') {
312 backticks += 1;
313 }
314
315 // Two backticks are an empty inline raw.
316 if backticks == 2 {
317 let nodes = vec![
318 SyntaxNode::leaf(SyntaxKind::RawDelim, "`", Span::detached()),
319 SyntaxNode::leaf(SyntaxKind::RawDelim, "`", Span::detached()),
320 ];
321 return (SyntaxKind::Raw, SyntaxNode::inner(SyntaxKind::Raw, nodes));
322 }
323
324 let mut found = 0;
325 while found < backticks {
326 match self.s.eat() {
327 Some('`') => found += 1,
328 Some(_) => found = 0,
329 None => {
330 let node = SyntaxNode::error(self.s.from(start), "unclosed raw text", Span::detached());
331 return (SyntaxKind::Error, node);
332 },
333 }
334 }
335 let end = self.s.cursor();
336
337 let mut inner = Scanner::new(self.s.get(start + backticks, end - backticks));
338 let inner_len = inner.string().len();
339
340 let delim = SyntaxNode::leaf(SyntaxKind::RawDelim, self.s.get(end - backticks, end), Span::detached());
341 let mut nodes = vec![delim.clone()];
342
343 let mut tag = None;
344 let mut future_len = None;
345 if backticks >= 3 {
346 let (t, f) = raw_lang_tag(&mut inner);
347 tag = t;
348 future_len = f;
349 if let Some(t) = tag {
350 nodes.push(SyntaxNode::leaf(SyntaxKind::RawLang, t, Span::detached()));
351 }
352 blocky_raw(&mut inner, &mut nodes);
353 } else {
354 inline_raw(&mut inner, &mut nodes);
355 }
356 nodes.push(delim);
357
358 let mut raw = SyntaxNode::inner(SyntaxKind::Raw, nodes);
359 add_raw_warnings(&mut raw, backticks, future_len, tag, inner_len);
360 (SyntaxKind::Raw, raw)
361 }
362}
363
364/// The language tag, if any, and the length the tag will have in the next version of Typst when that
365/// differs (all text up to whitespace or a backtick).
366fn raw_lang_tag<'a>(s: &mut Scanner<'a>) -> (Option<&'a str>, Option<usize>) {
367 let start = s.cursor();
368 let future = s.eat_until(|c: char| c.is_whitespace() || c == '`');
369 if future.is_empty() {
370 return (None, None);
371 }
372 s.jump(start);
373 let tag = if s.eat_if(is_id_start) {
374 s.eat_while(is_id_continue);
375 Some(s.from(start))
376 } else {
377 None
378 };
379 let differs = match tag {
380 Some(t) => t.len() != future.len(),
381 None => true,
382 };
383 (tag, if differs { Some(future.len()) } else { None })
384}
385
386// A blocky raw trims the rest of the opening line when it is blank (else one space), dedents the inner
387// lines by their common leading whitespace (the closing line counts, the opening line does not), and
388// drops a blank last line; a last line ending in a backtick loses one trailing space.
389fn blocky_raw(s: &mut Scanner, nodes: &mut Vec<SyntaxNode>) {
390 let mut lines = split_newlines(s.after());
391
392 let mut dedent = usize::MAX;
393 for line in lines.iter().skip(1) {
394 if !line.chars().all(char::is_whitespace) {
395 dedent = dedent.min(line.chars().take_while(|c| c.is_whitespace()).count());
396 }
397 }
398 if let Some(last) = lines.last() {
399 dedent = dedent.min(last.chars().take_while(|c| c.is_whitespace()).count());
400 }
401 if dedent == usize::MAX {
402 dedent = 0;
403 }
404
405 let last_blank = match lines.last() {
406 Some(last) => last.chars().all(char::is_whitespace),
407 None => false,
408 };
409 if last_blank {
410 lines.pop();
411 } else if let Some(last) = lines.last_mut() {
412 if last.trim_end().ends_with('`') {
413 if let Some(stripped) = last.strip_suffix(' ') {
414 *last = stripped;
415 }
416 }
417 }
418
419 let mut prev = s.cursor();
420 let mut push = |kind: SyntaxKind, s: &Scanner, prev: &mut usize| {
421 nodes.push(SyntaxNode::leaf(kind, s.from(*prev), Span::detached()));
422 *prev = s.cursor();
423 };
424
425 let mut lines = lines.into_iter();
426 if let Some(first) = lines.next() {
427 if first.chars().all(char::is_whitespace) {
428 // Folded into the next `RawTrimmed`.
429 s.advance(first.len());
430 } else {
431 let line_end = s.cursor() + first.len();
432 if s.eat_if(' ') {
433 push(SyntaxKind::RawTrimmed, s, &mut prev);
434 }
435 s.jump(line_end);
436 push(SyntaxKind::Text, s, &mut prev);
437 }
438 }
439
440 for line in lines {
441 let offset: usize = line.chars().take(dedent).map(char::len_utf8).sum();
442 s.eat_newline();
443 s.advance(offset);
444 push(SyntaxKind::RawTrimmed, s, &mut prev);
445 s.advance(line.len() - offset);
446 push(SyntaxKind::Text, s, &mut prev);
447 }
448
449 if !s.done() {
450 nodes.push(SyntaxNode::leaf(SyntaxKind::RawTrimmed, s.get(prev, s.string().len()), Span::detached()));
451 }
452}
453
454// Inline raw keeps all whitespace; each newline becomes a `RawTrimmed` between `Text` lines.
455fn inline_raw(s: &mut Scanner, nodes: &mut Vec<SyntaxNode>) {
456 let mut prev = s.cursor();
457 while !s.done() {
458 if s.at(is_newline) {
459 nodes.push(SyntaxNode::leaf(SyntaxKind::Text, s.from(prev), Span::detached()));
460 prev = s.cursor();
461 s.eat_newline();
462 nodes.push(SyntaxNode::leaf(SyntaxKind::RawTrimmed, s.from(prev), Span::detached()));
463 prev = s.cursor();
464 continue;
465 }
466 s.eat();
467 }
468 nodes.push(SyntaxNode::leaf(SyntaxKind::Text, s.from(prev), Span::detached()));
469}
470
471// Typst 0.15 warns where the next version will read the language tag differently, and on a tag with no
472// text after it.
473fn add_raw_warnings(
474 raw: &mut SyntaxNode,
475 backticks: usize,
476 future_len: Option<usize>,
477 tag: Option<&str>,
478 inner_len: usize,
479) {
480 if let Some(future_len) = future_len {
481 let (a, b) = (backticks, backticks + future_len);
482 match tag {
483 Some(tag) => {
484 raw.warn_at(a, b, "no whitespace between language tag and raw text");
485 raw.hint(&format!("currently, Typst is treating `{}` as the language tag", tag));
486 raw.hint("in the next version of Typst, this will change and we will treat all text \
487 until the first whitespace as the language tag");
488 let tag_end = backticks + tag.len();
489 raw.hint_at(a, tag_end, &format!("if the current behavior is correct, please add a space after `{}`", tag));
490 raw.hint_at(a, tag_end, "otherwise, add a space or newline after the initial backticks");
491 },
492 None => {
493 raw.warn_at(a, b, "no whitespace before raw text");
494 raw.hint("in the next version of Typst, this text will be treated as the language tag \
495 for this element");
496 raw.hint("to avoid this, add a space after the initial backticks");
497 },
498 }
499 } else if let Some(tag) = tag {
500 if inner_len == tag.len() {
501 raw.warn("empty raw text");
502 raw.hint(&format!("Typst is treating `{}` as the language tag", tag));
503 raw.hint_at(backticks, backticks + tag.len(), "to treat this as text, add a space after the initial backticks");
504 }
505 }
506}
507
508// Markup.
509impl Lexer<'_> {
510 fn markup(&mut self, start: usize, c: char) -> SyntaxKind {
511 match c {
512 '\\' => self.backslash(),
513 'h' if self.s.eat_if("ttp://") => self.link(),
514 'h' if self.s.eat_if("ttps://") => self.link(),
515 '<' if self.s.at(is_id_continue) => self.label(),
516 '@' if self.s.at(is_id_continue) => self.ref_marker(),
517
518 '.' if self.s.eat_if("..") => SyntaxKind::Shorthand,
519 '-' if self.s.eat_if("--") => SyntaxKind::Shorthand,
520 '-' if self.s.eat_if('-') => SyntaxKind::Shorthand,
521 '-' if self.s.eat_if('?') => SyntaxKind::Shorthand,
522 '-' if self.s.at(char::is_numeric) => SyntaxKind::Shorthand,
523 '*' if !self.in_word() => SyntaxKind::Star,
524 '_' if !self.in_word() => SyntaxKind::Underscore,
525
526 '#' => SyntaxKind::Hash,
527 '[' => SyntaxKind::LeftBracket,
528 ']' => SyntaxKind::RightBracket,
529 '\'' => SyntaxKind::SmartQuote,
530 '"' => SyntaxKind::SmartQuote,
531 '$' => SyntaxKind::Dollar,
532 '~' => SyntaxKind::Shorthand,
533 ':' => SyntaxKind::Colon,
534 '=' => {
535 self.s.eat_while('=');
536 if self.space_or_end() { SyntaxKind::HeadingMarker } else { self.text() }
537 },
538 '-' if self.space_or_end() => SyntaxKind::ListMarker,
539 '+' if self.space_or_end() => SyntaxKind::EnumMarker,
540 '/' if self.space_or_end() => SyntaxKind::TermMarker,
541 '0'..='9' => self.numbering(start),
542
543 _ => self.text(),
544 }
545 }
546
547 fn backslash(&mut self) -> SyntaxKind {
548 if self.s.eat_if("u{") {
549 let hex = self.s.eat_while(|c: char| c.is_ascii_alphanumeric());
550 if !self.s.eat_if('}') {
551 return self.error("unclosed Unicode escape sequence".to_string());
552 }
553 let valid = match u32::from_str_radix(hex, 16) {
554 Ok(n) => char::from_u32(n).is_some(),
555 Err(_) => false,
556 };
557 if !valid {
558 return self.error(format!("invalid Unicode codepoint: {}", hex));
559 }
560 return SyntaxKind::Escape;
561 }
562 if self.s.done() || self.s.at(char::is_whitespace) {
563 SyntaxKind::Linebreak
564 } else {
565 self.s.eat();
566 SyntaxKind::Escape
567 }
568 }
569
570 fn link(&mut self) -> SyntaxKind {
571 let (link, balanced) = link_prefix(self.s.after());
572 self.s.advance(link.len());
573 if !balanced {
574 return self.error("automatic links cannot contain unbalanced brackets, use the `link` function \
575 instead".to_string());
576 }
577 SyntaxKind::Link
578 }
579
580 fn numbering(&mut self, start: usize) -> SyntaxKind {
581 self.s.eat_while(|c: char| c.is_ascii_digit());
582 let read = self.s.from(start);
583 if self.s.eat_if('.') && self.space_or_end() && read.parse::<u64>().is_ok() {
584 return SyntaxKind::EnumMarker;
585 }
586 self.text()
587 }
588
589 fn ref_marker(&mut self) -> SyntaxKind {
590 self.s.eat_while(is_valid_in_label_literal);
591 // Trailing dots and colons are likely prose.
592 while matches!(self.s.scout(-1), Some('.' | ':')) {
593 self.s.uneat();
594 }
595 SyntaxKind::RefMarker
596 }
597
598 fn label(&mut self) -> SyntaxKind {
599 let label = self.s.eat_while(is_valid_in_label_literal);
600 if label.is_empty() {
601 return self.error("label cannot be empty".to_string());
602 }
603 if !self.s.eat_if('>') {
604 return self.error("unclosed label".to_string());
605 }
606 SyntaxKind::Label
607 }
608
609 fn text(&mut self) -> SyntaxKind {
610 loop {
611 self.s.eat_until(|c: char| {
612 if c.is_ascii() { TEXT_STOP[c as usize] } else { c.is_whitespace() }
613 });
614 // Carry on in the same text node where the stop char would become text anyway.
615 let mut s = self.s;
616 let go_on = match s.eat() {
617 Some(' ') => s.at(char::is_alphanumeric),
618 Some('/') => !s.at(['/', '*']),
619 Some('-') => !s.at(['-', '?']),
620 Some('.') => !s.at(".."),
621 Some('h') => !s.at("ttp://") && !s.at("ttps://"),
622 Some('@') => !s.at(is_valid_in_label_literal),
623 _ => false,
624 };
625 if !go_on {
626 break;
627 }
628 self.s = s;
629 }
630 SyntaxKind::Text
631 }
632
633 /// Is the char just eaten flanked by word characters (outside the CJK scripts)?
634 fn in_word(&self) -> bool {
635 let wordy = |c: Option<char>| match c {
636 Some(c) => c.is_alphanumeric() && !is_cjk(c),
637 None => false,
638 };
639 wordy(self.s.scout(-2)) && wordy(self.s.peek())
640 }
641
642 fn space_or_end(&self) -> bool {
643 self.s.done() || self.s.at(char::is_whitespace) || self.s.at("//") || self.s.at("/*")
644 }
645}
646
647// The ASCII chars at which a markup text token stops.
648const TEXT_STOP: [bool; 128] = {
649 let mut t = [false; 128];
650 let stops = b" \t\n\x0b\x0c\r\\/[]~-.'\"*_:h`$<>@#";
651 let mut i = 0;
652 while i < stops.len() {
653 t[stops[i] as usize] = true;
654 i += 1;
655 }
656 t
657};
658
659// Maths.
660impl Lexer<'_> {
661 fn math(&mut self, start: usize, c: char) -> (SyntaxKind, Option<SyntaxNode>) {
662 let kind = match c {
663 '\\' => self.backslash(),
664 '"' => self.string(),
665
666 '-' if self.s.eat_if(">>") => SyntaxKind::MathShorthand,
667 '-' if self.s.eat_if('>') => SyntaxKind::MathShorthand,
668 '-' if self.s.eat_if("->") => SyntaxKind::MathShorthand,
669 ':' if self.s.eat_if('=') => SyntaxKind::MathShorthand,
670 ':' if self.s.eat_if(":=") => SyntaxKind::MathShorthand,
671 '!' if self.s.eat_if('=') => SyntaxKind::MathShorthand,
672 '.' if self.s.eat_if("..") => SyntaxKind::MathShorthand,
673 '<' if self.s.eat_if("==>") => SyntaxKind::MathShorthand,
674 '<' if self.s.eat_if("-->") => SyntaxKind::MathShorthand,
675 '<' if self.s.eat_if("--") => SyntaxKind::MathShorthand,
676 '<' if self.s.eat_if("-<") => SyntaxKind::MathShorthand,
677 '<' if self.s.eat_if("->") => SyntaxKind::MathShorthand,
678 '<' if self.s.eat_if("<-") => SyntaxKind::MathShorthand,
679 '<' if self.s.eat_if("<<") => SyntaxKind::MathShorthand,
680 '<' if self.s.eat_if("=>") => SyntaxKind::MathShorthand,
681 '<' if self.s.eat_if("==") => SyntaxKind::MathShorthand,
682 '<' if self.s.eat_if("~~") => SyntaxKind::MathShorthand,
683 '<' if self.s.eat_if('=') => SyntaxKind::MathShorthand,
684 '<' if self.s.eat_if('<') => SyntaxKind::MathShorthand,
685 '<' if self.s.eat_if('-') => SyntaxKind::MathShorthand,
686 '<' if self.s.eat_if('~') => SyntaxKind::MathShorthand,
687 '>' if self.s.eat_if("->") => SyntaxKind::MathShorthand,
688 '>' if self.s.eat_if(">>") => SyntaxKind::MathShorthand,
689 '=' if self.s.eat_if("=>") => SyntaxKind::MathShorthand,
690 '=' if self.s.eat_if('>') => SyntaxKind::MathShorthand,
691 '=' if self.s.eat_if(':') => SyntaxKind::MathShorthand,
692 '>' if self.s.eat_if('=') => SyntaxKind::MathShorthand,
693 '>' if self.s.eat_if('>') => SyntaxKind::MathShorthand,
694 '|' if self.s.eat_if("->") => SyntaxKind::MathShorthand,
695 '|' if self.s.eat_if("=>") => SyntaxKind::MathShorthand,
696 '|' if self.s.eat_if('|') => SyntaxKind::MathShorthand,
697 '~' if self.s.eat_if("~>") => SyntaxKind::MathShorthand,
698 '~' if self.s.eat_if('>') => SyntaxKind::MathShorthand,
699 '*' | '-' | '~' => SyntaxKind::MathShorthand,
700
701 '.' => SyntaxKind::Dot,
702 ',' => SyntaxKind::Comma,
703 ';' => SyntaxKind::Semicolon,
704
705 '#' => SyntaxKind::Hash,
706 '_' => SyntaxKind::Underscore,
707 '$' => SyntaxKind::Dollar,
708 '/' => SyntaxKind::Slash,
709 '^' => SyntaxKind::Hat,
710 '&' => SyntaxKind::MathAlignPoint,
711 '√' | '∛' | '∜' => SyntaxKind::Root,
712 '!' => SyntaxKind::Bang,
713
714 '\'' => {
715 self.s.eat_while('\'');
716 SyntaxKind::MathPrimes
717 },
718
719 // Delimiters lex as braces and parens; the parser turns them back into text.
720 '(' => SyntaxKind::LeftParen,
721 ')' => SyntaxKind::RightParen,
722 '[' if self.s.eat_if('|') => SyntaxKind::LeftBrace,
723 '|' if self.s.eat_if(']') => SyntaxKind::RightBrace,
724 c if is_math_opening(c) => SyntaxKind::LeftBrace,
725 c if is_math_closing(c) => SyntaxKind::RightBrace,
726
727 c if is_math_id_start(c) && self.s.at(is_math_id_continue) => {
728 self.s.eat_while(is_math_id_continue);
729 let text = self.s.from(start);
730 if grapheme_len(text) == text.len() {
731 // One grapheme, a letter and its marks.
732 SyntaxKind::MathText
733 } else {
734 let (kind, node) = self.math_ident_or_field(start);
735 return (kind, Some(node));
736 }
737 },
738
739 _ => self.math_text(start, c),
740 };
741 (kind, None)
742 }
743
744 /// A `MathIdent`, or a whole `MathFieldAccess` chain when dots and identifiers follow.
745 fn math_ident_or_field(&mut self, start: usize) -> (SyntaxKind, SyntaxNode) {
746 let mut kind = SyntaxKind::MathIdent;
747 let mut node = SyntaxNode::leaf(kind, self.s.from(start), Span::detached());
748 while let Some(ident) = self.maybe_dot_ident() {
749 kind = SyntaxKind::MathFieldAccess;
750 node = SyntaxNode::inner(kind, vec![
751 node,
752 SyntaxNode::leaf(SyntaxKind::Dot, ".", Span::detached()),
753 SyntaxNode::leaf(SyntaxKind::MathIdent, ident, Span::detached()),
754 ]);
755 }
756 (kind, node)
757 }
758
759 fn maybe_dot_ident(&mut self) -> Option<&str> {
760 let next_starts = match self.s.scout(1) {
761 Some(c) => is_math_id_start(c),
762 None => false,
763 };
764 if next_starts && self.s.eat_if('.') {
765 let start = self.s.cursor();
766 self.s.eat();
767 self.s.eat_while(is_math_id_continue);
768 Some(self.s.from(start))
769 } else {
770 None
771 }
772 }
773
774 fn math_text(&mut self, start: usize, c: char) -> SyntaxKind {
775 if c.is_numeric() {
776 // A number keeps its digits and one decimal part together.
777 self.s.eat_while(char::is_numeric);
778 let mut s = self.s;
779 if s.eat_if('.') && !s.eat_while(char::is_numeric).is_empty() {
780 self.s = s;
781 }
782 } else {
783 let rest = self.s.get(start, self.s.string().len());
784 self.s.jump(start + grapheme_len(rest));
785 }
786 SyntaxKind::MathText
787 }
788
789 /// A named argument's name, `name:` but not `:=` or `::=`, at `start`; the cursor is restored when
790 /// there is none.
791 pub fn maybe_math_named_arg(&mut self, start: usize) -> Option<SyntaxNode> {
792 let cursor = self.s.cursor();
793 self.s.jump(start);
794 if self.s.eat_if(is_id_start) {
795 self.s.eat_while(is_id_continue);
796 if self.s.at(':') && !self.s.at(":=") && !self.s.at("::=") {
797 let text = self.s.from(start);
798 let node = if text != "_" {
799 SyntaxNode::leaf(SyntaxKind::Ident, text, Span::detached())
800 } else {
801 SyntaxNode::error(text, "expected identifier, found underscore", Span::detached())
802 };
803 return Some(node);
804 }
805 }
806 self.s.jump(cursor);
807 None
808 }
809
810 /// A spread's `..` at `start`, unless it spreads nothing or is the `...` shorthand.
811 pub fn maybe_math_spread_arg(&mut self, start: usize) -> Option<SyntaxNode> {
812 let cursor = self.s.cursor();
813 self.s.jump(start);
814 if self.s.eat_if("..") && !self.space_or_end() && !self.s.at(['.', ',', ';', ')', '$']) {
815 return Some(SyntaxNode::leaf(SyntaxKind::Dots, self.s.from(start), Span::detached()));
816 }
817 self.s.jump(cursor);
818 None
819 }
820}
821
822// Code.
823impl Lexer<'_> {
824 fn code(&mut self, start: usize, c: char) -> SyntaxKind {
825 match c {
826 '<' if self.s.at(is_id_continue) => self.label(),
827 '0'..='9' => self.number(start, c),
828 '.' if self.s.at(|c: char| c.is_ascii_digit()) => self.number(start, c),
829 '"' => self.string(),
830
831 '=' if self.s.eat_if('=') => SyntaxKind::EqEq,
832 '!' if self.s.eat_if('=') => SyntaxKind::ExclEq,
833 '<' if self.s.eat_if('=') => SyntaxKind::LtEq,
834 '>' if self.s.eat_if('=') => SyntaxKind::GtEq,
835 '+' if self.s.eat_if('=') => SyntaxKind::PlusEq,
836 '-' | '\u{2212}' if self.s.eat_if('=') => SyntaxKind::HyphEq,
837 '*' if self.s.eat_if('=') => SyntaxKind::StarEq,
838 '/' if self.s.eat_if('=') => SyntaxKind::SlashEq,
839 '.' if self.s.eat_if('.') => SyntaxKind::Dots,
840 '=' if self.s.eat_if('>') => SyntaxKind::Arrow,
841
842 '{' => SyntaxKind::LeftBrace,
843 '}' => SyntaxKind::RightBrace,
844 '[' => SyntaxKind::LeftBracket,
845 ']' => SyntaxKind::RightBracket,
846 '(' => SyntaxKind::LeftParen,
847 ')' => SyntaxKind::RightParen,
848 '$' => SyntaxKind::Dollar,
849 ',' => SyntaxKind::Comma,
850 ';' => SyntaxKind::Semicolon,
851 ':' => SyntaxKind::Colon,
852 '.' => SyntaxKind::Dot,
853 '+' => SyntaxKind::Plus,
854 '-' | '\u{2212}' => SyntaxKind::Minus,
855 '*' => SyntaxKind::Star,
856 '/' => SyntaxKind::Slash,
857 '=' => SyntaxKind::Eq,
858 '<' => SyntaxKind::Lt,
859 '>' => SyntaxKind::Gt,
860
861 c if is_id_start(c) => self.ident(start),
862
863 c => self.invalid_char_in_code(c),
864 }
865 }
866
867 // Typst's hints for the operators people bring from other languages.
868 fn invalid_char_in_code(&mut self, c: char) -> SyntaxKind {
869 let invalid_char = format!("the character `{}` is not valid in code", c);
870 match c {
871 _ if self.s.scout(-2) == Some('#') => {
872 self.error(invalid_char);
873 self.hint("the preceding hash is causing this to parse in code mode");
874 self.hint("try escaping the preceding hash: `\\#`");
875 },
876 '#' => {
877 self.error(invalid_char);
878 self.hint("you are already in code mode");
879 self.hint("try removing the `#`");
880 },
881 '&' if self.s.eat_if('&') => {
882 self.error("`&&` is not valid in code".to_string());
883 self.hint("in Typst, `and` is used for logical AND");
884 },
885 '|' if self.s.eat_if('|') => {
886 self.error("`||` is not valid in code".to_string());
887 self.hint("in Typst, `or` is used for logical OR");
888 },
889 '!' => {
890 self.error(invalid_char);
891 self.hint("in Typst, `not` is used for negation");
892 self.hint("or did you mean to write `!=` for not-equal?");
893 },
894 '~' if self.s.eat_if('=') => {
895 self.error("`~=` is not valid in code".to_string());
896 self.hint("in Typst, `!=` is used for not-equal");
897 },
898 _ => {
899 self.error(invalid_char);
900 },
901 }
902 SyntaxKind::Error
903 }
904
905 fn ident(&mut self, start: usize) -> SyntaxKind {
906 self.s.eat_while(is_id_continue);
907 let ident = self.s.from(start);
908 let prev = self.s.get(0, start);
909 // After a dot or an at sign the word is a field or a reference, never a keyword.
910 if !prev.ends_with(['.', '@']) || prev.ends_with("..") {
911 if let Some(k) = keyword(ident) {
912 return k;
913 }
914 }
915 if ident == "_" { SyntaxKind::Underscore } else { SyntaxKind::Ident }
916 }
917
918 /// An integer, a float or a numeric with a unit; only base-ten numbers may carry a unit.
919 fn number(&mut self, start: usize, first: char) -> SyntaxKind {
920 let base = match first {
921 '0' if self.s.eat_if('b') => 2,
922 '0' if self.s.eat_if('o') => 8,
923 '0' if self.s.eat_if('x') => 16,
924 _ => 10,
925 };
926
927 if base == 16 {
928 self.s.eat_while(|c: char| c.is_ascii_alphanumeric());
929 } else {
930 self.s.eat_while(|c: char| c.is_ascii_digit());
931 }
932
933 let mut is_float = false;
934 if base == 10 {
935 // Not a spread (`1..`) and not a method call (`1.abs`).
936 if first == '.' {
937 is_float = true;
938 } else {
939 let id_follows = match self.s.scout(1) {
940 Some(c) => is_id_start(c),
941 None => false,
942 };
943 if !self.s.at("..") && !id_follows && self.s.eat_if('.') {
944 is_float = true;
945 self.s.eat_while(|c: char| c.is_ascii_digit());
946 }
947 }
948 if !self.s.at("em") && self.s.eat_if(['e', 'E']) {
949 is_float = true;
950 self.s.eat_if(['+', '-']);
951 self.s.eat_while(|c: char| c.is_ascii_digit());
952 }
953 }
954
955 let number = self.s.from(start);
956 let suffix = self.s.eat_while(|c: char| c.is_ascii_alphanumeric() || c == '%');
957
958 // A base-ten integer too large for i64 reads as a float.
959 if base == 10 && !is_float && number.parse::<i64>().is_err() && number.parse::<f64>().is_ok() {
960 let overflow = number.chars().all(|c| c.is_ascii_digit()) && !number.is_empty();
961 if overflow {
962 is_float = true;
963 }
964 }
965
966 let mut suffix_res: Result<bool, String> = match suffix {
967 "" => Ok(false),
968 "pt" | "mm" | "cm" | "in" | "deg" | "rad" | "em" | "fr" | "%" => Ok(true),
969 _ =>
970 Err(format!("invalid number suffix: `{}`", suffix)),
971 };
972
973 let number_res: Result<(), String> = if is_float && number.parse::<f64>().is_err() {
974 Err(format!("invalid floating point number: `{}`", number))
975 } else if base == 10 {
976 Ok(())
977 } else {
978 let name = match base {
979 2 => "binary",
980 8 => "octal",
981 _ => "hexadecimal",
982 };
983 let digits = number.get(2..).unwrap_or("");
984 match i64::from_str_radix(digits, base) {
985 Ok(_) if suffix.is_empty() => Ok(()),
986 Ok(value) => {
987 if suffix_res.is_ok() {
988 suffix_res = Err(format!("try using a decimal number: `{}{}`", value, suffix));
989 }
990 Err(format!("{} numbers cannot have a suffix", name))
991 },
992 Err(_) if digits.is_empty() =>
993 Err(format!("expected a{} {} number", if base == 8 { "n" } else { "" }, name)),
994 Err(_) => Err(format!("invalid {} number: `{}`", name, number)),
995 }
996 };
997
998 match (number_res, suffix_res) {
999 (Ok(()), Ok(false)) if is_float => SyntaxKind::Float,
1000 (Ok(()), Ok(false)) => SyntaxKind::Int,
1001 (Ok(()), Ok(true)) => SyntaxKind::Numeric,
1002 (Err(n), Err(s)) => {
1003 let k = self.error(n);
1004 self.hint(&s);
1005 k
1006 },
1007 (Ok(()), Err(m)) => self.error(m),
1008 (Err(m), Ok(_)) => self.error(m),
1009 }
1010 }
1011
1012 fn string(&mut self) -> SyntaxKind {
1013 let mut escaped = false;
1014 self.s.eat_until(|c: char| {
1015 let stop = c == '"' && !escaped;
1016 escaped = c == '\\' && !escaped;
1017 stop
1018 });
1019 if !self.s.eat_if('"') {
1020 return self.error("unclosed string".to_string());
1021 }
1022 SyntaxKind::Str
1023 }
1024}
1025
1026fn keyword(ident: &str) -> Option<SyntaxKind> {
1027 Some(match ident {
1028 "none" => SyntaxKind::None,
1029 "auto" => SyntaxKind::Auto,
1030 "true" => SyntaxKind::Bool,
1031 "false" => SyntaxKind::Bool,
1032 "not" => SyntaxKind::Not,
1033 "and" => SyntaxKind::And,
1034 "or" => SyntaxKind::Or,
1035 "let" => SyntaxKind::Let,
1036 "set" => SyntaxKind::Set,
1037 "show" => SyntaxKind::Show,
1038 "context" => SyntaxKind::Context,
1039 "if" => SyntaxKind::If,
1040 "else" => SyntaxKind::Else,
1041 "for" => SyntaxKind::For,
1042 "in" => SyntaxKind::In,
1043 "while" => SyntaxKind::While,
1044 "break" => SyntaxKind::Break,
1045 "continue" => SyntaxKind::Continue,
1046 "return" => SyntaxKind::Return,
1047 "import" => SyntaxKind::Import,
1048 "include" => SyntaxKind::Include,
1049 "as" => SyntaxKind::As,
1050 _ => return None,
1051 })
1052}
1053
1054// ┌───────────────────────────────────────────────────────────────────────────────────────────┐
1055// │ Character classes │
1056// └───────────────────────────────────────────────────────────────────────────────────────────┘
1057
1058fn is_space(c: char, mode: LexMode) -> bool {
1059 match mode {
1060 LexMode::Markup => matches!(c, ' ' | '\t') || is_newline(c),
1061 _ => c.is_whitespace(),
1062 }
1063}
1064
1065/// Is the char one Typst reads as a newline (LF, VT, FF, CR, NEL, LS, PS)?
1066pub fn is_newline(c: char) -> bool {
1067 matches!(c, '\n' | '\x0B' | '\x0C' | '\r' | '\u{0085}' | '\u{2028}' | '\u{2029}')
1068}
1069
1070pub fn is_xid_start(c: char) -> bool { property::is_xid_start(c) }
1071
1072pub fn is_xid_continue(c: char) -> bool { property::is_xid_continue(c) }
1073
1074/// Can the char start an identifier? XID_Start plus the underscore.
1075pub fn is_id_start(c: char) -> bool { is_xid_start(c) || c == '_' }
1076
1077/// Can the char continue an identifier? XID_Continue plus the underscore and the hyphen.
1078pub fn is_id_continue(c: char) -> bool { is_xid_continue(c) || c == '_' || c == '-' }
1079
1080/// Is the string a valid Typst identifier?
1081pub fn is_ident(s: &str) -> bool {
1082 let mut chars = s.chars();
1083 match chars.next() {
1084 Some(c) => is_id_start(c) && chars.all(is_id_continue),
1085 None => false,
1086 }
1087}
1088
1089fn is_math_id_start(c: char) -> bool { is_xid_start(c) }
1090
1091fn is_math_id_continue(c: char) -> bool { is_xid_continue(c) && c != '_' }
1092
1093fn is_valid_in_label_literal(c: char) -> bool { is_id_continue(c) || matches!(c, ':' | '.') }
1094
1095/// Is the string usable as the name inside a `<label>` literal?
1096pub fn is_valid_label_literal_id(id: &str) -> bool {
1097 !id.is_empty() && id.chars().all(is_valid_in_label_literal)
1098}
1099
1100/// Is the char in the Han, Hiragana, Katakana or Hangul script, where `*` and `_` never sit inside a word?
1101fn is_cjk(c: char) -> bool {
1102 !c.is_ascii() && matches!(Script::of(c), Script::Han | Script::Hiragana | Script::Katakana | Script::Hangul)
1103}
1104
1105pub fn is_math_opening(c: char) -> bool { math::class(c) == Some(MathClass::Opening) }
1106
1107pub fn is_math_closing(c: char) -> bool { math::class(c) == Some(MathClass::Closing) }
1108
1109/// Does the char have the Unicode maths class Alphabetic? Typst's own class overrides touch no letter.
1110pub fn is_math_alphabetic_class(c: char) -> bool { math::class(c) == Some(MathClass::Alphabetic) }
1111
1112/// The byte length of the first extended grapheme cluster of `s`, reading only that cluster.
1113pub fn grapheme_len(s: &str) -> usize { segment::grapheme_len_at(s, 0) }
1114
1115/// The prefix of `text` that an automatic link takes, and whether its brackets balance. Trailing
1116/// punctuation likely to be prose is left out.
1117pub fn link_prefix(text: &str) -> (&str, bool) {
1118 let mut s = Scanner::new(text);
1119 let mut brackets: Vec<u8> = Vec::new();
1120 s.eat_while(|c: char| match c {
1121 '0'..='9' | 'a'..='z' | 'A'..='Z'
1122 | '!' | '#' | '$' | '%' | '&' | '*' | '+'
1123 | ',' | '-' | '.' | '/' | ':' | ';' | '='
1124 | '?' | '@' | '_' | '~' | '\'' => true,
1125 '[' => { brackets.push(b'['); true },
1126 '(' => { brackets.push(b'('); true },
1127 ']' => brackets.pop() == Some(b'['),
1128 ')' => brackets.pop() == Some(b'('),
1129 _ => false,
1130 });
1131 while matches!(s.scout(-1), Some('!' | ',' | '.' | ':' | ';' | '?' | '\'')) {
1132 s.uneat();
1133 }
1134 (s.before(), brackets.is_empty())
1135}
1136
1137/// Splits at newlines, a CR LF pair counting as one; the newlines are dropped.
1138pub fn split_newlines(text: &str) -> Vec<&str> {
1139 let mut s = Scanner::new(text);
1140 let mut lines = Vec::new();
1141 let mut start = 0;
1142 let mut end = 0;
1143 while let Some(c) = s.eat() {
1144 if is_newline(c) {
1145 if c == '\r' {
1146 s.eat_if('\n');
1147 }
1148 lines.push(s.get(start, end));
1149 start = s.cursor();
1150 }
1151 end = s.cursor();
1152 }
1153 lines.push(s.get(start, text.len()));
1154 lines
1155}
1156
1157fn count_newlines(text: &str) -> usize {
1158 let mut n = 0;
1159 let mut s = Scanner::new(text);
1160 while let Some(c) = s.eat() {
1161 if is_newline(c) {
1162 if c == '\r' {
1163 s.eat_if('\n');
1164 }
1165 n += 1;
1166 }
1167 }
1168 n
1169}