oxedyne/fe2o3/fe2o3_austenite/src/syntax/mod.rs
5.7 KiB, 7 runs
created by r1870400018:63566, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | // SPDX-License-Identifier: Apache-2.0 |
| 2 | // Original work copyright (c) the Typst project authors (typst-syntax `span.rs`, `file.rs` and `source.rs`, version 0.15.1). |
| 3 | // Modified for Austenite: ported to Hematite's types, IR and error handling. |
| 4 | //! Contract (U0, 2026-09-23): the concrete syntax tree is lossless -- concatenating every leaf's text |
| 5 | //! reproduces the source byte for byte -- and its kinds and child order follow typst-syntax 0.14/0.15, |
| 6 | //! so typst-syntax's own `ast.rs` is the reference for which child an accessor reads. A parse error is an |
| 7 | //! `Error` node inside the tree, never an `Err` and never a panic. U1 owns `kind.rs`, `node.rs`, |
| 8 | //! `lexer.rs`, `parser.rs` and `ast.rs`; this file, `Span`, `FileId` and `Source` are fixed. |
| 9 | |
| 10 | pub mod ast; |
| 11 | pub mod kind; |
| 12 | pub mod lexer; |
| 13 | pub mod node; |
| 14 | pub mod parser; |
| 15 | |
| 16 | pub use kind::SyntaxKind; |
| 17 | pub use node::SyntaxNode; |
| 18 | |
| 19 | |
| 20 | use std::path::PathBuf; |
| 21 | use std::sync::Arc; |
| 22 | |
| 23 | /// Which source file a span points into: an index into the evaluator's `World::sources`. |
| 24 | #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] |
| 25 | pub struct FileId(pub u16); |
| 26 | |
| 27 | impl FileId { |
| 28 | pub const DETACHED: FileId = FileId(u16::MAX); // no file: synthesised content |
| 29 | } |
| 30 | |
| 31 | impl Default for FileId { |
| 32 | fn default() -> Self { FileId::DETACHED } |
| 33 | } |
| 34 | |
| 35 | /// A byte range within one source file. Unlike [`crate::ir::Span`] it names its file, so a diagnostic |
| 36 | /// raised deep inside an imported module still reads back as `file:line:col`. |
| 37 | #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] |
| 38 | pub struct Span { |
| 39 | pub file: FileId, |
| 40 | pub start: u32, |
| 41 | pub end: u32, |
| 42 | } |
| 43 | |
| 44 | impl Span { |
| 45 | pub fn new(file: FileId, start: u32, end: u32) -> Self { Self { file, start, end } } |
| 46 | |
| 47 | pub const fn detached() -> Self { Self { file: FileId::DETACHED, start: 0, end: 0 } } |
| 48 | |
| 49 | /// Does the span point at no source (synthesised by native code)? |
| 50 | pub fn is_detached(&self) -> bool { self.file == FileId::DETACHED } |
| 51 | |
| 52 | /// The smallest span covering both, or `self` when they lie in different files. |
| 53 | pub fn join(self, other: Span) -> Span { |
| 54 | if self.is_detached() { |
| 55 | return other; |
| 56 | } |
| 57 | if other.is_detached() || other.file != self.file { |
| 58 | return self; |
| 59 | } |
| 60 | Span::new(self.file, self.start.min(other.start), self.end.max(other.end)) |
| 61 | } |
| 62 | |
| 63 | /// The byte range alone, for the back end's file-less [`crate::ir::Span`]. |
| 64 | pub fn range(&self) -> crate::ir::Span { crate::ir::Span::new(self.start, self.end) } |
| 65 | } |
| 66 | |
| 67 | /// One source file and its parsed tree. `path` is the vfs path it was read from, so a relative import |
| 68 | /// resolves against `path.parent()`. |
| 69 | #[derive(Clone, Debug)] |
| 70 | pub struct Source { |
| 71 | pub id: FileId, |
| 72 | pub path: PathBuf, |
| 73 | pub text: Arc<String>, |
| 74 | pub root: SyntaxNode, |
| 75 | } |
| 76 | |
| 77 | impl Source { |
| 78 | /// Parses `text` as markup, the mode a `.typ` file opens in. |
| 79 | pub fn new(id: FileId, path: PathBuf, text: String) -> Self { |
| 80 | let root = parser::parse(&text, id); |
| 81 | Self { id, path, text: Arc::new(text), root } |
| 82 | } |
| 83 | |
| 84 | /// The 1-based line and character column of a byte offset, as Typst counts them: a column is a count of |
| 85 | /// `char`s, so `é` is one column, not two. |
| 86 | pub fn line_col(&self, offset: u32) -> (usize, usize) { |
| 87 | self.line_col_in(offset, Cols::Chars) |
| 88 | } |
| 89 | |
| 90 | /// The 1-based line and column of a byte offset, the column in `cols`. A line ends at any of Typst's |
| 91 | /// line breaks: `\n`, a vertical tab, a form feed, a lone `\r` and the Unicode next-line, line and |
| 92 | /// paragraph separators, with `\r\n` one break. |
| 93 | pub fn line_col_in(&self, offset: u32, cols: Cols) -> (usize, usize) { |
| 94 | let mut end = (offset as usize).min(self.text.len()); |
| 95 | while !self.text.is_char_boundary(end) { |
| 96 | end -= 1; |
| 97 | } |
| 98 | let mut line = 1usize; |
| 99 | let mut col = 1usize; |
| 100 | let mut chars = self.text[..end].chars().peekable(); |
| 101 | while let Some(c) = chars.next() { |
| 102 | match c { |
| 103 | '\r' => { |
| 104 | if chars.peek() == Some(&'\n') { |
| 105 | chars.next(); |
| 106 | } |
| 107 | line += 1; |
| 108 | col = 1; |
| 109 | }, |
| 110 | '\n' | '\u{b}' | '\u{c}' | '\u{85}' | '\u{2028}' | '\u{2029}' => { |
| 111 | line += 1; |
| 112 | col = 1; |
| 113 | }, |
| 114 | c => col += match cols { |
| 115 | Cols::Chars => 1, |
| 116 | Cols::Utf16 => c.len_utf16(), |
| 117 | }, |
| 118 | } |
| 119 | } |
| 120 | (line, col) |
| 121 | } |
| 122 | } |
| 123 | |
| 124 | /// The unit a column is counted in: characters, as Typst's own diagnostics count them, or UTF-16 code units, |
| 125 | /// as JavaScript indexes a string. |
| 126 | #[derive(Clone, Copy, Debug, PartialEq, Eq)] |
| 127 | pub enum Cols { |
| 128 | Chars, |
| 129 | Utf16, |
| 130 | } |
| 131 | |
| 132 | #[cfg(test)] |
| 133 | mod tests { |
| 134 | use super::*; |
| 135 | |
| 136 | fn at(text: &str, needle: &str, cols: Cols) -> (usize, usize) { |
| 137 | let src = Source::new(FileId(0), PathBuf::from("/t.typ"), text.to_string()); |
| 138 | let off = text.find(needle).unwrap_or(0) as u32; |
| 139 | src.line_col_in(off, cols) |
| 140 | } |
| 141 | |
| 142 | /// Typst ends a line at `\n`, a vertical tab, a form feed, a lone `\r`, U+0085, U+2028 and U+2029, |
| 143 | /// and takes `\r\n` as one break; a column restarts after each. |
| 144 | #[test] |
| 145 | fn a_line_ends_at_every_break_typst_ends_one_at() { |
| 146 | for brk in ["\n", "\r", "\r\n", "\u{b}", "\u{c}", "\u{85}", "\u{2028}", "\u{2029}"] { |
| 147 | let text = format!("ab{}#x", brk); |
| 148 | assert_eq!(at(&text, "#x", Cols::Chars), (2, 1), "after {:?}", brk); |
| 149 | } |
| 150 | // Two breaks are two lines, and `\r\n` followed by `\n` is two. |
| 151 | assert_eq!(at("a\r\n\n#x", "#x", Cols::Chars), (3, 1)); |
| 152 | assert_eq!(at("a\r\r#x", "#x", Cols::Chars), (3, 1)); |
| 153 | } |
| 154 | |
| 155 | /// A UTF-16 column counts a character outside the BMP twice; a character column counts it once. |
| 156 | #[test] |
| 157 | fn a_utf16_column_counts_a_character_outside_the_bmp_twice() { |
| 158 | let text = "\u{3000}\u{1F600}x"; |
| 159 | assert_eq!(at(text, "x", Cols::Chars), (1, 3)); |
| 160 | assert_eq!(at(text, "x", Cols::Utf16), (1, 4)); |
| 161 | assert_eq!(at("abc", "c", Cols::Utf16), (1, 3)); |
| 162 | // A byte offset inside a character reads as the character's start. |
| 163 | let src = Source::new(FileId(0), PathBuf::from("/t.typ"), "\u{1F600}x".to_string()); |
| 164 | assert_eq!(src.line_col_in(2, Cols::Utf16), (1, 1)); |
| 165 | assert_eq!(src.line_col(4), (1, 2)); |
| 166 | } |
| 167 | } |