Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/src/syntax/mod.rs

5.7 KiB, 7 runs

created by r1870400018:63566, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// SPDX-License-Identifier: Apache-2.0
2// Original work copyright (c) the Typst project authors (typst-syntax `span.rs`, `file.rs` and `source.rs`, version 0.15.1).
3// Modified for Austenite: ported to Hematite's types, IR and error handling.
4//! Contract (U0, 2026-09-23): the concrete syntax tree is lossless -- concatenating every leaf's text
5//! reproduces the source byte for byte -- and its kinds and child order follow typst-syntax 0.14/0.15,
6//! so typst-syntax's own `ast.rs` is the reference for which child an accessor reads. A parse error is an
7//! `Error` node inside the tree, never an `Err` and never a panic. U1 owns `kind.rs`, `node.rs`,
8//! `lexer.rs`, `parser.rs` and `ast.rs`; this file, `Span`, `FileId` and `Source` are fixed.
9
10pub mod ast;
11pub mod kind;
12pub mod lexer;
13pub mod node;
14pub mod parser;
15
16pub use kind::SyntaxKind;
17pub use node::SyntaxNode;
18
19
20use std::path::PathBuf;
21use std::sync::Arc;
22
23/// Which source file a span points into: an index into the evaluator's `World::sources`.
24#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
25pub struct FileId(pub u16);
26
27impl FileId {
28 pub const DETACHED: FileId = FileId(u16::MAX); // no file: synthesised content
29}
30
31impl Default for FileId {
32 fn default() -> Self { FileId::DETACHED }
33}
34
35/// A byte range within one source file. Unlike [`crate::ir::Span`] it names its file, so a diagnostic
36/// raised deep inside an imported module still reads back as `file:line:col`.
37#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
38pub struct Span {
39 pub file: FileId,
40 pub start: u32,
41 pub end: u32,
42}
43
44impl Span {
45 pub fn new(file: FileId, start: u32, end: u32) -> Self { Self { file, start, end } }
46
47 pub const fn detached() -> Self { Self { file: FileId::DETACHED, start: 0, end: 0 } }
48
49 /// Does the span point at no source (synthesised by native code)?
50 pub fn is_detached(&self) -> bool { self.file == FileId::DETACHED }
51
52 /// The smallest span covering both, or `self` when they lie in different files.
53 pub fn join(self, other: Span) -> Span {
54 if self.is_detached() {
55 return other;
56 }
57 if other.is_detached() || other.file != self.file {
58 return self;
59 }
60 Span::new(self.file, self.start.min(other.start), self.end.max(other.end))
61 }
62
63 /// The byte range alone, for the back end's file-less [`crate::ir::Span`].
64 pub fn range(&self) -> crate::ir::Span { crate::ir::Span::new(self.start, self.end) }
65}
66
67/// One source file and its parsed tree. `path` is the vfs path it was read from, so a relative import
68/// resolves against `path.parent()`.
69#[derive(Clone, Debug)]
70pub struct Source {
71 pub id: FileId,
72 pub path: PathBuf,
73 pub text: Arc<String>,
74 pub root: SyntaxNode,
75}
76
77impl Source {
78 /// Parses `text` as markup, the mode a `.typ` file opens in.
79 pub fn new(id: FileId, path: PathBuf, text: String) -> Self {
80 let root = parser::parse(&text, id);
81 Self { id, path, text: Arc::new(text), root }
82 }
83
84 /// The 1-based line and character column of a byte offset, as Typst counts them: a column is a count of
85 /// `char`s, so `é` is one column, not two.
86 pub fn line_col(&self, offset: u32) -> (usize, usize) {
87 self.line_col_in(offset, Cols::Chars)
88 }
89
90 /// The 1-based line and column of a byte offset, the column in `cols`. A line ends at any of Typst's
91 /// line breaks: `\n`, a vertical tab, a form feed, a lone `\r` and the Unicode next-line, line and
92 /// paragraph separators, with `\r\n` one break.
93 pub fn line_col_in(&self, offset: u32, cols: Cols) -> (usize, usize) {
94 let mut end = (offset as usize).min(self.text.len());
95 while !self.text.is_char_boundary(end) {
96 end -= 1;
97 }
98 let mut line = 1usize;
99 let mut col = 1usize;
100 let mut chars = self.text[..end].chars().peekable();
101 while let Some(c) = chars.next() {
102 match c {
103 '\r' => {
104 if chars.peek() == Some(&'\n') {
105 chars.next();
106 }
107 line += 1;
108 col = 1;
109 },
110 '\n' | '\u{b}' | '\u{c}' | '\u{85}' | '\u{2028}' | '\u{2029}' => {
111 line += 1;
112 col = 1;
113 },
114 c => col += match cols {
115 Cols::Chars => 1,
116 Cols::Utf16 => c.len_utf16(),
117 },
118 }
119 }
120 (line, col)
121 }
122}
123
124/// The unit a column is counted in: characters, as Typst's own diagnostics count them, or UTF-16 code units,
125/// as JavaScript indexes a string.
126#[derive(Clone, Copy, Debug, PartialEq, Eq)]
127pub enum Cols {
128 Chars,
129 Utf16,
130}
131
132#[cfg(test)]
133mod tests {
134 use super::*;
135
136 fn at(text: &str, needle: &str, cols: Cols) -> (usize, usize) {
137 let src = Source::new(FileId(0), PathBuf::from("/t.typ"), text.to_string());
138 let off = text.find(needle).unwrap_or(0) as u32;
139 src.line_col_in(off, cols)
140 }
141
142 /// Typst ends a line at `\n`, a vertical tab, a form feed, a lone `\r`, U+0085, U+2028 and U+2029,
143 /// and takes `\r\n` as one break; a column restarts after each.
144 #[test]
145 fn a_line_ends_at_every_break_typst_ends_one_at() {
146 for brk in ["\n", "\r", "\r\n", "\u{b}", "\u{c}", "\u{85}", "\u{2028}", "\u{2029}"] {
147 let text = format!("ab{}#x", brk);
148 assert_eq!(at(&text, "#x", Cols::Chars), (2, 1), "after {:?}", brk);
149 }
150 // Two breaks are two lines, and `\r\n` followed by `\n` is two.
151 assert_eq!(at("a\r\n\n#x", "#x", Cols::Chars), (3, 1));
152 assert_eq!(at("a\r\r#x", "#x", Cols::Chars), (3, 1));
153 }
154
155 /// A UTF-16 column counts a character outside the BMP twice; a character column counts it once.
156 #[test]
157 fn a_utf16_column_counts_a_character_outside_the_bmp_twice() {
158 let text = "\u{3000}\u{1F600}x";
159 assert_eq!(at(text, "x", Cols::Chars), (1, 3));
160 assert_eq!(at(text, "x", Cols::Utf16), (1, 4));
161 assert_eq!(at("abc", "c", Cols::Utf16), (1, 3));
162 // A byte offset inside a character reads as the character's start.
163 let src = Source::new(FileId(0), PathBuf::from("/t.typ"), "\u{1F600}x".to_string());
164 assert_eq!(src.line_col_in(2, Cols::Utf16), (1, 1));
165 assert_eq!(src.line_col(4), (1, 2));
166 }
167}