Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tests/eval_oracle/markup.rs

4.9 KiB, 1 run

created by r1870400018:63620, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// A tag tokenizer for the three markup outputs the oracle is read through: Typst's HTML export, its
2// SVG pages and `pdftotext -bbox-layout`. None of them needs a DOM, only the tag stream in order,
3// so this is a scanner, not a parser; it does not validate nesting.
4
5#[derive(Clone, Debug)]
6pub enum Tok {
7 Open { name: String, attrs: Vec<(String, String)>, closed: bool }, // `closed` for `<x/>`
8 Close(String),
9 Text(String),
10}
11
12impl Tok {
13 pub fn attr<'a>(attrs: &'a [(String, String)], key: &str) -> Option<&'a str> {
14 attrs.iter().find(|(k, _)| k == key).map(|(_, v)| v.as_str())
15 }
16}
17
18/// Elements whose text is not document text.
19const RAW_TEXT: [&str; 2] = ["style", "script"];
20
21pub fn tokens(src: &str) -> Vec<Tok> {
22 let b = src.as_bytes();
23 let mut out = Vec::new();
24 let mut i = 0;
25 let mut text_start = 0;
26 while i < b.len() {
27 if b[i] != b'<' {
28 i += 1;
29 continue;
30 }
31 if i > text_start {
32 out.push(Tok::Text(decode(&src[text_start..i])));
33 }
34 // Comments, doctype and processing instructions carry nothing.
35 if src[i..].starts_with("<!--") {
36 i = match src[i..].find("-->") {
37 Some(k) => i + k + 3,
38 None => b.len(),
39 };
40 text_start = i;
41 continue;
42 }
43 if src[i..].starts_with("<!") || src[i..].starts_with("<?") {
44 i = match src[i..].find('>') {
45 Some(k) => i + k + 1,
46 None => b.len(),
47 };
48 text_start = i;
49 continue;
50 }
51 let end = match tag_end(b, i + 1) {
52 Some(e) => e,
53 None => b.len() - 1,
54 };
55 let inner = &src[i + 1..end];
56 i = end + 1;
57 text_start = i;
58 if let Some(name) = inner.strip_prefix('/') {
59 out.push(Tok::Close(name.trim().to_ascii_lowercase()));
60 continue;
61 }
62 let closed = inner.ends_with('/');
63 let inner = inner.trim_end_matches('/');
64 let (name, attrs) = split_tag(inner);
65 let raw = RAW_TEXT.contains(&name.as_str()) && !closed;
66 out.push(Tok::Open { name: name.clone(), attrs, closed });
67 if raw {
68 let close = fmt_close(&name);
69 let k = match src[i..].find(&close) {
70 Some(k) => i + k,
71 None => b.len(),
72 };
73 i = k;
74 text_start = k;
75 }
76 }
77 if text_start < b.len() {
78 out.push(Tok::Text(decode(&src[text_start..])));
79 }
80 out
81}
82
83fn fmt_close(name: &str) -> String {
84 let mut s = String::from("</");
85 s.push_str(name);
86 s
87}
88
89// The `>` that ends a tag opened just before `from`, skipping quoted attribute values.
90fn tag_end(b: &[u8], from: usize) -> Option<usize> {
91 let mut q: Option<u8> = None;
92 let mut i = from;
93 while i < b.len() {
94 let c = b[i];
95 match q {
96 Some(qc) if c == qc => q = None,
97 Some(_) => (),
98 None if c == b'"' || c == b'\'' => q = Some(c),
99 None if c == b'>' => return Some(i),
100 None => (),
101 }
102 i += 1;
103 }
104 None
105}
106
107fn split_tag(inner: &str) -> (String, Vec<(String, String)>) {
108 let inner = inner.trim();
109 let n = inner.find(|c: char| c.is_whitespace()).unwrap_or(inner.len());
110 let name = inner[..n].to_ascii_lowercase();
111 let mut attrs = Vec::new();
112 let rest = inner[n..].as_bytes();
113 let s = &inner[n..];
114 let mut i = 0;
115 while i < rest.len() {
116 while i < rest.len() && rest[i].is_ascii_whitespace() {
117 i += 1;
118 }
119 let k0 = i;
120 while i < rest.len() && rest[i] != b'=' && !rest[i].is_ascii_whitespace() {
121 i += 1;
122 }
123 let key = s[k0..i].to_ascii_lowercase();
124 if key.is_empty() {
125 i += 1;
126 continue;
127 }
128 while i < rest.len() && rest[i].is_ascii_whitespace() {
129 i += 1;
130 }
131 if i < rest.len() && rest[i] == b'=' {
132 i += 1;
133 while i < rest.len() && rest[i].is_ascii_whitespace() {
134 i += 1;
135 }
136 if i < rest.len() && (rest[i] == b'"' || rest[i] == b'\'') {
137 let q = rest[i];
138 let v0 = i + 1;
139 i = v0;
140 while i < rest.len() && rest[i] != q {
141 i += 1;
142 }
143 attrs.push((key, decode(&s[v0..i.min(rest.len())])));
144 i += 1;
145 } else {
146 let v0 = i;
147 while i < rest.len() && !rest[i].is_ascii_whitespace() {
148 i += 1;
149 }
150 attrs.push((key, decode(&s[v0..i])));
151 }
152 } else {
153 attrs.push((key, String::new()));
154 }
155 }
156 (name, attrs)
157}
158
159/// Decodes the character references HTML, SVG and poppler's XML emit.
160pub fn decode(s: &str) -> String {
161 if !s.contains('&') {
162 return s.to_string();
163 }
164 let mut out = String::with_capacity(s.len());
165 let mut rest = s;
166 while let Some(k) = rest.find('&') {
167 out.push_str(&rest[..k]);
168 let after = &rest[k + 1..];
169 let semi = match after.find(';') {
170 Some(p) if p <= 10 => p,
171 _ => {
172 out.push('&');
173 rest = after;
174 continue;
175 }
176 };
177 let ent = &after[..semi];
178 let ch = match ent {
179 "amp" => Some('&'),
180 "lt" => Some('<'),
181 "gt" => Some('>'),
182 "quot" => Some('"'),
183 "apos" => Some('\''),
184 "nbsp" => Some('\u{a0}'),
185 _ if ent.starts_with("#x") || ent.starts_with("#X") =>
186 u32::from_str_radix(&ent[2..], 16).ok().and_then(char::from_u32),
187 _ if ent.starts_with('#') => ent[1..].parse::<u32>().ok().and_then(char::from_u32),
188 _ => None,
189 };
190 match ch {
191 Some(c) => {
192 out.push(c);
193 rest = &after[semi + 1..];
194 }
195 None => {
196 out.push('&');
197 rest = after;
198 }
199 }
200 }
201 out.push_str(rest);
202 out
203}