oxedyne/fe2o3/fe2o3_austenite/tests/eval_oracle/markup.rs
4.9 KiB, 1 run
created by r1870400018:63620, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | // A tag tokenizer for the three markup outputs the oracle is read through: Typst's HTML export, its |
| 2 | // SVG pages and `pdftotext -bbox-layout`. None of them needs a DOM, only the tag stream in order, |
| 3 | // so this is a scanner, not a parser; it does not validate nesting. |
| 4 | |
| 5 | #[derive(Clone, Debug)] |
| 6 | pub enum Tok { |
| 7 | Open { name: String, attrs: Vec<(String, String)>, closed: bool }, // `closed` for `<x/>` |
| 8 | Close(String), |
| 9 | Text(String), |
| 10 | } |
| 11 | |
| 12 | impl Tok { |
| 13 | pub fn attr<'a>(attrs: &'a [(String, String)], key: &str) -> Option<&'a str> { |
| 14 | attrs.iter().find(|(k, _)| k == key).map(|(_, v)| v.as_str()) |
| 15 | } |
| 16 | } |
| 17 | |
| 18 | /// Elements whose text is not document text. |
| 19 | const RAW_TEXT: [&str; 2] = ["style", "script"]; |
| 20 | |
| 21 | pub fn tokens(src: &str) -> Vec<Tok> { |
| 22 | let b = src.as_bytes(); |
| 23 | let mut out = Vec::new(); |
| 24 | let mut i = 0; |
| 25 | let mut text_start = 0; |
| 26 | while i < b.len() { |
| 27 | if b[i] != b'<' { |
| 28 | i += 1; |
| 29 | continue; |
| 30 | } |
| 31 | if i > text_start { |
| 32 | out.push(Tok::Text(decode(&src[text_start..i]))); |
| 33 | } |
| 34 | // Comments, doctype and processing instructions carry nothing. |
| 35 | if src[i..].starts_with("<!--") { |
| 36 | i = match src[i..].find("-->") { |
| 37 | Some(k) => i + k + 3, |
| 38 | None => b.len(), |
| 39 | }; |
| 40 | text_start = i; |
| 41 | continue; |
| 42 | } |
| 43 | if src[i..].starts_with("<!") || src[i..].starts_with("<?") { |
| 44 | i = match src[i..].find('>') { |
| 45 | Some(k) => i + k + 1, |
| 46 | None => b.len(), |
| 47 | }; |
| 48 | text_start = i; |
| 49 | continue; |
| 50 | } |
| 51 | let end = match tag_end(b, i + 1) { |
| 52 | Some(e) => e, |
| 53 | None => b.len() - 1, |
| 54 | }; |
| 55 | let inner = &src[i + 1..end]; |
| 56 | i = end + 1; |
| 57 | text_start = i; |
| 58 | if let Some(name) = inner.strip_prefix('/') { |
| 59 | out.push(Tok::Close(name.trim().to_ascii_lowercase())); |
| 60 | continue; |
| 61 | } |
| 62 | let closed = inner.ends_with('/'); |
| 63 | let inner = inner.trim_end_matches('/'); |
| 64 | let (name, attrs) = split_tag(inner); |
| 65 | let raw = RAW_TEXT.contains(&name.as_str()) && !closed; |
| 66 | out.push(Tok::Open { name: name.clone(), attrs, closed }); |
| 67 | if raw { |
| 68 | let close = fmt_close(&name); |
| 69 | let k = match src[i..].find(&close) { |
| 70 | Some(k) => i + k, |
| 71 | None => b.len(), |
| 72 | }; |
| 73 | i = k; |
| 74 | text_start = k; |
| 75 | } |
| 76 | } |
| 77 | if text_start < b.len() { |
| 78 | out.push(Tok::Text(decode(&src[text_start..]))); |
| 79 | } |
| 80 | out |
| 81 | } |
| 82 | |
| 83 | fn fmt_close(name: &str) -> String { |
| 84 | let mut s = String::from("</"); |
| 85 | s.push_str(name); |
| 86 | s |
| 87 | } |
| 88 | |
| 89 | // The `>` that ends a tag opened just before `from`, skipping quoted attribute values. |
| 90 | fn tag_end(b: &[u8], from: usize) -> Option<usize> { |
| 91 | let mut q: Option<u8> = None; |
| 92 | let mut i = from; |
| 93 | while i < b.len() { |
| 94 | let c = b[i]; |
| 95 | match q { |
| 96 | Some(qc) if c == qc => q = None, |
| 97 | Some(_) => (), |
| 98 | None if c == b'"' || c == b'\'' => q = Some(c), |
| 99 | None if c == b'>' => return Some(i), |
| 100 | None => (), |
| 101 | } |
| 102 | i += 1; |
| 103 | } |
| 104 | None |
| 105 | } |
| 106 | |
| 107 | fn split_tag(inner: &str) -> (String, Vec<(String, String)>) { |
| 108 | let inner = inner.trim(); |
| 109 | let n = inner.find(|c: char| c.is_whitespace()).unwrap_or(inner.len()); |
| 110 | let name = inner[..n].to_ascii_lowercase(); |
| 111 | let mut attrs = Vec::new(); |
| 112 | let rest = inner[n..].as_bytes(); |
| 113 | let s = &inner[n..]; |
| 114 | let mut i = 0; |
| 115 | while i < rest.len() { |
| 116 | while i < rest.len() && rest[i].is_ascii_whitespace() { |
| 117 | i += 1; |
| 118 | } |
| 119 | let k0 = i; |
| 120 | while i < rest.len() && rest[i] != b'=' && !rest[i].is_ascii_whitespace() { |
| 121 | i += 1; |
| 122 | } |
| 123 | let key = s[k0..i].to_ascii_lowercase(); |
| 124 | if key.is_empty() { |
| 125 | i += 1; |
| 126 | continue; |
| 127 | } |
| 128 | while i < rest.len() && rest[i].is_ascii_whitespace() { |
| 129 | i += 1; |
| 130 | } |
| 131 | if i < rest.len() && rest[i] == b'=' { |
| 132 | i += 1; |
| 133 | while i < rest.len() && rest[i].is_ascii_whitespace() { |
| 134 | i += 1; |
| 135 | } |
| 136 | if i < rest.len() && (rest[i] == b'"' || rest[i] == b'\'') { |
| 137 | let q = rest[i]; |
| 138 | let v0 = i + 1; |
| 139 | i = v0; |
| 140 | while i < rest.len() && rest[i] != q { |
| 141 | i += 1; |
| 142 | } |
| 143 | attrs.push((key, decode(&s[v0..i.min(rest.len())]))); |
| 144 | i += 1; |
| 145 | } else { |
| 146 | let v0 = i; |
| 147 | while i < rest.len() && !rest[i].is_ascii_whitespace() { |
| 148 | i += 1; |
| 149 | } |
| 150 | attrs.push((key, decode(&s[v0..i]))); |
| 151 | } |
| 152 | } else { |
| 153 | attrs.push((key, String::new())); |
| 154 | } |
| 155 | } |
| 156 | (name, attrs) |
| 157 | } |
| 158 | |
| 159 | /// Decodes the character references HTML, SVG and poppler's XML emit. |
| 160 | pub fn decode(s: &str) -> String { |
| 161 | if !s.contains('&') { |
| 162 | return s.to_string(); |
| 163 | } |
| 164 | let mut out = String::with_capacity(s.len()); |
| 165 | let mut rest = s; |
| 166 | while let Some(k) = rest.find('&') { |
| 167 | out.push_str(&rest[..k]); |
| 168 | let after = &rest[k + 1..]; |
| 169 | let semi = match after.find(';') { |
| 170 | Some(p) if p <= 10 => p, |
| 171 | _ => { |
| 172 | out.push('&'); |
| 173 | rest = after; |
| 174 | continue; |
| 175 | } |
| 176 | }; |
| 177 | let ent = &after[..semi]; |
| 178 | let ch = match ent { |
| 179 | "amp" => Some('&'), |
| 180 | "lt" => Some('<'), |
| 181 | "gt" => Some('>'), |
| 182 | "quot" => Some('"'), |
| 183 | "apos" => Some('\''), |
| 184 | "nbsp" => Some('\u{a0}'), |
| 185 | _ if ent.starts_with("#x") || ent.starts_with("#X") => |
| 186 | u32::from_str_radix(&ent[2..], 16).ok().and_then(char::from_u32), |
| 187 | _ if ent.starts_with('#') => ent[1..].parse::<u32>().ok().and_then(char::from_u32), |
| 188 | _ => None, |
| 189 | }; |
| 190 | match ch { |
| 191 | Some(c) => { |
| 192 | out.push(c); |
| 193 | rest = &after[semi + 1..]; |
| 194 | } |
| 195 | None => { |
| 196 | out.push('&'); |
| 197 | rest = after; |
| 198 | } |
| 199 | } |
| 200 | } |
| 201 | out.push_str(rest); |
| 202 | out |
| 203 | } |