oxedyne/fe2o3/fe2o3_austenite/src/eval/lib/string.rs
13.2 KiB, 12 runs
created by r1870400018:63488, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | // U3 owns this file. Methods on `str` (and `str.to-unicode`/`str.from-unicode`). Indices are byte |
| 2 | // offsets and `first`/`last`/`at`/`rev` work in grapheme clusters, as in Typst. Regex matching walks |
| 3 | // `fe2o3_text::regex`'s iterators once over the whole text, so `^` and `\b` see the true context and a |
| 4 | // long text is not searched afresh from every match. |
| 5 | |
| 6 | use crate::diag::DiagnosticKind; |
| 7 | use crate::eval::args::Args; |
| 8 | use crate::eval::lib::foundations::{ |
| 9 | finish, |
| 10 | int_of, |
| 11 | locate_bound, |
| 12 | mismatch, |
| 13 | need, |
| 14 | receiver, |
| 15 | slice_bounds, |
| 16 | str_of, |
| 17 | }; |
| 18 | use crate::eval::scope::Scope; |
| 19 | use crate::eval::value::{ |
| 20 | Alignment, |
| 21 | Dict, |
| 22 | HAlign, |
| 23 | RegexValue, |
| 24 | Value, |
| 25 | }; |
| 26 | use crate::eval::Engine; |
| 27 | use crate::syntax::Span; |
| 28 | |
| 29 | use oxedyne_fe2o3_core::prelude::*; |
| 30 | use oxedyne_fe2o3_text::unicode::norm; |
| 31 | use oxedyne_fe2o3_text::unicode::segment; |
| 32 | |
| 33 | use std::sync::Arc; |
| 34 | |
| 35 | native_fns! { |
| 36 | pub enum StrFn { |
| 37 | Len => "len", |
| 38 | First => "first", |
| 39 | Last => "last", |
| 40 | At => "at", |
| 41 | Slice => "slice", |
| 42 | Clusters => "clusters", |
| 43 | Codepoints => "codepoints", |
| 44 | Contains => "contains", |
| 45 | StartsWith => "starts-with", |
| 46 | EndsWith => "ends-with", |
| 47 | Find => "find", |
| 48 | Position => "position", |
| 49 | Match => "match", |
| 50 | Matches => "matches", |
| 51 | Replace => "replace", |
| 52 | Rev => "rev", |
| 53 | Split => "split", |
| 54 | Trim => "trim", |
| 55 | ToUnicode => "to-unicode", |
| 56 | FromUnicode => "from-unicode", |
| 57 | Normalize => "normalize", |
| 58 | } |
| 59 | } |
| 60 | |
| 61 | pub fn define(_scope: &mut Scope) {} |
| 62 | |
| 63 | pub fn method(name: &str) -> Option<StrFn> { |
| 64 | StrFn::ALL.iter().copied().find(|f| f.name() == name |
| 65 | && !matches!(f, StrFn::ToUnicode | StrFn::FromUnicode)) |
| 66 | } |
| 67 | |
| 68 | /// `str.to-unicode` and `str.from-unicode`, reached through the type rather than a receiver. |
| 69 | pub fn static_fn(name: &str) -> Option<StrFn> { |
| 70 | match name { |
| 71 | "to-unicode" => Some(StrFn::ToUnicode), |
| 72 | "from-unicode" => Some(StrFn::FromUnicode), |
| 73 | _ => None, |
| 74 | } |
| 75 | } |
| 76 | |
| 77 | // A pattern argument: a string matched literally or a regex. |
| 78 | enum Pattern { |
| 79 | Str(Arc<String>), |
| 80 | Regex(Arc<RegexValue>), |
| 81 | } |
| 82 | |
| 83 | fn pattern(engine: &mut Engine, span: Span, v: Value) -> Outcome<Pattern> { |
| 84 | if let Some(s) = v.str_cast() { |
| 85 | return Ok(Pattern::Str(s)); |
| 86 | } |
| 87 | match v { |
| 88 | Value::Regex(r) => Ok(Pattern::Regex(r)), |
| 89 | other => Err(mismatch(engine, span, "string or regular expression", &other)), |
| 90 | } |
| 91 | } |
| 92 | |
| 93 | /// A match: its byte range and, for a regex, its groups' texts (`none` for a group that took no part). |
| 94 | struct Hit { |
| 95 | a: usize, |
| 96 | b: usize, |
| 97 | caps: Vec<Value>, |
| 98 | } |
| 99 | |
| 100 | fn regex_error(engine: &mut Engine, span: Span, e: Error<ErrTag>) -> Error<ErrTag> { |
| 101 | engine.error(DiagnosticKind::Type, span, fmt!("regex search failed: {}", e.msgs().last().cloned().unwrap_or_default())) |
| 102 | } |
| 103 | |
| 104 | fn hit(c: &oxedyne_fe2o3_text::regex::Captures) -> Hit { |
| 105 | let w = c.whole(); |
| 106 | let caps = (1..c.len()).map(|i| match c.text(i) { |
| 107 | Some(t) => Value::str(t), |
| 108 | None => Value::None, |
| 109 | }).collect(); |
| 110 | Hit { a: w.start, b: w.end, caps } |
| 111 | } |
| 112 | |
| 113 | /// The first `limit` non-overlapping matches, left to right, by the `regex` crate's iteration rule (an |
| 114 | /// empty match directly after the previous match is passed over). One pass over the text. |
| 115 | fn find_all(engine: &mut Engine, span: Span, pat: &Pattern, hay: &str, limit: usize) -> Outcome<Vec<Hit>> { |
| 116 | let mut out = Vec::new(); |
| 117 | match pat { |
| 118 | Pattern::Str(p) => for (i, m) in hay.match_indices(p.as_str()).take(limit) { |
| 119 | out.push(Hit { a: i, b: i + m.len(), caps: Vec::new() }); |
| 120 | }, |
| 121 | Pattern::Regex(re) => for c in re.re.captures_iter(hay).take(limit) { |
| 122 | match c { |
| 123 | Ok(c) => out.push(hit(&c)), |
| 124 | Err(e) => return Err(regex_error(engine, span, e)), |
| 125 | } |
| 126 | }, |
| 127 | } |
| 128 | Ok(out) |
| 129 | } |
| 130 | |
| 131 | fn first_match(engine: &mut Engine, span: Span, pat: &Pattern, hay: &str) -> Outcome<Option<Hit>> { |
| 132 | match pat { |
| 133 | Pattern::Str(p) => Ok(hay.find(p.as_str()).map(|i| Hit { a: i, b: i + p.len(), caps: Vec::new() })), |
| 134 | Pattern::Regex(re) => match re.re.captures(hay) { |
| 135 | Ok(c) => Ok(c.map(|c| hit(&c))), |
| 136 | Err(e) => Err(regex_error(engine, span, e)), |
| 137 | }, |
| 138 | } |
| 139 | } |
| 140 | |
| 141 | fn match_dict(hay: &str, h: Hit) -> Value { |
| 142 | let mut d = Dict::new(); |
| 143 | d.insert("start", Value::Int(h.a as i64)); |
| 144 | d.insert("end", Value::Int(h.b as i64)); |
| 145 | d.insert("text", Value::str(hay.get(h.a..h.b).unwrap_or(""))); |
| 146 | d.insert("captures", Value::array(h.caps)); |
| 147 | Value::dict(d) |
| 148 | } |
| 149 | |
| 150 | fn text_recv(engine: &mut Engine, args: &mut Args) -> Outcome<Arc<String>> { |
| 151 | let v = res!(receiver(args)); |
| 152 | str_of(engine, args.span, v) |
| 153 | } |
| 154 | |
| 155 | pub fn call(f: StrFn, engine: &mut Engine, mut args: Args) -> Outcome<Value> { |
| 156 | let span = args.span; |
| 157 | if let StrFn::FromUnicode = f { |
| 158 | let v = res!(need(engine, &mut args, "value")); |
| 159 | let i = res!(int_of(engine, span, v)); |
| 160 | res!(finish(engine, args)); |
| 161 | if i < 0 { |
| 162 | return Err(engine.error(DiagnosticKind::Type, span, "number must be at least zero")); |
| 163 | } |
| 164 | return match u32::try_from(i).ok().and_then(char::from_u32) { |
| 165 | Some(c) => Ok(Value::str(c.to_string())), |
| 166 | None => Err(engine.error(DiagnosticKind::Type, span, fmt!("{:#x} is not a valid codepoint", i))), |
| 167 | }; |
| 168 | } |
| 169 | if let StrFn::ToUnicode = f { |
| 170 | let s = res!(text_recv(engine, &mut args)); |
| 171 | res!(finish(engine, args)); |
| 172 | let mut cs = s.chars(); |
| 173 | return match (cs.next(), cs.next()) { |
| 174 | (Some(c), None) => Ok(Value::Int(c as i64)), |
| 175 | _ => Err(engine.error(DiagnosticKind::Type, span, "expected exactly one character")), |
| 176 | }; |
| 177 | } |
| 178 | let s = res!(text_recv(engine, &mut args)); |
| 179 | let out = match f { |
| 180 | StrFn::Len => Value::Int(s.len() as i64), |
| 181 | StrFn::First | StrFn::Last => { |
| 182 | let default = res!(args.named::<Value>("default")); |
| 183 | let gs = segment::graphemes(&s); |
| 184 | let g = if f == StrFn::First { gs.first() } else { gs.last() }; |
| 185 | match (g, default) { |
| 186 | (Some(g), _) => Value::str(*g), |
| 187 | (None, Some(d)) => d, |
| 188 | (None, None) => return Err(engine.error(DiagnosticKind::Type, span, "string is empty")), |
| 189 | } |
| 190 | } |
| 191 | StrFn::At => { |
| 192 | let i = res!(need(engine, &mut args, "index")); |
| 193 | let i = res!(int_of(engine, span, i)); |
| 194 | let default = res!(args.named::<Value>("default")); |
| 195 | let at = res!(locate_bound(engine, span, i, s.len(), &|k| s.is_char_boundary(k))); |
| 196 | let g = at.and_then(|k| segment::graphemes(&s[k..]).first().map(|g| g.to_string())); |
| 197 | match (g, default) { |
| 198 | (Some(g), _) => Value::str(g), |
| 199 | (None, Some(d)) => d, |
| 200 | (None, None) => return Err(engine.error(DiagnosticKind::Type, span, fmt!( |
| 201 | "no default value was specified and string index out of bounds (index: {}, len: {})", |
| 202 | i, s.len()))), |
| 203 | } |
| 204 | } |
| 205 | StrFn::Slice => { |
| 206 | let (a, b) = res!(slice_bounds(engine, &mut args, s.len(), "string", |k| s.is_char_boundary(k))); |
| 207 | Value::str(&s[a..b]) |
| 208 | } |
| 209 | StrFn::Clusters => Value::array(segment::graphemes(&s).into_iter().map(Value::str).collect()), |
| 210 | StrFn::Codepoints => Value::array(s.chars().map(|c| Value::str(c.to_string())).collect()), |
| 211 | StrFn::Contains | StrFn::StartsWith | StrFn::EndsWith | StrFn::Find | StrFn::Position | StrFn::Match => { |
| 212 | let p = res!(need(engine, &mut args, "pattern")); |
| 213 | let p = res!(pattern(engine, span, p)); |
| 214 | match f { |
| 215 | StrFn::Contains => Value::Bool(res!(first_match(engine, span, &p, &s)).is_some()), |
| 216 | StrFn::StartsWith => Value::Bool(match &p { |
| 217 | Pattern::Str(q) => s.starts_with(q.as_str()), |
| 218 | Pattern::Regex(_) => matches!(res!(first_match(engine, span, &p, &s)), Some(Hit { a: 0, .. })), |
| 219 | }), |
| 220 | // A regex ends the string when a search from some position finds first a match ending |
| 221 | // there, as Typst restarts its search one character past each match's start. |
| 222 | StrFn::EndsWith => Value::Bool(match &p { |
| 223 | Pattern::Str(q) => s.ends_with(q.as_str()), |
| 224 | Pattern::Regex(re) => { |
| 225 | let mut found = false; |
| 226 | for m in re.re.overlapping_iter(&s) { |
| 227 | match m { |
| 228 | Ok(m) if m.end == s.len() => { found = true; break; } |
| 229 | Ok(_) => (), |
| 230 | Err(e) => return Err(regex_error(engine, span, e)), |
| 231 | } |
| 232 | } |
| 233 | found |
| 234 | } |
| 235 | }), |
| 236 | StrFn::Find => match res!(first_match(engine, span, &p, &s)) { |
| 237 | Some(h) => Value::str(&s[h.a..h.b]), |
| 238 | None => Value::None, |
| 239 | }, |
| 240 | StrFn::Position => match res!(first_match(engine, span, &p, &s)) { |
| 241 | Some(h) => Value::Int(h.a as i64), |
| 242 | None => Value::None, |
| 243 | }, |
| 244 | _ => match res!(first_match(engine, span, &p, &s)) { |
| 245 | Some(m) => match_dict(&s, m), |
| 246 | None => Value::None, |
| 247 | }, |
| 248 | } |
| 249 | } |
| 250 | StrFn::Matches => { |
| 251 | let p = res!(need(engine, &mut args, "pattern")); |
| 252 | let p = res!(pattern(engine, span, p)); |
| 253 | let ms = res!(find_all(engine, span, &p, &s, usize::MAX)); |
| 254 | Value::array(ms.into_iter().map(|m| match_dict(&s, m)).collect()) |
| 255 | } |
| 256 | StrFn::Replace => { |
| 257 | let p = res!(need(engine, &mut args, "pattern")); |
| 258 | let p = res!(pattern(engine, span, p)); |
| 259 | let with = res!(need(engine, &mut args, "replacement")); |
| 260 | let count = match res!(args.named::<Value>("count")) { |
| 261 | None => usize::MAX, |
| 262 | Some(v) => { |
| 263 | let c = res!(int_of(engine, span, v)); |
| 264 | if c < 0 { |
| 265 | return Err(engine.error(DiagnosticKind::Type, span, "number must be at least zero")); |
| 266 | } |
| 267 | c as usize |
| 268 | } |
| 269 | }; |
| 270 | res!(finish(engine, args)); |
| 271 | return replace(engine, span, &s, &p, with, count); |
| 272 | } |
| 273 | StrFn::Rev => { |
| 274 | let mut gs = segment::graphemes(&s); |
| 275 | gs.reverse(); |
| 276 | Value::str(gs.concat()) |
| 277 | } |
| 278 | StrFn::Split => { |
| 279 | let p = res!(args.eat::<Value>()); |
| 280 | match p { |
| 281 | None | Some(Value::None) => Value::array(s.split_whitespace().map(Value::str).collect()), |
| 282 | Some(v) => { |
| 283 | let p = res!(pattern(engine, span, v)); |
| 284 | match &p { |
| 285 | Pattern::Str(q) => Value::array(s.split(q.as_str()).map(Value::str).collect()), |
| 286 | Pattern::Regex(_) => { |
| 287 | let ms = res!(find_all(engine, span, &p, &s, usize::MAX)); |
| 288 | let mut out = Vec::with_capacity(ms.len() + 1); |
| 289 | let mut last = 0; |
| 290 | for h in ms { |
| 291 | out.push(Value::str(&s[last..h.a])); |
| 292 | last = h.b; |
| 293 | } |
| 294 | out.push(Value::str(&s[last..])); |
| 295 | Value::array(out) |
| 296 | } |
| 297 | } |
| 298 | } |
| 299 | } |
| 300 | } |
| 301 | StrFn::Trim => { |
| 302 | let p = match res!(args.eat::<Value>()) { |
| 303 | None | Some(Value::None) => None, |
| 304 | Some(v) => Some(res!(pattern(engine, span, v))), |
| 305 | }; |
| 306 | let at = match res!(args.named::<Value>("at")) { |
| 307 | None | Some(Value::None) => None, |
| 308 | Some(Value::Alignment(Alignment { x: Some(HAlign::Start), y: None })) => Some(true), |
| 309 | Some(Value::Alignment(Alignment { x: Some(HAlign::End), y: None })) => Some(false), |
| 310 | Some(_) => return Err(engine.error(DiagnosticKind::Type, span, "expected either `start` or `end`")), |
| 311 | }; |
| 312 | let repeat = match res!(args.named::<Value>("repeat")) { |
| 313 | None => true, |
| 314 | Some(Value::Bool(b)) => b, |
| 315 | Some(other) => return Err(mismatch(engine, span, "boolean", &other)), |
| 316 | }; |
| 317 | Value::str(res!(trim(engine, span, &s, p, at, repeat))) |
| 318 | } |
| 319 | StrFn::Normalize => { |
| 320 | let form = match res!(args.named::<Value>("form")) { |
| 321 | None => norm::Form::Nfc, |
| 322 | Some(Value::Str(x)) => match x.as_str() { |
| 323 | "nfc" => norm::Form::Nfc, |
| 324 | "nfd" => norm::Form::Nfd, |
| 325 | "nfkc" => norm::Form::Nfkc, |
| 326 | "nfkd" => norm::Form::Nfkd, |
| 327 | _ => return Err(engine.error(DiagnosticKind::Type, span, |
| 328 | "expected \"nfc\", \"nfd\", \"nfkc\", or \"nfkd\"")), |
| 329 | }, |
| 330 | Some(other) => return Err(mismatch(engine, span, "string", &other)), |
| 331 | }; |
| 332 | Value::str(norm::normalise(&s, form)) |
| 333 | } |
| 334 | StrFn::ToUnicode | StrFn::FromUnicode => Value::None, |
| 335 | }; |
| 336 | res!(finish(engine, args)); |
| 337 | Ok(out) |
| 338 | } |
| 339 | |
| 340 | fn replace(engine: &mut Engine, span: Span, s: &str, p: &Pattern, with: Value, count: usize) -> Outcome<Value> { |
| 341 | let ms = res!(find_all(engine, span, p, s, count)); |
| 342 | let mut out = String::with_capacity(s.len()); |
| 343 | let mut last = 0; |
| 344 | for h in ms { |
| 345 | let (a, b) = (h.a, h.b); |
| 346 | out.push_str(&s[last..a]); |
| 347 | match &with { |
| 348 | Value::Str(_) | Value::Symbol(_) => match with.str_cast() { |
| 349 | Some(r) => out.push_str(&r), |
| 350 | None => return Err(mismatch(engine, span, "string or function", &with)), |
| 351 | }, |
| 352 | Value::Func(_) | Value::Type(_) => { |
| 353 | let f = res!(crate::eval::lib::array::func_of(engine, span, with.clone())); |
| 354 | let f = &f; |
| 355 | let mut fa = Args::new(span); |
| 356 | fa.push(span, match_dict(s, h)); |
| 357 | let r = res!(engine.call_func(f, fa)); |
| 358 | match r.str_cast() { |
| 359 | Some(r) => out.push_str(&r), |
| 360 | None => return Err(mismatch(engine, span, "string", &r)), |
| 361 | } |
| 362 | } |
| 363 | other => return Err(mismatch(engine, span, "string or function", other)), |
| 364 | } |
| 365 | last = b; |
| 366 | } |
| 367 | out.push_str(&s[last..]); |
| 368 | Ok(Value::str(out)) |
| 369 | } |
| 370 | |
| 371 | fn trim(engine: &mut Engine, span: Span, s: &str, p: Option<Pattern>, at: Option<bool>, repeat: bool) -> Outcome<String> { |
| 372 | let mut start = at != Some(false); |
| 373 | let end = at != Some(true); |
| 374 | let r = match p { |
| 375 | None => match at { |
| 376 | None => s.trim(), |
| 377 | Some(true) => s.trim_start(), |
| 378 | Some(false) => s.trim_end(), |
| 379 | }, |
| 380 | Some(Pattern::Str(q)) => { |
| 381 | let q = q.as_str(); |
| 382 | let mut t = s; |
| 383 | if repeat { |
| 384 | if start { t = t.trim_start_matches(q); } |
| 385 | if end { t = t.trim_end_matches(q); } |
| 386 | } else { |
| 387 | if start { t = t.strip_prefix(q).unwrap_or(t); } |
| 388 | if end { t = t.strip_suffix(q).unwrap_or(t); } |
| 389 | } |
| 390 | t |
| 391 | } |
| 392 | Some(p @ Pattern::Regex(_)) => { |
| 393 | let ms = res!(find_all(engine, span, &p, s, usize::MAX)); |
| 394 | let mut last: Option<usize> = None; |
| 395 | let (mut lo, mut hi) = (0, s.len()); |
| 396 | for Hit { a, b, .. } in ms { |
| 397 | let consecutive = last == Some(a); |
| 398 | start &= a == 0 || consecutive; |
| 399 | if start { |
| 400 | lo = b; |
| 401 | start &= repeat; |
| 402 | } |
| 403 | if end && (!consecutive || !repeat) { |
| 404 | hi = a; |
| 405 | } |
| 406 | last = Some(b); |
| 407 | } |
| 408 | if last.map(|l| l < s.len()).unwrap_or(false) { |
| 409 | hi = s.len(); |
| 410 | } |
| 411 | &s[lo..lo.max(hi)] |
| 412 | } |
| 413 | }; |
| 414 | Ok(r.to_string()) |
| 415 | } |