Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/src/eval/lib/string.rs

13.2 KiB, 12 runs

created by r1870400018:63488, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// U3 owns this file. Methods on `str` (and `str.to-unicode`/`str.from-unicode`). Indices are byte
2// offsets and `first`/`last`/`at`/`rev` work in grapheme clusters, as in Typst. Regex matching walks
3// `fe2o3_text::regex`'s iterators once over the whole text, so `^` and `\b` see the true context and a
4// long text is not searched afresh from every match.
5
6use crate::diag::DiagnosticKind;
7use crate::eval::args::Args;
8use crate::eval::lib::foundations::{
9 finish,
10 int_of,
11 locate_bound,
12 mismatch,
13 need,
14 receiver,
15 slice_bounds,
16 str_of,
17};
18use crate::eval::scope::Scope;
19use crate::eval::value::{
20 Alignment,
21 Dict,
22 HAlign,
23 RegexValue,
24 Value,
25};
26use crate::eval::Engine;
27use crate::syntax::Span;
28
29use oxedyne_fe2o3_core::prelude::*;
30use oxedyne_fe2o3_text::unicode::norm;
31use oxedyne_fe2o3_text::unicode::segment;
32
33use std::sync::Arc;
34
35native_fns! {
36 pub enum StrFn {
37 Len => "len",
38 First => "first",
39 Last => "last",
40 At => "at",
41 Slice => "slice",
42 Clusters => "clusters",
43 Codepoints => "codepoints",
44 Contains => "contains",
45 StartsWith => "starts-with",
46 EndsWith => "ends-with",
47 Find => "find",
48 Position => "position",
49 Match => "match",
50 Matches => "matches",
51 Replace => "replace",
52 Rev => "rev",
53 Split => "split",
54 Trim => "trim",
55 ToUnicode => "to-unicode",
56 FromUnicode => "from-unicode",
57 Normalize => "normalize",
58 }
59}
60
61pub fn define(_scope: &mut Scope) {}
62
63pub fn method(name: &str) -> Option<StrFn> {
64 StrFn::ALL.iter().copied().find(|f| f.name() == name
65 && !matches!(f, StrFn::ToUnicode | StrFn::FromUnicode))
66}
67
68/// `str.to-unicode` and `str.from-unicode`, reached through the type rather than a receiver.
69pub fn static_fn(name: &str) -> Option<StrFn> {
70 match name {
71 "to-unicode" => Some(StrFn::ToUnicode),
72 "from-unicode" => Some(StrFn::FromUnicode),
73 _ => None,
74 }
75}
76
77// A pattern argument: a string matched literally or a regex.
78enum Pattern {
79 Str(Arc<String>),
80 Regex(Arc<RegexValue>),
81}
82
83fn pattern(engine: &mut Engine, span: Span, v: Value) -> Outcome<Pattern> {
84 if let Some(s) = v.str_cast() {
85 return Ok(Pattern::Str(s));
86 }
87 match v {
88 Value::Regex(r) => Ok(Pattern::Regex(r)),
89 other => Err(mismatch(engine, span, "string or regular expression", &other)),
90 }
91}
92
93/// A match: its byte range and, for a regex, its groups' texts (`none` for a group that took no part).
94struct Hit {
95 a: usize,
96 b: usize,
97 caps: Vec<Value>,
98}
99
100fn regex_error(engine: &mut Engine, span: Span, e: Error<ErrTag>) -> Error<ErrTag> {
101 engine.error(DiagnosticKind::Type, span, fmt!("regex search failed: {}", e.msgs().last().cloned().unwrap_or_default()))
102}
103
104fn hit(c: &oxedyne_fe2o3_text::regex::Captures) -> Hit {
105 let w = c.whole();
106 let caps = (1..c.len()).map(|i| match c.text(i) {
107 Some(t) => Value::str(t),
108 None => Value::None,
109 }).collect();
110 Hit { a: w.start, b: w.end, caps }
111}
112
113/// The first `limit` non-overlapping matches, left to right, by the `regex` crate's iteration rule (an
114/// empty match directly after the previous match is passed over). One pass over the text.
115fn find_all(engine: &mut Engine, span: Span, pat: &Pattern, hay: &str, limit: usize) -> Outcome<Vec<Hit>> {
116 let mut out = Vec::new();
117 match pat {
118 Pattern::Str(p) => for (i, m) in hay.match_indices(p.as_str()).take(limit) {
119 out.push(Hit { a: i, b: i + m.len(), caps: Vec::new() });
120 },
121 Pattern::Regex(re) => for c in re.re.captures_iter(hay).take(limit) {
122 match c {
123 Ok(c) => out.push(hit(&c)),
124 Err(e) => return Err(regex_error(engine, span, e)),
125 }
126 },
127 }
128 Ok(out)
129}
130
131fn first_match(engine: &mut Engine, span: Span, pat: &Pattern, hay: &str) -> Outcome<Option<Hit>> {
132 match pat {
133 Pattern::Str(p) => Ok(hay.find(p.as_str()).map(|i| Hit { a: i, b: i + p.len(), caps: Vec::new() })),
134 Pattern::Regex(re) => match re.re.captures(hay) {
135 Ok(c) => Ok(c.map(|c| hit(&c))),
136 Err(e) => Err(regex_error(engine, span, e)),
137 },
138 }
139}
140
141fn match_dict(hay: &str, h: Hit) -> Value {
142 let mut d = Dict::new();
143 d.insert("start", Value::Int(h.a as i64));
144 d.insert("end", Value::Int(h.b as i64));
145 d.insert("text", Value::str(hay.get(h.a..h.b).unwrap_or("")));
146 d.insert("captures", Value::array(h.caps));
147 Value::dict(d)
148}
149
150fn text_recv(engine: &mut Engine, args: &mut Args) -> Outcome<Arc<String>> {
151 let v = res!(receiver(args));
152 str_of(engine, args.span, v)
153}
154
155pub fn call(f: StrFn, engine: &mut Engine, mut args: Args) -> Outcome<Value> {
156 let span = args.span;
157 if let StrFn::FromUnicode = f {
158 let v = res!(need(engine, &mut args, "value"));
159 let i = res!(int_of(engine, span, v));
160 res!(finish(engine, args));
161 if i < 0 {
162 return Err(engine.error(DiagnosticKind::Type, span, "number must be at least zero"));
163 }
164 return match u32::try_from(i).ok().and_then(char::from_u32) {
165 Some(c) => Ok(Value::str(c.to_string())),
166 None => Err(engine.error(DiagnosticKind::Type, span, fmt!("{:#x} is not a valid codepoint", i))),
167 };
168 }
169 if let StrFn::ToUnicode = f {
170 let s = res!(text_recv(engine, &mut args));
171 res!(finish(engine, args));
172 let mut cs = s.chars();
173 return match (cs.next(), cs.next()) {
174 (Some(c), None) => Ok(Value::Int(c as i64)),
175 _ => Err(engine.error(DiagnosticKind::Type, span, "expected exactly one character")),
176 };
177 }
178 let s = res!(text_recv(engine, &mut args));
179 let out = match f {
180 StrFn::Len => Value::Int(s.len() as i64),
181 StrFn::First | StrFn::Last => {
182 let default = res!(args.named::<Value>("default"));
183 let gs = segment::graphemes(&s);
184 let g = if f == StrFn::First { gs.first() } else { gs.last() };
185 match (g, default) {
186 (Some(g), _) => Value::str(*g),
187 (None, Some(d)) => d,
188 (None, None) => return Err(engine.error(DiagnosticKind::Type, span, "string is empty")),
189 }
190 }
191 StrFn::At => {
192 let i = res!(need(engine, &mut args, "index"));
193 let i = res!(int_of(engine, span, i));
194 let default = res!(args.named::<Value>("default"));
195 let at = res!(locate_bound(engine, span, i, s.len(), &|k| s.is_char_boundary(k)));
196 let g = at.and_then(|k| segment::graphemes(&s[k..]).first().map(|g| g.to_string()));
197 match (g, default) {
198 (Some(g), _) => Value::str(g),
199 (None, Some(d)) => d,
200 (None, None) => return Err(engine.error(DiagnosticKind::Type, span, fmt!(
201 "no default value was specified and string index out of bounds (index: {}, len: {})",
202 i, s.len()))),
203 }
204 }
205 StrFn::Slice => {
206 let (a, b) = res!(slice_bounds(engine, &mut args, s.len(), "string", |k| s.is_char_boundary(k)));
207 Value::str(&s[a..b])
208 }
209 StrFn::Clusters => Value::array(segment::graphemes(&s).into_iter().map(Value::str).collect()),
210 StrFn::Codepoints => Value::array(s.chars().map(|c| Value::str(c.to_string())).collect()),
211 StrFn::Contains | StrFn::StartsWith | StrFn::EndsWith | StrFn::Find | StrFn::Position | StrFn::Match => {
212 let p = res!(need(engine, &mut args, "pattern"));
213 let p = res!(pattern(engine, span, p));
214 match f {
215 StrFn::Contains => Value::Bool(res!(first_match(engine, span, &p, &s)).is_some()),
216 StrFn::StartsWith => Value::Bool(match &p {
217 Pattern::Str(q) => s.starts_with(q.as_str()),
218 Pattern::Regex(_) => matches!(res!(first_match(engine, span, &p, &s)), Some(Hit { a: 0, .. })),
219 }),
220 // A regex ends the string when a search from some position finds first a match ending
221 // there, as Typst restarts its search one character past each match's start.
222 StrFn::EndsWith => Value::Bool(match &p {
223 Pattern::Str(q) => s.ends_with(q.as_str()),
224 Pattern::Regex(re) => {
225 let mut found = false;
226 for m in re.re.overlapping_iter(&s) {
227 match m {
228 Ok(m) if m.end == s.len() => { found = true; break; }
229 Ok(_) => (),
230 Err(e) => return Err(regex_error(engine, span, e)),
231 }
232 }
233 found
234 }
235 }),
236 StrFn::Find => match res!(first_match(engine, span, &p, &s)) {
237 Some(h) => Value::str(&s[h.a..h.b]),
238 None => Value::None,
239 },
240 StrFn::Position => match res!(first_match(engine, span, &p, &s)) {
241 Some(h) => Value::Int(h.a as i64),
242 None => Value::None,
243 },
244 _ => match res!(first_match(engine, span, &p, &s)) {
245 Some(m) => match_dict(&s, m),
246 None => Value::None,
247 },
248 }
249 }
250 StrFn::Matches => {
251 let p = res!(need(engine, &mut args, "pattern"));
252 let p = res!(pattern(engine, span, p));
253 let ms = res!(find_all(engine, span, &p, &s, usize::MAX));
254 Value::array(ms.into_iter().map(|m| match_dict(&s, m)).collect())
255 }
256 StrFn::Replace => {
257 let p = res!(need(engine, &mut args, "pattern"));
258 let p = res!(pattern(engine, span, p));
259 let with = res!(need(engine, &mut args, "replacement"));
260 let count = match res!(args.named::<Value>("count")) {
261 None => usize::MAX,
262 Some(v) => {
263 let c = res!(int_of(engine, span, v));
264 if c < 0 {
265 return Err(engine.error(DiagnosticKind::Type, span, "number must be at least zero"));
266 }
267 c as usize
268 }
269 };
270 res!(finish(engine, args));
271 return replace(engine, span, &s, &p, with, count);
272 }
273 StrFn::Rev => {
274 let mut gs = segment::graphemes(&s);
275 gs.reverse();
276 Value::str(gs.concat())
277 }
278 StrFn::Split => {
279 let p = res!(args.eat::<Value>());
280 match p {
281 None | Some(Value::None) => Value::array(s.split_whitespace().map(Value::str).collect()),
282 Some(v) => {
283 let p = res!(pattern(engine, span, v));
284 match &p {
285 Pattern::Str(q) => Value::array(s.split(q.as_str()).map(Value::str).collect()),
286 Pattern::Regex(_) => {
287 let ms = res!(find_all(engine, span, &p, &s, usize::MAX));
288 let mut out = Vec::with_capacity(ms.len() + 1);
289 let mut last = 0;
290 for h in ms {
291 out.push(Value::str(&s[last..h.a]));
292 last = h.b;
293 }
294 out.push(Value::str(&s[last..]));
295 Value::array(out)
296 }
297 }
298 }
299 }
300 }
301 StrFn::Trim => {
302 let p = match res!(args.eat::<Value>()) {
303 None | Some(Value::None) => None,
304 Some(v) => Some(res!(pattern(engine, span, v))),
305 };
306 let at = match res!(args.named::<Value>("at")) {
307 None | Some(Value::None) => None,
308 Some(Value::Alignment(Alignment { x: Some(HAlign::Start), y: None })) => Some(true),
309 Some(Value::Alignment(Alignment { x: Some(HAlign::End), y: None })) => Some(false),
310 Some(_) => return Err(engine.error(DiagnosticKind::Type, span, "expected either `start` or `end`")),
311 };
312 let repeat = match res!(args.named::<Value>("repeat")) {
313 None => true,
314 Some(Value::Bool(b)) => b,
315 Some(other) => return Err(mismatch(engine, span, "boolean", &other)),
316 };
317 Value::str(res!(trim(engine, span, &s, p, at, repeat)))
318 }
319 StrFn::Normalize => {
320 let form = match res!(args.named::<Value>("form")) {
321 None => norm::Form::Nfc,
322 Some(Value::Str(x)) => match x.as_str() {
323 "nfc" => norm::Form::Nfc,
324 "nfd" => norm::Form::Nfd,
325 "nfkc" => norm::Form::Nfkc,
326 "nfkd" => norm::Form::Nfkd,
327 _ => return Err(engine.error(DiagnosticKind::Type, span,
328 "expected \"nfc\", \"nfd\", \"nfkc\", or \"nfkd\"")),
329 },
330 Some(other) => return Err(mismatch(engine, span, "string", &other)),
331 };
332 Value::str(norm::normalise(&s, form))
333 }
334 StrFn::ToUnicode | StrFn::FromUnicode => Value::None,
335 };
336 res!(finish(engine, args));
337 Ok(out)
338}
339
340fn replace(engine: &mut Engine, span: Span, s: &str, p: &Pattern, with: Value, count: usize) -> Outcome<Value> {
341 let ms = res!(find_all(engine, span, p, s, count));
342 let mut out = String::with_capacity(s.len());
343 let mut last = 0;
344 for h in ms {
345 let (a, b) = (h.a, h.b);
346 out.push_str(&s[last..a]);
347 match &with {
348 Value::Str(_) | Value::Symbol(_) => match with.str_cast() {
349 Some(r) => out.push_str(&r),
350 None => return Err(mismatch(engine, span, "string or function", &with)),
351 },
352 Value::Func(_) | Value::Type(_) => {
353 let f = res!(crate::eval::lib::array::func_of(engine, span, with.clone()));
354 let f = &f;
355 let mut fa = Args::new(span);
356 fa.push(span, match_dict(s, h));
357 let r = res!(engine.call_func(f, fa));
358 match r.str_cast() {
359 Some(r) => out.push_str(&r),
360 None => return Err(mismatch(engine, span, "string", &r)),
361 }
362 }
363 other => return Err(mismatch(engine, span, "string or function", other)),
364 }
365 last = b;
366 }
367 out.push_str(&s[last..]);
368 Ok(Value::str(out))
369}
370
371fn trim(engine: &mut Engine, span: Span, s: &str, p: Option<Pattern>, at: Option<bool>, repeat: bool) -> Outcome<String> {
372 let mut start = at != Some(false);
373 let end = at != Some(true);
374 let r = match p {
375 None => match at {
376 None => s.trim(),
377 Some(true) => s.trim_start(),
378 Some(false) => s.trim_end(),
379 },
380 Some(Pattern::Str(q)) => {
381 let q = q.as_str();
382 let mut t = s;
383 if repeat {
384 if start { t = t.trim_start_matches(q); }
385 if end { t = t.trim_end_matches(q); }
386 } else {
387 if start { t = t.strip_prefix(q).unwrap_or(t); }
388 if end { t = t.strip_suffix(q).unwrap_or(t); }
389 }
390 t
391 }
392 Some(p @ Pattern::Regex(_)) => {
393 let ms = res!(find_all(engine, span, &p, s, usize::MAX));
394 let mut last: Option<usize> = None;
395 let (mut lo, mut hi) = (0, s.len());
396 for Hit { a, b, .. } in ms {
397 let consecutive = last == Some(a);
398 start &= a == 0 || consecutive;
399 if start {
400 lo = b;
401 start &= repeat;
402 }
403 if end && (!consecutive || !repeat) {
404 hi = a;
405 }
406 last = Some(b);
407 }
408 if last.map(|l| l < s.len()).unwrap_or(false) {
409 hi = s.len();
410 }
411 &s[lo..lo.max(hi)]
412 }
413 };
414 Ok(r.to_string())
415}