Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_text/src/secret/scrub.rs

24.9 KiB, 28 runs

created by r1870400018:65664, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! Daimond's content scrubber, in Rust: every credential in a string replaced by a marker.
2//!
3//! `www/js/debugshare.js` carries a scrubber that searches each string of a debug feed for the
4//! shapes a credential wears and writes `[redacted <shape> #<hash>/<length>]` in the place of
5//! each, so that a reader still sees that a key was there and can tell two sightings apart. It is
6//! one block of that file, copied into `dev/lens.mjs` and `fe2o3_net/lens/js/redact.js`. This
7//! module is the same block in Rust, so that a peer's feed is guarded as a browser's is, and
8//! `fe2o3_net/tests/data/lens_scrub.tsv` holds the two together: a table of inputs, each stored
9//! as a recipe so that no credential is, and what the JavaScript made of every one.
10//!
11//! It is a table of its own, beside the one [`super::scan`] reads and not an extension of it.
12//! `scan` reads files a person wrote and refuses a commit, so a shape added to it is a shape that
13//! stops work, and it excuses a placeholder. This answers for text bound for a log, where all a
14//! false hit costs is a word, and it excuses nothing.
15//!
16//! # Why UTF-16, and why no regular expressions
17//!
18//! The JavaScript counts and hashes in UTF-16 code units, and a marker's hash and length are how
19//! a reader tells one sighting from another, so a string is walked here as `u16`s and a minimum
20//! length means what it means there: an astral character counts as two. For the same reason no
21//! `regex` stands in this file. JavaScript's `\b` and `/i` are ASCII-only, its `\s` holds U+FEFF
22//! and not U+0085, which is the reverse of `char::is_whitespace`, and the crate's own engine
23//! follows the `regex` crate on every one of the three.
24//!
25//! # No run is measured twice
26//!
27//! Almost every rule is a literal opening and a run of one class of character. A string with an
28//! opening every few bytes and one long run behind them would cost a rule that re-measured the run
29//! at each opening the square of its length, and a feed's author may be anyone. `Text::run`
30//! therefore remembers the last run it measured of each class, and the two rules that look again
31//! after a miss step over what they have already decided. The one place the JavaScript backtracks
32//! in earnest, an AWS secret beside its key id, is solved outright: its greedy run and its lazy
33//! gap together come to "the first separator within 200 units after the run".
34
35use oxedyne_fe2o3_core::prelude::*;
36
37use std::{
38 collections::HashMap,
39 f64::consts::LN_2,
40};
41
42
43pub const MIN_RUN: usize = 32; // shortest unbroken run the entropy catch weighs
44pub const MIN_BITS: f64 = 4.0; // bits a character that run must exceed
45
46// Fields whose strings are correlation ids, which a reader joins rows by and a provider may mint
47// in an alphabet no different from a key's. They are spared the entropy catch and nothing else.
48pub const ID_KEYS: &[&str] = &[
49 "id", "mid", "callId", "call_id", "tool_call_id", "toolCallId", "turn", "chat", "chatId",
50 "device", "deviceId", "build", "d", "b", "n", "w",
51];
52
53const NO_MAX: usize = usize::MAX;
54
55// The characters of a pem header's words, which are held apart so that this file does not itself
56// carry a header a scanner looks for.
57const BEGIN: &str = "-----BEGIN ";
58const END: &str = "-----END ";
59const KEY: &str = "PRIVATE KEY";
60const DASHES: &str = "-----";
61
62
63// What a run of characters after an opening may hold.
64#[derive(Clone, Copy)]
65enum Class {
66 Alnum, // [A-Za-z0-9]
67 Word, // [A-Za-z0-9_]
68 Token, // [A-Za-z0-9_-]
69 Dash, // [A-Za-z0-9-]
70 Upper, // [0-9A-Z]
71 Hex, // [0-9a-f]
72 Std, // [A-Za-z0-9/+=]
73 Bearer, // [A-Za-z0-9._~+/=-]
74 Named, // [A-Za-z0-9_./+=-]
75 Url, // anything but & " ' # < > [ ] and white space
76 Caps, // [A-Z0-9 ]
77 Quote, // a quote mark or white space
78 Space, // white space
79 Blank, // a space or a tab, `[ \t]`
80}
81
82const CLASSES: usize = 14;
83
84impl Class {
85
86 fn admits(self, c: u16) -> bool {
87 match self {
88 Self::Url => !matches!(c, 0x26 | 0x22 | 0x27 | 0x23 | 0x3c | 0x3e | 0x5b | 0x5d)
89 && !js_space(c),
90 Self::Quote => c == 0x22 || c == 0x27 || js_space(c),
91 Self::Space => js_space(c),
92 Self::Blank => c == 0x20 || c == 0x09,
93 _ if c > 127 => false,
94 _ => {
95 let b = c as u8;
96 let alnum = b.is_ascii_alphanumeric();
97 match self {
98 Self::Alnum => alnum,
99 Self::Word => alnum || b == b'_',
100 Self::Token => alnum || b == b'_' || b == b'-',
101 Self::Dash => alnum || b == b'-',
102 Self::Upper => b.is_ascii_digit() || b.is_ascii_uppercase(),
103 Self::Hex => b.is_ascii_digit() || (b'a'..=b'f').contains(&b),
104 Self::Std => alnum || matches!(b, b'/' | b'+' | b'='),
105 Self::Bearer => alnum || matches!(b, b'.' | b'_' | b'~' | b'+' | b'/' | b'=' | b'-'),
106 Self::Named => alnum || matches!(b, b'_' | b'-' | b'.' | b'/' | b'+' | b'='),
107 Self::Caps => b.is_ascii_digit() || b.is_ascii_uppercase() || b == b' ',
108 _ => false,
109 }
110 },
111 }
112 }
113}
114
115// JavaScript's `\s`.
116fn js_space(c: u16) -> bool {
117 matches!(c,
118 0x09..=0x0d | 0x20 | 0xa0 | 0x1680 | 0x2000..=0x200a | 0x2028 | 0x2029 | 0x202f | 0x205f
119 | 0x3000 | 0xfeff)
120}
121
122// What JavaScript's `.` does not match: the four line terminators.
123fn js_line_end(c: u16) -> bool {
124 matches!(c, 0x0a | 0x0d | 0x2028 | 0x2029)
125}
126
127// JavaScript's `\w`, which is ASCII alone.
128fn is_word(c: u16) -> bool {
129 c < 128 && ((c as u8).is_ascii_alphanumeric() || c == b'_' as u16)
130}
131
132fn is_digit(c: u16) -> bool {
133 (0x30..=0x39).contains(&c)
134}
135
136fn is_letter(c: u16) -> bool {
137 c < 128 && (c as u8).is_ascii_alphabetic()
138}
139
140fn is_hex(c: u16) -> bool {
141 c < 128 && (c as u8).is_ascii_hexdigit()
142}
143
144
145// The string being scanned, as UTF-16 units, with the last run measured of each class.
146struct Text<'a> {
147 s: &'a [u16],
148 runs: [(usize, usize); CLASSES], // from and to, every unit between being of the class
149}
150
151impl<'a> Text<'a> {
152
153 fn new(s: &'a [u16]) -> Self {
154 Self { s, runs: [(0, 0); CLASSES] }
155 }
156
157 fn at(&self, i: usize) -> Option<u16> {
158 self.s.get(i).copied()
159 }
160
161 fn is(&self, i: usize, b: u8) -> bool {
162 self.at(i) == Some(b as u16)
163 }
164
165 // Does this ASCII text stand here, exactly?
166 fn lit(&self, i: usize, lit: &str) -> bool {
167 lit.bytes().enumerate().all(|(k, b)| self.is(i + k, b))
168 }
169
170 // Does this ASCII text stand here, in either case? JavaScript's `/i` folds ASCII letters only.
171 fn lit_ci(&self, i: usize, lit: &str) -> bool {
172 lit.bytes().enumerate().all(|(k, b)| match self.at(i + k) {
173 Some(c) => c < 128 && (c as u8).eq_ignore_ascii_case(&b),
174 None => false,
175 })
176 }
177
178 // Where a name ends if it stands here, in either case. A `~` in it is `[_-]?`.
179 fn name(&self, i: usize, pat: &str) -> Option<usize> {
180 let mut p = i;
181 for b in pat.bytes() {
182 let c = self.at(p);
183 if b == b'~' {
184 if c == Some(b'_' as u16) || c == Some(b'-' as u16) {
185 p += 1;
186 }
187 } else {
188 match c {
189 Some(c) if c < 128 && (c as u8).eq_ignore_ascii_case(&b) => p += 1,
190 _ => return None,
191 }
192 }
193 }
194 Some(p)
195 }
196
197 // Is there a `\b` before a word character at this position?
198 fn boundary(&self, i: usize) -> bool {
199 i == 0 || !is_word(self.s[i - 1])
200 }
201
202 // How many units from here are of the class.
203 fn run(&mut self, i: usize, class: Class) -> usize {
204 let k = class as usize;
205 let (from, to) = self.runs[k];
206 if from <= i && i < to {
207 return to - i;
208 }
209 let mut e = i;
210 while e < self.s.len() && class.admits(self.s[e]) {
211 e += 1;
212 }
213 if e > i {
214 self.runs[k] = (i, e);
215 }
216 e - i
217 }
218
219 // As `run`, counting no further than `cap`.
220 fn count(&self, i: usize, class: Class, cap: usize) -> usize {
221 let mut e = i;
222 while e < self.s.len() && e - i < cap && class.admits(self.s[e]) {
223 e += 1;
224 }
225 e - i
226 }
227}
228
229
230// How a shape is recognised.
231enum How {
232 Pem,
233 Jwt,
234 Lead {
235 leads: &'static [&'static str], // the literal openings, any one
236 class: Class, // what follows
237 min: usize, // fewest of them
238 max: usize, // and most, which `{n}` makes the same
239 },
240}
241
242struct Shape {
243 kind: &'static str,
244 how: How,
245}
246
247// Daimond's `SCRUB_SHAPES`, in its order. Each is matched whole and replaced whole, and each
248// but the first opens at a word boundary.
249const SHAPES: &[Shape] = &[
250 Shape { kind: "pem", how: How::Pem },
251 Shape { kind: "gh", how: How::Lead {
252 leads: &["ghp_", "gho_", "ghu_", "ghs_", "ghr_"], class: Class::Alnum, min: 16, max: NO_MAX } },
253 Shape { kind: "ghpat", how: How::Lead {
254 leads: &["github_pat_"], class: Class::Word, min: 20, max: NO_MAX } },
255 Shape { kind: "stripe", how: How::Lead {
256 leads: &["sk_live_", "sk_test_", "rk_live_", "rk_test_"], class: Class::Alnum, min: 10, max: NO_MAX } },
257 Shape { kind: "whsec", how: How::Lead {
258 leads: &["whsec_"], class: Class::Alnum, min: 16, max: NO_MAX } },
259 // `sk-`, `sk-or-v1-`, `sk-ant-api03-`: every provider that took the prefix kept the alphabet.
260 Shape { kind: "sk", how: How::Lead {
261 leads: &["sk-"], class: Class::Token, min: 16, max: NO_MAX } },
262 Shape { kind: "aws", how: How::Lead {
263 leads: &["AKIA", "ASIA", "ABIA", "ACCA", "AGPA", "AIDA", "AIPA", "ANPA", "ANVA", "AROA", "APKA"],
264 class: Class::Upper, min: 12, max: NO_MAX } },
265 Shape { kind: "gcp", how: How::Lead {
266 leads: &["AIza"], class: Class::Token, min: 30, max: NO_MAX } },
267 Shape { kind: "slack", how: How::Lead {
268 leads: &["xoxa-", "xoxb-", "xoxe-", "xoxp-", "xoxr-", "xoxs-"], class: Class::Dash, min: 10, max: NO_MAX } },
269 Shape { kind: "jwt", how: How::Jwt },
270 // The tune relay mints one a run and a page carries it as its provider key. Exactly 32.
271 Shape { kind: "tune", how: How::Lead {
272 leads: &["tune-"], class: Class::Hex, min: 32, max: 32 } },
273];
274
275// Where a shape stands at `i`, as the span it replaces.
276fn shape_at(shape: &Shape, t: &mut Text, i: usize) -> Option<(usize, usize)> {
277 match &shape.how {
278 How::Pem => pem(t, i),
279 How::Jwt => jwt(t, i),
280 How::Lead { leads, class, min, max } => {
281 if !t.boundary(i) {
282 return None;
283 }
284 for lead in leads.iter() {
285 if t.lit(i, lead) {
286 let from = i + lead.len();
287 let n = t.run(from, *class);
288 if n >= *min {
289 return Some((i, from + n.min(*max)));
290 }
291 }
292 }
293 None
294 },
295 }
296}
297
298// A header's opening, a run of capitals, digits and spaces, the two words and the dashes. The
299// run ends where the first dash stands, so the words are looked for just before it.
300fn pem_head(t: &mut Text, i: usize, open: &str) -> Option<usize> {
301 if !t.lit(i, open) {
302 return None;
303 }
304 let from = i + open.len();
305 let n = t.run(from, Class::Caps);
306 let end = from + n;
307 if n >= KEY.len() && t.lit(end - KEY.len(), KEY) && t.lit(end, DASHES) {
308 return Some(end + DASHES.len());
309 }
310 None
311}
312
313// A private key, whole or clipped by elision: to its footer, or else to the end of the string.
314fn pem(t: &mut Text, i: usize) -> Option<(usize, usize)> {
315 let mut q = match pem_head(t, i, BEGIN) {
316 Some(q) => q,
317 None => return None,
318 };
319 let n = t.s.len();
320 while q < n {
321 if t.is(q, b'-') {
322 if let Some(e) = pem_head(t, q, END) {
323 return Some((i, e));
324 }
325 }
326 q += 1;
327 }
328 Some((i, n))
329}
330
331// Three runs of the url-safe alphabet joined by dots: the first opens `eyJ` and holds six more,
332// the second six, the third four.
333fn jwt(t: &mut Text, i: usize) -> Option<(usize, usize)> {
334 if !t.boundary(i) || !t.lit(i, "eyJ") {
335 return None;
336 }
337 let mut p = i + 3;
338 for (k, min) in [6usize, 6, 4].iter().enumerate() {
339 let n = t.run(p, Class::Token);
340 if n < *min {
341 return None;
342 }
343 p += n;
344 if k < 2 {
345 if !t.is(p, b'.') {
346 return None;
347 }
348 p += 1;
349 }
350 }
351 Some((i, p))
352}
353
354
355// Daimond's `SCRUB_PAIRS`, where the name survives and only the value goes. Each answers with
356// the span of the value alone.
357
358// An `Authorization` header, however it was spelled into the text. `bearer` in any case, since a
359// header name and its scheme are case-free on the wire; `Basic` and `Token` capitalised only, as
360// prose uses both words lower case.
361fn bearer(t: &mut Text, i: usize) -> Option<(usize, usize)> {
362 if !t.boundary(i) {
363 return None;
364 }
365 for word in ["Bearer", "Basic", "Token"] {
366 let here = if word == "Bearer" { t.lit_ci(i, word) } else { t.lit(i, word) };
367 if here {
368 let at = i + word.len();
369 let sp = t.run(at, Class::Space);
370 if sp == 0 {
371 continue;
372 }
373 let v = at + sp;
374 let n = t.run(v, Class::Bearer);
375 if n >= 8 {
376 return Some((v, v + n));
377 }
378 }
379 }
380 None
381}
382
383const URLARG: &[&str] = &[
384 "access_token", "refresh_token", "id_token", "token", "api~key", "apikey", "key", "auth",
385 "secret", "password", "passwd", "sig", "signature",
386];
387
388// A credential in a URL's query or fragment. A value never holds `[` or `]`, so that a marker
389// already in its place is not taken for a value and marked a second time.
390fn urlarg(t: &mut Text, i: usize) -> Option<(usize, usize)> {
391 if !(t.is(i, b'?') || t.is(i, b'&') || t.is(i, b'#')) {
392 return None;
393 }
394 for name in URLARG {
395 if let Some(e) = t.name(i + 1, name) {
396 if t.is(e, b'=') {
397 let n = t.run(e + 1, Class::Url);
398 if n >= 4 {
399 return Some((e + 1, e + 1 + n));
400 }
401 }
402 }
403 }
404 None
405}
406
407// Daimond's `SCRUB_NAME`, in its order: every name the `named` rules look for against a value.
408const NAMED: &[&str] = &[
409 "api~key", "apikey", "auth~token", "access~token", "token", "secret", "master~key",
410 "private~key", "salt", "sealed", "wrapped", "passphrase", "password", "passwd", "mnemonic",
411 "seed~phrase",
412];
413
414// Its `SCRUB_NAME_TYPED`: the names whose value a person types, and which runs to the end of the
415// line, since a passphrase is words.
416const TYPED: &[&str] = &["passphrase", "password", "passwd", "mnemonic", "seed~phrase"];
417
418// What a marker opens with. A value that opens on one is not taken again, so a second pass over
419// scrubbed text changes nothing. JavaScript's `(?!\[redacted )` under `/i`.
420const MARKED: &str = "[redacted ";
421
422// The optional quote after a name, up to three units of `class`, a colon or an equals sign, and
423// up to three more: where the value starts, if all of that stands here. Each step is decided
424// without giving anything back, since what follows a run of `class` is never of the class.
425fn joint(t: &mut Text, e: usize, class: Class) -> Option<usize> {
426 let mut p = e;
427 if t.is(p, b'"') || t.is(p, b'\'') {
428 p += 1;
429 }
430 let n = t.count(p, class, 4);
431 if n > 3 {
432 return None;
433 }
434 p += n;
435 if !(t.is(p, b':') || t.is(p, b'=')) {
436 return None;
437 }
438 p += 1;
439 let n = t.count(p, class, 4);
440 if n > 3 {
441 return None;
442 }
443 Some(p + n)
444}
445
446// A secret's name and a quoted value, whole up to the closing quote, at any length, with an
447// escaped quote kept inside. The JavaScript is, for `q` a double quote,
448// `NAME["']?\s{0,3}[:=]\s{0,3}"((?!\[redacted )(?:[^"\\\r\n]|\\.)+)(?=")`. Nothing it takes is
449// an unescaped `q`, so the closing quote is the first unit its loop cannot take, and a stop on
450// anything else is a miss that giving back cannot mend.
451fn quoted(t: &mut Text, i: usize, q: u8) -> Option<(usize, usize)> {
452 for name in NAMED {
453 let e = match t.name(i, name) {
454 Some(e) => e,
455 None => continue,
456 };
457 let o = match joint(t, e, Class::Space) {
458 Some(o) => o,
459 None => continue,
460 };
461 if !t.is(o, q) {
462 continue;
463 }
464 let v = o + 1;
465 if t.lit_ci(v, MARKED) {
466 continue;
467 }
468 let mut p = v;
469 let closed = loop {
470 match t.at(p) {
471 Some(c) if c == q as u16 => break true,
472 Some(0x5c) => match t.at(p + 1) {
473 Some(c) if !js_line_end(c) => p += 2,
474 _ => break false,
475 },
476 Some(0x0a) | Some(0x0d) | None => break false,
477 Some(_) => p += 1,
478 }
479 };
480 if closed && p > v {
481 return Some((v, p));
482 }
483 }
484 None
485}
486
487fn quoted_double(t: &mut Text, i: usize) -> Option<(usize, usize)> {
488 quoted(t, i, b'"')
489}
490
491fn quoted_single(t: &mut Text, i: usize) -> Option<(usize, usize)> {
492 quoted(t, i, b'\'')
493}
494
495// A typed secret's name and the rest of its line, which is what a passphrase of words needs. The
496// JavaScript is `TYPED["']?[ \t]{0,3}[:=][ \t]{0,3}((?!["']|\[redacted )\S(?:[^\r\n]*\S)?)`: a
497// first unit that is no space, then up to the last unit before the line's end that is no space.
498fn typed(t: &mut Text, i: usize) -> Option<(usize, usize)> {
499 for name in TYPED {
500 let e = match t.name(i, name) {
501 Some(e) => e,
502 None => continue,
503 };
504 let v = match joint(t, e, Class::Blank) {
505 Some(v) => v,
506 None => continue,
507 };
508 match t.at(v) {
509 Some(c) if !js_space(c) && c != 0x22 && c != 0x27 && !t.lit_ci(v, MARKED) => (),
510 _ => continue,
511 }
512 let mut end = v + 1;
513 let mut p = v + 1;
514 while let Some(c) = t.at(p) {
515 if c == 0x0a || c == 0x0d {
516 break;
517 }
518 p += 1;
519 if !js_space(c) {
520 end = p;
521 }
522 }
523 return Some((v, end));
524 }
525 None
526}
527
528// What follows a name: up to three quote marks or spaces, a colon or an equals sign, and up to
529// three more. Where more than three stand, the JavaScript's greedy count takes three and meets
530// another, and giving some back leaves a quote where its value should start, so it is a miss.
531fn after_name(t: &mut Text, i: usize) -> Option<usize> {
532 let q = t.run(i, Class::Quote);
533 if q > 3 {
534 return None;
535 }
536 let c = i + q;
537 if !(t.is(c, b':') || t.is(c, b'=')) {
538 return None;
539 }
540 let q2 = t.run(c + 1, Class::Quote);
541 if q2 > 3 {
542 return None;
543 }
544 Some(c + 1 + q2)
545}
546
547// A secret's name sitting against a run of eight or more of a token's characters, which is what a
548// bare `looksSecret` used to answer yes or no to, with the value cut out so that the message stays
549// readable. It runs after the quoted and the typed rules, which take what it once let through.
550fn named(t: &mut Text, i: usize) -> Option<(usize, usize)> {
551 for name in NAMED {
552 let e = match t.name(i, name) {
553 Some(e) => e,
554 None => continue,
555 };
556 if let Some(v) = after_name(t, e) {
557 let n = t.run(v, Class::Named);
558 if n >= 8 {
559 return Some((v, v + n));
560 }
561 }
562 }
563 None
564}
565
566fn separator(c: u16) -> bool {
567 matches!(c, 0x22 | 0x27 | 0x3a | 0x3d | 0x2c) || js_space(c)
568}
569
570// An AWS secret is forty characters of base64 wearing no prefix, so it is recognisable only beside
571// the key id, or the name, it belongs to. This one is beside a key id: the id, twelve or more
572// capitals and digits, any 200 units or fewer, a separator, and the forty. The JavaScript is
573// `[0-9A-Z]{12,}[\s\S]{0,200}?["'\s:=,]`, and the run before the gap can give units back, but a
574// unit it gives is a capital or a digit and so no separator, so the first separator at or after
575// the run's end is the only one the pattern can use. Every start within one run meets that same
576// separator, which is why a miss is all of them and `skip` steps past the run.
577fn aws_id(t: &mut Text, i: usize, skip: &mut usize) -> Option<(usize, usize)> {
578 if i < *skip || !(t.lit(i, "AKIA") || t.lit(i, "ASIA")) {
579 return None;
580 }
581 let n = t.run(i + 4, Class::Upper);
582 let end = i + 4 + n;
583 *skip = end;
584 if n < 12 {
585 return None;
586 }
587 for p in end..=end + 200 {
588 match t.at(p) {
589 Some(c) if separator(c) => if t.count(p + 1, Class::Std, 41) == 40 {
590 return Some((p + 1, p + 41));
591 },
592 Some(_) => (),
593 None => break,
594 }
595 }
596 None
597}
598
599// The same secret beside its name: `aws_secret_access_key`, with or without the first word and
600// each underscore, in either case, then the usual separators and forty characters exactly.
601fn aws_name(t: &mut Text, i: usize) -> Option<(usize, usize)> {
602 let mut p = i;
603 if t.lit_ci(p, "aws") {
604 p += 3;
605 }
606 for word in ["secret", "access", "key"] {
607 if t.is(p, b'_') {
608 p += 1;
609 }
610 if !t.lit_ci(p, word) {
611 return None;
612 }
613 p += word.len();
614 }
615 let v = match after_name(t, p) {
616 Some(v) => v,
617 None => return None,
618 };
619 if t.count(v, Class::Std, 41) == 40 {
620 return Some((v, v + 40));
621 }
622 None
623}
624
625
626// One rule over the whole string, left to right, replacing each span it finds with a marker. A
627// span is where the rule says its value stands, and what lies between the last replacement and
628// it is kept, which is how a pair keeps its name. Answers whether it found anything.
629fn pass<F>(cur: &mut Vec<u16>, kind: &str, mut rule: F) -> bool
630where
631 F: FnMut(&mut Text, usize) -> Option<(usize, usize)>,
632{
633 let n = cur.len();
634 let mut text = Text::new(cur);
635 let mut out: Option<Vec<u16>> = None;
636 let mut done = 0;
637 let mut i = 0;
638 while i < n {
639 match rule(&mut text, i) {
640 Some((v, e)) => {
641 let o = out.get_or_insert_with(|| Vec::with_capacity(n + 32));
642 o.extend_from_slice(&cur[done..v]);
643 o.extend(mark_units(kind, &cur[v..e]));
644 done = e;
645 i = e;
646 },
647 None => i += 1,
648 }
649 }
650 match out {
651 Some(mut o) => {
652 o.extend_from_slice(&cur[done..]);
653 *cur = o;
654 true
655 },
656 None => false,
657 }
658}
659
660fn units(s: &str) -> Vec<u16> {
661 s.encode_utf16().collect()
662}
663
664fn mark_units(kind: &str, v: &[u16]) -> Vec<u16> {
665 // djb2 over the UTF-16 units, as the JavaScript's `scrubMark` and `fingerprint` do.
666 let mut h: u32 = 5381;
667 for &c in v {
668 h = (h << 5).wrapping_add(h).wrapping_add(c as u32);
669 }
670 units(&fmt!("[redacted {} #{:x}/{}]", kind, h, v.len()))
671}
672
673/// The marker for one scrubbed value: its shape, a stable hash and its length in UTF-16 units.
674pub fn mark(kind: &str, v: &str) -> String {
675 String::from_utf16_lossy(&mark_units(kind, &units(v)))
676}
677
678/// Shannon entropy of a string in bits a unit. Hex tops out at 4.0 by construction and
679/// base64url at 6.0, which is where the threshold sits: a digest stops and a key begins.
680///
681/// The terms are summed in the order JavaScript's `Object.keys` gives the counts, the digits
682/// ascending and then the rest as they were first met. `f64::ln` is the platform's and V8's is
683/// its own, so the two can differ in the last bit or two. That matters only for a run whose
684/// entropy lies within 2e-15 of the threshold, and on 2026-10-02 a differential run over 10,000
685/// runs, most of them built to sit on it, found no decision that differed.
686pub fn entropy(s: &str) -> f64 {
687 entropy_units(&units(s))
688}
689
690fn entropy_units(s: &[u16]) -> f64 {
691 let mut at: HashMap<u16, usize> = HashMap::new();
692 let mut seen: Vec<(u16, usize)> = Vec::new();
693 for &c in s {
694 match at.get(&c) {
695 Some(&k) => seen[k].1 += 1,
696 None => {
697 at.insert(c, seen.len());
698 seen.push((c, 1));
699 },
700 }
701 }
702 let mut order: Vec<(u16, usize)> = seen.iter().filter(|(c, _)| is_digit(*c)).copied().collect();
703 order.sort_by_key(|(c, _)| *c);
704 order.extend(seen.iter().filter(|(c, _)| !is_digit(*c)).copied());
705 let len = s.len() as f64;
706 let mut h = 0.0;
707 for (_, n) in order {
708 let p = n as f64 / len;
709 h -= p * (p.ln() / LN_2);
710 }
711 h
712}
713
714/// Is this unbroken run one the feed is supposed to carry? A digest, a build id and a device id
715/// are hex, a correlation id is hex joined by `-` or `_`, a count is decimal, a stretch of prose
716/// is letters, and an identifier a person wrote is every segment a word or a number. Named
717/// rather than left to the entropy threshold, so that the feed stays legible by rule.
718pub fn safe_run(s: &str) -> bool {
719 safe_units(&units(s))
720}
721
722fn safe_units(s: &[u16]) -> bool {
723 let sep = |c: u16| c == b'-' as u16 || c == b'_' as u16;
724 let bare: Vec<u16> = s.iter().copied().filter(|c| !sep(*c)).collect();
725 if !bare.is_empty() {
726 if bare.iter().all(|c| is_hex(*c)) || bare.iter().all(|c| is_digit(*c))
727 || bare.iter().all(|c| is_letter(*c))
728 {
729 return true;
730 }
731 }
732 let parts: Vec<&[u16]> = s.split(|c| sep(*c)).collect();
733 if parts.len() > 1 {
734 return parts.iter().all(|p|
735 (p.len() >= 2 && p.iter().all(|c| is_letter(*c)))
736 || (!p.is_empty() && p.iter().all(|c| is_digit(*c))));
737 }
738 false
739}
740
741// The scrubbed units, or nothing where no rule found anything.
742fn go(s: &str, is_id: bool) -> Option<Vec<u16>> {
743 let mut cur = units(s);
744 // Nothing in the list above is shorter than a dozen.
745 if cur.len() < 8 {
746 return None;
747 }
748 let mut hit = false;
749 for shape in SHAPES {
750 hit |= pass(&mut cur, shape.kind, |t, i| shape_at(shape, t, i));
751 }
752 hit |= pass(&mut cur, "bearer", bearer);
753 hit |= pass(&mut cur, "urlarg", urlarg);
754 hit |= pass(&mut cur, "named", quoted_double);
755 hit |= pass(&mut cur, "named", quoted_single);
756 hit |= pass(&mut cur, "named", typed);
757 hit |= pass(&mut cur, "named", named);
758 let mut skip = 0;
759 hit |= pass(&mut cur, "awssec", |t, i| aws_id(t, i, &mut skip));
760 hit |= pass(&mut cur, "awssec", aws_name);
761 // The catch-all runs last, so a run already turned into a marker is not weighed again. It is
762 // a shape nobody has listed: a long unbroken run that is neither one of the shapes a feed
763 // carries nor low in entropy.
764 if !is_id {
765 let mut skip = 0;
766 hit |= pass(&mut cur, "hi", |t, i| {
767 if i < skip {
768 return None;
769 }
770 let n = t.run(i, Class::Token);
771 if n == 0 {
772 return None;
773 }
774 // Every start in this run sees the same run, shorter, so it is decided once.
775 skip = i + n;
776 if n < MIN_RUN {
777 return None;
778 }
779 let s = t.s;
780 let run = &s[i..i + n];
781 if safe_units(run) || entropy_units(run) <= MIN_BITS {
782 return None;
783 }
784 Some((i, i + n))
785 });
786 }
787 if hit { Some(cur) } else { None }
788}
789
790/// The string with every credential in it replaced by `[redacted <shape> #<hash>/<length>]`,
791/// exactly as the JavaScript scrubber writes it, and the same string where there is none.
792///
793/// `is_id` is for a string found in a field that names a correlation id ([`ID_KEYS`]): the
794/// entropy catch is not applied to it, and every shape and pair still is.
795pub fn text(s: &str, is_id: bool) -> String {
796 match go(s, is_id) {
797 Some(u) => String::from_utf16_lossy(&u),
798 None => s.to_string(),
799 }
800}
801
802/// Does `text` find anything in this string?
803pub fn hit(s: &str, is_id: bool) -> bool {
804 go(s, is_id).is_some()
805}