oxedyne/fe2o3/fe2o3_text/src/secret/scrub.rs
24.9 KiB, 28 runs
created by r1870400018:65664, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! Daimond's content scrubber, in Rust: every credential in a string replaced by a marker. |
| 2 | //! |
| 3 | //! `www/js/debugshare.js` carries a scrubber that searches each string of a debug feed for the |
| 4 | //! shapes a credential wears and writes `[redacted <shape> #<hash>/<length>]` in the place of |
| 5 | //! each, so that a reader still sees that a key was there and can tell two sightings apart. It is |
| 6 | //! one block of that file, copied into `dev/lens.mjs` and `fe2o3_net/lens/js/redact.js`. This |
| 7 | //! module is the same block in Rust, so that a peer's feed is guarded as a browser's is, and |
| 8 | //! `fe2o3_net/tests/data/lens_scrub.tsv` holds the two together: a table of inputs, each stored |
| 9 | //! as a recipe so that no credential is, and what the JavaScript made of every one. |
| 10 | //! |
| 11 | //! It is a table of its own, beside the one [`super::scan`] reads and not an extension of it. |
| 12 | //! `scan` reads files a person wrote and refuses a commit, so a shape added to it is a shape that |
| 13 | //! stops work, and it excuses a placeholder. This answers for text bound for a log, where all a |
| 14 | //! false hit costs is a word, and it excuses nothing. |
| 15 | //! |
| 16 | //! # Why UTF-16, and why no regular expressions |
| 17 | //! |
| 18 | //! The JavaScript counts and hashes in UTF-16 code units, and a marker's hash and length are how |
| 19 | //! a reader tells one sighting from another, so a string is walked here as `u16`s and a minimum |
| 20 | //! length means what it means there: an astral character counts as two. For the same reason no |
| 21 | //! `regex` stands in this file. JavaScript's `\b` and `/i` are ASCII-only, its `\s` holds U+FEFF |
| 22 | //! and not U+0085, which is the reverse of `char::is_whitespace`, and the crate's own engine |
| 23 | //! follows the `regex` crate on every one of the three. |
| 24 | //! |
| 25 | //! # No run is measured twice |
| 26 | //! |
| 27 | //! Almost every rule is a literal opening and a run of one class of character. A string with an |
| 28 | //! opening every few bytes and one long run behind them would cost a rule that re-measured the run |
| 29 | //! at each opening the square of its length, and a feed's author may be anyone. `Text::run` |
| 30 | //! therefore remembers the last run it measured of each class, and the two rules that look again |
| 31 | //! after a miss step over what they have already decided. The one place the JavaScript backtracks |
| 32 | //! in earnest, an AWS secret beside its key id, is solved outright: its greedy run and its lazy |
| 33 | //! gap together come to "the first separator within 200 units after the run". |
| 34 | |
| 35 | use oxedyne_fe2o3_core::prelude::*; |
| 36 | |
| 37 | use std::{ |
| 38 | collections::HashMap, |
| 39 | f64::consts::LN_2, |
| 40 | }; |
| 41 | |
| 42 | |
| 43 | pub const MIN_RUN: usize = 32; // shortest unbroken run the entropy catch weighs |
| 44 | pub const MIN_BITS: f64 = 4.0; // bits a character that run must exceed |
| 45 | |
| 46 | // Fields whose strings are correlation ids, which a reader joins rows by and a provider may mint |
| 47 | // in an alphabet no different from a key's. They are spared the entropy catch and nothing else. |
| 48 | pub const ID_KEYS: &[&str] = &[ |
| 49 | "id", "mid", "callId", "call_id", "tool_call_id", "toolCallId", "turn", "chat", "chatId", |
| 50 | "device", "deviceId", "build", "d", "b", "n", "w", |
| 51 | ]; |
| 52 | |
| 53 | const NO_MAX: usize = usize::MAX; |
| 54 | |
| 55 | // The characters of a pem header's words, which are held apart so that this file does not itself |
| 56 | // carry a header a scanner looks for. |
| 57 | const BEGIN: &str = "-----BEGIN "; |
| 58 | const END: &str = "-----END "; |
| 59 | const KEY: &str = "PRIVATE KEY"; |
| 60 | const DASHES: &str = "-----"; |
| 61 | |
| 62 | |
| 63 | // What a run of characters after an opening may hold. |
| 64 | #[derive(Clone, Copy)] |
| 65 | enum Class { |
| 66 | Alnum, // [A-Za-z0-9] |
| 67 | Word, // [A-Za-z0-9_] |
| 68 | Token, // [A-Za-z0-9_-] |
| 69 | Dash, // [A-Za-z0-9-] |
| 70 | Upper, // [0-9A-Z] |
| 71 | Hex, // [0-9a-f] |
| 72 | Std, // [A-Za-z0-9/+=] |
| 73 | Bearer, // [A-Za-z0-9._~+/=-] |
| 74 | Named, // [A-Za-z0-9_./+=-] |
| 75 | Url, // anything but & " ' # < > [ ] and white space |
| 76 | Caps, // [A-Z0-9 ] |
| 77 | Quote, // a quote mark or white space |
| 78 | Space, // white space |
| 79 | Blank, // a space or a tab, `[ \t]` |
| 80 | } |
| 81 | |
| 82 | const CLASSES: usize = 14; |
| 83 | |
| 84 | impl Class { |
| 85 | |
| 86 | fn admits(self, c: u16) -> bool { |
| 87 | match self { |
| 88 | Self::Url => !matches!(c, 0x26 | 0x22 | 0x27 | 0x23 | 0x3c | 0x3e | 0x5b | 0x5d) |
| 89 | && !js_space(c), |
| 90 | Self::Quote => c == 0x22 || c == 0x27 || js_space(c), |
| 91 | Self::Space => js_space(c), |
| 92 | Self::Blank => c == 0x20 || c == 0x09, |
| 93 | _ if c > 127 => false, |
| 94 | _ => { |
| 95 | let b = c as u8; |
| 96 | let alnum = b.is_ascii_alphanumeric(); |
| 97 | match self { |
| 98 | Self::Alnum => alnum, |
| 99 | Self::Word => alnum || b == b'_', |
| 100 | Self::Token => alnum || b == b'_' || b == b'-', |
| 101 | Self::Dash => alnum || b == b'-', |
| 102 | Self::Upper => b.is_ascii_digit() || b.is_ascii_uppercase(), |
| 103 | Self::Hex => b.is_ascii_digit() || (b'a'..=b'f').contains(&b), |
| 104 | Self::Std => alnum || matches!(b, b'/' | b'+' | b'='), |
| 105 | Self::Bearer => alnum || matches!(b, b'.' | b'_' | b'~' | b'+' | b'/' | b'=' | b'-'), |
| 106 | Self::Named => alnum || matches!(b, b'_' | b'-' | b'.' | b'/' | b'+' | b'='), |
| 107 | Self::Caps => b.is_ascii_digit() || b.is_ascii_uppercase() || b == b' ', |
| 108 | _ => false, |
| 109 | } |
| 110 | }, |
| 111 | } |
| 112 | } |
| 113 | } |
| 114 | |
| 115 | // JavaScript's `\s`. |
| 116 | fn js_space(c: u16) -> bool { |
| 117 | matches!(c, |
| 118 | 0x09..=0x0d | 0x20 | 0xa0 | 0x1680 | 0x2000..=0x200a | 0x2028 | 0x2029 | 0x202f | 0x205f |
| 119 | | 0x3000 | 0xfeff) |
| 120 | } |
| 121 | |
| 122 | // What JavaScript's `.` does not match: the four line terminators. |
| 123 | fn js_line_end(c: u16) -> bool { |
| 124 | matches!(c, 0x0a | 0x0d | 0x2028 | 0x2029) |
| 125 | } |
| 126 | |
| 127 | // JavaScript's `\w`, which is ASCII alone. |
| 128 | fn is_word(c: u16) -> bool { |
| 129 | c < 128 && ((c as u8).is_ascii_alphanumeric() || c == b'_' as u16) |
| 130 | } |
| 131 | |
| 132 | fn is_digit(c: u16) -> bool { |
| 133 | (0x30..=0x39).contains(&c) |
| 134 | } |
| 135 | |
| 136 | fn is_letter(c: u16) -> bool { |
| 137 | c < 128 && (c as u8).is_ascii_alphabetic() |
| 138 | } |
| 139 | |
| 140 | fn is_hex(c: u16) -> bool { |
| 141 | c < 128 && (c as u8).is_ascii_hexdigit() |
| 142 | } |
| 143 | |
| 144 | |
| 145 | // The string being scanned, as UTF-16 units, with the last run measured of each class. |
| 146 | struct Text<'a> { |
| 147 | s: &'a [u16], |
| 148 | runs: [(usize, usize); CLASSES], // from and to, every unit between being of the class |
| 149 | } |
| 150 | |
| 151 | impl<'a> Text<'a> { |
| 152 | |
| 153 | fn new(s: &'a [u16]) -> Self { |
| 154 | Self { s, runs: [(0, 0); CLASSES] } |
| 155 | } |
| 156 | |
| 157 | fn at(&self, i: usize) -> Option<u16> { |
| 158 | self.s.get(i).copied() |
| 159 | } |
| 160 | |
| 161 | fn is(&self, i: usize, b: u8) -> bool { |
| 162 | self.at(i) == Some(b as u16) |
| 163 | } |
| 164 | |
| 165 | // Does this ASCII text stand here, exactly? |
| 166 | fn lit(&self, i: usize, lit: &str) -> bool { |
| 167 | lit.bytes().enumerate().all(|(k, b)| self.is(i + k, b)) |
| 168 | } |
| 169 | |
| 170 | // Does this ASCII text stand here, in either case? JavaScript's `/i` folds ASCII letters only. |
| 171 | fn lit_ci(&self, i: usize, lit: &str) -> bool { |
| 172 | lit.bytes().enumerate().all(|(k, b)| match self.at(i + k) { |
| 173 | Some(c) => c < 128 && (c as u8).eq_ignore_ascii_case(&b), |
| 174 | None => false, |
| 175 | }) |
| 176 | } |
| 177 | |
| 178 | // Where a name ends if it stands here, in either case. A `~` in it is `[_-]?`. |
| 179 | fn name(&self, i: usize, pat: &str) -> Option<usize> { |
| 180 | let mut p = i; |
| 181 | for b in pat.bytes() { |
| 182 | let c = self.at(p); |
| 183 | if b == b'~' { |
| 184 | if c == Some(b'_' as u16) || c == Some(b'-' as u16) { |
| 185 | p += 1; |
| 186 | } |
| 187 | } else { |
| 188 | match c { |
| 189 | Some(c) if c < 128 && (c as u8).eq_ignore_ascii_case(&b) => p += 1, |
| 190 | _ => return None, |
| 191 | } |
| 192 | } |
| 193 | } |
| 194 | Some(p) |
| 195 | } |
| 196 | |
| 197 | // Is there a `\b` before a word character at this position? |
| 198 | fn boundary(&self, i: usize) -> bool { |
| 199 | i == 0 || !is_word(self.s[i - 1]) |
| 200 | } |
| 201 | |
| 202 | // How many units from here are of the class. |
| 203 | fn run(&mut self, i: usize, class: Class) -> usize { |
| 204 | let k = class as usize; |
| 205 | let (from, to) = self.runs[k]; |
| 206 | if from <= i && i < to { |
| 207 | return to - i; |
| 208 | } |
| 209 | let mut e = i; |
| 210 | while e < self.s.len() && class.admits(self.s[e]) { |
| 211 | e += 1; |
| 212 | } |
| 213 | if e > i { |
| 214 | self.runs[k] = (i, e); |
| 215 | } |
| 216 | e - i |
| 217 | } |
| 218 | |
| 219 | // As `run`, counting no further than `cap`. |
| 220 | fn count(&self, i: usize, class: Class, cap: usize) -> usize { |
| 221 | let mut e = i; |
| 222 | while e < self.s.len() && e - i < cap && class.admits(self.s[e]) { |
| 223 | e += 1; |
| 224 | } |
| 225 | e - i |
| 226 | } |
| 227 | } |
| 228 | |
| 229 | |
| 230 | // How a shape is recognised. |
| 231 | enum How { |
| 232 | Pem, |
| 233 | Jwt, |
| 234 | Lead { |
| 235 | leads: &'static [&'static str], // the literal openings, any one |
| 236 | class: Class, // what follows |
| 237 | min: usize, // fewest of them |
| 238 | max: usize, // and most, which `{n}` makes the same |
| 239 | }, |
| 240 | } |
| 241 | |
| 242 | struct Shape { |
| 243 | kind: &'static str, |
| 244 | how: How, |
| 245 | } |
| 246 | |
| 247 | // Daimond's `SCRUB_SHAPES`, in its order. Each is matched whole and replaced whole, and each |
| 248 | // but the first opens at a word boundary. |
| 249 | const SHAPES: &[Shape] = &[ |
| 250 | Shape { kind: "pem", how: How::Pem }, |
| 251 | Shape { kind: "gh", how: How::Lead { |
| 252 | leads: &["ghp_", "gho_", "ghu_", "ghs_", "ghr_"], class: Class::Alnum, min: 16, max: NO_MAX } }, |
| 253 | Shape { kind: "ghpat", how: How::Lead { |
| 254 | leads: &["github_pat_"], class: Class::Word, min: 20, max: NO_MAX } }, |
| 255 | Shape { kind: "stripe", how: How::Lead { |
| 256 | leads: &["sk_live_", "sk_test_", "rk_live_", "rk_test_"], class: Class::Alnum, min: 10, max: NO_MAX } }, |
| 257 | Shape { kind: "whsec", how: How::Lead { |
| 258 | leads: &["whsec_"], class: Class::Alnum, min: 16, max: NO_MAX } }, |
| 259 | // `sk-`, `sk-or-v1-`, `sk-ant-api03-`: every provider that took the prefix kept the alphabet. |
| 260 | Shape { kind: "sk", how: How::Lead { |
| 261 | leads: &["sk-"], class: Class::Token, min: 16, max: NO_MAX } }, |
| 262 | Shape { kind: "aws", how: How::Lead { |
| 263 | leads: &["AKIA", "ASIA", "ABIA", "ACCA", "AGPA", "AIDA", "AIPA", "ANPA", "ANVA", "AROA", "APKA"], |
| 264 | class: Class::Upper, min: 12, max: NO_MAX } }, |
| 265 | Shape { kind: "gcp", how: How::Lead { |
| 266 | leads: &["AIza"], class: Class::Token, min: 30, max: NO_MAX } }, |
| 267 | Shape { kind: "slack", how: How::Lead { |
| 268 | leads: &["xoxa-", "xoxb-", "xoxe-", "xoxp-", "xoxr-", "xoxs-"], class: Class::Dash, min: 10, max: NO_MAX } }, |
| 269 | Shape { kind: "jwt", how: How::Jwt }, |
| 270 | // The tune relay mints one a run and a page carries it as its provider key. Exactly 32. |
| 271 | Shape { kind: "tune", how: How::Lead { |
| 272 | leads: &["tune-"], class: Class::Hex, min: 32, max: 32 } }, |
| 273 | ]; |
| 274 | |
| 275 | // Where a shape stands at `i`, as the span it replaces. |
| 276 | fn shape_at(shape: &Shape, t: &mut Text, i: usize) -> Option<(usize, usize)> { |
| 277 | match &shape.how { |
| 278 | How::Pem => pem(t, i), |
| 279 | How::Jwt => jwt(t, i), |
| 280 | How::Lead { leads, class, min, max } => { |
| 281 | if !t.boundary(i) { |
| 282 | return None; |
| 283 | } |
| 284 | for lead in leads.iter() { |
| 285 | if t.lit(i, lead) { |
| 286 | let from = i + lead.len(); |
| 287 | let n = t.run(from, *class); |
| 288 | if n >= *min { |
| 289 | return Some((i, from + n.min(*max))); |
| 290 | } |
| 291 | } |
| 292 | } |
| 293 | None |
| 294 | }, |
| 295 | } |
| 296 | } |
| 297 | |
| 298 | // A header's opening, a run of capitals, digits and spaces, the two words and the dashes. The |
| 299 | // run ends where the first dash stands, so the words are looked for just before it. |
| 300 | fn pem_head(t: &mut Text, i: usize, open: &str) -> Option<usize> { |
| 301 | if !t.lit(i, open) { |
| 302 | return None; |
| 303 | } |
| 304 | let from = i + open.len(); |
| 305 | let n = t.run(from, Class::Caps); |
| 306 | let end = from + n; |
| 307 | if n >= KEY.len() && t.lit(end - KEY.len(), KEY) && t.lit(end, DASHES) { |
| 308 | return Some(end + DASHES.len()); |
| 309 | } |
| 310 | None |
| 311 | } |
| 312 | |
| 313 | // A private key, whole or clipped by elision: to its footer, or else to the end of the string. |
| 314 | fn pem(t: &mut Text, i: usize) -> Option<(usize, usize)> { |
| 315 | let mut q = match pem_head(t, i, BEGIN) { |
| 316 | Some(q) => q, |
| 317 | None => return None, |
| 318 | }; |
| 319 | let n = t.s.len(); |
| 320 | while q < n { |
| 321 | if t.is(q, b'-') { |
| 322 | if let Some(e) = pem_head(t, q, END) { |
| 323 | return Some((i, e)); |
| 324 | } |
| 325 | } |
| 326 | q += 1; |
| 327 | } |
| 328 | Some((i, n)) |
| 329 | } |
| 330 | |
| 331 | // Three runs of the url-safe alphabet joined by dots: the first opens `eyJ` and holds six more, |
| 332 | // the second six, the third four. |
| 333 | fn jwt(t: &mut Text, i: usize) -> Option<(usize, usize)> { |
| 334 | if !t.boundary(i) || !t.lit(i, "eyJ") { |
| 335 | return None; |
| 336 | } |
| 337 | let mut p = i + 3; |
| 338 | for (k, min) in [6usize, 6, 4].iter().enumerate() { |
| 339 | let n = t.run(p, Class::Token); |
| 340 | if n < *min { |
| 341 | return None; |
| 342 | } |
| 343 | p += n; |
| 344 | if k < 2 { |
| 345 | if !t.is(p, b'.') { |
| 346 | return None; |
| 347 | } |
| 348 | p += 1; |
| 349 | } |
| 350 | } |
| 351 | Some((i, p)) |
| 352 | } |
| 353 | |
| 354 | |
| 355 | // Daimond's `SCRUB_PAIRS`, where the name survives and only the value goes. Each answers with |
| 356 | // the span of the value alone. |
| 357 | |
| 358 | // An `Authorization` header, however it was spelled into the text. `bearer` in any case, since a |
| 359 | // header name and its scheme are case-free on the wire; `Basic` and `Token` capitalised only, as |
| 360 | // prose uses both words lower case. |
| 361 | fn bearer(t: &mut Text, i: usize) -> Option<(usize, usize)> { |
| 362 | if !t.boundary(i) { |
| 363 | return None; |
| 364 | } |
| 365 | for word in ["Bearer", "Basic", "Token"] { |
| 366 | let here = if word == "Bearer" { t.lit_ci(i, word) } else { t.lit(i, word) }; |
| 367 | if here { |
| 368 | let at = i + word.len(); |
| 369 | let sp = t.run(at, Class::Space); |
| 370 | if sp == 0 { |
| 371 | continue; |
| 372 | } |
| 373 | let v = at + sp; |
| 374 | let n = t.run(v, Class::Bearer); |
| 375 | if n >= 8 { |
| 376 | return Some((v, v + n)); |
| 377 | } |
| 378 | } |
| 379 | } |
| 380 | None |
| 381 | } |
| 382 | |
| 383 | const URLARG: &[&str] = &[ |
| 384 | "access_token", "refresh_token", "id_token", "token", "api~key", "apikey", "key", "auth", |
| 385 | "secret", "password", "passwd", "sig", "signature", |
| 386 | ]; |
| 387 | |
| 388 | // A credential in a URL's query or fragment. A value never holds `[` or `]`, so that a marker |
| 389 | // already in its place is not taken for a value and marked a second time. |
| 390 | fn urlarg(t: &mut Text, i: usize) -> Option<(usize, usize)> { |
| 391 | if !(t.is(i, b'?') || t.is(i, b'&') || t.is(i, b'#')) { |
| 392 | return None; |
| 393 | } |
| 394 | for name in URLARG { |
| 395 | if let Some(e) = t.name(i + 1, name) { |
| 396 | if t.is(e, b'=') { |
| 397 | let n = t.run(e + 1, Class::Url); |
| 398 | if n >= 4 { |
| 399 | return Some((e + 1, e + 1 + n)); |
| 400 | } |
| 401 | } |
| 402 | } |
| 403 | } |
| 404 | None |
| 405 | } |
| 406 | |
| 407 | // Daimond's `SCRUB_NAME`, in its order: every name the `named` rules look for against a value. |
| 408 | const NAMED: &[&str] = &[ |
| 409 | "api~key", "apikey", "auth~token", "access~token", "token", "secret", "master~key", |
| 410 | "private~key", "salt", "sealed", "wrapped", "passphrase", "password", "passwd", "mnemonic", |
| 411 | "seed~phrase", |
| 412 | ]; |
| 413 | |
| 414 | // Its `SCRUB_NAME_TYPED`: the names whose value a person types, and which runs to the end of the |
| 415 | // line, since a passphrase is words. |
| 416 | const TYPED: &[&str] = &["passphrase", "password", "passwd", "mnemonic", "seed~phrase"]; |
| 417 | |
| 418 | // What a marker opens with. A value that opens on one is not taken again, so a second pass over |
| 419 | // scrubbed text changes nothing. JavaScript's `(?!\[redacted )` under `/i`. |
| 420 | const MARKED: &str = "[redacted "; |
| 421 | |
| 422 | // The optional quote after a name, up to three units of `class`, a colon or an equals sign, and |
| 423 | // up to three more: where the value starts, if all of that stands here. Each step is decided |
| 424 | // without giving anything back, since what follows a run of `class` is never of the class. |
| 425 | fn joint(t: &mut Text, e: usize, class: Class) -> Option<usize> { |
| 426 | let mut p = e; |
| 427 | if t.is(p, b'"') || t.is(p, b'\'') { |
| 428 | p += 1; |
| 429 | } |
| 430 | let n = t.count(p, class, 4); |
| 431 | if n > 3 { |
| 432 | return None; |
| 433 | } |
| 434 | p += n; |
| 435 | if !(t.is(p, b':') || t.is(p, b'=')) { |
| 436 | return None; |
| 437 | } |
| 438 | p += 1; |
| 439 | let n = t.count(p, class, 4); |
| 440 | if n > 3 { |
| 441 | return None; |
| 442 | } |
| 443 | Some(p + n) |
| 444 | } |
| 445 | |
| 446 | // A secret's name and a quoted value, whole up to the closing quote, at any length, with an |
| 447 | // escaped quote kept inside. The JavaScript is, for `q` a double quote, |
| 448 | // `NAME["']?\s{0,3}[:=]\s{0,3}"((?!\[redacted )(?:[^"\\\r\n]|\\.)+)(?=")`. Nothing it takes is |
| 449 | // an unescaped `q`, so the closing quote is the first unit its loop cannot take, and a stop on |
| 450 | // anything else is a miss that giving back cannot mend. |
| 451 | fn quoted(t: &mut Text, i: usize, q: u8) -> Option<(usize, usize)> { |
| 452 | for name in NAMED { |
| 453 | let e = match t.name(i, name) { |
| 454 | Some(e) => e, |
| 455 | None => continue, |
| 456 | }; |
| 457 | let o = match joint(t, e, Class::Space) { |
| 458 | Some(o) => o, |
| 459 | None => continue, |
| 460 | }; |
| 461 | if !t.is(o, q) { |
| 462 | continue; |
| 463 | } |
| 464 | let v = o + 1; |
| 465 | if t.lit_ci(v, MARKED) { |
| 466 | continue; |
| 467 | } |
| 468 | let mut p = v; |
| 469 | let closed = loop { |
| 470 | match t.at(p) { |
| 471 | Some(c) if c == q as u16 => break true, |
| 472 | Some(0x5c) => match t.at(p + 1) { |
| 473 | Some(c) if !js_line_end(c) => p += 2, |
| 474 | _ => break false, |
| 475 | }, |
| 476 | Some(0x0a) | Some(0x0d) | None => break false, |
| 477 | Some(_) => p += 1, |
| 478 | } |
| 479 | }; |
| 480 | if closed && p > v { |
| 481 | return Some((v, p)); |
| 482 | } |
| 483 | } |
| 484 | None |
| 485 | } |
| 486 | |
| 487 | fn quoted_double(t: &mut Text, i: usize) -> Option<(usize, usize)> { |
| 488 | quoted(t, i, b'"') |
| 489 | } |
| 490 | |
| 491 | fn quoted_single(t: &mut Text, i: usize) -> Option<(usize, usize)> { |
| 492 | quoted(t, i, b'\'') |
| 493 | } |
| 494 | |
| 495 | // A typed secret's name and the rest of its line, which is what a passphrase of words needs. The |
| 496 | // JavaScript is `TYPED["']?[ \t]{0,3}[:=][ \t]{0,3}((?!["']|\[redacted )\S(?:[^\r\n]*\S)?)`: a |
| 497 | // first unit that is no space, then up to the last unit before the line's end that is no space. |
| 498 | fn typed(t: &mut Text, i: usize) -> Option<(usize, usize)> { |
| 499 | for name in TYPED { |
| 500 | let e = match t.name(i, name) { |
| 501 | Some(e) => e, |
| 502 | None => continue, |
| 503 | }; |
| 504 | let v = match joint(t, e, Class::Blank) { |
| 505 | Some(v) => v, |
| 506 | None => continue, |
| 507 | }; |
| 508 | match t.at(v) { |
| 509 | Some(c) if !js_space(c) && c != 0x22 && c != 0x27 && !t.lit_ci(v, MARKED) => (), |
| 510 | _ => continue, |
| 511 | } |
| 512 | let mut end = v + 1; |
| 513 | let mut p = v + 1; |
| 514 | while let Some(c) = t.at(p) { |
| 515 | if c == 0x0a || c == 0x0d { |
| 516 | break; |
| 517 | } |
| 518 | p += 1; |
| 519 | if !js_space(c) { |
| 520 | end = p; |
| 521 | } |
| 522 | } |
| 523 | return Some((v, end)); |
| 524 | } |
| 525 | None |
| 526 | } |
| 527 | |
| 528 | // What follows a name: up to three quote marks or spaces, a colon or an equals sign, and up to |
| 529 | // three more. Where more than three stand, the JavaScript's greedy count takes three and meets |
| 530 | // another, and giving some back leaves a quote where its value should start, so it is a miss. |
| 531 | fn after_name(t: &mut Text, i: usize) -> Option<usize> { |
| 532 | let q = t.run(i, Class::Quote); |
| 533 | if q > 3 { |
| 534 | return None; |
| 535 | } |
| 536 | let c = i + q; |
| 537 | if !(t.is(c, b':') || t.is(c, b'=')) { |
| 538 | return None; |
| 539 | } |
| 540 | let q2 = t.run(c + 1, Class::Quote); |
| 541 | if q2 > 3 { |
| 542 | return None; |
| 543 | } |
| 544 | Some(c + 1 + q2) |
| 545 | } |
| 546 | |
| 547 | // A secret's name sitting against a run of eight or more of a token's characters, which is what a |
| 548 | // bare `looksSecret` used to answer yes or no to, with the value cut out so that the message stays |
| 549 | // readable. It runs after the quoted and the typed rules, which take what it once let through. |
| 550 | fn named(t: &mut Text, i: usize) -> Option<(usize, usize)> { |
| 551 | for name in NAMED { |
| 552 | let e = match t.name(i, name) { |
| 553 | Some(e) => e, |
| 554 | None => continue, |
| 555 | }; |
| 556 | if let Some(v) = after_name(t, e) { |
| 557 | let n = t.run(v, Class::Named); |
| 558 | if n >= 8 { |
| 559 | return Some((v, v + n)); |
| 560 | } |
| 561 | } |
| 562 | } |
| 563 | None |
| 564 | } |
| 565 | |
| 566 | fn separator(c: u16) -> bool { |
| 567 | matches!(c, 0x22 | 0x27 | 0x3a | 0x3d | 0x2c) || js_space(c) |
| 568 | } |
| 569 | |
| 570 | // An AWS secret is forty characters of base64 wearing no prefix, so it is recognisable only beside |
| 571 | // the key id, or the name, it belongs to. This one is beside a key id: the id, twelve or more |
| 572 | // capitals and digits, any 200 units or fewer, a separator, and the forty. The JavaScript is |
| 573 | // `[0-9A-Z]{12,}[\s\S]{0,200}?["'\s:=,]`, and the run before the gap can give units back, but a |
| 574 | // unit it gives is a capital or a digit and so no separator, so the first separator at or after |
| 575 | // the run's end is the only one the pattern can use. Every start within one run meets that same |
| 576 | // separator, which is why a miss is all of them and `skip` steps past the run. |
| 577 | fn aws_id(t: &mut Text, i: usize, skip: &mut usize) -> Option<(usize, usize)> { |
| 578 | if i < *skip || !(t.lit(i, "AKIA") || t.lit(i, "ASIA")) { |
| 579 | return None; |
| 580 | } |
| 581 | let n = t.run(i + 4, Class::Upper); |
| 582 | let end = i + 4 + n; |
| 583 | *skip = end; |
| 584 | if n < 12 { |
| 585 | return None; |
| 586 | } |
| 587 | for p in end..=end + 200 { |
| 588 | match t.at(p) { |
| 589 | Some(c) if separator(c) => if t.count(p + 1, Class::Std, 41) == 40 { |
| 590 | return Some((p + 1, p + 41)); |
| 591 | }, |
| 592 | Some(_) => (), |
| 593 | None => break, |
| 594 | } |
| 595 | } |
| 596 | None |
| 597 | } |
| 598 | |
| 599 | // The same secret beside its name: `aws_secret_access_key`, with or without the first word and |
| 600 | // each underscore, in either case, then the usual separators and forty characters exactly. |
| 601 | fn aws_name(t: &mut Text, i: usize) -> Option<(usize, usize)> { |
| 602 | let mut p = i; |
| 603 | if t.lit_ci(p, "aws") { |
| 604 | p += 3; |
| 605 | } |
| 606 | for word in ["secret", "access", "key"] { |
| 607 | if t.is(p, b'_') { |
| 608 | p += 1; |
| 609 | } |
| 610 | if !t.lit_ci(p, word) { |
| 611 | return None; |
| 612 | } |
| 613 | p += word.len(); |
| 614 | } |
| 615 | let v = match after_name(t, p) { |
| 616 | Some(v) => v, |
| 617 | None => return None, |
| 618 | }; |
| 619 | if t.count(v, Class::Std, 41) == 40 { |
| 620 | return Some((v, v + 40)); |
| 621 | } |
| 622 | None |
| 623 | } |
| 624 | |
| 625 | |
| 626 | // One rule over the whole string, left to right, replacing each span it finds with a marker. A |
| 627 | // span is where the rule says its value stands, and what lies between the last replacement and |
| 628 | // it is kept, which is how a pair keeps its name. Answers whether it found anything. |
| 629 | fn pass<F>(cur: &mut Vec<u16>, kind: &str, mut rule: F) -> bool |
| 630 | where |
| 631 | F: FnMut(&mut Text, usize) -> Option<(usize, usize)>, |
| 632 | { |
| 633 | let n = cur.len(); |
| 634 | let mut text = Text::new(cur); |
| 635 | let mut out: Option<Vec<u16>> = None; |
| 636 | let mut done = 0; |
| 637 | let mut i = 0; |
| 638 | while i < n { |
| 639 | match rule(&mut text, i) { |
| 640 | Some((v, e)) => { |
| 641 | let o = out.get_or_insert_with(|| Vec::with_capacity(n + 32)); |
| 642 | o.extend_from_slice(&cur[done..v]); |
| 643 | o.extend(mark_units(kind, &cur[v..e])); |
| 644 | done = e; |
| 645 | i = e; |
| 646 | }, |
| 647 | None => i += 1, |
| 648 | } |
| 649 | } |
| 650 | match out { |
| 651 | Some(mut o) => { |
| 652 | o.extend_from_slice(&cur[done..]); |
| 653 | *cur = o; |
| 654 | true |
| 655 | }, |
| 656 | None => false, |
| 657 | } |
| 658 | } |
| 659 | |
| 660 | fn units(s: &str) -> Vec<u16> { |
| 661 | s.encode_utf16().collect() |
| 662 | } |
| 663 | |
| 664 | fn mark_units(kind: &str, v: &[u16]) -> Vec<u16> { |
| 665 | // djb2 over the UTF-16 units, as the JavaScript's `scrubMark` and `fingerprint` do. |
| 666 | let mut h: u32 = 5381; |
| 667 | for &c in v { |
| 668 | h = (h << 5).wrapping_add(h).wrapping_add(c as u32); |
| 669 | } |
| 670 | units(&fmt!("[redacted {} #{:x}/{}]", kind, h, v.len())) |
| 671 | } |
| 672 | |
| 673 | /// The marker for one scrubbed value: its shape, a stable hash and its length in UTF-16 units. |
| 674 | pub fn mark(kind: &str, v: &str) -> String { |
| 675 | String::from_utf16_lossy(&mark_units(kind, &units(v))) |
| 676 | } |
| 677 | |
| 678 | /// Shannon entropy of a string in bits a unit. Hex tops out at 4.0 by construction and |
| 679 | /// base64url at 6.0, which is where the threshold sits: a digest stops and a key begins. |
| 680 | /// |
| 681 | /// The terms are summed in the order JavaScript's `Object.keys` gives the counts, the digits |
| 682 | /// ascending and then the rest as they were first met. `f64::ln` is the platform's and V8's is |
| 683 | /// its own, so the two can differ in the last bit or two. That matters only for a run whose |
| 684 | /// entropy lies within 2e-15 of the threshold, and on 2026-10-02 a differential run over 10,000 |
| 685 | /// runs, most of them built to sit on it, found no decision that differed. |
| 686 | pub fn entropy(s: &str) -> f64 { |
| 687 | entropy_units(&units(s)) |
| 688 | } |
| 689 | |
| 690 | fn entropy_units(s: &[u16]) -> f64 { |
| 691 | let mut at: HashMap<u16, usize> = HashMap::new(); |
| 692 | let mut seen: Vec<(u16, usize)> = Vec::new(); |
| 693 | for &c in s { |
| 694 | match at.get(&c) { |
| 695 | Some(&k) => seen[k].1 += 1, |
| 696 | None => { |
| 697 | at.insert(c, seen.len()); |
| 698 | seen.push((c, 1)); |
| 699 | }, |
| 700 | } |
| 701 | } |
| 702 | let mut order: Vec<(u16, usize)> = seen.iter().filter(|(c, _)| is_digit(*c)).copied().collect(); |
| 703 | order.sort_by_key(|(c, _)| *c); |
| 704 | order.extend(seen.iter().filter(|(c, _)| !is_digit(*c)).copied()); |
| 705 | let len = s.len() as f64; |
| 706 | let mut h = 0.0; |
| 707 | for (_, n) in order { |
| 708 | let p = n as f64 / len; |
| 709 | h -= p * (p.ln() / LN_2); |
| 710 | } |
| 711 | h |
| 712 | } |
| 713 | |
| 714 | /// Is this unbroken run one the feed is supposed to carry? A digest, a build id and a device id |
| 715 | /// are hex, a correlation id is hex joined by `-` or `_`, a count is decimal, a stretch of prose |
| 716 | /// is letters, and an identifier a person wrote is every segment a word or a number. Named |
| 717 | /// rather than left to the entropy threshold, so that the feed stays legible by rule. |
| 718 | pub fn safe_run(s: &str) -> bool { |
| 719 | safe_units(&units(s)) |
| 720 | } |
| 721 | |
| 722 | fn safe_units(s: &[u16]) -> bool { |
| 723 | let sep = |c: u16| c == b'-' as u16 || c == b'_' as u16; |
| 724 | let bare: Vec<u16> = s.iter().copied().filter(|c| !sep(*c)).collect(); |
| 725 | if !bare.is_empty() { |
| 726 | if bare.iter().all(|c| is_hex(*c)) || bare.iter().all(|c| is_digit(*c)) |
| 727 | || bare.iter().all(|c| is_letter(*c)) |
| 728 | { |
| 729 | return true; |
| 730 | } |
| 731 | } |
| 732 | let parts: Vec<&[u16]> = s.split(|c| sep(*c)).collect(); |
| 733 | if parts.len() > 1 { |
| 734 | return parts.iter().all(|p| |
| 735 | (p.len() >= 2 && p.iter().all(|c| is_letter(*c))) |
| 736 | || (!p.is_empty() && p.iter().all(|c| is_digit(*c)))); |
| 737 | } |
| 738 | false |
| 739 | } |
| 740 | |
| 741 | // The scrubbed units, or nothing where no rule found anything. |
| 742 | fn go(s: &str, is_id: bool) -> Option<Vec<u16>> { |
| 743 | let mut cur = units(s); |
| 744 | // Nothing in the list above is shorter than a dozen. |
| 745 | if cur.len() < 8 { |
| 746 | return None; |
| 747 | } |
| 748 | let mut hit = false; |
| 749 | for shape in SHAPES { |
| 750 | hit |= pass(&mut cur, shape.kind, |t, i| shape_at(shape, t, i)); |
| 751 | } |
| 752 | hit |= pass(&mut cur, "bearer", bearer); |
| 753 | hit |= pass(&mut cur, "urlarg", urlarg); |
| 754 | hit |= pass(&mut cur, "named", quoted_double); |
| 755 | hit |= pass(&mut cur, "named", quoted_single); |
| 756 | hit |= pass(&mut cur, "named", typed); |
| 757 | hit |= pass(&mut cur, "named", named); |
| 758 | let mut skip = 0; |
| 759 | hit |= pass(&mut cur, "awssec", |t, i| aws_id(t, i, &mut skip)); |
| 760 | hit |= pass(&mut cur, "awssec", aws_name); |
| 761 | // The catch-all runs last, so a run already turned into a marker is not weighed again. It is |
| 762 | // a shape nobody has listed: a long unbroken run that is neither one of the shapes a feed |
| 763 | // carries nor low in entropy. |
| 764 | if !is_id { |
| 765 | let mut skip = 0; |
| 766 | hit |= pass(&mut cur, "hi", |t, i| { |
| 767 | if i < skip { |
| 768 | return None; |
| 769 | } |
| 770 | let n = t.run(i, Class::Token); |
| 771 | if n == 0 { |
| 772 | return None; |
| 773 | } |
| 774 | // Every start in this run sees the same run, shorter, so it is decided once. |
| 775 | skip = i + n; |
| 776 | if n < MIN_RUN { |
| 777 | return None; |
| 778 | } |
| 779 | let s = t.s; |
| 780 | let run = &s[i..i + n]; |
| 781 | if safe_units(run) || entropy_units(run) <= MIN_BITS { |
| 782 | return None; |
| 783 | } |
| 784 | Some((i, i + n)) |
| 785 | }); |
| 786 | } |
| 787 | if hit { Some(cur) } else { None } |
| 788 | } |
| 789 | |
| 790 | /// The string with every credential in it replaced by `[redacted <shape> #<hash>/<length>]`, |
| 791 | /// exactly as the JavaScript scrubber writes it, and the same string where there is none. |
| 792 | /// |
| 793 | /// `is_id` is for a string found in a field that names a correlation id ([`ID_KEYS`]): the |
| 794 | /// entropy catch is not applied to it, and every shape and pair still is. |
| 795 | pub fn text(s: &str, is_id: bool) -> String { |
| 796 | match go(s, is_id) { |
| 797 | Some(u) => String::from_utf16_lossy(&u), |
| 798 | None => s.to_string(), |
| 799 | } |
| 800 | } |
| 801 | |
| 802 | /// Does `text` find anything in this string? |
| 803 | pub fn hit(s: &str, is_id: bool) -> bool { |
| 804 | go(s, is_id).is_some() |
| 805 | } |