oxedyne/fe2o3/fe2o3_net/src/lens/redact.rs
29.6 KiB, 15 runs
created by r1870400018:65642, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! The redactor of Daimond's feed, over a `Dat`. |
| 2 | //! |
| 3 | //! `secret_name` is `SECRET_RE` and `fingerprint` is `fingerprint` from `www/js/debugshare.js` |
| 4 | //! (the lines marked `SECRET_RE`, `SECRET_RE_LOOSE` and `function fingerprint` there), and a |
| 5 | //! test holds each to the JavaScript over a table that JavaScript generated. `Redact` is its |
| 6 | //! `redact` walk: a deep copy in which every field whose name looks secret has its value |
| 7 | //! replaced by a fingerprint, so the operator still sees which fields were present and how they |
| 8 | //! relate, and only the value is gone. Two hooks are added for a caller's own knowledge: a |
| 9 | //! deny-list of names and paths, and a test over strings. |
| 10 | //! |
| 11 | //! Free text is the caller's string test, and `Shapes` is a stock one for the shapes a credential |
| 12 | //! wears. It is the union of two things in `fe2o3_text::secret`: the scanner behind the commit |
| 13 | //! hook, and a port of the content scrubber of `debugshare.js` (its `SCRUB_SHAPES`, its pairs and |
| 14 | //! its entropy catch), so a feed is at least as well guarded here as it is in Daimond's browser. |
| 15 | //! A caller adds what only it knows, as Oxegen adds its eight-word passphrase, by passing its own |
| 16 | //! test. |
| 17 | |
| 18 | use super::row::Row; |
| 19 | |
| 20 | use oxedyne_fe2o3_core::prelude::*; |
| 21 | use oxedyne_fe2o3_jdat::prelude::*; |
| 22 | use oxedyne_fe2o3_text::secret::{ |
| 23 | self, |
| 24 | scrub, |
| 25 | }; |
| 26 | |
| 27 | use std::collections::HashSet; |
| 28 | |
| 29 | |
| 30 | // The walk's bound, as in `redact`. A `Dat` cannot hold a cycle, so this is a depth bound alone. |
| 31 | const MAX_DEPTH: usize = 40; |
| 32 | const DEEP: &str = "[redacted:cycle-or-deep]"; |
| 33 | const OBJECT: &str = "[redacted:object]"; |
| 34 | const ABSENT: &str = "[redacted:absent]"; |
| 35 | |
| 36 | // SECRET_RE's words, in its order. |
| 37 | const WORDS: [&str; 17] = [ |
| 38 | "apikey", "api_key", "key", "token", "secret", "passphrase", "password", "salt", "wrapped", |
| 39 | "wrappedpriv", "sealed", "seal", "privatekey", "priv", "mnemonic", "seed", "masterkey", |
| 40 | ]; |
| 41 | |
| 42 | fn sep(b: u8) -> bool { |
| 43 | matches!(b, b'_' | b'.' | b'-') |
| 44 | } |
| 45 | |
| 46 | // JavaScript's `\w`, which is ASCII alone. |
| 47 | fn word(b: u8) -> bool { |
| 48 | b.is_ascii_alphanumeric() || b == b'_' |
| 49 | } |
| 50 | |
| 51 | /// Does the name look like a secret's? This is Daimond's `SECRET_RE`, |
| 52 | /// |
| 53 | /// `/(?:^|[_.-])(?:apikey|api_key|key|token|secret|passphrase|password|salt|wrapped|wrappedpriv| |
| 54 | /// sealed|seal|privatekey|priv|mnemonic|seed|masterkey)(?:$|[_.-]|enc\b)/i`, |
| 55 | /// |
| 56 | /// tested anywhere in the name. A word counts only at the start of the name or after a `_`, `.` |
| 57 | /// or `-`, so a camel-joined `pushToken` slides past it; `secret_name_loose` catches those. |
| 58 | /// |
| 59 | /// This is a hand matcher and not a `fe2o3_text::regex::Regex` on purpose. JavaScript's `\b` and |
| 60 | /// `/i` are ASCII-only, while the crate's engine follows the `regex` crate, whose `\b` and case |
| 61 | /// folding are Unicode, so the two would disagree on a name such as `keyenc` followed by `é`. A |
| 62 | /// test holds the matcher to the JavaScript over generated names, `é` included, and to the engine |
| 63 | /// over the ASCII ones. |
| 64 | pub fn secret_name(name: &str) -> bool { |
| 65 | let b = name.as_bytes(); |
| 66 | let mut starts = vec![0usize]; |
| 67 | for (i, &c) in b.iter().enumerate() { |
| 68 | if sep(c) { |
| 69 | starts.push(i + 1); |
| 70 | } |
| 71 | } |
| 72 | for s in starts { |
| 73 | for w in WORDS { |
| 74 | let n = w.len(); |
| 75 | if b.len() < s + n || !b[s..s + n].eq_ignore_ascii_case(w.as_bytes()) { |
| 76 | continue; |
| 77 | } |
| 78 | let e = s + n; |
| 79 | // `$`, a separator, or `enc` ending at a word boundary. |
| 80 | let tail = e == b.len() |
| 81 | || sep(b[e]) |
| 82 | || (b.len() >= e + 3 |
| 83 | && b[e..e + 3].eq_ignore_ascii_case(b"enc") |
| 84 | && (b.len() == e + 3 || !word(b[e + 3]))); |
| 85 | if tail { |
| 86 | return true; |
| 87 | } |
| 88 | } |
| 89 | } |
| 90 | false |
| 91 | } |
| 92 | |
| 93 | /// `secret_name`, or Daimond's `SECRET_RE_LOOSE`, `/token|secret|passphrase|password|key$/i`. |
| 94 | /// |
| 95 | /// The loose pattern has no boundary, so it also takes a real, non-secret field such as |
| 96 | /// `tokenStats`. Daimond uses it only on its free-form `config`, and so should a caller that |
| 97 | /// knows its names are camel-joined. |
| 98 | pub fn secret_name_loose(name: &str) -> bool { |
| 99 | if secret_name(name) { |
| 100 | return true; |
| 101 | } |
| 102 | let l = name.to_ascii_lowercase(); |
| 103 | l.contains("token") || l.contains("secret") || l.contains("passphrase") || l.contains("password") |
| 104 | || l.ends_with("key") |
| 105 | } |
| 106 | |
| 107 | /// Is this the name of a field a person types into? This is Daimond's `SECRET_KEY_TYPED`, |
| 108 | /// |
| 109 | /// `/(?:pass(?:word|wd|phrase)|mnemonic|seed[_-]?phrase)$|^pass$/i`: |
| 110 | /// |
| 111 | /// a passphrase, a password, a mailbox's app password (Daimond's mail posts it as `password` and |
| 112 | /// keeps it sealed as `pass`), a recovery phrase. It is matched at the END of the name, so |
| 113 | /// `smtpPassword` and `newPassphrase` count while `passAt` and `passwordHint` do not, and `pass` |
| 114 | /// only whole, so `bypass` and `compass` do not. `Redact` covers such a field whatever else its |
| 115 | /// name is, and marks its value with no head (SCRUB-ONE, D-20261006-24): the first six characters |
| 116 | /// a fingerprint keeps to tell two keys apart are the first word of a passphrase. |
| 117 | pub fn typed_name(name: &str) -> bool { |
| 118 | let l = name.to_ascii_lowercase(); |
| 119 | l == "pass" |
| 120 | || ["password", "passwd", "passphrase", "mnemonic", "seedphrase", "seed_phrase", "seed-phrase"] |
| 121 | .iter() |
| 122 | .any(|w| l.ends_with(w)) |
| 123 | } |
| 124 | |
| 125 | /// A short, non-reversible rendering of a secret for a person's eye: its first `head` |
| 126 | /// characters, a stable hash and its length. With `head` of 6 this is Daimond's `fingerprint` |
| 127 | /// exactly, which hashes and counts UTF-16 code units as JavaScript does. |
| 128 | /// |
| 129 | /// The hash is djb2, not a cryptographic hash. It only has to tell two secrets apart and match |
| 130 | /// two sightings of one. The head is itself a disclosure of up to `head` characters, and `0` |
| 131 | /// withholds it. |
| 132 | pub fn fingerprint(v: &str, head: usize) -> String { |
| 133 | if v.is_empty() { |
| 134 | return "[redacted:empty]".to_string(); |
| 135 | } |
| 136 | let u: Vec<u16> = v.encode_utf16().collect(); |
| 137 | let mut h: u32 = 5381; |
| 138 | for &c in &u { |
| 139 | h = (h << 5).wrapping_add(h).wrapping_add(c as u32); |
| 140 | } |
| 141 | let hd = String::from_utf16_lossy(&u[..head.min(u.len())]); |
| 142 | fmt!("[redacted {}…#{:x}/{}]", hd, h, u.len()) |
| 143 | } |
| 144 | |
| 145 | // Is this a marker the redactor itself wrote? A second pass leaves it alone, since a |
| 146 | // fingerprint of a fingerprint tells the reader nothing and hides the first. |
| 147 | pub(super) fn marker(s: &str) -> bool { |
| 148 | s.starts_with("[redacted") && s.ends_with(']') |
| 149 | } |
| 150 | |
| 151 | /// A caller's test over strings: does this one hold a secret? |
| 152 | /// |
| 153 | /// Any closure over `&str` is one. A string that passes is replaced whole by its fingerprint, |
| 154 | /// so the test should look *inside* a string (Oxegen's asks whether eight words of the EFF list |
| 155 | /// stand in a row) and not only at all of it. |
| 156 | pub trait StrTest: Send + Sync { |
| 157 | fn hit(&self, s: &str) -> bool; |
| 158 | |
| 159 | /// The same question of a string found under a correlation-id field (see |
| 160 | /// [`Redact::id_key`]). A test may excuse an id from its heuristics here and keep its shapes; |
| 161 | /// by default an id is held to the same test as any other string. |
| 162 | fn hit_id(&self, s: &str) -> bool { self.hit(s) } |
| 163 | |
| 164 | /// The byte ranges of `s`, the strings of a structure joined in order with a line break |
| 165 | /// between each, that this test calls secret for standing together. A word run is one: a |
| 166 | /// passphrase kept a word to a field is a run only once the fields are read as one text, and |
| 167 | /// the keys between the words would break it were the text the structure's own. A test of |
| 168 | /// shapes judges each string alone, as `hit` does, and answers `Vec::new()`. |
| 169 | /// |
| 170 | /// There is no default, so that a test that holds a word run cannot leave this out and have |
| 171 | /// a bundle's words go unseen without anything saying so. A test that wraps a `Phrase` hands |
| 172 | /// its `spans` on. |
| 173 | fn joined_spans(&self, s: &str) -> Vec<(usize, usize)>; |
| 174 | } |
| 175 | |
| 176 | /// The test that never fires, for a redactor that has no string test of its own. |
| 177 | #[derive(Clone, Copy, Debug, Default)] |
| 178 | pub struct NoTest; |
| 179 | |
| 180 | impl StrTest for NoTest { |
| 181 | fn hit(&self, _s: &str) -> bool { false } |
| 182 | fn joined_spans(&self, _s: &str) -> Vec<(usize, usize)> { Vec::new() } |
| 183 | } |
| 184 | |
| 185 | impl<F: Fn(&str) -> bool + Send + Sync> StrTest for F { |
| 186 | fn hit(&self, s: &str) -> bool { self(s) } |
| 187 | fn joined_spans(&self, _s: &str) -> Vec<(usize, usize)> { Vec::new() } |
| 188 | } |
| 189 | |
| 190 | /// The stock string test: a string that holds a credential is covered whole. |
| 191 | /// |
| 192 | /// A credential here is anything `fe2o3_text::secret::holds` finds, which honours no `allowlist |
| 193 | /// secret` marker and skips no text for a NUL since a feed's text is the very thing in doubt, or |
| 194 | /// anything Daimond's content scrubber would mark: every shape in its `SCRUB_SHAPES`, a secret's |
| 195 | /// name standing against a value (`token=...`, an `Authorization` header, a URL's `?key=`), and |
| 196 | /// a long unbroken run of high entropy. That last is a heuristic, so a string found under a |
| 197 | /// correlation-id field is excused from it alone ([`StrTest::hit_id`]); a shape still hits there. |
| 198 | /// |
| 199 | /// `hit` says whether, and `scrub` is the in-place form Daimond writes, which cuts only the |
| 200 | /// credential out of a string and keeps the words round it. A [`Redact`] covers a hit whole, which |
| 201 | /// is the stronger of the two. Pair it with `Redact::with_head(0)` where the first characters of a |
| 202 | /// credential must not show in its fingerprint. |
| 203 | #[derive(Clone, Copy, Debug, Default)] |
| 204 | pub struct Shapes; |
| 205 | |
| 206 | impl Shapes { |
| 207 | |
| 208 | /// The string with each credential replaced by `[redacted <shape> #<hash>/<length>]`, exactly |
| 209 | /// as the JavaScript scrubber writes it. `is_id` suppresses the entropy catch alone. |
| 210 | pub fn scrub(&self, s: &str, is_id: bool) -> String { |
| 211 | scrub::text(s, is_id) |
| 212 | } |
| 213 | } |
| 214 | |
| 215 | impl StrTest for Shapes { |
| 216 | fn hit(&self, s: &str) -> bool { secret::holds(s) || scrub::hit(s, false) } |
| 217 | fn hit_id(&self, s: &str) -> bool { secret::holds(s) || scrub::hit(s, true) } |
| 218 | fn joined_spans(&self, _s: &str) -> Vec<(usize, usize)> { Vec::new() } |
| 219 | } |
| 220 | |
| 221 | /// A test that fires when either of two does, so an app can add its own to the stock shapes: |
| 222 | /// `Or(Shapes, Phrase::new(words, 8))`. |
| 223 | #[derive(Clone, Copy, Debug, Default)] |
| 224 | pub struct Or<A, B>(pub A, pub B); |
| 225 | |
| 226 | impl<A: StrTest, B: StrTest> StrTest for Or<A, B> { |
| 227 | fn hit(&self, s: &str) -> bool { self.0.hit(s) || self.1.hit(s) } |
| 228 | fn hit_id(&self, s: &str) -> bool { self.0.hit_id(s) || self.1.hit_id(s) } |
| 229 | fn joined_spans(&self, s: &str) -> Vec<(usize, usize)> { |
| 230 | let mut out = self.0.joined_spans(s); |
| 231 | out.extend(self.1.joined_spans(s)); |
| 232 | out |
| 233 | } |
| 234 | } |
| 235 | |
| 236 | /// A run of words from a caller's list: `n` of them in a row are a secret. |
| 237 | /// |
| 238 | /// An app whose passphrases are drawn from a list of words (Oxegen's are eight words of the EFF |
| 239 | /// large list) cannot name the secret by shape, since nothing about a word is secret, but a string |
| 240 | /// in which `n` of the list's words stand together is a passphrase or near enough to one that no |
| 241 | /// row may keep it. Words are the runs of ASCII letters in the text, whatever separates them |
| 242 | /// (a space, a hyphen, a comma, a line break, a digit, a letter outside ASCII) and whatever the |
| 243 | /// case; any run of letters that is not on the list breaks the run. The list is the caller's, so a |
| 244 | /// second app passes its own and the test is the same. |
| 245 | /// |
| 246 | /// A list entry may hold hyphens (the EFF list has `drop-down`, `felt-tip`, `t-shirt` and |
| 247 | /// `yo-yo`). Where the text spells such an entry, hyphens and all, it counts as the one word it is, |
| 248 | /// and the longest entry wins. Splitting on every non-letter cannot see those four entries, and |
| 249 | /// breaks a passphrase that holds one into runs shorter than eight. An entry whose every part is |
| 250 | /// a list word counts for its parts, since a dashed passphrase joins its words with the same hyphen. |
| 251 | /// |
| 252 | /// This is the Rust half of `createRunGuard` in `redact.js`, and `tests/data/lens_phrase.tsv` holds |
| 253 | /// the two to the same answers. |
| 254 | #[derive(Clone, Debug)] |
| 255 | pub struct Phrase { |
| 256 | words: HashSet<String>, // lower case |
| 257 | n: usize, // words in a row that make a secret |
| 258 | parts: usize, // the most hyphen-joined parts any entry has |
| 259 | } |
| 260 | |
| 261 | impl Phrase { |
| 262 | |
| 263 | /// A test over `words`, any case, firing on `n` in a row (at least one). |
| 264 | pub fn new<'a, I: IntoIterator<Item = &'a str>>(words: I, n: usize) -> Self { |
| 265 | let words: HashSet<String> = words.into_iter() |
| 266 | .map(|w| w.trim().to_ascii_lowercase()) |
| 267 | .filter(|w| !w.is_empty()) |
| 268 | .collect(); |
| 269 | let parts = words.iter().map(|w| w.matches('-').count() + 1).max().unwrap_or(1); |
| 270 | Self { words, n: n.max(1), parts } |
| 271 | } |
| 272 | |
| 273 | // The runs of ASCII letters in `b`, as byte ranges. |
| 274 | fn tokens(b: &[u8]) -> Vec<(usize, usize)> { |
| 275 | let mut out = Vec::new(); |
| 276 | let mut at = None; |
| 277 | for (i, c) in b.iter().enumerate() { |
| 278 | match (c.is_ascii_alphabetic(), at) { |
| 279 | (true, None) => at = Some(i), |
| 280 | (false, Some(from)) => { |
| 281 | out.push((from, i)); |
| 282 | at = None; |
| 283 | }, |
| 284 | _ => {}, |
| 285 | } |
| 286 | } |
| 287 | if let Some(from) = at { |
| 288 | out.push((from, b.len())); |
| 289 | } |
| 290 | out |
| 291 | } |
| 292 | |
| 293 | // The word starting at token `i`, as (tokens it takes up, words it counts for); (0, 0) if none. |
| 294 | // The longest hyphen-joined entry of the list that the text spells wins. It counts for one |
| 295 | // word, or for as many as it has parts where each part is itself a word, since hyphens also |
| 296 | // join the words of a dashed passphrase and the safer count is the larger. |
| 297 | fn word_at(&self, s: &str, t: &[(usize, usize)], i: usize) -> (usize, usize) { |
| 298 | let mut chain = 1; |
| 299 | while chain < self.parts && i + chain < t.len() && &s[t[i + chain - 1].1..t[i + chain].0] == "-" { |
| 300 | chain += 1; |
| 301 | } |
| 302 | for k in (2..=chain).rev() { |
| 303 | if self.words.contains(&s[t[i].0..t[i + k - 1].1].to_ascii_lowercase()) { |
| 304 | let each = (i..i + k).all(|j| self.words.contains(&s[t[j].0..t[j].1].to_ascii_lowercase())); |
| 305 | return (k, if each { k } else { 1 }); |
| 306 | } |
| 307 | } |
| 308 | if self.words.contains(&s[t[i].0..t[i].1].to_ascii_lowercase()) { (1, 1) } else { (0, 0) } |
| 309 | } |
| 310 | |
| 311 | /// The byte ranges of `s` that hold `n` words or more in a row, each running from the first |
| 312 | /// word's first letter to the last word's last, with whatever stood between the words. |
| 313 | pub fn spans(&self, s: &str) -> Vec<(usize, usize)> { |
| 314 | let mut out = Vec::new(); |
| 315 | if s.len() < self.n.saturating_mul(2).saturating_sub(1) { |
| 316 | return out; |
| 317 | } |
| 318 | let t = Self::tokens(s.as_bytes()); |
| 319 | let (mut from, mut to, mut run) = (0usize, 0usize, 0usize); |
| 320 | let mut i = 0; |
| 321 | while i < t.len() { |
| 322 | let (took, count) = self.word_at(s, &t, i); |
| 323 | if took == 0 { |
| 324 | if run >= self.n { |
| 325 | out.push((from, to)); |
| 326 | } |
| 327 | run = 0; |
| 328 | i += 1; |
| 329 | continue; |
| 330 | } |
| 331 | if run == 0 { |
| 332 | from = t[i].0; |
| 333 | } |
| 334 | to = t[i + took - 1].1; |
| 335 | run += count; |
| 336 | i += took; |
| 337 | } |
| 338 | if run >= self.n { |
| 339 | out.push((from, to)); |
| 340 | } |
| 341 | out |
| 342 | } |
| 343 | |
| 344 | /// Does `s` hold `n` words of the list in a row? |
| 345 | pub fn has(&self, s: &str) -> bool { |
| 346 | !self.spans(s).is_empty() |
| 347 | } |
| 348 | } |
| 349 | |
| 350 | impl StrTest for Phrase { |
| 351 | fn hit(&self, s: &str) -> bool { self.has(s) } |
| 352 | fn joined_spans(&self, s: &str) -> Vec<(usize, usize)> { self.spans(s) } |
| 353 | } |
| 354 | |
| 355 | /// A place a caller knows holds a secret, whatever it is called. |
| 356 | #[derive(Clone, Debug)] |
| 357 | pub enum Deny { |
| 358 | Name(String), // a field of this exact name, at any depth |
| 359 | Path(Vec<String>), // the fields along this path from the root, where `*` is any one name |
| 360 | } |
| 361 | |
| 362 | /// Walks a `Dat` and covers what looks secret. |
| 363 | /// |
| 364 | /// A value is covered when its field's name is a secret's (`secret_name`, or the loose pattern if |
| 365 | /// asked), when its name or its path is in the deny-list, or, for a string, when the caller's |
| 366 | /// test fires. A covered scalar becomes its fingerprint, a covered object or list becomes |
| 367 | /// `[redacted:object]` without being walked, and an empty value becomes `[redacted:absent]`. |
| 368 | /// Lists are transparent to a deny path, so `["sess", "nonce"]` reaches the members of a list of |
| 369 | /// sessions. Map keys that are strings meet the caller's test as well, and a string under a |
| 370 | /// correlation-id field meets its `hit_id` instead (see [`Redact::id_key`]). |
| 371 | pub struct Redact<T: StrTest = NoTest> { |
| 372 | deny: Vec<Deny>, |
| 373 | test: T, |
| 374 | head: usize, |
| 375 | loose: bool, |
| 376 | ids: Vec<String>, // fields whose strings are correlation ids |
| 377 | } |
| 378 | |
| 379 | impl Redact<NoTest> { |
| 380 | |
| 381 | pub fn new() -> Self { |
| 382 | Self { |
| 383 | deny: Vec::new(), |
| 384 | test: NoTest, |
| 385 | head: Self::DEFAULT_HEAD, |
| 386 | loose: false, |
| 387 | ids: scrub::ID_KEYS.iter().map(|k| k.to_string()).collect(), |
| 388 | } |
| 389 | } |
| 390 | } |
| 391 | |
| 392 | impl Default for Redact<NoTest> { |
| 393 | fn default() -> Self { Self::new() } |
| 394 | } |
| 395 | |
| 396 | impl<T: StrTest> Redact<T> { |
| 397 | |
| 398 | pub const DEFAULT_HEAD: usize = 6; |
| 399 | |
| 400 | pub fn deny(mut self, d: Deny) -> Self { |
| 401 | self.deny.push(d); |
| 402 | self |
| 403 | } |
| 404 | |
| 405 | pub fn deny_name(self, name: &str) -> Self { |
| 406 | self.deny(Deny::Name(name.to_string())) |
| 407 | } |
| 408 | |
| 409 | pub fn deny_path(self, path: &[&str]) -> Self { |
| 410 | self.deny(Deny::Path(path.iter().map(|s| s.to_string()).collect())) |
| 411 | } |
| 412 | |
| 413 | /// Takes `secret_name_loose` for the names, which catches `pushToken` and `refreshSecret`. |
| 414 | pub fn loose(mut self) -> Self { |
| 415 | self.loose = true; |
| 416 | self |
| 417 | } |
| 418 | |
| 419 | /// How many leading characters a fingerprint shows. Daimond shows 6. |
| 420 | pub fn with_head(mut self, head: usize) -> Self { |
| 421 | self.head = head; |
| 422 | self |
| 423 | } |
| 424 | |
| 425 | /// Names a field whose strings are correlation ids, which a reader joins rows by: a string |
| 426 | /// under it, or under it through any lists, meets the test's `hit_id` and not its `hit`. |
| 427 | /// Daimond's own list (`id`, `callId`, `turn`, `device`, `d`, `b`, `n` and the rest of |
| 428 | /// `SCRUB_ID_KEYS`) is there from the start; an application adds the ids it stamps, such as a |
| 429 | /// session or a connection. |
| 430 | pub fn id_key(mut self, name: &str) -> Self { |
| 431 | self.ids.push(name.to_string()); |
| 432 | self |
| 433 | } |
| 434 | |
| 435 | pub fn with_test<U: StrTest>(self, test: U) -> Redact<U> { |
| 436 | Redact { |
| 437 | deny: self.deny, |
| 438 | test, |
| 439 | head: self.head, |
| 440 | loose: self.loose, |
| 441 | ids: self.ids, |
| 442 | } |
| 443 | } |
| 444 | |
| 445 | /// A deep copy with the secrets covered. |
| 446 | pub fn dat(&self, d: &Dat) -> Dat { |
| 447 | self.walk(d).0 |
| 448 | } |
| 449 | |
| 450 | /// `dat`, and whether anything was covered. A caller that finds nothing changed can keep the |
| 451 | /// text it parsed the `Dat` from, byte for byte. |
| 452 | pub fn walk(&self, d: &Dat) -> (Dat, bool) { |
| 453 | let mut path = Vec::new(); |
| 454 | self.value(d, &mut path, 0, false) |
| 455 | } |
| 456 | |
| 457 | /// A copy of the row with its payload covered. The payload's field names are the root's. |
| 458 | pub fn row(&self, row: &Row) -> Row { |
| 459 | let mut out = row.clone(); |
| 460 | let mut path = Vec::new(); |
| 461 | for (k, v) in out.body.iter_mut() { |
| 462 | path.push(k.clone()); |
| 463 | let (nv, _) = self.member(k, v, &mut path, 1); |
| 464 | path.pop(); |
| 465 | *v = nv; |
| 466 | } |
| 467 | out |
| 468 | } |
| 469 | |
| 470 | /// The caller's string test. |
| 471 | pub fn test(&self) -> &T { &self.test } |
| 472 | |
| 473 | /// The fingerprint this redactor writes over a string it covers, with its own head. |
| 474 | pub fn mark(&self, s: &str) -> String { |
| 475 | fingerprint(s, self.head) |
| 476 | } |
| 477 | |
| 478 | /// The fingerprint of a free string if the caller's test fires on it. |
| 479 | pub fn text(&self, s: &str) -> Option<String> { |
| 480 | if self.test.hit(s) { |
| 481 | Some(fingerprint(s, self.head)) |
| 482 | } else { |
| 483 | None |
| 484 | } |
| 485 | } |
| 486 | |
| 487 | fn secret(&self, name: &str) -> bool { |
| 488 | if self.loose { secret_name_loose(name) } else { secret_name(name) } |
| 489 | } |
| 490 | |
| 491 | fn denied(&self, path: &[String]) -> bool { |
| 492 | self.deny.iter().any(|d| match d { |
| 493 | Deny::Name(n) => path.last().map_or(false, |l| l == n), |
| 494 | Deny::Path(p) => p.len() == path.len() |
| 495 | && p.iter().zip(path.iter()).all(|(a, b)| a == "*" || a == b), |
| 496 | }) |
| 497 | } |
| 498 | |
| 499 | // One field of a map: covered by name or place, or walked. |
| 500 | fn member(&self, name: &str, v: &Dat, path: &mut Vec<String>, depth: usize) -> (Dat, bool) { |
| 501 | let typed = typed_name(name); |
| 502 | if typed || self.secret(name) || self.denied(path) { |
| 503 | return self.cover(v, typed); |
| 504 | } |
| 505 | let id = self.ids.iter().any(|k| k == name); |
| 506 | self.value(v, path, depth, id) |
| 507 | } |
| 508 | |
| 509 | // The caller's test, in the form that suits a correlation id or any other string. |
| 510 | fn hit(&self, s: &str, id: bool) -> bool { |
| 511 | if id { self.test.hit_id(s) } else { self.test.hit(s) } |
| 512 | } |
| 513 | |
| 514 | // A value covered by its field: a scalar's fingerprint, or a note of what it was. A typed |
| 515 | // field's scalar is marked with no head, `[redacted named #<hash>/<length>]`, as the scrubber |
| 516 | // marks a value it cuts from text, whatever head this redactor shows. |
| 517 | fn cover(&self, v: &Dat, typed: bool) -> (Dat, bool) { |
| 518 | let s = match v { |
| 519 | Dat::Empty => return (Dat::Str(ABSENT.to_string()), true), |
| 520 | Dat::Opt(o) => match &**o { |
| 521 | Some(x) => return self.cover(x, typed), |
| 522 | None => return (Dat::Str(ABSENT.to_string()), true), |
| 523 | }, |
| 524 | Dat::Box(b) => return self.cover(b, typed), |
| 525 | Dat::Str(s) => s.clone(), |
| 526 | Dat::Bool(b) => fmt!("{}", b), |
| 527 | Dat::U8(n) => fmt!("{}", n), |
| 528 | Dat::U16(n) => fmt!("{}", n), |
| 529 | Dat::U32(n) => fmt!("{}", n), |
| 530 | Dat::U64(n) => fmt!("{}", n), |
| 531 | Dat::U128(n) => fmt!("{}", n), |
| 532 | Dat::I8(n) => fmt!("{}", n), |
| 533 | Dat::I16(n) => fmt!("{}", n), |
| 534 | Dat::I32(n) => fmt!("{}", n), |
| 535 | Dat::I64(n) => fmt!("{}", n), |
| 536 | Dat::I128(n) => fmt!("{}", n), |
| 537 | Dat::F32(n) => fmt!("{}", n), |
| 538 | Dat::F64(n) => fmt!("{}", n), |
| 539 | Dat::C64(n) => fmt!("{}", n), |
| 540 | // A list, a map, bytes: noted as present, never shown, as `[redacted:object]` is. |
| 541 | _ => return (Dat::Str(OBJECT.to_string()), true), |
| 542 | }; |
| 543 | if marker(&s) { |
| 544 | return (Dat::Str(s), false); |
| 545 | } |
| 546 | if typed && !s.is_empty() { |
| 547 | return (Dat::Str(scrub::mark("named", &s)), true); |
| 548 | } |
| 549 | (Dat::Str(fingerprint(&s, self.head)), true) |
| 550 | } |
| 551 | |
| 552 | fn key(&self, k: &Dat) -> (Dat, bool) { |
| 553 | match k { |
| 554 | Dat::Str(s) if self.test.hit(s) && !marker(s) => (Dat::Str(fingerprint(s, self.head)), true), |
| 555 | _ => (k.clone(), false), |
| 556 | } |
| 557 | } |
| 558 | |
| 559 | fn value(&self, v: &Dat, path: &mut Vec<String>, depth: usize, id: bool) -> (Dat, bool) { |
| 560 | let deep = depth > MAX_DEPTH; |
| 561 | match v { |
| 562 | Dat::Str(s) => { |
| 563 | if !marker(s) && self.hit(s, id) { |
| 564 | return (Dat::Str(fingerprint(s, self.head)), true); |
| 565 | } |
| 566 | }, |
| 567 | Dat::Map(m) => { |
| 568 | if deep { |
| 569 | return (Dat::Str(DEEP.to_string()), true); |
| 570 | } |
| 571 | let mut out = DaticleMap::new(); |
| 572 | let mut changed = false; |
| 573 | for (k, val) in m { |
| 574 | let name = key_text(k); |
| 575 | let (mut nk, kc) = self.key(k); |
| 576 | while out.contains_key(&nk) { |
| 577 | nk = Dat::Str(fmt!("{}~", key_text(&nk))); |
| 578 | } |
| 579 | path.push(name.clone()); |
| 580 | let (nv, vc) = self.member(&name, val, path, depth + 1); |
| 581 | path.pop(); |
| 582 | changed |= kc | vc; |
| 583 | out.insert(nk, nv); |
| 584 | } |
| 585 | return (Dat::Map(out), changed); |
| 586 | }, |
| 587 | Dat::OrdMap(m) => { |
| 588 | if deep { |
| 589 | return (Dat::Str(DEEP.to_string()), true); |
| 590 | } |
| 591 | let mut out = OrdDaticleMap::new(); |
| 592 | let mut changed = false; |
| 593 | for (mk, val) in m { |
| 594 | let name = key_text(mk.dat()); |
| 595 | let (nk, kc) = self.key(mk.dat()); |
| 596 | path.push(name.clone()); |
| 597 | let (nv, vc) = self.member(&name, val, path, depth + 1); |
| 598 | path.pop(); |
| 599 | changed |= kc | vc; |
| 600 | out.insert(MapKey::new(mk.ord(), nk), nv); |
| 601 | } |
| 602 | return (Dat::OrdMap(out), changed); |
| 603 | }, |
| 604 | Dat::List(l) => { |
| 605 | if deep { |
| 606 | return (Dat::Str(DEEP.to_string()), true); |
| 607 | } |
| 608 | let mut out = Vec::with_capacity(l.len()); |
| 609 | let mut changed = false; |
| 610 | for x in l { |
| 611 | let (nx, c) = self.value(x, path, depth + 1, id); |
| 612 | changed |= c; |
| 613 | out.push(nx); |
| 614 | } |
| 615 | return (Dat::List(out), changed); |
| 616 | }, |
| 617 | Dat::Box(b) => { |
| 618 | let (nb, c) = self.value(b, path, depth, id); |
| 619 | return (Dat::Box(Box::new(nb)), c); |
| 620 | }, |
| 621 | Dat::Opt(o) => { |
| 622 | if let Some(x) = &**o { |
| 623 | let (nx, c) = self.value(x, path, depth, id); |
| 624 | return (Dat::Opt(Box::new(Some(nx))), c); |
| 625 | } |
| 626 | }, |
| 627 | Dat::Usr(uid, Some(b)) => { |
| 628 | let (nb, c) = self.value(b, path, depth, id); |
| 629 | return (Dat::Usr(uid.clone(), Some(Box::new(nb))), c); |
| 630 | }, |
| 631 | _ => {}, |
| 632 | } |
| 633 | // A tuple is a list that has a length of its own. |
| 634 | macro_rules! tuples { |
| 635 | ($($t:ident),*) => { |
| 636 | match v { |
| 637 | $( |
| 638 | Dat::$t(a) => { |
| 639 | if deep { |
| 640 | return (Dat::Str(DEEP.to_string()), true); |
| 641 | } |
| 642 | let mut b = a.clone(); |
| 643 | let mut changed = false; |
| 644 | for x in b.iter_mut() { |
| 645 | let (nx, c) = self.value(x, path, depth + 1, id); |
| 646 | *x = nx; |
| 647 | changed |= c; |
| 648 | } |
| 649 | return (Dat::$t(b), changed); |
| 650 | }, |
| 651 | )* |
| 652 | _ => {}, |
| 653 | } |
| 654 | }; |
| 655 | } |
| 656 | tuples!(Tup2, Tup3, Tup4, Tup5, Tup6, Tup7, Tup8, Tup9, Tup10); |
| 657 | (v.clone(), false) |
| 658 | } |
| 659 | } |
| 660 | |
| 661 | fn key_text(k: &Dat) -> String { |
| 662 | match k { |
| 663 | Dat::Str(s) => s.clone(), |
| 664 | other => fmt!("{}", other), |
| 665 | } |
| 666 | } |
| 667 | |
| 668 | |
| 669 | #[cfg(test)] |
| 670 | mod tests { |
| 671 | use super::*; |
| 672 | |
| 673 | #[test] |
| 674 | fn enc_needs_a_word_boundary_after_it() { |
| 675 | // `enc\b`: the end of the name or a non-word character, and not `_`, a digit or a letter. |
| 676 | assert!(secret_name("keyenc")); |
| 677 | assert!(secret_name("keyENC")); |
| 678 | assert!(secret_name("key.enc")); |
| 679 | assert!(!secret_name("keyenc_x")); |
| 680 | assert!(!secret_name("keyenc1")); |
| 681 | assert!(!secret_name("keyencx")); |
| 682 | assert!(secret_name("keyenc é")); |
| 683 | } |
| 684 | |
| 685 | #[test] |
| 686 | fn a_word_counts_only_at_the_start_or_after_a_separator() { |
| 687 | assert!(secret_name("key")); |
| 688 | assert!(secret_name("x_key")); |
| 689 | assert!(secret_name("x.token-y")); |
| 690 | assert!(!secret_name("xkey")); |
| 691 | assert!(!secret_name("monkey")); |
| 692 | assert!(!secret_name("keys")); |
| 693 | assert!(!secret_name("")); |
| 694 | assert!(secret_name_loose("monkey"), "key$ is boundary-free"); |
| 695 | assert!(!secret_name_loose("keyboard")); |
| 696 | } |
| 697 | |
| 698 | #[test] |
| 699 | fn a_phrase_is_n_list_words_in_a_row_whatever_stands_between() { |
| 700 | let p = Phrase::new(["red", "green", "blue"], 3); |
| 701 | assert!(p.hit("Red, GREEN; blue")); |
| 702 | assert!(p.hit("red1green2blue")); |
| 703 | assert!(!p.hit("red green")); |
| 704 | assert!(!p.hit("red green purple blue"), "a stranger breaks the run"); |
| 705 | assert_eq!(p.spans("x red green blue y red"), vec![(2, 16)]); |
| 706 | } |
| 707 | |
| 708 | #[test] |
| 709 | fn a_hyphenated_entry_is_one_word_where_the_text_spells_it() { |
| 710 | let p = Phrase::new(["red", "t-shirt", "blue"], 3); |
| 711 | assert!(p.hit("red t-shirt blue"), "the entry is one word"); |
| 712 | assert!(!p.hit("red t shirt blue"), "and spelled otherwise it is not"); |
| 713 | assert!(!p.hit("red t-shirts blue")); |
| 714 | } |
| 715 | |
| 716 | #[test] |
| 717 | fn either_of_two_tests_is_a_test() { |
| 718 | let t = Or(|s: &str| s.contains("one"), Phrase::new(["a", "b"], 2)); |
| 719 | assert!(t.hit("one") && t.hit("a b") && !t.hit("a c b")); |
| 720 | } |
| 721 | |
| 722 | #[test] |
| 723 | fn a_marker_is_recognised_so_it_is_not_covered_twice() { |
| 724 | assert!(marker(&fingerprint("abcdefgh", 6))); |
| 725 | assert!(marker(OBJECT) && marker(ABSENT) && marker(DEEP)); |
| 726 | assert!(!marker("[redacted")); |
| 727 | assert!(!marker("not [redacted x]")); |
| 728 | } |
| 729 | |
| 730 | #[test] |
| 731 | fn the_stock_test_is_the_commit_hooks_and_ignores_its_excuses() { |
| 732 | // A credential's shape, in two pieces so that this file passes the hook it tests. |
| 733 | let k = fmt!("{}{}", "sk-ant", "-api03-AbCdEfGhIjKlMnOpQrStUvWx"); |
| 734 | assert!(Shapes.hit(&k)); |
| 735 | assert!(Shapes.hit(&fmt!("{} # allowlist secret", k))); |
| 736 | assert!(!Shapes.hit("sk-ant-short")); |
| 737 | // Covered whole, with none of its characters left standing in the fingerprint. |
| 738 | let red = Redact::new().with_head(0).with_test(Shapes); |
| 739 | let out = red.text(&fmt!("deployed with {}", k)).unwrap_or_default(); |
| 740 | assert!(out.starts_with("[redacted") && !out.contains("sk-ant")); |
| 741 | } |
| 742 | } |