oxedyne/fe2o3/fe2o3_net/src/lens/chunk.rs
25.6 KiB, 49 runs
created by r1870400018:65638, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! Chunk rows: a bundle sent as base64 in `ds <kind> <id> i/N` rows, and the guard that reads them. |
| 2 | //! |
| 3 | //! Daimond sends its telemetry and its snapshots, and Oxegen's test page its snapshots, as minified |
| 4 | //! JSON that the page has already scrubbed whole, base64-encoded and cut into rows of about three |
| 5 | //! hundred characters. A content guard that looks at a row's text cannot be left to judge such a |
| 6 | //! row as it stands: base64 is a long, unbroken, high-entropy run by construction, which is exactly |
| 7 | //! what the entropy catch of [`Shapes`](super::Shapes) fingerprints, so every chunk of every bundle |
| 8 | //! was stored as a marker and no bundle could be read back (found on the first end-to-end run of the |
| 9 | //! Oxegen test site, 2026-10-03). |
| 10 | //! |
| 11 | //! The fix is not to exempt the rows from the guard but to put the guard where the text is. A row |
| 12 | //! whose tag is strictly a chunk's ([`ChunkTag`]) and whose payload is verifiably the base64 of text |
| 13 | //! ([`text`]) is judged by the decoded text, which meets the caller's whole string test: the shapes, |
| 14 | //! the entropy catch, and whatever the app adds (Oxegen's eight words). A hit covers the payload by |
| 15 | //! its fingerprint, and a reader then finds the bundle broken. Anything else about a tagged row is |
| 16 | //! guarded exactly as any row is: a payload that is not base64, or whose bytes are not text (an |
| 17 | //! image sent under a snapshot's tag), is not exempt from anything. |
| 18 | //! |
| 19 | //! A secret can straddle a cut, so the chunks of one bundle in one post are judged together, in |
| 20 | //! order, as the run of text they make ([`judge`]). A bundle that crosses two posts is judged |
| 21 | //! chunk by chunk and run by run within each, which is how the pages send it: a bundle is one post. |
| 22 | //! |
| 23 | //! A bundle whose text parses as JSON meets the sink's whole [`Redact`] as well, and not the string |
| 24 | //! test alone (2026-10-04, F5 of the Fable QA). A plain row is held to its field names, its |
| 25 | //! deny-list and its strings, so a `passphrase` field inside a snapshot was kept where the same |
| 26 | //! field in a row would have been covered; and a run of words was missed whenever a letter-only key |
| 27 | //! stood between them, since the test read the JSON's own text. The words are now also sought in |
| 28 | //! the bundle's string values joined in order, which no key can break, and what is found is |
| 29 | //! covered where it stands. The covered JSON is encoded again in the alphabet it came in, and cut |
| 30 | //! into the rows it came in ([`Verdict::Cover`]). Where it will not fit them, or the rows are not |
| 31 | //! the whole bundle, the whole is covered instead, which is the safe side of the same choice. |
| 32 | //! |
| 33 | //! Two things were added the same day (N2 of the Opus QA). A bundle that is the whole of its post |
| 34 | //! and opens as an object or an array, but that the strict reader refuses (a member named twice, |
| 35 | //! half a surrogate pair, nesting past the limit), cannot be walked, and is now covered whole where |
| 36 | //! it had been stored as it came: another reader takes all three, so a reader's refusing them was |
| 37 | //! no defence. And the runs of words are sought before the walk, across the values, across the names |
| 38 | //! and across both, since the walk covers a field named for a secret in the middle of a run and |
| 39 | //! leaves the two halves too short to be one. |
| 40 | |
| 41 | use super::{ |
| 42 | redact::{ |
| 43 | marker, |
| 44 | Redact, |
| 45 | StrTest, |
| 46 | }, |
| 47 | row::compact, |
| 48 | }; |
| 49 | |
| 50 | use oxedyne_fe2o3_core::prelude::*; |
| 51 | use oxedyne_fe2o3_jdat::{ |
| 52 | bdat::limits::DecodeLimits, |
| 53 | prelude::*, |
| 54 | }; |
| 55 | use oxedyne_fe2o3_text::base64; |
| 56 | |
| 57 | use std::collections::HashMap; |
| 58 | |
| 59 | |
| 60 | // The most bytes of a partial character that may stand at either end of a chunk's text. |
| 61 | const EDGE: usize = 3; |
| 62 | |
| 63 | /// What a sink should do with one row of a post. |
| 64 | #[derive(Clone, Debug, Eq, PartialEq)] |
| 65 | pub enum Verdict { |
| 66 | Row, // not a chunk, or a chunk that is not the base64 of text: guarded as any row is |
| 67 | Clean, // the base64 of text that the guard passes: kept as it stands |
| 68 | Hit, // the base64 of text that the string test fires on: covered whole |
| 69 | Cover(String), // the base64 of JSON with something covered in it: the row's data, replaced |
| 70 | } |
| 71 | |
| 72 | /// The tag of a chunk row, `ds snapshot|telemetry <id> <i>/<N>`. |
| 73 | #[derive(Clone, Debug, Eq, PartialEq)] |
| 74 | pub struct ChunkTag { |
| 75 | pub kind: String, // snapshot or telemetry |
| 76 | pub id: String, // the bundle's id, one word |
| 77 | pub i: u32, // this chunk, from 1 |
| 78 | pub n: u32, // chunks in the bundle |
| 79 | } |
| 80 | |
| 81 | impl ChunkTag { |
| 82 | |
| 83 | /// The tag, if it is exactly a chunk's: four words split by single spaces, a kind of |
| 84 | /// `snapshot` or `telemetry`, an id, and `i/N` in plain digits with `1 <= i <= N`. Anything |
| 85 | /// else (another kind, a double space, a `0/3`, a `4/3`, a trailing word, a clipped tag) is not. |
| 86 | pub fn parse(tag: &str) -> Option<Self> { |
| 87 | let mut it = tag.split(' '); |
| 88 | if it.next() != Some("ds") { |
| 89 | return None; |
| 90 | } |
| 91 | let kind = it.next()?; |
| 92 | if kind != "snapshot" && kind != "telemetry" { |
| 93 | return None; |
| 94 | } |
| 95 | let id = it.next()?; |
| 96 | let frac = it.next()?; |
| 97 | if id.is_empty() || it.next().is_some() { |
| 98 | return None; |
| 99 | } |
| 100 | let (i, n) = frac.split_once('/')?; |
| 101 | let num = |s: &str| -> Option<u32> { |
| 102 | if s.is_empty() || s.len() > 9 || !s.bytes().all(|b| b.is_ascii_digit()) { |
| 103 | return None; |
| 104 | } |
| 105 | s.parse::<u32>().ok() |
| 106 | }; |
| 107 | let (i, n) = (num(i)?, num(n)?); |
| 108 | if i < 1 || i > n { |
| 109 | return None; |
| 110 | } |
| 111 | Some(Self { kind: kind.to_string(), id: id.to_string(), i, n }) |
| 112 | } |
| 113 | } |
| 114 | |
| 115 | /// The text a chunk's payload encodes, if it is verifiably the base64 of text. |
| 116 | /// |
| 117 | /// The payload may be Daimond's padded `A-Za-z0-9+/` or the page's unpadded `A-Za-z0-9-_`, and |
| 118 | /// white space in it is ignored, which is how the first Oxegen pages wrapped it (words of sixteen, |
| 119 | /// as MIME does). The decoders are `fe2o3_text::base64`'s, which are strict, so a payload that |
| 120 | /// is nearly base64 is not. The bytes must also be UTF-8, allowing a partial character at either |
| 121 | /// end where a cut fell, so that a plain-text payload made of letters, which decodes to bytes |
| 122 | /// that are anything but text, is not mistaken for base64 and let through unread. |
| 123 | pub fn text(payload: &str) -> Option<String> { |
| 124 | let flat: String = payload.chars().filter(|c| !c.is_ascii_whitespace()).collect(); |
| 125 | let std_only = flat.bytes().any(|b| matches!(b, b'+' | b'/' | b'=')); |
| 126 | let url_only = flat.bytes().any(|b| matches!(b, b'-' | b'_')); |
| 127 | let bytes = match (std_only, url_only) { |
| 128 | (true, true) => return None, |
| 129 | (true, false) => base64::decode(&flat).ok()?, |
| 130 | (false, _) => base64::decode_url(&flat).ok()?, |
| 131 | }; |
| 132 | // Text, give or take the edges a cut can fall inside. |
| 133 | let mut lo = 0; |
| 134 | while lo < bytes.len().min(EDGE) && bytes[lo] & 0xC0 == 0x80 { |
| 135 | lo += 1; |
| 136 | } |
| 137 | let body = &bytes[lo..]; |
| 138 | match std::str::from_utf8(body) { |
| 139 | Ok(_) => {}, |
| 140 | Err(e) => { |
| 141 | // Only an unfinished character at the very end is allowed. |
| 142 | if e.error_len().is_some() || body.len() - e.valid_up_to() > EDGE { |
| 143 | return None; |
| 144 | } |
| 145 | }, |
| 146 | } |
| 147 | Some(String::from_utf8_lossy(&bytes).into_owned()) |
| 148 | } |
| 149 | |
| 150 | /// The verdict on each row of a post, given as `(tag, data)` exactly as they will be stored. |
| 151 | /// |
| 152 | /// The chunks of one bundle are taken in order of index and cut into runs of consecutive indices. |
| 153 | /// A run is decoded whole and its text meets the guard, so a secret that straddles a cut is seen and |
| 154 | /// the chunks on either side of it are all covered. A run that will not decode as one (a chunk |
| 155 | /// among them is not base64 of text) is judged chunk by chunk, and a chunk that will not decode |
| 156 | /// alone, such as one cut off a four character boundary and sent with no neighbour, is a plain row. |
| 157 | /// |
| 158 | /// The guard is `red`'s: its string test over the text as it stands, then, if the text is JSON, |
| 159 | /// its names and strings over the parsed value and its word runs over the strings joined in |
| 160 | /// order. `cap` is the most bytes a row's data may hold, which a covered re-encoding is cut to fit. |
| 161 | pub fn judge<T: StrTest>(red: &Redact<T>, cap: usize, rows: &[(String, String)]) -> Vec<Verdict> { |
| 162 | let mut out = vec![Verdict::Row; rows.len()]; |
| 163 | // The chunk rows by bundle, in the order the bundles first appear. |
| 164 | let mut sets: Vec<(String, String)> = Vec::new(); |
| 165 | let mut by: HashMap<(String, String), Vec<(u32, usize, u32)>> = HashMap::new(); |
| 166 | for (at, (tag, _)) in rows.iter().enumerate() { |
| 167 | if let Some(t) = ChunkTag::parse(tag) { |
| 168 | let key = (t.kind, t.id); |
| 169 | if !by.contains_key(&key) { |
| 170 | sets.push(key.clone()); |
| 171 | } |
| 172 | by.entry(key).or_default().push((t.i, at, t.n)); |
| 173 | } |
| 174 | } |
| 175 | for key in sets { |
| 176 | let mut members = match by.remove(&key) { |
| 177 | Some(m) => m, |
| 178 | None => continue, |
| 179 | }; |
| 180 | members.sort(); |
| 181 | // Runs of consecutive indices; a repeated index starts a run of its own. |
| 182 | let mut runs: Vec<Vec<(u32, usize, u32)>> = Vec::new(); |
| 183 | let mut last = 0u32; |
| 184 | for m in members { |
| 185 | if runs.is_empty() || m.0 != last + 1 { |
| 186 | runs.push(Vec::new()); |
| 187 | } |
| 188 | last = m.0; |
| 189 | if let Some(run) = runs.last_mut() { |
| 190 | run.push(m); |
| 191 | } |
| 192 | } |
| 193 | for run in runs { |
| 194 | let at: Vec<usize> = run.iter().map(|&(_, at, _)| at).collect(); |
| 195 | let k = at.len(); |
| 196 | // Whole if it starts at the first chunk and every row of it says the set is this long. |
| 197 | let entire = run.first().map_or(false, |m| m.0 == 1) && run.iter().all(|m| m.2 as usize == k); |
| 198 | let flat: String = at.iter().map(|&a| rows[a].1.as_str()).collect::<Vec<&str>>().concat(); |
| 199 | match text(&flat) { |
| 200 | Some(t) => { |
| 201 | let found = of_text(red, cap, &t, &flat, k, entire); |
| 202 | for (&a, v) in at.iter().zip(found) { |
| 203 | out[a] = v; |
| 204 | } |
| 205 | }, |
| 206 | None => { |
| 207 | for &a in &at { |
| 208 | if let Some(t) = text(&rows[a].1) { |
| 209 | out[a] = if red.test().hit(&t) { Verdict::Hit } else { Verdict::Clean }; |
| 210 | } |
| 211 | } |
| 212 | }, |
| 213 | } |
| 214 | } |
| 215 | } |
| 216 | out |
| 217 | } |
| 218 | |
| 219 | // What is to be done with a text that is verifiably a run of chunks, `k` rows whose data was `flat`. |
| 220 | fn of_text<T: StrTest>(red: &Redact<T>, cap: usize, text: &str, flat: &str, k: usize, entire: bool) -> Vec<Verdict> { |
| 221 | if red.test().hit(text) { |
| 222 | return vec![Verdict::Hit; k]; |
| 223 | } |
| 224 | let covered = match cover(red, text, entire) { |
| 225 | Seen::Plain => return vec![Verdict::Clean; k], |
| 226 | Seen::Whole => return vec![Verdict::Hit; k], |
| 227 | Seen::Covered(c) => c, |
| 228 | }; |
| 229 | // Covering changes the text, so it cannot be re-cut where some of the set is elsewhere. |
| 230 | if !entire { |
| 231 | return vec![Verdict::Hit; k]; |
| 232 | } |
| 233 | match recut(&covered, flat, k, cap) { |
| 234 | Some(rows) => rows.into_iter().map(Verdict::Cover).collect(), |
| 235 | None => vec![Verdict::Hit; k], |
| 236 | } |
| 237 | } |
| 238 | |
| 239 | // What a text asks of the redactor. |
| 240 | enum Seen { |
| 241 | Plain, // not JSON, or JSON with nothing to cover |
| 242 | Covered(String), // JSON with something covered, as compact JSON |
| 243 | Whole, // JSON that cannot be covered in part |
| 244 | } |
| 245 | |
| 246 | // Which of a structure's strings a run of words is read across. |
| 247 | #[derive(Clone, Copy, Eq, PartialEq)] |
| 248 | enum Reads { |
| 249 | Values, // the string values: no member name stands between the words |
| 250 | Names, // the member names: no value stands between the words |
| 251 | Both, // the names and the values, in the order the document gives them |
| 252 | } |
| 253 | |
| 254 | // A text that parses as JSON, put through `red`: the runs of words first, across its strings joined |
| 255 | // in order, then its names and strings by the walk. The runs come first because the walk covers a |
| 256 | // field named for a secret in the middle of a run, and the two halves left over are each too short |
| 257 | // to be one. `entire` says the text is the whole of a bundle. |
| 258 | fn cover<T: StrTest>(red: &Redact<T>, text: &str, entire: bool) -> Seen { |
| 259 | let parsed = match Dat::decode_json_strict_ordered(text, &DecodeLimits::default()) { |
| 260 | Ok(d) => d, |
| 261 | // A whole bundle that the strict reader refuses (a member named twice, half a surrogate pair, |
| 262 | // nesting past the limit) is one that a lenient reader takes, and it cannot be walked, so it |
| 263 | // is covered as a whole and never stored as it came. A text that does not open as a bundle is |
| 264 | // no bundle: the string test has had it. |
| 265 | Err(_) => return if entire && opens(text) { Seen::Whole } else { Seen::Plain }, |
| 266 | }; |
| 267 | let mut d = parsed; |
| 268 | let mut changed = false; |
| 269 | for reads in [Reads::Values, Reads::Names, Reads::Both] { |
| 270 | let strs = strings_of(&d, reads); |
| 271 | let spans = red.test().joined_spans(&strs.join("\n")); |
| 272 | if spans.is_empty() { |
| 273 | continue; |
| 274 | } |
| 275 | // Each string that a run touches is covered, as a plain row's string would be. |
| 276 | let mut at = 0; |
| 277 | let mut take = Vec::with_capacity(strs.len()); |
| 278 | for l in &strs { |
| 279 | take.push(spans.iter().any(|&(a, b)| a < at + l.len() && b > at)); |
| 280 | at += l.len() + 1; |
| 281 | } |
| 282 | let mut i = 0; |
| 283 | d = map_strings(&d, reads, &mut |s| { |
| 284 | i += 1; |
| 285 | if take[i - 1] && !marker(s) { Some(red.mark(s)) } else { None } |
| 286 | }); |
| 287 | changed = true; |
| 288 | } |
| 289 | // Covering must leave nothing of a run, or it has not covered it. |
| 290 | for reads in [Reads::Values, Reads::Names, Reads::Both] { |
| 291 | if !red.test().joined_spans(&strings_of(&d, reads).join("\n")).is_empty() { |
| 292 | return Seen::Whole; |
| 293 | } |
| 294 | } |
| 295 | let (walked, c) = red.walk(&d); |
| 296 | if !changed && !c { |
| 297 | return Seen::Plain; |
| 298 | } |
| 299 | match compact(&walked) { |
| 300 | Ok(c) => Seen::Covered(c), |
| 301 | Err(_) => Seen::Whole, |
| 302 | } |
| 303 | } |
| 304 | |
| 305 | // Does the text open as a JSON object or array, as a bundle does, whatever follows? |
| 306 | fn opens(text: &str) -> bool { |
| 307 | text.trim_start_matches([' ', '\t', '\n', '\r']).starts_with(['{', '[']) |
| 308 | } |
| 309 | |
| 310 | fn strings_of(d: &Dat, reads: Reads) -> Vec<String> { |
| 311 | let mut out = Vec::new(); |
| 312 | map_strings(d, reads, &mut |s| { |
| 313 | out.push(s.to_string()); |
| 314 | None |
| 315 | }); |
| 316 | out |
| 317 | } |
| 318 | |
| 319 | // A copy of `d` in which each string that `reads` names, in the order written, is given to `f`, which |
| 320 | // may answer a replacement. A member's name comes before its value. JSON makes only maps, lists and |
| 321 | // strings of the kinds that hold others, so those are all that is walked. |
| 322 | fn map_strings<F: FnMut(&str) -> Option<String>>(d: &Dat, reads: Reads, f: &mut F) -> Dat { |
| 323 | let name = |k: &Dat, f: &mut F| match k { |
| 324 | Dat::Str(s) if reads != Reads::Values => Dat::Str(f(s).unwrap_or_else(|| s.clone())), |
| 325 | other => other.clone(), |
| 326 | }; |
| 327 | match d { |
| 328 | Dat::Str(s) if reads == Reads::Names => Dat::Str(s.clone()), |
| 329 | Dat::Str(s) => Dat::Str(f(s).unwrap_or_else(|| s.clone())), |
| 330 | Dat::List(l) => Dat::List(l.iter().map(|x| map_strings(x, reads, f)).collect()), |
| 331 | Dat::Map(m) => { |
| 332 | let mut out = DaticleMap::new(); |
| 333 | for (k, v) in m { |
| 334 | let mut nk = name(k, f); |
| 335 | // Two names covered the same way must stay two members. |
| 336 | while out.contains_key(&nk) { |
| 337 | nk = Dat::Str(fmt!("{}~", match &nk { Dat::Str(s) => s.as_str(), _ => "" })); |
| 338 | } |
| 339 | out.insert(nk, map_strings(v, reads, f)); |
| 340 | } |
| 341 | Dat::Map(out) |
| 342 | }, |
| 343 | Dat::OrdMap(m) => { |
| 344 | let mut at: Vec<(&MapKey, &Dat)> = m.iter().collect(); |
| 345 | at.sort_by_key(|(k, _)| k.ord()); |
| 346 | let mut out = OrdDaticleMap::new(); |
| 347 | for (k, v) in at { |
| 348 | let nk = name(k.dat(), f); |
| 349 | out.insert(MapKey::new(k.ord(), nk), map_strings(v, reads, f)); |
| 350 | } |
| 351 | Dat::OrdMap(out) |
| 352 | }, |
| 353 | other => other.clone(), |
| 354 | } |
| 355 | } |
| 356 | |
| 357 | // The text as base64 in the alphabet `like` is written in, cut into `k` rows of `cap` bytes at most, |
| 358 | // or none if it will not go. A row takes whole quanta, as the pages cut them, so every row but the |
| 359 | // last decodes alone, and the rows are filled evenly. A text of fewer quanta than rows leaves the |
| 360 | // last ones empty, which a reader joins as nothing. |
| 361 | fn recut(text: &str, like: &str, k: usize, cap: usize) -> Option<Vec<String>> { |
| 362 | let std = like.bytes().any(|b| matches!(b, b'+' | b'/' | b'=')); |
| 363 | let enc = if std { base64::encode(text.as_bytes()) } else { base64::encode_url(text.as_bytes()) }; |
| 364 | let per = (enc.len().div_ceil(4)).div_ceil(k.max(1)) * 4; |
| 365 | if per == 0 || per > cap { |
| 366 | return None; |
| 367 | } |
| 368 | let mut rows: Vec<String> = enc.as_bytes().chunks(per).map(|c| String::from_utf8_lossy(c).into_owned()).collect(); |
| 369 | rows.resize(k, String::new()); |
| 370 | Some(rows) |
| 371 | } |
| 372 | |
| 373 | |
| 374 | #[cfg(test)] |
| 375 | mod tests { |
| 376 | use super::*; |
| 377 | use super::super::redact::Phrase; |
| 378 | |
| 379 | // Any closure is a string test, so the judge is tried with a plain word. |
| 380 | fn secret(s: &str) -> bool { s.contains("SECRET") } |
| 381 | |
| 382 | fn red() -> Redact<fn(&str) -> bool> { Redact::new().with_head(0).with_test(secret as fn(&str) -> bool) } |
| 383 | |
| 384 | fn row(tag: &str, data: &str) -> (String, String) { (tag.to_string(), data.to_string()) } |
| 385 | |
| 386 | #[test] |
| 387 | fn a_tag_is_a_chunks_only_when_it_is_exactly_so() { |
| 388 | assert_eq!(ChunkTag::parse("ds snapshot a-1 3/7").map(|t| (t.i, t.n)), Some((3, 7))); |
| 389 | for bad in ["ds snapshot a 0/7", "ds snapshot a 8/7", "ds snapshot a 3/7 x", "ds snapshot a 3/7", "ds shot a 3/7", "ds snapshot a 3-7"] { |
| 390 | assert_eq!(ChunkTag::parse(bad), None, "{}", bad); |
| 391 | } |
| 392 | } |
| 393 | |
| 394 | #[test] |
| 395 | fn a_payload_is_text_or_it_is_nothing() { |
| 396 | assert_eq!(text("U0VDUkVU").as_deref(), Some("SECRET")); |
| 397 | assert_eq!(text("U0VD UkVU").as_deref(), Some("SECRET"), "white space is skipped"); |
| 398 | assert_eq!(text("U0VDUkVU=="), None, "padding that is not there is refused"); |
| 399 | // Letters alone, taken for base64, make bytes that are not text. |
| 400 | assert_eq!(text("thequickbrownfoxjumpsoverthelazydog"), None); |
| 401 | } |
| 402 | |
| 403 | #[test] |
| 404 | fn a_recut_text_reads_back_in_the_alphabet_it_came_in() { |
| 405 | let t = "{\"a\":\"\u{e9}\u{e9}\u{e9}?>>\"}"; |
| 406 | for (like, std) in [("Zm9v", false), ("Zm9v+/", true), ("Zm9v=", true)] { |
| 407 | let rows = recut(t, like, 3, 400).unwrap_or_default(); |
| 408 | assert_eq!(rows.len(), 3, "{}", like); |
| 409 | // Whole quanta to a row, so the padding of the standard form stands in the last alone. |
| 410 | assert!(rows[..2].iter().all(|r| r.len() % 4 == 0 && !r.contains('=')), "{}", like); |
| 411 | let flat = rows.concat(); |
| 412 | let back = if std { base64::decode(&flat) } else { base64::decode_url(&flat) }; |
| 413 | assert_eq!(back.map(|b| String::from_utf8_lossy(&b).into_owned()).unwrap_or_default(), t, "{}", like); |
| 414 | } |
| 415 | // A short text still fills the rows it came in, with nothing at the end. |
| 416 | let rows = recut("{}", "Zm9v", 4, 400).unwrap_or_default(); |
| 417 | assert_eq!(rows, vec!["e30".to_string(), String::new(), String::new(), String::new()]); |
| 418 | // And one that will not go in the rows is refused, to the byte. |
| 419 | let big = "x".repeat(300); |
| 420 | assert!(recut(&big, "Zm9v", 1, 400).is_some()); |
| 421 | assert!(recut(&big, "Zm9v", 1, 399).is_none()); |
| 422 | assert!(recut(&big, "Zm9v", 2, 400).is_some(), "two rows hold what one cannot"); |
| 423 | } |
| 424 | |
| 425 | #[test] |
| 426 | fn json_is_covered_by_its_names_and_its_strings_joined_in_order() { |
| 427 | let enc = |t: &str| base64::encode_url(t.as_bytes()); |
| 428 | let one = |t: &str| vec![("ds snapshot j 1/1".to_string(), enc(t))]; |
| 429 | let phrase = Redact::new().with_head(0).with_test(Phrase::new(["red", "green", "blue"], 3)); |
| 430 | let verdicts = |t: &str| judge(&phrase, 400, &one(t)); |
| 431 | // Keys of letters between the words, which broke the run of the text as it stands. |
| 432 | assert!(!phrase.test().has("{\"a\":\"red\",\"b\":\"green\",\"c\":\"blue\"}")); |
| 433 | assert!(matches!(verdicts("{\"a\":\"red\",\"b\":\"green\",\"c\":\"blue\"}")[..], [Verdict::Cover(_)])); |
| 434 | // In the order of the document and not of the names, which a plain map would sort: `bb` is |
| 435 | // after `c` where it was written, and between `b` and `c` where it would be sorted. |
| 436 | assert!(matches!(verdicts("{\"a\":\"red\",\"b\":\"green\",\"c\":\"blue\",\"bb\":\"x\"}")[..], [Verdict::Cover(_)])); |
| 437 | assert!(matches!(verdicts("{\"a\":\"red\",\"b\":\"green\",\"bb\":\"x\",\"c\":\"blue\"}")[..], [Verdict::Clean])); |
| 438 | // A name, with nothing for the string test to find. |
| 439 | assert!(matches!(verdicts("{\"seed\":\"x\"}")[..], [Verdict::Cover(_)])); |
| 440 | assert_eq!(verdicts("{\"state\":\"live\"}"), vec![Verdict::Clean]); |
| 441 | assert_eq!(verdicts("not json {"), vec![Verdict::Clean]); |
| 442 | } |
| 443 | |
| 444 | #[test] |
| 445 | fn a_bundle_the_strict_reader_refuses_is_covered_whole() { |
| 446 | let enc = |t: &str| base64::encode_url(t.as_bytes()); |
| 447 | let one = |t: &str| vec![("ds snapshot r 1/1".to_string(), enc(t))]; |
| 448 | let red = Redact::new().with_head(0).with_test(Phrase::new(["ant", "bee", "cow"], 3)); |
| 449 | let deep = ["{\"passphrase\":\"hunter2\",\"d\":", &"[".repeat(65), &"]".repeat(65), "}"].concat(); |
| 450 | // Each of these is a bundle to a lenient reader, and to none that is strict: a name twice, |
| 451 | // half a surrogate pair (which a page makes by cutting a string inside an emoji), and nesting |
| 452 | // one level past the limit. Held as it came, the field named for a secret would be stored. |
| 453 | let refused = [ |
| 454 | ("a name twice", "{\"passphrase\":\"hunter2\",\"passphrase\":\"x\"}"), |
| 455 | ("half a surrogate pair", "{\"passphrase\":\"hunter2\",\"t\":\"\\ud83d\"}"), |
| 456 | ("nesting past the limit", deep.as_str()), |
| 457 | ("blanks before it", " \n\t{\"passphrase\":\"hunter2\",\"passphrase\":\"x\"}"), |
| 458 | ("an array", "[\"a\",{\"passphrase\":\"hunter2\",\"passphrase\":\"x\"}]"), |
| 459 | ]; |
| 460 | for (what, t) in refused { |
| 461 | assert!(Dat::decode_json_strict_ordered(t, &DecodeLimits::default()).is_err(), "{}: refused", what); |
| 462 | assert_eq!(judge(&red, 400, &one(t)), vec![Verdict::Hit], "{}: covered whole", what); |
| 463 | } |
| 464 | // The same field in a bundle that is read is covered where it stands. |
| 465 | assert!(matches!(judge(&red, 400, &one("{\"passphrase\":\"hunter2\"}"))[..], [Verdict::Cover(_)])); |
| 466 | // What does not open as an object or an array is no bundle, and is left to the string test. |
| 467 | for t in ["not json {", "\"\\ud83d\""] { |
| 468 | assert_eq!(judge(&red, 400, &one(t)), vec![Verdict::Clean], "{}", t); |
| 469 | } |
| 470 | // A run of chunks that is not the whole bundle cannot be read as one, so it is not refused as one. |
| 471 | let half = vec![("ds snapshot r 1/2".to_string(), enc("{\"passphrase\":\"hunter2\",\"passphrase\":"))]; |
| 472 | assert_eq!(judge(&red, 400, &half), vec![Verdict::Clean]); |
| 473 | } |
| 474 | |
| 475 | #[test] |
| 476 | fn a_word_run_is_sought_before_the_walk_and_in_the_names_as_well() { |
| 477 | let enc = |t: &str| base64::encode_url(t.as_bytes()); |
| 478 | let stored = |red: &Redact<Phrase>, t: &str| -> String { |
| 479 | match judge(red, 400, &[("ds snapshot w 1/1".to_string(), enc(t))]).pop() { |
| 480 | Some(Verdict::Cover(d)) => text(&d).unwrap_or_default(), |
| 481 | other => fmt!("{:?}", other), |
| 482 | } |
| 483 | }; |
| 484 | let words = ["ant", "bee", "cow", "dog", "eel"]; |
| 485 | let five = Redact::new().with_head(0).with_test(Phrase::new(words, 5)); |
| 486 | let three = Redact::new().with_head(0).with_test(Phrase::new(words, 3)); |
| 487 | let gone = |t: &str| words.iter().all(|w| !t.contains(&fmt!("\"{}\"", w))); |
| 488 | // A field named for a secret, inside the run: the walk covers its value, and the run is then |
| 489 | // two short ones that no longer reach five, so the other four were left standing. |
| 490 | let t = stored(&five, "{\"a\":\"ant\",\"b\":\"bee\",\"seed\":\"cow\",\"c\":\"dog\",\"d\":\"eel\"}"); |
| 491 | assert!(gone(&t) && t.matches("[redacted").count() == 5, "all five are covered: {}", t); |
| 492 | // Words that are the names, with nothing between them that is a string. |
| 493 | let t = stored(&three, "{\"ant\":true,\"bee\":true,\"cow\":true}"); |
| 494 | assert!(gone(&t) && t.matches("[redacted").count() == 3, "the three names are covered: {}", t); |
| 495 | // Names and values in turn, which neither alone makes a run of. The `true` between breaks the |
| 496 | // run in the text as it stands, and no string is in its way. |
| 497 | let t = stored(&five, "{\"ant\":\"bee\",\"cow\":true,\"dog\":\"eel\"}"); |
| 498 | assert!(gone(&t) && t.matches("[redacted").count() == 5, "the five strings are covered: {}", t); |
| 499 | // Letters that are not on the list still break a run, as they do in a row's text. |
| 500 | assert_eq!(judge(&three, 400, &[("ds snapshot w 1/1".to_string(), enc("{\"ant\":\"bee\",\"x\":1,\"cow\":\"y\"}"))]), vec![Verdict::Clean]); |
| 501 | } |
| 502 | |
| 503 | #[test] |
| 504 | fn the_chunks_of_a_bundle_are_judged_as_one_text() { |
| 505 | // The word is cut in two on a quantum boundary: neither half holds it, together they do. |
| 506 | let enc = base64::encode_url(b"xxxSECRETxxx"); |
| 507 | let (a, b) = enc.split_at(8); |
| 508 | let alone = [row("ds snapshot s 1/2", a)]; |
| 509 | assert_eq!(judge(&red(), 400, &alone), vec![Verdict::Clean]); |
| 510 | let both = [row("ds snapshot s 2/2", b), row("ds snapshot s 1/2", a), row("diag", "SECRET")]; |
| 511 | // The order of the post does not matter, and an untagged row is left to the sink. |
| 512 | assert_eq!(judge(&red(), 400, &both), vec![Verdict::Hit, Verdict::Hit, Verdict::Row]); |
| 513 | } |
| 514 | } |