Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_net/tests/lens.rs

105 KiB, 24 runs

created by r1870400018:65662, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! `fe2o3_net::lens`, the generic half of a debug lens.
2//!
3//! The gates, the envelope and the redaction rule are Daimond's (`gateway/src/handlers/
4//! debug_trace.rs` and `www/js/debugshare.js`), so each test here either re-states a Daimond test
5//! against the Rust code or pins a value to the JavaScript it mirrors. The two oracle tables in
6//! `tests/data` are generated by running `SECRET_RE` and `fingerprint` themselves (see
7//! `tests/data/lens_oracle.mjs`), so a transcription error in the Rust cannot hide.
8
9use oxedyne_fe2o3_core::prelude::*;
10use oxedyne_fe2o3_jdat::{
11 bdat::limits::DecodeLimits,
12 prelude::*,
13};
14use oxedyne_fe2o3_net::{
15 guard::floor::KeyFloor,
16 lens::{
17 chunk,
18 compact,
19 fingerprint,
20 is_stamp,
21 judge,
22 sanitise,
23 secret_name,
24 secret_name_loose,
25 stamp,
26 ChunkTag,
27 Deny,
28 Gates,
29 Or,
30 Phrase,
31 Posted,
32 Redact,
33 Refusal,
34 Row,
35 Shapes,
36 Sink,
37 StrTest,
38 Verdict,
39 },
40};
41use oxedyne_fe2o3_text::{
42 base64,
43 regex::Regex,
44 secret,
45};
46
47use std::{
48 fs,
49 path::{
50 Path,
51 PathBuf,
52 },
53 sync::atomic::{
54 AtomicU64,
55 Ordering,
56 },
57 time::{
58 Duration,
59 SystemTime,
60 UNIX_EPOCH,
61 },
62};
63
64
65// Scratch directories ====================================================================
66
67static SERIAL: AtomicU64 = AtomicU64::new(0);
68
69/// A directory of its own, removed again when the test ends.
70struct Scratch(PathBuf);
71
72impl Scratch {
73 fn new(tag: &str) -> Self {
74 let n = SERIAL.fetch_add(1, Ordering::SeqCst);
75 let dir = std::env::temp_dir().join(fmt!("lens-test-{}-{}-{}", std::process::id(), tag, n));
76 let _ = fs::remove_dir_all(&dir);
77 Self(dir)
78 }
79 fn dir(&self) -> PathBuf { self.0.join("lens") }
80}
81
82impl Drop for Scratch {
83 fn drop(&mut self) {
84 let _ = fs::remove_dir_all(&self.0);
85 }
86}
87
88const T0: u64 = 1_757_635_200_000; // A fixed instant, so rotated names are predictable.
89const DAY: u64 = 86_400_000;
90
91// Builders ===============================================================================
92
93fn omap(kv: Vec<(&str, Dat)>) -> Dat {
94 create_dat_ordmap(kv.into_iter().map(|(k, v)| (Dat::Str(k.to_string()), v)).collect())
95}
96
97fn st(s: &str) -> Dat { Dat::Str(s.to_string()) }
98
99/// The member of a map at a dotted path, lists excluded.
100fn at<'a>(d: &'a Dat, path: &str) -> Option<&'a Dat> {
101 let mut cur = d;
102 for seg in path.split('.') {
103 cur = cur.map_get(&st(seg)).ok().flatten()?;
104 }
105 Some(cur)
106}
107
108fn text_of(d: &Dat) -> String {
109 match d {
110 Dat::Str(s) => s.clone(),
111 other => fmt!("<{:?}>", other),
112 }
113}
114
115fn text_at(d: &Dat, path: &str) -> String {
116 match at(d, path) {
117 Some(Dat::Str(s)) => s.clone(),
118 other => fmt!("<{:?}>", other),
119 }
120}
121
122/// A trace body in Daimond's request shape, `{v, device, rows:[{ts, tag, data}]}`.
123fn body(device: &str, rows: &[(i64, &str, &str)]) -> Vec<u8> {
124 let j = |s: &str| Dat::Str(s.to_string()).json().unwrap_or_default();
125 let list: Vec<String> = rows.iter()
126 .map(|(ts, tag, data)| fmt!("{{\"ts\":{},\"tag\":{},\"data\":{}}}", ts, j(tag), j(data)))
127 .collect();
128 fmt!("{{\"v\":1,\"device\":{},\"rows\":[{}]}}", j(device), list.join(",")).into_bytes()
129}
130
131fn lines(path: &Path) -> Outcome<Vec<Dat>> {
132 let text = res!(fs::read_to_string(path));
133 let mut out = Vec::new();
134 for line in text.lines() {
135 out.push(res!(Dat::decode_json_strict(line, &DecodeLimits::default())));
136 }
137 Ok(out)
138}
139
140fn names(dir: &Path) -> Vec<String> {
141 let mut v: Vec<String> = match fs::read_dir(dir) {
142 Ok(rd) => rd.filter_map(|e| e.ok()).filter_map(|e| e.file_name().into_string().ok()).collect(),
143 Err(_) => Vec::new(),
144 };
145 v.sort();
146 v
147}
148
149fn all_bytes(dir: &Path) -> Outcome<Vec<u8>> {
150 let mut out = Vec::new();
151 for n in names(dir) {
152 out.extend(res!(fs::read(dir.join(n))));
153 }
154 Ok(out)
155}
156
157fn has(hay: &[u8], needle: &str) -> bool {
158 let n = needle.as_bytes();
159 !n.is_empty() && hay.windows(n.len()).any(|w| w == n)
160}
161
162fn stored(p: Posted) -> usize {
163 match p {
164 Posted::Stored(n) => n,
165 Posted::Refused(r) => panic!("expected the post to be stored, but it was refused: {:?}", r),
166 }
167}
168
169fn refused(p: Posted) -> Refusal {
170 match p {
171 Posted::Refused(r) => r,
172 Posted::Stored(n) => panic!("expected the post to be refused, but {} row(s) were stored", n),
173 }
174}
175
176/// Pushes a live file past `max_file` so the next write rotates it.
177fn overfill(sink: &Sink, key: &[&str]) -> Outcome<()> {
178 res!(fs::create_dir_all(sink.dir()));
179 res!(fs::write(sink.path(key), vec![b'x'; (sink.gates().max_file + 1) as usize]));
180 Ok(())
181}
182
183fn small_gates() -> Gates {
184 Gates { max_file: 600, ..Gates::default() }
185}
186
187fn rotated_name(stem: &str, at_ms: u64) -> String {
188 fmt!("{}.ndjson.{}", stem, stamp(at_ms))
189}
190
191fn now_real_ms() -> u64 {
192 SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0)
193}
194
195// The envelope ===========================================================================
196
197#[test]
198fn the_row_envelope_is_daimonds_v_d_n_b_t() -> Outcome<()> {
199 let row = Row::new("dev1", 7, "b1", T0, "console")
200 .with("lvl", st("warn"))
201 // The envelope wins over a payload key of the same name, as it does in `emit`.
202 .with("n", Dat::U64(99))
203 .with("t", Dat::U64(1))
204 .with("x", Dat::U64(3));
205 let j = res!(row.json());
206 assert_eq!(j, fmt!(
207 "{{\"v\":1,\"d\":\"dev1\",\"n\":7,\"b\":\"b1\",\"t\":{},\"lvl\":\"warn\",\"x\":3}}", T0));
208 assert_eq!(row.tag(), "ev console");
209 Ok(())
210}
211
212#[test]
213fn a_small_row_is_not_trimmed() -> Outcome<()> {
214 let row = Row::new("d", 1, "b", T0, "k").with("msg", st("short"));
215 let j = res!(row.fit(360));
216 assert_eq!(j, res!(row.json()));
217 assert!(!j.contains("\"tr\""));
218 Ok(())
219}
220
221#[test]
222fn fit_trims_the_largest_string_and_marks_tr() -> Outcome<()> {
223 let row = Row::new("dev1", 7, "b1", T0, "fetch.fail")
224 .with("tool", st("bash"))
225 .with("err", st(&"e".repeat(1000)));
226 let j = res!(row.fit(360));
227 assert!(j.len() <= 360, "the fitted event is {} bytes", j.len());
228 let d = res!(Dat::decode_json_strict(&j, &DecodeLimits::default()));
229 assert_eq!(text_at(&d, "tool"), "bash", "the short field must survive the trimming of the long one");
230 let err = text_at(&d, "err");
231 assert!(!err.is_empty() && err.len() < 1000 && "e".repeat(err.len()) == err);
232 assert!(at(&d, "tr").is_some(), "tr must be present");
233 assert!(j.ends_with(",\"tr\":1}"), "tr:1 is the last key: {}", j);
234 // The envelope is never sacrificed.
235 assert!(j.starts_with(&fmt!("{{\"v\":1,\"d\":\"dev1\",\"n\":7,\"b\":\"b1\",\"t\":{},", T0)));
236 Ok(())
237}
238
239#[test]
240fn fit_counts_bytes_and_still_converges_on_multibyte_text() -> Outcome<()> {
241 // 200 characters of two bytes each are trimmed by characters, and the result is whole JSON.
242 let row = Row::new("d", 1, "b", T0, "k").with("msg", st(&"é".repeat(200)));
243 let j = res!(row.fit(360));
244 assert!(j.len() <= 360, "{} bytes", j.len());
245 let d = res!(Dat::decode_json_strict(&j, &DecodeLimits::default()));
246 let msg = text_at(&d, "msg");
247 assert!(!msg.is_empty() && msg.chars().count() < 200 && msg.chars().all(|c| c == 'é'), "{:?}", msg);
248 // With 500 the excess in bytes is larger than the characters there are, so the field goes
249 // whole, which is what `fit` in debugshare.js does too (`keep > 0` else `delete`).
250 let row = Row::new("d", 1, "b", T0, "k").with("msg", st(&"é".repeat(500)));
251 let j = res!(row.fit(360));
252 assert!(j.len() <= 360, "{} bytes", j.len());
253 let d = res!(Dat::decode_json_strict(&j, &DecodeLimits::default()));
254 assert!(at(&d, "msg").is_none() && at(&d, "tr").is_some(), "{}", j);
255 Ok(())
256}
257
258#[test]
259fn fit_falls_back_to_the_bare_envelope() -> Outcome<()> {
260 // Nothing here is a string, so nothing can be trimmed.
261 let big = Dat::List((0..300).map(|i| Dat::U64(1_000_000 + i)).collect());
262 let row = Row::new("dev1", 7, "b1", T0, "k").with("big", big);
263 let j = res!(row.fit(360));
264 assert_eq!(j, fmt!("{{\"v\":1,\"d\":\"dev1\",\"n\":7,\"b\":\"b1\",\"t\":{},\"tr\":1}}", T0));
265 Ok(())
266}
267
268#[test]
269fn an_entry_is_one_ndjson_line_carrying_the_row() -> Outcome<()> {
270 let row = Row::new("dev1", 7, "b1", T0, "console").with("msg", st("a\nb\"c\\d"));
271 let entry = res!(row.entry(360));
272 assert_eq!(entry.ts, T0 as i64);
273 assert_eq!(entry.tag, "ev console");
274 let line = entry.line("devABC", T0 + 5);
275 assert!(line.ends_with('\n'));
276 assert_eq!(line.matches('\n').count(), 1, "a row is one line however its text is built");
277 let d = res!(Dat::decode_json_strict(line.trim_end(), &DecodeLimits::default()));
278 assert_eq!(at(&d, "bt").and_then(|x| x.get_i64()), Some((T0 + 5) as i64));
279 assert_eq!(text_at(&d, "device"), "devABC");
280 assert_eq!(at(&d, "ts").and_then(|x| x.get_i64()), Some(T0 as i64));
281 assert_eq!(text_at(&d, "tag"), "ev console");
282 // The data is the event's own JSON, which parses again to the same fields.
283 let ev = res!(Dat::decode_json_strict(&text_at(&d, "data"), &DecodeLimits::default()));
284 assert_eq!(text_at(&ev, "msg"), "a\nb\"c\\d");
285 assert_eq!(text_at(&ev, "d"), "dev1");
286 Ok(())
287}
288
289// The gates ==============================================================================
290
291#[test]
292fn the_gates_carry_daimonds_values_and_the_plans_defaults() {
293 let d = Gates::daimond();
294 assert_eq!(d.max_body, 256 * 1024);
295 assert_eq!(d.max_rows, 1000);
296 assert_eq!(d.max_tag, 64);
297 assert_eq!(d.max_data, 400);
298 assert_eq!(d.min_interval_ms, 2_000);
299 assert_eq!(d.max_file, 2 * 1024 * 1024);
300 assert_eq!(d.retain_days, 14);
301 assert_eq!(d.max_rotated, 200);
302 assert_eq!(d.max_total, u64::MAX, "Daimond bounds a device, not the directory");
303 assert!(!d.sweep_idle);
304 // The plan's defaults (§6): 2 MB files, 7 days or 20 files, 200 MB in all.
305 let p = Gates::default();
306 assert_eq!((p.max_body, p.max_rows, p.max_tag, p.max_data), (d.max_body, d.max_rows, d.max_tag, d.max_data));
307 assert_eq!(p.min_interval_ms, d.min_interval_ms);
308 assert_eq!(p.max_file, 2 * 1024 * 1024);
309 assert_eq!(p.retain_days, 7);
310 assert_eq!(p.max_rotated, 20);
311 assert_eq!(p.max_total, 200 * 1024 * 1024);
312 assert!(p.sweep_idle);
313}
314
315#[test]
316fn an_authed_post_appends_a_file_the_operator_can_read() -> Outcome<()> {
317 let s = Scratch::new("post");
318 let sink = Sink::new(s.dir(), Gates::default());
319 let b = body("devABC", &[
320 (1000, "dispatch election", "self=aa nominee=bb PRESENT"),
321 (1001, "collect CLAIMED", "turn=t1"),
322 ]);
323 assert_eq!(stored(res!(sink.post(&["acct-diag"], &b, T0))), 2);
324 let path = s.dir().join("acct-diag.ndjson");
325 assert_eq!(sink.path(&["acct-diag"]), path);
326 let got = res!(lines(&path));
327 assert_eq!(got.len(), 2);
328 assert_eq!(text_at(&got[0], "tag"), "dispatch election");
329 assert_eq!(text_at(&got[0], "data"), "self=aa nominee=bb PRESENT");
330 assert_eq!(text_at(&got[0], "device"), "devABC");
331 assert_eq!(at(&got[0], "bt").and_then(|x| x.get_i64()), Some(T0 as i64));
332 assert_eq!(text_at(&got[1], "tag"), "collect CLAIMED");
333 Ok(())
334}
335
336#[test]
337fn a_body_over_the_cap_is_refused_413_and_writes_nothing() -> Outcome<()> {
338 let s = Scratch::new("body");
339 let sink = Sink::new(s.dir(), Gates::default());
340 let mut b = body("d", &[(1, "t", "x")]);
341 b.resize(sink.gates().max_body + 1, b' ');
342 let r = refused(res!(sink.post(&["k"], &b, T0)));
343 assert_eq!(r, Refusal::TooLarge);
344 assert_eq!(r.status(), 413);
345 assert_eq!(r.message(), "Diagnostic trace too large.");
346 assert!(names(&s.dir()).is_empty(), "a refused body must not create a file");
347 Ok(())
348}
349
350#[test]
351fn a_body_exactly_at_the_cap_is_not_refused_for_size() -> Outcome<()> {
352 let s = Scratch::new("bodyat");
353 let sink = Sink::new(s.dir(), Gates::default());
354 let mut b = body("d", &[(1, "t", "x")]);
355 b.resize(sink.gates().max_body, b' ');
356 assert_eq!(stored(res!(sink.post(&["k"], &b, T0))), 1);
357 Ok(())
358}
359
360#[test]
361fn more_rows_than_the_cap_are_refused_not_truncated() -> Outcome<()> {
362 let s = Scratch::new("rows");
363 let sink = Sink::new(s.dir(), Gates::default());
364 let many: Vec<(i64, &str, &str)> = (0..1001).map(|i| (i as i64, "t", "x")).collect();
365 let r = refused(res!(sink.post(&["k"], &body("d", &many), T0)));
366 assert_eq!(r, Refusal::TooManyRows);
367 assert_eq!(r.status(), 400);
368 assert_eq!(r.message(), "Too many rows in one trace.");
369 assert!(names(&s.dir()).is_empty(), "a disagreement is loud: nothing is written");
370 // Exactly the cap is accepted, whole.
371 assert_eq!(stored(res!(sink.post(&["k"], &body("d", &many[..1000]), T0))), 1000);
372 assert_eq!(res!(lines(&sink.path(&["k"]))).len(), 1000);
373 Ok(())
374}
375
376#[test]
377fn a_body_with_no_rows_list_or_no_json_is_refused_400() -> Outcome<()> {
378 let s = Scratch::new("shape");
379 let sink = Sink::new(s.dir(), Gates::default());
380 let r = refused(res!(sink.post(&["k"], br#"{"v":1,"device":"d"}"#, T0)));
381 assert_eq!(r, Refusal::NoRows);
382 assert_eq!(r.status(), 400);
383 assert_eq!(r.message(), "A trace must carry a `rows` list.");
384 let r = refused(res!(sink.post(&["k"], b"not json at all", T0 + 10_000)));
385 assert!(matches!(r, Refusal::Malformed(_)));
386 assert_eq!(r.status(), 400);
387 let r = refused(res!(sink.post(&["k"], b"", T0 + 20_000)));
388 assert!(matches!(r, Refusal::Malformed(_)));
389 assert!(names(&s.dir()).is_empty());
390 Ok(())
391}
392
393#[test]
394fn a_tag_and_a_data_field_are_clipped_on_a_character_boundary() -> Outcome<()> {
395 let s = Scratch::new("clip");
396 let sink = Sink::new(s.dir(), Gates::default());
397 let long_tag = "t".repeat(100);
398 let long_data = "d".repeat(500);
399 let wide = "é".repeat(300); // 600 bytes
400 let b = body("d", &[(1, &long_tag, &long_data), (2, "wide", &wide)]);
401 assert_eq!(stored(res!(sink.post(&["k"], &b, T0))), 2);
402 let got = res!(lines(&sink.path(&["k"])));
403 assert_eq!(text_at(&got[0], "tag").len(), 64);
404 assert_eq!(text_at(&got[0], "data").len(), 400);
405 let w = text_at(&got[1], "data");
406 assert!(w.len() <= 400 && w.len() >= 398, "{} bytes", w.len());
407 assert!(w.chars().all(|c| c == 'é'), "the clip must not cut a character");
408 Ok(())
409}
410
411#[test]
412fn a_row_cannot_forge_another_by_carrying_a_line_break() -> Outcome<()> {
413 let s = Scratch::new("forge");
414 let sink = Sink::new(s.dir(), Gates::default());
415 let b = body("d", &[(1, "tag", "line1\n{\"forged\":1}\r\n\tend")]);
416 assert_eq!(stored(res!(sink.post(&["k"], &b, T0))), 1);
417 let text = res!(fs::read_to_string(sink.path(&["k"])));
418 assert_eq!(text.matches('\n').count(), 1, "one row, one line: {:?}", text);
419 let got = res!(lines(&sink.path(&["k"])));
420 assert_eq!(text_at(&got[0], "data"), "line1 {\"forged\":1} end");
421 Ok(())
422}
423
424#[test]
425fn a_row_with_no_tag_is_marked_not_dropped() -> Outcome<()> {
426 let s = Scratch::new("malformed");
427 let sink = Sink::new(s.dir(), Gates::default());
428 let b = br#"{"v":1,"device":"d","rows":[{"ts":1},{"ts":2,"tag":"ok","data":"x"}]}"#;
429 assert_eq!(stored(res!(sink.post(&["k"], b, T0))), 2);
430 let got = res!(lines(&sink.path(&["k"])));
431 assert_eq!(got.len(), 2);
432 assert_eq!(text_at(&got[0], "tag"), "(malformed row)");
433 assert_eq!(text_at(&got[1], "tag"), "ok");
434 Ok(())
435}
436
437// The rate floor =========================================================================
438
439#[test]
440fn the_rate_floor_refuses_under_two_seconds_and_accepts_over() -> Outcome<()> {
441 let s = Scratch::new("rate");
442 let sink = Sink::new(s.dir(), Gates::default());
443 let b = body("devR", &[(1, "pull", "x")]);
444 assert_eq!(stored(res!(sink.post(&["acct-rate"], &b, T0))), 1);
445 let r = refused(res!(sink.post(&["acct-rate"], &b, T0 + 1_500)));
446 assert_eq!(r, Refusal::TooFast);
447 assert_eq!(r.status(), 429);
448 assert_eq!(r.message(), "Wait a moment before sharing again.");
449 // The refusal did not hold the floor open: 2.5 s after the first is 1 s after the refusal.
450 assert_eq!(stored(res!(sink.post(&["acct-rate"], &b, T0 + 2_500))), 1);
451 // Exactly the interval is enough, as `now - last < MIN_INTERVAL_MS` is the refusal.
452 assert_eq!(stored(res!(sink.post(&["acct-rate"], &b, T0 + 4_500))), 1);
453 // Another key is another floor.
454 assert_eq!(stored(res!(sink.post(&["acct-other"], &b, T0 + 4_600))), 1);
455 assert_eq!(res!(lines(&sink.path(&["acct-rate"]))).len(), 3);
456 Ok(())
457}
458
459#[test]
460fn the_floor_is_keyed_by_the_session_key_never_by_the_body() -> Outcome<()> {
461 let s = Scratch::new("bodykey");
462 let sink = Sink::new(s.dir(), Gates::default());
463 assert_eq!(stored(res!(sink.post(&["acct"], &body("devX", &[(1, "a", "x")]), T0))), 1);
464 // The same session claiming another device in the body is still the same floor.
465 let r = refused(res!(sink.post(&["acct"], &body("devY", &[(1, "a", "x")]), T0 + 100)));
466 assert_eq!(r, Refusal::TooFast);
467 // The body's device is a label on the line, and names no file.
468 assert_eq!(names(&s.dir()), vec!["acct.ndjson".to_string()]);
469 // A hostile label cannot walk out of the directory, or split a line.
470 let h = body("../../etc/passwd\nx", &[(1, "a", "x")]);
471 assert_eq!(stored(res!(sink.post(&["acct2"], &h, T0))), 1);
472 assert_eq!(names(&s.dir()), vec!["acct.ndjson".to_string(), "acct2.ndjson".to_string()]);
473 assert_eq!(res!(lines(&sink.path(&["acct2"]))).len(), 1);
474 Ok(())
475}
476
477#[test]
478fn the_key_floor_remembers_the_last_admitted_call_only() -> Outcome<()> {
479 let f = KeyFloor::new(2_000);
480 assert!(res!(f.admit("a", 10_000)));
481 assert!(!res!(f.admit("a", 11_999)));
482 assert!(!res!(f.admit("a", 11_999)), "a refusal leaves the record alone");
483 assert!(res!(f.admit("a", 12_000)), "the interval itself is enough");
484 assert!(res!(f.admit("b", 12_001)), "keys are independent");
485 // A clock that steps back never underflows into a long refusal.
486 assert!(!res!(f.admit("b", 1)));
487 Ok(())
488}
489
490#[test]
491fn the_key_floor_prunes_stale_keys_only_once_it_has_grown() -> Outcome<()> {
492 let f = KeyFloor::new(2_000).with_ttl_ms(10_000).with_prune_at(3);
493 for (i, k) in ["a", "b", "c"].iter().enumerate() {
494 assert!(res!(f.admit(k, 1_000 + i as u64)));
495 }
496 // Three entries, at the bound, not past it: nothing is walked.
497 assert!(res!(f.admit("d", 20_000)));
498 // Now past the bound, and a, b, c are older than the time to live.
499 assert!(res!(f.admit("e", 20_001)));
500 assert_eq!(res!(f.len()), 2, "only d and e are younger than the time to live");
501 Ok(())
502}
503
504// Hostile keys ===========================================================================
505
506#[test]
507fn a_key_is_reduced_to_a_safe_filename_alphabet() {
508 assert_eq!(sanitise("acct_AB-12"), "acct_AB-12");
509 assert_eq!(sanitise("../../etc/passwd"), "______etc_passwd");
510 assert!(!sanitise("../../etc/passwd").contains('/'));
511 assert_eq!(sanitise(""), "unknown");
512 assert!(sanitise(&"x".repeat(100)).len() <= 48);
513}
514
515#[test]
516fn a_hostile_key_cannot_escape_the_directory() -> Outcome<()> {
517 let s = Scratch::new("escape");
518 let sink = Sink::new(s.dir(), Gates::default());
519 let p = sink.path(&["../../etc", "passwd"]);
520 assert_eq!(p, s.dir().join("______etc-passwd.ndjson"));
521 assert_eq!(stored(res!(sink.post(&["../../etc", "passwd"], &body("d", &[(1, "t", "x")]), T0))), 1);
522 assert_eq!(names(&s.dir()), vec!["______etc-passwd.ndjson".to_string()]);
523 // Daimond's key is account and device, each reduced on its own and joined by a hyphen.
524 assert_eq!(sink.path(&["acct-diag", "devABC"]), s.dir().join("acct-diag-devABC.ndjson"));
525 Ok(())
526}
527
528// Rotation ===============================================================================
529
530#[test]
531fn a_stamp_is_fixed_width_utc_and_sorts_as_a_string() {
532 assert_eq!(stamp(0), "19700101T000000Z");
533 assert_eq!(stamp(86_399_000), "19700101T235959Z");
534 assert_eq!(stamp(T0), "20250912T000000Z");
535 assert!(is_stamp(&stamp(T0)));
536 assert!(!is_stamp("1"), "a pre-stamp `.log.1` suffix is not a stamp");
537 assert!(!is_stamp("1970010 T000000Z"));
538 assert!(stamp(0) < stamp(86_399_000));
539}
540
541#[test]
542fn rotation_by_size_stamps_the_closed_file_and_starts_fresh() -> Outcome<()> {
543 let s = Scratch::new("size");
544 let sink = Sink::new(s.dir(), small_gates());
545 let live = sink.path(&["acct"]);
546 res!(fs::create_dir_all(s.dir()));
547 // A file of exactly the cap is not past it.
548 res!(fs::write(&live, vec![b'x'; 600]));
549 assert_eq!(stored(res!(sink.post(&["acct"], &body("d", &[(1, "first", "a")]), T0))), 1);
550 assert_eq!(names(&s.dir()), vec!["acct.ndjson".to_string()], "600 bytes is not over a cap of 600");
551 // Now it is past the cap, so the next append closes it under the instant of the rotation.
552 let rot_ms = T0 + 6_000;
553 assert_eq!(stored(res!(sink.post(&["acct"], &body("d", &[(2, "after-rotate", "b")]), rot_ms))), 1);
554 let rotated = s.dir().join(rotated_name("acct", rot_ms));
555 assert!(rotated.exists(), "the oversized file should have rotated to a stamped name: {:?}", names(&s.dir()));
556 let old = res!(fs::read_to_string(&rotated));
557 assert!(old.contains("first"), "the rotated file must keep the history, not discard it");
558 let fresh = res!(fs::read_to_string(&live));
559 assert!(fresh.contains("after-rotate"));
560 assert!(!fresh.contains("first"), "the fresh file must not hold pre-rotation content");
561 Ok(())
562}
563
564#[test]
565fn two_rotations_in_one_second_do_not_overwrite_each_other() -> Outcome<()> {
566 let s = Scratch::new("collide");
567 let sink = Sink::new(s.dir(), small_gates());
568 let row = Row::new("d", 1, "b", T0, "k").with("m", st("x"));
569 res!(sink.write(&["acct"], "dev", &[row.clone()], T0));
570 res!(overfill(&sink, &["acct"]));
571 res!(sink.write(&["acct"], "dev", &[row.clone()], T0 + 1_000));
572 res!(overfill(&sink, &["acct"]));
573 res!(sink.write(&["acct"], "dev", &[row.clone()], T0 + 1_000));
574 let n = names(&s.dir());
575 let first = rotated_name("acct", T0 + 1_000);
576 assert!(n.contains(&first), "{:?}", n);
577 assert!(n.contains(&fmt!("{}-1", first)), "an occupied stamp takes a -1 suffix: {:?}", n);
578 Ok(())
579}
580
581#[test]
582fn rotation_by_age_drops_the_over_age_files_and_leaves_the_rest() -> Outcome<()> {
583 let s = Scratch::new("age");
584 let sink = Sink::new(s.dir(), small_gates());
585 let now = 40 * DAY;
586 res!(fs::create_dir_all(s.dir()));
587 let over_age = rotated_name("acct", now - 8 * DAY); // Past the plan's 7 days.
588 let in_age = rotated_name("acct", now - 6 * DAY);
589 let other = rotated_name("other", now - 20 * DAY); // Past the age, and another key's.
590 let other_in = rotated_name("other", now - 2 * DAY);
591 for n in [&over_age, &in_age, &other, &other_in, &"acct.ndjson.1".to_string()] {
592 res!(fs::write(s.dir().join(n), b"rotated\n"));
593 }
594 res!(overfill(&sink, &["acct"]));
595 res!(sink.write(&["acct"], "dev", &[Row::new("d", 1, "b", now, "k")], now));
596 let n = names(&s.dir());
597 assert!(!n.contains(&over_age), "a rotation past retain_days should be gone: {:?}", n);
598 assert!(n.contains(&in_age));
599 // The sweep covers the directory, where Daimond's prune covered one device's files, because
600 // a directory keyed by session grows a key at a time and the others never rotate again.
601 assert!(!n.contains(&other), "another key's file past the age goes too: {:?}", n);
602 assert!(n.contains(&other_in), "and one inside it stays");
603 assert!(n.contains(&"acct.ndjson.1".to_string()), "a pre-stamp rotation is left alone");
604 assert!(n.contains(&rotated_name("acct", now)), "the rotation that triggered the prune is kept");
605 // Daimond's own age is 14 days, which keeps the 8-day-old file.
606 let s2 = Scratch::new("age14");
607 let sink2 = Sink::new(s2.dir(), Gates { max_file: 600, ..Gates::daimond() });
608 res!(fs::create_dir_all(s2.dir()));
609 res!(fs::write(s2.dir().join(&over_age), b"rotated\n"));
610 res!(fs::write(s2.dir().join(rotated_name("acct", now - 15 * DAY)), b"rotated\n"));
611 res!(overfill(&sink2, &["acct"]));
612 res!(sink2.write(&["acct"], "dev", &[Row::new("d", 1, "b", now, "k")], now));
613 let n2 = names(&s2.dir());
614 assert!(n2.contains(&over_age), "an 8-day-old file is inside Daimond's 14 days: {:?}", n2);
615 assert!(!n2.contains(&rotated_name("acct", now - 15 * DAY)));
616 Ok(())
617}
618
619#[test]
620fn rotation_by_count_keeps_the_newest_max_rotated() -> Outcome<()> {
621 let s = Scratch::new("count");
622 let sink = Sink::new(s.dir(), Gates { max_file: 600, max_rotated: 5, ..Gates::default() });
623 let now = 40 * DAY;
624 res!(fs::create_dir_all(s.dir()));
625 // Seven rotations inside the age, a minute apart, newest first in `in_age`.
626 let mut in_age = Vec::new();
627 for k in 1..=7u64 {
628 let n = rotated_name("acct", now - k * 60_000);
629 res!(fs::write(s.dir().join(&n), b"rotated\n"));
630 in_age.push(n);
631 }
632 res!(overfill(&sink, &["acct"]));
633 res!(sink.write(&["acct"], "dev", &[Row::new("d", 1, "b", now, "k")], now));
634 let n = names(&s.dir());
635 // Eight rotations now stand (the seven and the one just made); five are kept, newest first.
636 assert!(n.contains(&rotated_name("acct", now)));
637 for kept in &in_age[..4] {
638 assert!(n.contains(kept), "{} should survive: {:?}", kept, n);
639 }
640 for gone in &in_age[4..] {
641 assert!(!n.contains(gone), "{} is over the count and should be gone: {:?}", gone, n);
642 }
643 assert!(n.contains(&"acct.ndjson".to_string()), "the live file is not a rotation");
644 Ok(())
645}
646
647#[test]
648fn the_total_cap_deletes_the_oldest_rotations_across_keys() -> Outcome<()> {
649 let s = Scratch::new("total");
650 let sink = Sink::new(s.dir(), Gates {
651 max_file: 200,
652 max_total: 3_000,
653 max_rotated: 100,
654 ..Gates::default()
655 });
656 let now = 40 * DAY;
657 res!(fs::create_dir_all(s.dir()));
658 let kb = vec![b'r'; 1_000];
659 let order = [("a", 6), ("b", 5), ("a", 4), ("b", 3), ("a", 2)]; // Oldest first, in days back.
660 let mut made = Vec::new();
661 for (stem, back) in order {
662 let n = rotated_name(stem, now - back * DAY / 2);
663 res!(fs::write(s.dir().join(&n), &kb));
664 made.push(n);
665 }
666 res!(overfill(&sink, &["c"]));
667 res!(sink.write(&["c"], "dev", &[Row::new("d", 1, "b", now, "k")], now));
668 let n = names(&s.dir());
669 for gone in &made[..3] {
670 assert!(!n.contains(gone), "{} is among the oldest and should be gone: {:?}", gone, n);
671 }
672 for kept in &made[3..] {
673 assert!(n.contains(kept), "{} should survive: {:?}", kept, n);
674 }
675 assert!(n.contains(&rotated_name("c", now)));
676 assert!(n.contains(&"c.ndjson".to_string()), "the file being written is never swept");
677 let total: u64 = n.iter().map(|f| fs::metadata(s.dir().join(f)).map(|m| m.len()).unwrap_or(0)).sum();
678 assert!(total <= 3_000, "{} bytes remain", total);
679 Ok(())
680}
681
682#[test]
683fn idle_live_files_age_out_when_the_gates_say_so() -> Outcome<()> {
684 let now = now_real_ms();
685 let make = |gates: Gates, tag: &str| -> Outcome<(Scratch, Sink)> {
686 let s = Scratch::new(tag);
687 let sink = Sink::new(s.dir(), gates);
688 res!(fs::create_dir_all(s.dir()));
689 for (name, back) in [("old.ndjson", 8 * DAY), ("fresh.ndjson", 3_600_000)] {
690 let f = res!(fs::File::create(s.dir().join(name)));
691 res!(f.set_modified(UNIX_EPOCH + Duration::from_millis(now - back)));
692 }
693 Ok((s, sink))
694 };
695 // A first row for a new key is what sets the sweep off.
696 let (s, sink) = res!(make(Gates::default(), "idle"));
697 assert_eq!(stored(res!(sink.post(&["new"], &body("d", &[(1, "t", "x")]), now))), 1);
698 assert_eq!(names(&s.dir()), vec!["fresh.ndjson".to_string(), "new.ndjson".to_string()],
699 "an idle file past retain_days goes, a recent one and the new one stay");
700 // Daimond never swept a device that had gone quiet.
701 let (s, sink) = res!(make(Gates::daimond(), "idle-daimond"));
702 assert_eq!(stored(res!(sink.post(&["new"], &body("d", &[(1, "t", "x")]), now))), 1);
703 assert_eq!(names(&s.dir()).len(), 3, "{:?}", names(&s.dir()));
704 Ok(())
705}
706
707#[cfg(unix)]
708#[test]
709fn the_directory_is_created_0700_and_the_files_0600() -> Outcome<()> {
710 use std::os::unix::fs::PermissionsExt;
711 let s = Scratch::new("perm");
712 let sink = Sink::new(s.dir(), Gates::default());
713 assert_eq!(stored(res!(sink.post(&["k"], &body("d", &[(1, "t", "x")]), T0))), 1);
714 let mode = |p: &Path| fs::metadata(p).map(|m| m.permissions().mode() & 0o777).unwrap_or(0);
715 assert_eq!(mode(&s.dir()), 0o700);
716 assert_eq!(mode(&sink.path(&["k"])), 0o600);
717 // A directory that already exists, and is open to others, is closed.
718 let s2 = Scratch::new("perm2");
719 res!(fs::create_dir_all(s2.dir()));
720 res!(fs::set_permissions(s2.dir(), fs::Permissions::from_mode(0o755)));
721 let sink2 = Sink::new(s2.dir(), Gates::default());
722 assert_eq!(stored(res!(sink2.post(&["k"], &body("d", &[(1, "t", "x")]), T0))), 1);
723 assert_eq!(mode(&s2.dir()), 0o700);
724 Ok(())
725}
726
727// The secret rule ========================================================================
728
729#[test]
730fn secret_name_is_secret_re_over_three_thousand_names() -> Outcome<()> {
731 let table = res!(fs::read_to_string(
732 Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/data/lens_secret_re.tsv")));
733 let mut n = 0;
734 let mut hits = 0;
735 for line in table.lines() {
736 let cols: Vec<&str> = line.split('\t').collect();
737 assert_eq!(cols.len(), 3, "bad oracle line {:?}", line);
738 let (name, strict, loose) = (cols[0], cols[1] == "1", cols[2] == "1");
739 assert_eq!(secret_name(name), strict, "SECRET_RE on {:?}", name);
740 assert_eq!(secret_name_loose(name), loose, "SECRET_RE or SECRET_RE_LOOSE on {:?}", name);
741 n += 1;
742 if strict { hits += 1; }
743 }
744 assert!(n > 3_000, "the oracle holds {} names", n);
745 assert!(hits > 500 && hits < n, "a table that is all one answer proves nothing: {} of {}", hits, n);
746 Ok(())
747}
748
749#[test]
750fn a_fingerprint_is_what_the_javascript_makes() -> Outcome<()> {
751 let table = res!(fs::read_to_string(
752 Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/data/lens_fingerprint.tsv")));
753 let red = Redact::new();
754 let mut n = 0;
755 for line in table.lines() {
756 let (input, want) = match line.split_once('\t') {
757 Some(p) => p,
758 None => panic!("bad oracle line {:?}", line),
759 };
760 let want = text_of(&res!(Dat::decode_json_strict(want, &DecodeLimits::default())));
761 let want = want.as_str();
762 // Each input is JSON, so null and a number take the route a field value takes.
763 let v = res!(Dat::decode_json_strict(input, &DecodeLimits::default()));
764 let out = red.dat(&omap(vec![("secret", v.clone())]));
765 assert_eq!(text_at(&out, "secret"), want, "fingerprint of {}", input);
766 if let Dat::Str(s) = &v {
767 assert_eq!(fingerprint(s, 6), want, "fingerprint() of {}", input);
768 }
769 n += 1;
770 }
771 assert!(n >= 20);
772 Ok(())
773}
774
775#[test]
776fn the_head_of_a_fingerprint_can_be_withheld() {
777 let fp = fingerprint("hunter2hunter2", 0);
778 assert!(fp.starts_with("[redacted …#") && fp.ends_with("/14]"), "{}", fp);
779 assert!(!fp.contains("hunter"));
780 assert_eq!(fingerprint("", 0), "[redacted:empty]");
781 // The same hash whatever the head, so two occurrences of one secret still match.
782 let a = fingerprint("hunter2hunter2", 6);
783 assert_eq!(a.split('#').nth(1), fp.split('#').nth(1));
784 // And a redactor can be set to withhold it everywhere.
785 let r = Redact::new().with_head(0).dat(&omap(vec![("token", st("hunter2hunter2"))]));
786 assert_eq!(text_at(&r, "token"), fp);
787 // A typed field withholds it whatever the head, and is marked as the scrubber marks.
788 let r = Redact::new().with_head(6).dat(&omap(vec![("password", st("hunter2hunter2"))]));
789 assert_eq!(text_at(&r, "password"), secret::scrub::mark("named", "hunter2hunter2"));
790}
791
792// The redactor ===========================================================================
793
794#[test]
795fn secret_named_fields_are_fingerprinted_at_any_depth() -> Outcome<()> {
796 let d = omap(vec![
797 ("a", omap(vec![("b", Dat::List(vec![
798 omap(vec![("password", st("p1")), ("ok", st("fine"))]),
799 st("a bare string in a list is not a field"),
800 ]))])),
801 ("token", omap(vec![("x", Dat::U64(1))])), // A secret-named object is noted, not walked.
802 ("key", Dat::List(vec![Dat::U64(1), Dat::U64(2)])),
803 ("salt", Dat::U64(77)),
804 ("n", Dat::U64(5)),
805 ("apiKey", st("sk-live-abcdef")),
806 ("gone", Dat::Opt(Box::new(None))),
807 ("seed", Dat::Opt(Box::new(None))),
808 ]);
809 let r = Redact::new().dat(&d);
810 assert_eq!(text_at(&r, "token"), "[redacted:object]");
811 assert_eq!(text_at(&r, "key"), "[redacted:object]");
812 assert_eq!(text_at(&r, "salt"), fingerprint("77", 6));
813 assert_eq!(text_at(&r, "apiKey"), fingerprint("sk-live-abcdef", 6));
814 assert_eq!(text_at(&r, "seed"), "[redacted:absent]");
815 assert_eq!(at(&r, "n"), Some(&Dat::U64(5)));
816 assert_eq!(at(&r, "gone"), Some(&Dat::Opt(Box::new(None))), "an empty field not named secret is left alone");
817 let j = res!(compact(&r));
818 assert!(!j.contains("\"p1\"") && !j.contains("sk-live-abcdef"), "{}", j);
819 assert!(j.contains("\"ok\":\"fine\"") && j.contains("a bare string in a list"), "{}", j);
820 Ok(())
821}
822
823#[test]
824fn a_camel_cased_name_needs_the_loose_companion() -> Outcome<()> {
825 let d = omap(vec![("pushToken", st("t0k3n-value")), ("tokenStats", Dat::U64(3)), ("note", st("n"))]);
826 let strict = Redact::new().dat(&d);
827 assert_eq!(text_at(&strict, "pushToken"), "t0k3n-value", "SECRET_RE misses a camel-joined name");
828 let loose = Redact::new().loose().dat(&d);
829 assert_eq!(text_at(&loose, "pushToken"), fingerprint("t0k3n-value", 6));
830 assert_eq!(text_at(&loose, "tokenStats"), fingerprint("3", 6), "the loose pattern is boundary-free");
831 assert_eq!(text_at(&loose, "note"), "n");
832 Ok(())
833}
834
835#[test]
836fn deny_names_and_paths_are_redacted_and_anchored_paths_stay_anchored() -> Outcome<()> {
837 let red = Redact::new()
838 .deny_name("durable")
839 .deny(Deny::Path(vec!["state".to_string(), "wallet".to_string(), "coins".to_string()]))
840 .deny_path(&["sess", "*", "nonce"]);
841 let d = omap(vec![
842 ("x", omap(vec![("durable", st("D1")), ("fine", st("F"))])),
843 ("state", omap(vec![("wallet", omap(vec![("coins", st("C1")), ("name", st("w"))]))])),
844 ("other", omap(vec![("state", omap(vec![("wallet", omap(vec![("coins", st("C2"))]))]))])),
845 // A list is transparent to a path, so the elements are the object's own members.
846 ("sess", Dat::List(vec![omap(vec![("a", omap(vec![("nonce", st("N1"))]))])])),
847 ]);
848 let r = red.dat(&d);
849 assert_eq!(text_at(&r, "x.durable"), fingerprint("D1", 6));
850 assert_eq!(text_at(&r, "x.fine"), "F");
851 assert_eq!(text_at(&r, "state.wallet.coins"), fingerprint("C1", 6));
852 assert_eq!(text_at(&r, "state.wallet.name"), "w");
853 assert_eq!(text_at(&r, "other.state.wallet.coins"), "C2", "a path from the root does not match deeper down");
854 let j = res!(compact(&r));
855 assert!(j.contains(&fingerprint("N1", 6)) && !j.contains("\"N1\""), "{}", j);
856 Ok(())
857}
858
859#[test]
860fn a_caller_string_test_redacts_values_keys_and_list_members() -> Outcome<()> {
861 let eight = "alpha bravo charlie delta echo foxtrot golf hotel";
862 let seven = "alpha bravo charlie delta echo foxtrot golf";
863 let red = Redact::new().with_test(|s: &str| s.split(' ').count() == 8);
864 let d = omap(vec![
865 ("msg", st(&fmt!("{} done", "boot"))),
866 ("phrase", st(eight)),
867 ("near", st(seven)),
868 ("list", Dat::List(vec![st("x"), st(eight)])),
869 ("nest", omap(vec![("deep", st(eight))])),
870 (eight, st("a value under a key that is itself the phrase")),
871 ]);
872 let r = red.dat(&d);
873 assert_eq!(text_at(&r, "phrase"), fingerprint(eight, 6));
874 assert_eq!(text_at(&r, "near"), seven, "seven words are not the pattern");
875 assert_eq!(text_at(&r, "msg"), "boot done");
876 assert_eq!(text_at(&r, "nest.deep"), fingerprint(eight, 6));
877 let j = res!(compact(&r));
878 assert!(!j.contains(eight), "the phrase survives somewhere: {}", j);
879 assert!(j.contains(&fingerprint(eight, 6)));
880 Ok(())
881}
882
883#[test]
884fn the_walk_is_bounded_and_a_marker_stands_where_it_stops() -> Outcome<()> {
885 let mut d = omap(vec![("leaf", st("DEEP-PLANT"))]);
886 for _ in 0..45 {
887 d = omap(vec![("n", d)]);
888 }
889 let r = Redact::new().dat(&d);
890 let j = res!(compact(&r));
891 assert!(j.contains("[redacted:cycle-or-deep]"), "{}", j);
892 assert!(!j.contains("DEEP-PLANT"));
893 Ok(())
894}
895
896#[test]
897fn redacting_twice_does_not_fingerprint_a_fingerprint() -> Outcome<()> {
898 let d = omap(vec![("passphrase", st("correct horse")), ("n", Dat::U64(1))]);
899 let red = Redact::new();
900 let once = red.dat(&d);
901 let (twice, changed) = red.walk(&once);
902 assert_eq!(res!(compact(&once)), res!(compact(&twice)));
903 assert!(!changed, "a marker already in place is not a change");
904 let (_, changed) = red.walk(&d);
905 assert!(changed);
906 let (_, changed) = red.walk(&omap(vec![("ok", st("fine"))]));
907 assert!(!changed);
908 Ok(())
909}
910
911// Nothing planted reaches a byte =========================================================
912
913const PHRASE: &str = "abandon ability able about above absent absorb abstract";
914
915/// Does the text hold eight lowercase words in a row? Oxegen's own test asks the same of the EFF list.
916fn eight_words(s: &str) -> bool {
917 let w: Vec<bool> = s.split(' ')
918 .map(|x| !x.is_empty() && x.chars().all(|c| c.is_ascii_lowercase()))
919 .collect();
920 w.windows(8).any(|win| win.iter().all(|&b| b))
921}
922
923#[test]
924fn a_planted_secret_is_absent_from_every_byte_written() -> Outcome<()> {
925 let plants = [
926 "PLANT-passphrase-1", "PLANT-wallet-seedling-2", "PLANT-durable-3", "PLANT-token-4",
927 "PLANT-apikey-5", "PLANT-listmember-6",
928 ];
929 let s = Scratch::new("planted");
930 let sink = Sink::new(s.dir(), Gates { max_file: 900, ..Gates::default() })
931 .with_redact(Redact::new()
932 .deny_name("durable_blob")
933 .deny_path(&["state", "wallet", "seedling"])
934 .with_test(eight_words));
935 let dev = "peer";
936 // The typed door: rows built in process, as the peer's own taps will.
937 for i in 0..12u64 {
938 let row = Row::new(dev, i, "b1", T0 + i, "state")
939 .with("auth", omap(vec![("passphrase", st(plants[0])), ("user", st("ann"))]))
940 .with("state", omap(vec![("wallet", omap(vec![("seedling", st(plants[1])), ("open", Dat::Bool(true))]))]))
941 .with("durable_blob", st(plants[2]))
942 .with("console", st(&fmt!("boot {} done", PHRASE)))
943 .with("list", Dat::List(vec![st("x"), st(PHRASE)]))
944 .with("tokenStats", Dat::U64(5));
945 assert_eq!(res!(sink.write(&["sid-1"], dev, &[row], T0 + 1_000 * i)), 1);
946 }
947 // The posted door: a client's batch, whose events are JSON inside a string.
948 let ev = fmt!(
949 "{{\"v\":1,\"d\":\"dev1\",\"n\":1,\"b\":\"b\",\"t\":1,\"token\":\"{}\",\"nested\":{{\"api_key\":\"{}\",\"list\":[\"{}\"],\"keep\":\"kept\"}}}}",
950 plants[3], plants[4], PHRASE);
951 let text_row = fmt!("saw {} in a plain text row", PHRASE);
952 for i in 0..6i64 {
953 let b = body("devX", &[(i, "ev console", &ev), (i, "diag", &text_row)]);
954 assert_eq!(stored(res!(sink.post(&["sid-2"], &b, T0 + 3_000 * i as u64))), 2);
955 }
956 // An event whose data is no JSON cannot be examined by name, so it is not kept.
957 let broken = fmt!("{{\"token\":\"{}\" oops", plants[5]);
958 assert_eq!(stored(res!(sink.post(&["sid-2"], &body("devX", &[(1, "ev broken", &broken)]), T0 + 60_000))), 1);
959
960 let n = names(&s.dir());
961 assert!(n.iter().any(|f| f.contains(".ndjson.2025")), "the run must have rotated at least once: {:?}", n);
962 let bytes = res!(all_bytes(&s.dir()));
963 for p in plants.iter().chain([PHRASE, "abandon"].iter()) {
964 assert!(!has(&bytes, p), "{:?} reached the disk", p);
965 }
966 // Not vacuous: the rows are there, with the markers and the innocent fields.
967 assert!(has(&bytes, "ann") && has(&bytes, "kept") && has(&bytes, "tokenStats"));
968 assert!(has(&bytes, &fingerprint(PHRASE, 6)) || has(&bytes, "[redacted"), "markers are expected");
969 // A typed field keeps no head of its value, so the passphrase's mark is the scrubber's.
970 assert!(has(&bytes, &secret::scrub::mark("named", plants[0])), "the passphrase's mark is expected");
971
972 // The control: the same fields written without the sink do carry the plants, so a pass above
973 // is the redactor's doing and not a search that cannot see.
974 let raw = res!(Row::new(dev, 0, "b1", T0, "state")
975 .with("auth", omap(vec![("passphrase", st(plants[0]))]))
976 .with("durable_blob", st(plants[2]))
977 .with("console", st(&fmt!("boot {} done", PHRASE)))
978 .entry(4096)).line(dev, T0);
979 for p in [plants[0], plants[2], PHRASE] {
980 assert!(has(raw.as_bytes(), p), "the control line should hold {:?}: {}", p, raw);
981 }
982 Ok(())
983}
984
985
986// The stock string test ==================================================================
987
988// One credential of each shape `fe2o3_text::secret` knows, in two pieces joined at run time, so
989// that the scanners that read this file find nothing in it to refuse.
990const SHAPED: &[(&str, &str)] = &[
991 ("fw", "_3ZjKq81mAbCdEfGhIjKlMnOpQrSt"),
992 ("sk-ant", "-api03-AbCdEfGhIjKlMnOpQrStUvWx"),
993 ("sk-proj", "-AbCdEfGhIjKlMnOpQrStUvWxYz01"),
994 ("sk-or", "-v1-0123456789abcdef0123456789abcdef"),
995 ("sk-", "AbCdEfGhIjKlMnOpQrStUvWxYz0123456789"),
996 ("AKIA", "IOSFODNN7EXAMPLE"),
997 ("ghp", "_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789"),
998 ("github_pat", "_11ABCDEFG0AbCdEfGhIjKlMnOpQrStUvWxYz0123456789"),
999 ("xoxb", "-1234567890-abcdefghij"),
1000 ("sk_live", "_AbCdEfGhIjKlMnOpQrStUv"),
1001 ("AIza", "SyA0123456789abcdefghijklmnopqrstuv"),
1002 ("-----BEGIN ", "OPENSSH PRIVATE KEY-----"),
1003];
1004
1005fn cred(i: usize) -> String {
1006 fmt!("{}{}", SHAPED[i].0, SHAPED[i].1)
1007}
1008
1009#[test]
1010fn shapes_cover_each_credential_shape_whole_and_leave_prose_alone() -> Outcome<()> {
1011 // Head 0, so that no character of a credential stands in its own fingerprint.
1012 let red = Redact::new().with_head(0).with_test(Shapes);
1013 for i in 0..SHAPED.len() {
1014 let c = cred(i);
1015 let doc = omap(vec![
1016 ("alone", st(&c)),
1017 ("note", st(&fmt!("deployed with {} this morning", c))),
1018 ("list", Dat::List(vec![st("fine"), st(&fmt!("k={}\n", c))])),
1019 ("deep", omap(vec![("x", omap(vec![("y", st(&fmt!("a\nb {} c\nd", c)))]))])),
1020 ]);
1021 let out = red.dat(&doc);
1022 let json = res!(compact(&out));
1023 assert!(!json.contains(&c), "{:?} survived the redactor: {}", SHAPED[i].0, json);
1024 for p in ["alone", "note", "deep.x.y"] {
1025 assert!(text_at(&out, p).starts_with("[redacted"), "{} for {:?}: {:?}", p, SHAPED[i].0, text_at(&out, p));
1026 }
1027 assert!(json.contains("fine"), "the innocent list member stays: {}", json);
1028 }
1029 // An assigned literal under a secret-looking name is a shape too.
1030 let lit = fmt!("password = \"{}{}\"", "9f3Bq7", "ZmR4tYuIoPkLjHgFdS");
1031 assert!(Shapes.hit(&lit));
1032 for ok in [
1033 "the quick brown fox", "sk-nope", "skip-tracing and key lore", "AKIA", "fw_short",
1034 "an api_key = \"\" field", "key = \"your-key-here\"", "", "ghp_tooshort",
1035 ] {
1036 assert!(!Shapes.hit(ok), "{:?} is not a credential", ok);
1037 }
1038 Ok(())
1039}
1040
1041#[test]
1042fn a_marker_and_a_nul_do_not_excuse_a_credential_in_a_feed() -> Outcome<()> {
1043 let k = cred(0);
1044 // `scan` is a file reader and takes the author's word; the control shows it does.
1045 let excused = [
1046 fmt!("{} # {}", k, secret::MARKER),
1047 fmt!("{}\n{}", secret::MARKER, k),
1048 fmt!("\0{}", k),
1049 ];
1050 for t in &excused {
1051 assert!(secret::scan(t.as_bytes()).is_empty(), "scan should excuse {:?}", t);
1052 assert!(secret::holds(t), "holds must not excuse {:?}", t);
1053 assert!(Shapes.hit(t), "Shapes must not excuse {:?}", t);
1054 }
1055 // And the same through both doors of a sink: nothing of the key reaches a byte.
1056 let s = Scratch::new("holds");
1057 let sink = Sink::new(s.dir(), Gates::default())
1058 .with_redact(Redact::new().with_head(0).with_test(Shapes));
1059 let row = Row::new("peer", 1, "b", T0, "console")
1060 .with("line", st(&excused[0]))
1061 .with("other", st(&excused[2]));
1062 assert_eq!(res!(sink.write(&["sid"], "peer", &[row], T0)), 1);
1063 let ev = fmt!("{{\"v\":1,\"d\":\"x\",\"n\":1,\"b\":\"b\",\"t\":1,\"line\":\"{}\"}}", excused[0]);
1064 let b = body("devX", &[(1, "ev console", &ev), (2, "diag", &excused[1])]);
1065 assert_eq!(stored(res!(sink.post(&["sid2"], &b, T0 + 5_000))), 2);
1066 let bytes = res!(all_bytes(&s.dir()));
1067 assert!(!has(&bytes, &k), "the key reached the disk");
1068 assert!(has(&bytes, "[redacted"), "markers are expected");
1069 Ok(())
1070}
1071
1072#[test]
1073fn secret_name_agrees_with_the_regex_engine_on_every_ascii_name() -> Outcome<()> {
1074 // The hand matcher stands for the JavaScript, whose `\b` and `/i` are ASCII-only. On ASCII
1075 // names the crate's own engine, which follows the `regex` crate, must say the same.
1076 let re = res!(Regex::new(
1077 r"(?i)(?:^|[_.-])(?:apikey|api_key|key|token|secret|passphrase|password|salt|wrapped|wrappedpriv|sealed|seal|privatekey|priv|mnemonic|seed|masterkey)(?:$|[_.-]|enc\b)"));
1078 let table = res!(fs::read_to_string(
1079 Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/data/lens_secret_re.tsv")));
1080 let mut n = 0;
1081 for line in table.lines() {
1082 let name = line.split('\t').next().unwrap_or("");
1083 if !name.is_ascii() {
1084 continue;
1085 }
1086 assert_eq!(secret_name(name), res!(re.is_match(name)), "on {:?}", name);
1087 n += 1;
1088 }
1089 assert!(n > 2_500, "only {} ASCII names were compared", n);
1090 Ok(())
1091}
1092
1093
1094// The content scrubber ===================================================================
1095
1096// `tests/data/lens_scrub.tsv` holds each input of the JavaScript scrubber as a recipe and never as a
1097// credential, so that nothing paste-able is in the repository. `{{n:seed:alphabet}}` is n
1098// characters: the state starts at `seed * 2654435761` mod 2^32, each character steps it by
1099// `h * 1103515245 + 12345` in u32 wrapping arithmetic and takes `alphabet[(h >> 16) mod len]`.
1100// `{{dash5}}` is five hyphens.
1101
1102fn alphabet(name: &str) -> &'static str {
1103 match name {
1104 "alnum" => "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789",
1105 "hex" => "0123456789abcdef",
1106 "b64" => "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_",
1107 "b64std" => "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789/+",
1108 "upper36" => "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789",
1109 "digit" => "0123456789",
1110 other => panic!("no alphabet {:?}", other),
1111 }
1112}
1113
1114/// `n` characters of one alphabet, from one seed.
1115fn run(n: usize, seed: u32, alpha: &str) -> String {
1116 let a: Vec<char> = alphabet(alpha).chars().collect();
1117 let mut h = (seed as u64 * 2_654_435_761u64) as u32;
1118 let mut s = String::new();
1119 for _ in 0..n {
1120 h = h.wrapping_mul(1_103_515_245).wrapping_add(12_345);
1121 s.push(a[((h >> 16) as usize) % a.len()]);
1122 }
1123 s
1124}
1125
1126/// A template of the table, with its recipes expanded.
1127fn expand(t: &str) -> Outcome<String> {
1128 let mut out = String::new();
1129 let mut rest = t;
1130 while let Some(i) = rest.find("{{") {
1131 out.push_str(&rest[..i]);
1132 let tail = &rest[i + 2..];
1133 let j = res!(tail.find("}}").ok_or_else(|| err!("An unclosed recipe in {:?}.", t; Invalid, Input)));
1134 let body = &tail[..j];
1135 if body == "dash5" {
1136 out.push_str("-----");
1137 } else {
1138 let p: Vec<&str> = body.split(':').collect();
1139 if p.len() != 3 {
1140 return Err(err!("The recipe {:?} is not n:seed:alphabet.", body; Invalid, Input));
1141 }
1142 out.push_str(&run(res!(p[0].parse::<usize>()), res!(p[1].parse::<u32>()), p[2]));
1143 }
1144 rest = &tail[j + 2..];
1145 }
1146 out.push_str(rest);
1147 Ok(out)
1148}
1149
1150struct Case {
1151 template: String,
1152 input: String, // the template, expanded
1153 want: String, // what the JavaScript made of it
1154 id: bool, // scrubbed as a correlation id, so with no entropy catch
1155}
1156
1157fn scrub_cases() -> Outcome<Vec<Case>> {
1158 let table = res!(fs::read_to_string(
1159 Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/data/lens_scrub.tsv")));
1160 let mut cases = Vec::new();
1161 for line in table.lines() {
1162 let cols: Vec<&str> = line.split('\t').collect();
1163 assert_eq!(cols.len(), 3, "bad oracle line {:?}", line);
1164 let template = text_of(&res!(Dat::decode_json_strict(cols[0], &DecodeLimits::default())));
1165 let want = text_of(&res!(Dat::decode_json_strict(cols[1], &DecodeLimits::default())));
1166 let input = res!(expand(&template));
1167 cases.push(Case { template, input, want, id: cols[2] == "1" });
1168 }
1169 Ok(cases)
1170}
1171
1172#[test]
1173fn the_expander_gives_the_pinned_vectors() -> Outcome<()> {
1174 // The port is held to the JavaScript's before it is trusted with the table.
1175 for (t, want) in [
1176 ("{{8:1:hex}}", "8388dbc2"),
1177 ("{{12:7:alnum}}", "Dn8QmkIRy6vU"),
1178 ("{{10:3:digit}}", "0086285001"),
1179 ("{{6:5:upper36}}", "PJMEIR"),
1180 ("a{{dash5}}b", "a-----b"),
1181 ] {
1182 assert_eq!(res!(expand(t)), want, "{}", t);
1183 }
1184 Ok(())
1185}
1186
1187#[test]
1188fn shapes_scrub_the_javascripts_table_to_the_same_text() -> Outcome<()> {
1189 let cases = res!(scrub_cases());
1190 assert_eq!(cases.len(), 165, "the table has changed size");
1191 let mut changed = 0;
1192 for c in &cases {
1193 let got = Shapes.scrub(&c.input, c.id);
1194 assert_eq!(got, c.want, "recipe {:?}", c.template);
1195 // A second pass finds nothing in the markers of the first.
1196 assert_eq!(Shapes.scrub(&got, c.id), got, "not idempotent, recipe {:?}", c.template);
1197 if got != c.input {
1198 changed += 1;
1199 }
1200 }
1201 let unchanged = cases.len() - changed;
1202 assert!(changed >= 100 && unchanged >= 20,
1203 "a table that is nearly one answer proves little: {} changed, {} not", changed, unchanged);
1204 // Every shape, every pair and the entropy catch is exercised.
1205 for kind in [
1206 "pem", "gh", "ghpat", "stripe", "whsec", "sk", "aws", "gcp", "slack", "jwt", "tune",
1207 "bearer", "urlarg", "named", "awssec", "hi",
1208 ] {
1209 let tag = fmt!("[redacted {} #", kind);
1210 assert!(cases.iter().any(|c| c.want.contains(&tag)), "no case yields {:?}", kind);
1211 }
1212 Ok(())
1213}
1214
1215// SCRUB-ONE (D-20261006-24): `tests/data/lens_scrub_leaks.tsv` is the corpus's `LEAKS`, each a
1216// template and what of it must be gone once scrubbed (a multi-word passphrase, a short password, a
1217// quoted value, a lower-case `bearer`). It holds the scrubber to the absence itself, so a table
1218// generated by a scrubber that leaks cannot make it pass.
1219#[test]
1220fn no_leak_of_the_corpus_survives_the_scrubber() -> Outcome<()> {
1221 let table = res!(fs::read_to_string(
1222 Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/data/lens_scrub_leaks.tsv")));
1223 let mut kept = Vec::new();
1224 let mut rows = 0;
1225 for line in table.lines() {
1226 let cols: Vec<&str> = line.split('\t').collect();
1227 assert_eq!(cols.len(), 2, "bad leak line {:?}", line);
1228 let template = text_of(&res!(Dat::decode_json_strict(cols[0], &DecodeLimits::default())));
1229 let gone = text_of(&res!(Dat::decode_json_strict(cols[1], &DecodeLimits::default())));
1230 let got = Shapes.scrub(&res!(expand(&template)), false);
1231 if got.contains(&res!(expand(&gone))) {
1232 kept.push(template.clone());
1233 }
1234 assert_eq!(Shapes.scrub(&got, false), got, "not idempotent, recipe {:?}", template);
1235 assert!(Shapes.hit(&res!(expand(&template))), "not a hit, recipe {:?}", template);
1236 rows += 1;
1237 }
1238 assert!(rows >= 20, "the leak table has {} rows", rows);
1239 assert!(kept.is_empty(), "{} leaks kept: {:?}", kept.len(), kept);
1240 Ok(())
1241}
1242
1243// A field a person types into (mail's `password`, its sealed `pass`, a passphrase) is marked whole
1244// by the name walk, with none of its head, since a passphrase's first six characters are its first
1245// word. A key keeps its six, and a field merely starting `pass` reads as it was.
1246#[test]
1247fn a_typed_password_field_keeps_no_head() -> Outcome<()> {
1248 let app = "abcd efgh ijkl mnop";
1249 let phrase = "correct horse battery staple";
1250 let d = omap(vec![
1251 ("password", st(phrase)), ("pass", st(app)), ("smtpPassword", st(app)),
1252 ("seed_phrase", st(phrase)), ("apiKey", st("sk-or-v1-abcdefghij")),
1253 ("passAt", st("1789300000000")), ("passwordHint", st("the usual")), ("bypass", st("on")),
1254 ]);
1255 // The loose rule covers `passwordHint` by its own word, as Daimond's `redactConfig` does, so it
1256 // is held to the typed fields alone.
1257 let loose = fmt!("{:?}", Redact::new().loose().dat(&d));
1258 assert!(!loose.contains("correc") && !loose.contains("abcd"), "a typed field kept its head: {}", loose);
1259 for red in [Redact::new().dat(&d), Redact::new().with_test(Shapes).dat(&d)] {
1260 let txt = fmt!("{:?}", red);
1261 assert!(!txt.contains("correc") && !txt.contains("abcd"), "a typed field kept its head: {}", txt);
1262 assert!(txt.contains("[redacted sk-or-"), "a key lost its head: {}", txt);
1263 assert!(txt.contains("1789300000000") && txt.contains("the usual") && txt.contains("\"on\""),
1264 "a field beside them was covered: {}", txt);
1265 assert!(txt.contains(&"[redacted named #"), "no mark: {}", txt);
1266 assert_eq!(Redact::new().dat(&red), red, "a second walk changed it");
1267 }
1268 Ok(())
1269}
1270
1271#[test]
1272fn a_hit_is_a_string_the_scrubber_would_change() -> Outcome<()> {
1273 for c in res!(scrub_cases()) {
1274 let changed = c.want != c.input;
1275 let hit = if c.id { Shapes.hit_id(&c.input) } else { Shapes.hit(&c.input) };
1276 assert_eq!(hit, changed, "recipe {:?}", c.template);
1277 // What the scrubber wrote is not a hit, so a row the client already scrubbed passes the
1278 // redactor with its own bytes.
1279 let again = if c.id { Shapes.hit_id(&c.want) } else { Shapes.hit(&c.want) };
1280 assert!(!again, "the output of recipe {:?} is itself a hit", c.template);
1281 let red = Redact::new().with_test(Shapes);
1282 let (out, covered) = red.walk(&omap(vec![(if c.id { "callId" } else { "msg" }, st(&c.want))]));
1283 assert!(!covered, "recipe {:?} was covered a second time: {:?}", c.template, out);
1284 }
1285 Ok(())
1286}
1287
1288#[test]
1289fn the_shapes_the_commit_hook_does_not_know_are_caught_and_its_scan_is_unchanged() -> Outcome<()> {
1290 // Each is Daimond's and not the hook's, built from recipes so that no literal is in this file.
1291 let jwt = fmt!("eyJ{}.{}.{}", run(8, 301, "b64"), run(12, 302, "b64"), run(6, 303, "b64"));
1292 let cases = [
1293 ("a JWT", jwt),
1294 ("a webhook secret", fmt!("whsec_{}", run(20, 304, "alnum"))),
1295 ("a test-mode Stripe key", fmt!("sk_test_{}", run(14, 305, "alnum"))),
1296 ("a test-mode restricted", fmt!("rk_test_{}", run(14, 306, "alnum"))),
1297 ("a relay key", fmt!("tune-{}", run(32, 307, "hex"))),
1298 ("a temporary AWS id", fmt!("ASIA{}", run(14, 308, "upper36"))),
1299 ("a role AWS id", fmt!("AROA{}", run(14, 309, "upper36"))),
1300 ("a short GitHub run", fmt!("ghp_{}", run(16, 310, "alnum"))),
1301 ("a new Slack token", fmt!("xoxe-{}", run(14, 311, "alnum"))),
1302 ("a short Google key", fmt!("AIza{}", run(32, 312, "b64"))),
1303 ("an `sk-` key of 16", fmt!("sk-{}", run(16, 313, "b64"))),
1304 ("a bearer header", fmt!("Authorization: Bearer {}", run(20, 314, "b64"))),
1305 ("a URL's token", fmt!("https://h.example/cb?token={}&z=1", run(12, 315, "alnum"))),
1306 ("a named secret", fmt!("config password: {}", run(12, 316, "alnum"))),
1307 ("an unnamed blob", fmt!("blob {} end", run(40, 317, "b64"))),
1308 ];
1309 for (what, text) in &cases {
1310 assert!(secret::scan(text.as_bytes()).is_empty(), "the hook would now refuse {}: {:?}", what, text);
1311 assert!(!secret::holds(text), "holds, which is the hook's shapes, now sees {}", what);
1312 assert!(Shapes.hit(text), "Shapes misses {}: {:?}", what, text);
1313 assert_ne!(Shapes.scrub(text, false), *text, "scrub misses {}", what);
1314 let out = Redact::new().with_head(0).with_test(Shapes).dat(&omap(vec![("msg", st(text))]));
1315 assert!(text_at(&out, "msg").starts_with("[redacted"), "{}: {:?}", what, text_at(&out, "msg"));
1316 }
1317 Ok(())
1318}
1319
1320#[test]
1321fn a_placeholder_under_a_secret_name_is_scrubbed_as_daimond_scrubs_it() -> Outcome<()> {
1322 // The hook excuses a placeholder, being noisy over source; Daimond's scrubber does not, being
1323 // over a feed, and `Shapes` is no weaker than it.
1324 let t = "password: \"your-key-here\"";
1325 assert!(secret::scan(t.as_bytes()).is_empty() && !secret::holds(t));
1326 assert!(Shapes.hit(t));
1327 assert!(Shapes.scrub(t, false).starts_with("password: \"[redacted named #"));
1328 Ok(())
1329}
1330
1331#[test]
1332fn a_correlation_id_is_excused_from_the_entropy_catch_and_nothing_else() -> Outcome<()> {
1333 let blob = run(40, 401, "b64");
1334 let key = fmt!("sk-{}", run(20, 402, "b64"));
1335 let doc = omap(vec![
1336 ("callId", st(&blob)),
1337 ("note", st(&blob)),
1338 ("id", st(&key)),
1339 ("sid", st(&blob)),
1340 ("inner", omap(vec![("turn", st(&blob)), ("x", st(&blob))])),
1341 ]);
1342 let red = Redact::new().with_test(Shapes);
1343 let out = red.dat(&doc);
1344 assert_eq!(text_at(&out, "callId"), blob, "Daimond's own id fields keep their values");
1345 assert!(text_at(&out, "note").starts_with("[redacted"), "any other field is held to the catch");
1346 assert!(text_at(&out, "id").starts_with("[redacted"), "an id holding a key is still a key");
1347 assert!(text_at(&out, "sid").starts_with("[redacted"), "sid is the application's to name");
1348 assert_eq!(text_at(&out, "inner.turn"), blob, "at any depth");
1349 assert!(text_at(&out, "inner.x").starts_with("[redacted"), "an id does not excuse its neighbour");
1350 // The application names its own, and a list under an id field is ids as well.
1351 let red = Redact::new().with_test(Shapes).id_key("sid");
1352 let out = red.dat(&omap(vec![
1353 ("sid", st(&blob)),
1354 ("turn", Dat::List(vec![st(&blob), st(&blob)])),
1355 ("other", Dat::List(vec![st(&blob)])),
1356 ]));
1357 assert_eq!(text_at(&out, "sid"), blob);
1358 let j = res!(compact(&out));
1359 assert_eq!(j.matches(&blob).count(), 3, "two list members under turn, and sid: {}", j);
1360 assert!(j.contains("[redacted"), "the list under another name is still covered: {}", j);
1361 // A row meets the same rule through its payload's field names.
1362 let row = Row::new("peer", 1, "b", T0, "console").with("callId", st(&blob)).with("msg", st(&blob));
1363 let out = red.row(&row);
1364 let get = |n: &str| out.body.iter().find(|(k, _)| k == n).map(|(_, v)| text_of(v)).unwrap_or_default();
1365 assert_eq!(get("callId"), blob);
1366 assert!(get("msg").starts_with("[redacted"), "{:?}", get("msg"));
1367 // And a caller's own test is not excused for an id: the default is the same answer.
1368 let own = Redact::new().with_test(|s: &str| s.contains("zzz"));
1369 let out = own.dat(&omap(vec![("callId", st("a zzz b"))]));
1370 assert!(text_at(&out, "callId").starts_with("[redacted"));
1371 Ok(())
1372}
1373
1374
1375// The tap on a websocket's messages ======================================================
1376
1377#[cfg(feature = "async")]
1378mod tap {
1379 use super::*;
1380
1381 use oxedyne_fe2o3_iop_db::api::NoDatabase;
1382 use oxedyne_fe2o3_net::{
1383 lens::{
1384 Dir,
1385 Frame,
1386 Tap,
1387 },
1388 ws::{
1389 encode_message,
1390 handler::WebSocketEchoHandler,
1391 read_message,
1392 tap::{
1393 encode_message_tapped,
1394 read_message_tapped,
1395 FrameTap,
1396 NoTap,
1397 },
1398 WebSocket,
1399 WebSocketLimits,
1400 WebSocketMessage,
1401 },
1402 };
1403
1404 use std::sync::Arc;
1405
1406 use tokio::io::{
1407 duplex,
1408 AsyncWriteExt,
1409 };
1410
1411 const CHUNK: usize = 1_024;
1412
1413 /// What a connection knows about itself, which is what a filter gets to see.
1414 struct Conn {
1415 sid: String,
1416 dev: String,
1417 }
1418
1419 fn conn(sid: &str) -> Conn {
1420 Conn { sid: sid.to_string(), dev: "peer".to_string() }
1421 }
1422
1423 /// The safe filter: every frame, reduced to its kind and its size.
1424 fn shape_only(_dir: Dir, c: &Conn, msg: &WebSocketMessage) -> Option<Frame> {
1425 Some(Frame::new(&[&c.sid], &c.dev).shape(msg))
1426 }
1427
1428 fn num_at(d: &Dat, path: &str) -> Option<u64> {
1429 match at(d, path)? {
1430 Dat::U8(n) => Some(*n as u64),
1431 Dat::U16(n) => Some(*n as u64),
1432 Dat::U32(n) => Some(*n as u64),
1433 Dat::U64(n) => Some(*n),
1434 Dat::I8(n) => u64::try_from(*n).ok(),
1435 Dat::I16(n) => u64::try_from(*n).ok(),
1436 Dat::I32(n) => u64::try_from(*n).ok(),
1437 Dat::I64(n) => u64::try_from(*n).ok(),
1438 _ => None,
1439 }
1440 }
1441
1442 /// The events a key holds, each the JSON a row carried in its `data`.
1443 fn events<T: StrTest>(sink: &Sink<T>, key: &[&str]) -> Outcome<Vec<Dat>> {
1444 let mut out = Vec::new();
1445 for l in res!(lines(&sink.path(key))) {
1446 out.push(res!(Dat::decode_json_strict(&text_at(&l, "data"), &DecodeLimits::default())));
1447 }
1448 Ok(out)
1449 }
1450
1451 fn text(s: &str) -> WebSocketMessage { WebSocketMessage::Text(s.to_string()) }
1452
1453 /// Frames a message as a client would, masked.
1454 fn from_client(m: &WebSocketMessage) -> Outcome<Vec<u8>> {
1455 encode_message(m, true, CHUNK, CHUNK)
1456 }
1457
1458 /// One tapped read.
1459 async fn read_one<X: FrameTap>(bytes: &[u8], tap: &X, ctx: &X::Ctx) -> Outcome<Option<WebSocketMessage>> {
1460 let mut stream = bytes;
1461 let mut buf = Vec::new();
1462 read_message_tapped(&mut stream, &mut buf, CHUNK, WebSocketLimits::default(), tap, ctx).await
1463 }
1464
1465 #[tokio::test]
1466 async fn frames_in_both_directions_reach_the_sink_as_kind_and_size_only() -> Outcome<()> {
1467 let s = Scratch::new("tap-both");
1468 let sink = Arc::new(Sink::new(s.dir(), Gates::default()));
1469 let tap = Tap::new(sink.clone(), "b9", shape_only);
1470 let c = conn("sess-1");
1471
1472 // In: a text frame, then a binary one, then a ping.
1473 let secret_text = "do-not-keep-this-chat-line";
1474 let got = res!(read_one(&res!(from_client(&text(secret_text))), &tap, &c).await);
1475 assert!(matches!(got, Some(WebSocketMessage::Text(ref t)) if t == secret_text));
1476 let got = res!(read_one(&res!(from_client(&WebSocketMessage::Binary(vec![7u8; 300]))), &tap, &c).await);
1477 assert!(matches!(got, Some(WebSocketMessage::Binary(ref b)) if b.len() == 300));
1478 res!(read_one(&res!(from_client(&WebSocketMessage::Ping(vec![1, 2, 3]))), &tap, &c).await);
1479
1480 // Out: a text reply and a close, framed unmasked as a server does.
1481 let out = res!(encode_message_tapped(&text("pong!"), false, CHUNK, CHUNK, &tap, &c));
1482 assert_eq!(out, res!(encode_message(&text("pong!"), false, CHUNK, CHUNK)));
1483 res!(encode_message_tapped(&WebSocketMessage::Close(None, Some("bye".to_string())), false, CHUNK, CHUNK, &tap, &c));
1484
1485 let ev = res!(events(&sink, &["sess-1"]));
1486 assert_eq!(ev.len(), 5, "{:?}", ev);
1487 let want = [
1488 ("frame.in", "text", secret_text.len() as u64),
1489 ("frame.in", "binary", 300),
1490 ("frame.in", "ping", 3),
1491 ("frame.out", "text", 5),
1492 ("frame.out", "close", 3),
1493 ];
1494 for (i, (kind, op, bytes)) in want.iter().enumerate() {
1495 let e = &ev[i];
1496 assert_eq!(num_at(e, "n"), Some(i as u64 + 1), "n counts from 1, as Daimond's does: {:?}", e);
1497 assert_eq!(text_at(e, "d"), "peer");
1498 assert_eq!(text_at(e, "b"), "b9");
1499 assert_eq!(text_at(e, "op"), *op, "{:?}", e);
1500 assert_eq!(num_at(e, "bytes"), Some(*bytes), "{:?}", e);
1501 // The tag carries the direction.
1502 let tag = text_at(&res!(lines(&sink.path(&["sess-1"])))[i], "tag");
1503 assert_eq!(tag, fmt!("ev {}", kind));
1504 }
1505 // The default arm never carried content.
1506 let bytes = res!(all_bytes(&s.dir()));
1507 assert!(!has(&bytes, secret_text), "a frame's text reached the file");
1508 assert_eq!(tap.lost(), 0);
1509 Ok(())
1510 }
1511
1512 #[tokio::test]
1513 async fn the_filters_reduction_is_what_lands_and_a_forgetful_filter_meets_the_redactor() -> Outcome<()> {
1514 let s = Scratch::new("tap-reduce");
1515 let sink = Arc::new(Sink::new(s.dir(), Gates::default())
1516 .with_redact(Redact::new().with_head(0)));
1517 // Oxegen's case: an enrol bundle carries a biometric, and only its shape may be kept.
1518 let filter = |dir: Dir, c: &Conn, msg: &WebSocketMessage| -> Option<Frame> {
1519 let t = match msg {
1520 WebSocketMessage::Text(t) => t,
1521 _ => return Some(Frame::new(&[&c.sid], &c.dev).shape(msg)),
1522 };
1523 let d = match Dat::decode_json_strict(t, &DecodeLimits::default()) {
1524 Ok(d) => d,
1525 Err(_) => return None,
1526 };
1527 let kind = text_at(&d, "kind");
1528 if kind == "EnrolBundle" {
1529 return Some(Frame::new(&[&c.sid], &c.dev)
1530 .with("kind", st(&kind))
1531 .with("bundle", st(&text_at(&d, "bundle")))
1532 .with("item", st(&text_at(&d, "item")))
1533 .with("bytes", Dat::U64(t.len() as u64))
1534 .with("dir", st(if dir == Dir::In { "in" } else { "out" })));
1535 }
1536 if kind == "Forgetful" {
1537 // A filter that keeps a field it should not: the sink's redactor still covers it.
1538 return Some(Frame::new(&[&c.sid], &c.dev).with("token", st(&text_at(&d, "token"))));
1539 }
1540 None
1541 };
1542 let tap = Tap::new(sink.clone(), "b9", filter);
1543 let c = conn("sess-2");
1544 let raw = "RAW-FACE-TEMPLATE-0123456789";
1545 let enrol = fmt!("{{\"kind\":\"EnrolBundle\",\"bundle\":\"b-7\",\"item\":\"face-3\",\"blob\":\"{}\"}}", raw);
1546 res!(read_one(&res!(from_client(&text(&enrol))), &tap, &c).await);
1547 res!(encode_message_tapped(&text(&enrol), false, CHUNK, CHUNK, &tap, &c));
1548 let leaky = "PLANT-token-from-a-filter";
1549 let forgot = fmt!("{{\"kind\":\"Forgetful\",\"token\":\"{}\"}}", leaky);
1550 res!(read_one(&res!(from_client(&text(&forgot))), &tap, &c).await);
1551
1552 let ev = res!(events(&sink, &["sess-2"]));
1553 assert_eq!(ev.len(), 3, "{:?}", ev);
1554 for (i, dir) in ["in", "out"].iter().enumerate() {
1555 assert_eq!(text_at(&ev[i], "kind"), "EnrolBundle");
1556 assert_eq!(text_at(&ev[i], "bundle"), "b-7");
1557 assert_eq!(text_at(&ev[i], "item"), "face-3");
1558 assert_eq!(num_at(&ev[i], "bytes"), Some(enrol.len() as u64));
1559 assert_eq!(text_at(&ev[i], "dir"), *dir);
1560 }
1561 assert!(text_at(&ev[2], "token").starts_with("[redacted"), "{:?}", ev[2]);
1562 let bytes = res!(all_bytes(&s.dir()));
1563 assert!(!has(&bytes, raw), "the raw payload reached a file");
1564 assert!(!has(&bytes, leaky), "a forgetful filter leaked past the sink's redactor");
1565 assert!(has(&bytes, "face-3"), "the reduction is there");
1566 Ok(())
1567 }
1568
1569 #[tokio::test]
1570 async fn none_drops_a_frame_and_leaves_no_gap_in_the_sequence() -> Outcome<()> {
1571 let s = Scratch::new("tap-none");
1572 let sink = Arc::new(Sink::new(s.dir(), Gates::default()));
1573 // Keeps the text frames only.
1574 let tap = Tap::new(sink.clone(), "b", |_d: Dir, c: &Conn, m: &WebSocketMessage| match m {
1575 WebSocketMessage::Text(_) => Some(Frame::new(&[&c.sid], &c.dev).shape(m)),
1576 _ => None,
1577 });
1578 let c = conn("sess-3");
1579 for m in [text("a"), WebSocketMessage::Binary(vec![0]), text("bb"), WebSocketMessage::Ping(vec![]), text("ccc")] {
1580 res!(read_one(&res!(from_client(&m)), &tap, &c).await);
1581 }
1582 let ev = res!(events(&sink, &["sess-3"]));
1583 let ns: Vec<Option<u64>> = ev.iter().map(|e| num_at(e, "n")).collect();
1584 assert_eq!(ns, vec![Some(1), Some(2), Some(3)], "{:?}", ev);
1585 let sizes: Vec<Option<u64>> = ev.iter().map(|e| num_at(e, "bytes")).collect();
1586 assert_eq!(sizes, vec![Some(1), Some(2), Some(3)]);
1587 Ok(())
1588 }
1589
1590 #[tokio::test]
1591 async fn a_sink_that_cannot_write_counts_the_loss_and_the_frame_still_flows() -> Outcome<()> {
1592 let s = Scratch::new("tap-lost");
1593 res!(fs::create_dir_all(&s.0));
1594 // A regular file where the directory must go.
1595 let blocker = s.0.join("blocker");
1596 res!(fs::write(&blocker, b"x"));
1597 let sink = Arc::new(Sink::new(blocker.join("lens"), Gates::default()));
1598 let tap = Tap::new(sink.clone(), "b", shape_only);
1599 let c = conn("sess-4");
1600 for want in ["one", "two", "three"] {
1601 let got = res!(read_one(&res!(from_client(&text(want))), &tap, &c).await);
1602 assert!(matches!(got, Some(WebSocketMessage::Text(ref t)) if t == want), "the frame must flow: {:?}", got);
1603 }
1604 let out = res!(encode_message_tapped(&text("x"), false, CHUNK, CHUNK, &tap, &c));
1605 assert!(!out.is_empty());
1606 assert_eq!(tap.lost(), 4, "each write that failed is counted");
1607 Ok(())
1608 }
1609
1610 type Echo<'a, S, X> = WebSocket<'a, 8, u64, (), (), NoDatabase<8, u64, (), ()>, S, WebSocketEchoHandler, X>;
1611
1612 #[tokio::test]
1613 async fn a_websocket_taps_its_own_read_and_send() -> Outcome<()> {
1614 let s = Scratch::new("tap-ws");
1615 let sink = Arc::new(Sink::new(s.dir(), Gates::default()));
1616 let tap = Tap::new(sink.clone(), "b", shape_only);
1617 let (mut server_end, mut client_end) = duplex(8_192);
1618 let mut ws = Echo::new_server(&mut server_end, WebSocketEchoHandler, CHUNK, CHUNK)
1619 .with_tap(&tap, conn("sess-5"));
1620 res!(client_end.write_all(&res!(from_client(&text("hello")))).await);
1621 let got = res!(ws.read().await);
1622 assert!(matches!(got, Some(WebSocketMessage::Text(ref t)) if t == "hello"));
1623 res!(ws.send(&text("hello back")).await);
1624 // The peer hears the same bytes the tap saw framed.
1625 let mut buf = Vec::new();
1626 let heard = res!(read_message(&mut client_end, &mut buf, CHUNK, WebSocketLimits::default()).await);
1627 assert!(matches!(heard, Some(WebSocketMessage::Text(ref t)) if t == "hello back"));
1628 let ev = res!(events(&sink, &["sess-5"]));
1629 assert_eq!(ev.len(), 2, "{:?}", ev);
1630 assert_eq!((text_at(&ev[0], "op"), num_at(&ev[0], "bytes")), ("text".to_string(), Some(5)));
1631 assert_eq!((text_at(&ev[1], "op"), num_at(&ev[1], "bytes")), ("text".to_string(), Some(10)));
1632 let tags: Vec<String> = res!(lines(&sink.path(&["sess-5"]))).iter().map(|l| text_at(l, "tag")).collect();
1633 assert_eq!(tags, vec!["ev frame.in".to_string(), "ev frame.out".to_string()]);
1634 Ok(())
1635 }
1636
1637 #[tokio::test]
1638 async fn a_websocket_with_no_tap_compiles_unchanged_and_behaves() -> Outcome<()> {
1639 let (mut server_end, mut client_end) = duplex(8_192);
1640 // The eight-argument spelling, as `fe2o3_steel` writes it, still names a complete type.
1641 let mut ws: WebSocket<'_, 8, u64, (), (), NoDatabase<8, u64, (), ()>, _, WebSocketEchoHandler> =
1642 WebSocket::new_server(&mut server_end, WebSocketEchoHandler, CHUNK, CHUNK);
1643 res!(client_end.write_all(&res!(from_client(&text("plain")))).await);
1644 let got = res!(ws.read().await);
1645 assert!(matches!(got, Some(WebSocketMessage::Text(ref t)) if t == "plain"));
1646 res!(ws.send(&text("plain back")).await);
1647 let mut buf = Vec::new();
1648 let heard = res!(read_message(&mut client_end, &mut buf, CHUNK, WebSocketLimits::default()).await);
1649 assert!(matches!(heard, Some(WebSocketMessage::Text(ref t)) if t == "plain back"));
1650 Ok(())
1651 }
1652
1653 // Zero cost with no tap is shown by type, not by counting allocations: a counting global
1654 // allocator needs `unsafe`, which this tree forbids. `NoTap` and its context occupy no bytes,
1655 // its hooks are empty and inlined, and the tapped free functions give what the untapped ones
1656 // give, byte for byte.
1657 #[tokio::test]
1658 async fn no_tap_costs_nothing_and_changes_nothing() -> Outcome<()> {
1659 assert_eq!(std::mem::size_of::<NoTap>(), 0);
1660 assert_eq!(std::mem::size_of::<<NoTap as FrameTap>::Ctx>(), 0);
1661 for m in [text("same"), WebSocketMessage::Binary(vec![9u8; 2_000]), WebSocketMessage::Ping(vec![1])] {
1662 // Unmasked, so the framing is deterministic.
1663 let plain = res!(encode_message(&m, false, 512, 512));
1664 let tapped = res!(encode_message_tapped(&m, false, 512, 512, &NoTap, &()));
1665 assert_eq!(plain, tapped);
1666 let mut a: &[u8] = &plain;
1667 let mut b: &[u8] = &plain;
1668 let (mut ba, mut bb) = (Vec::new(), Vec::new());
1669 let x = res!(read_message(&mut a, &mut ba, 512, WebSocketLimits::default()).await);
1670 let y = res!(read_message_tapped(&mut b, &mut bb, 512, WebSocketLimits::default(), &NoTap, &()).await);
1671 assert_eq!(fmt!("{:?}", x), fmt!("{:?}", y));
1672 assert_eq!(a.len(), b.len(), "both consumed the same bytes");
1673 }
1674 // The same holds for a tap behind a reference or an `Arc`.
1675 let s = Scratch::new("tap-arc");
1676 let sink = Arc::new(Sink::new(s.dir(), Gates::default()));
1677 let tap = Arc::new(Tap::new(sink.clone(), "b", shape_only));
1678 let c = conn("sess-6");
1679 res!(read_one(&res!(from_client(&text("via arc"))), &tap, &c).await);
1680 assert_eq!(res!(events(&sink, &["sess-6"])).len(), 1);
1681 Ok(())
1682 }
1683
1684 #[test]
1685 fn a_direction_names_its_kind() {
1686 assert_eq!(Dir::In.kind(), "frame.in");
1687 assert_eq!(Dir::Out.kind(), "frame.out");
1688 }
1689}
1690
1691/// A device label is a string a client sends like any other: one that holds a credential is covered
1692/// before it is written beside each line, by the post and by a caller's own write.
1693#[test]
1694fn a_device_label_that_holds_a_credential_is_covered() -> Outcome<()> {
1695 let s = Scratch::new("devsecret");
1696 let sink = Sink::new(s.dir(), Gates::default()).with_redact(Redact::new().with_head(0).with_test(Shapes));
1697 // Assembled here and not written down, so the source holds no credential for a hook to find.
1698 let key = fmt!("{}{}{}", "sk_", "live_", "Qm9xZ3RwYWJjZGVmZ2hpamtsbW5v");
1699 assert_eq!(stored(res!(sink.post(&["k"], &body(&key, &[(1, "a", "x")]), T0))), 1);
1700 let rows = vec![Row::new("src", 1, "b", T0, "note")];
1701 res!(sink.write(&["w"], &key, &rows, T0 + 5_000));
1702 for name in ["k", "w"] {
1703 let text = res!(fs::read_to_string(sink.path(&[name])), IO, File, Read);
1704 assert!(!text.contains(&key), "the label is covered in {}: {}", name, text);
1705 assert!(text.contains("[redacted"), "and says it was covered in {}: {}", name, text);
1706 }
1707 // An ordinary label is left alone.
1708 assert_eq!(stored(res!(sink.post(&["plain"], &body("devABC", &[(1, "a", "x")]), T0))), 1);
1709 let text = res!(fs::read_to_string(sink.path(&["plain"])), IO, File, Read);
1710 assert!(text.contains("\"device\":\"devABC\""), "{}", text);
1711 Ok(())
1712}
1713
1714
1715// The word-run test ======================================================================
1716
1717struct PhraseCase {
1718 text: String,
1719 n: usize,
1720 hit: bool,
1721 spans: Vec<String>,
1722}
1723
1724fn json_text(col: &str) -> Outcome<String> {
1725 Ok(text_of(&res!(Dat::decode_json_strict(col, &DecodeLimits::default()))))
1726}
1727
1728/// `tests/data/lens_phrase.tsv`: the list, then each case as input, n, hit and the spans found. The
1729/// JavaScript's `createRunGuard` is held to the same file (`lens/js/test/phrase.test.mjs`).
1730fn phrase_table() -> Outcome<(Vec<String>, Vec<PhraseCase>)> {
1731 let table = res!(fs::read_to_string(
1732 Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/data/lens_phrase.tsv")));
1733 let mut list = Vec::new();
1734 let mut cases = Vec::new();
1735 for line in table.lines() {
1736 if let Some(rest) = line.strip_prefix("#list\t") {
1737 list = rest.split(' ').map(|w| w.to_string()).collect();
1738 continue;
1739 }
1740 let cols: Vec<&str> = line.split('\t').collect();
1741 assert_eq!(cols.len(), 4, "bad case line {:?}", line);
1742 let spans = match res!(Dat::decode_json_strict(cols[3], &DecodeLimits::default())) {
1743 Dat::List(l) => l.iter().map(text_of).collect(),
1744 other => panic!("spans are a list, not {:?}", other),
1745 };
1746 cases.push(PhraseCase {
1747 text: res!(json_text(cols[0])),
1748 n: res!(cols[1].parse::<usize>()),
1749 hit: cols[2] == "1",
1750 spans,
1751 });
1752 }
1753 Ok((list, cases))
1754}
1755
1756#[test]
1757fn a_word_run_agrees_with_the_shared_case_file() -> Outcome<()> {
1758 let (list, cases) = res!(phrase_table());
1759 assert_eq!(cases.len(), 40, "the table has changed size");
1760 assert!(list.iter().any(|w| w.contains('-')), "the list holds hyphenated entries");
1761 let (mut hits, mut misses) = (0, 0);
1762 for c in &cases {
1763 let p = Phrase::new(list.iter().map(|w| w.as_str()), c.n);
1764 let got: Vec<String> = p.spans(&c.text).iter().map(|&(a, b)| c.text[a..b].to_string()).collect();
1765 assert_eq!(got, c.spans, "spans of {:?} at n={}", c.text, c.n);
1766 assert_eq!(p.has(&c.text), c.hit, "has {:?} at n={}", c.text, c.n);
1767 assert_eq!(p.hit(&c.text), c.hit);
1768 assert_eq!(p.hit_id(&c.text), c.hit, "an id is held to the same test");
1769 if c.hit { hits += 1 } else { misses += 1 }
1770 }
1771 assert!(hits >= 20 && misses >= 10, "a table that is nearly one answer proves little: {} hits, {} misses", hits, misses);
1772 Ok(())
1773}
1774
1775// What the peer's test did before this one: a word is a run of letters, and nothing else joins them.
1776fn split_only(list: &[String], n: usize, s: &str) -> bool {
1777 let mut run = 0;
1778 for tok in s.split(|c: char| !c.is_ascii_alphabetic()) {
1779 if tok.is_empty() {
1780 continue;
1781 }
1782 if list.iter().any(|w| w == &tok.to_ascii_lowercase()) {
1783 run += 1;
1784 if run >= n {
1785 return true;
1786 }
1787 } else {
1788 run = 0;
1789 }
1790 }
1791 false
1792}
1793
1794#[test]
1795fn splitting_on_every_non_letter_misses_a_passphrase_with_a_hyphenated_word() -> Outcome<()> {
1796 // The control for the case file: the split-only test agrees with it everywhere except where a
1797 // hyphenated list entry is spelled, which is the hole the entry rule closes.
1798 let (list, cases) = res!(phrase_table());
1799 let mut missed = Vec::new();
1800 for c in &cases {
1801 if split_only(&list, c.n, &c.text) != c.hit {
1802 missed.push(c.text.clone());
1803 }
1804 }
1805 assert!(missed.len() >= 5, "the hyphenated entries are in the table: {:?}", missed);
1806 for m in &missed {
1807 assert!(["drop-down", "t-shirt", "yo-yo", "felt-tip", "Drop-Down", "T-SHIRT"].iter().any(|e| m.contains(e)),
1808 "the only disagreement is a hyphenated entry: {:?}", m);
1809 }
1810 Ok(())
1811}
1812
1813#[test]
1814fn one_test_can_stand_for_two() {
1815 let w = Phrase::new(["alpha", "beta"], 2);
1816 let either = Or(Shapes, w);
1817 assert!(either.hit("alpha beta"));
1818 assert!(either.hit(&fmt!("{}{}", "sk-ant", "-api03-AbCdEfGhIjKlMnOpQrStUvWx")));
1819 assert!(!either.hit("alpha gamma beta"));
1820 assert!(either.hit_id(&fmt!("{}{}", "sk-ant", "-api03-AbCdEfGhIjKlMnOpQrStUvWx")), "an id is still held to a shape");
1821}
1822
1823
1824// Chunk rows =============================================================================
1825
1826// The eight words that the tests below plant, and the stock string test with them added.
1827fn chunk_test() -> Or<Shapes, Phrase> {
1828 Or(Shapes, Phrase::new(PHRASE.split(' '), 8))
1829}
1830
1831fn chunk_sink(s: &Scratch) -> Sink<Or<Shapes, Phrase>> {
1832 Sink::new(s.dir(), Gates { min_interval_ms: 0, ..Gates::default() })
1833 .with_redact(Redact::new().with_head(0).with_test(chunk_test()))
1834}
1835
1836// A snapshot as the page makes it: scrubbed JSON, plain prose in its free fields. The filler is
1837// there to make a few chunks of it.
1838fn snapshot_json(note: &str) -> String {
1839 fmt!(
1840 "{{\"ts\":1790000000000,\"iso\":\"2026-10-03T08:00:00.000Z\",\"sid\":\"k3j9x0a1qz\",\"device\":\"p4m8w2zr\",\
1841\"build\":\"387d393\",\"why\":\"screen\",\"vis\":\"visible\",\"w\":390,\"screen\":\"oxegen_home\",\
1842\"app\":{{\"phase\":\"home\",\"busy\":false,\"lang\":\"en\"}},\"member\":{{\"known\":true,\"fp\":\"[redacted …#1f3a9c/44]\"}},\
1843\"wallet\":{{\"coins\":3,\"epoch\":19}},\"net\":{{\"state\":\"live\",\"wire\":\"ws\",\"may_count\":true}},\
1844\"ls\":{{\"keys\":9}},\"idb\":{{\"dbs\":1}},\"lens\":{{\"depth\":4,\"health\":\"ok\",\"fault\":false}},\
1845\"note\":\"{}\",\"more\":\"the page sat on the home screen while the member read the notice and the net stayed live\"}}",
1846 note)
1847}
1848
1849fn rows_of(kind: &str, id: &str, enc: &str, size: usize, wrap: bool) -> Vec<(i64, String, String)> {
1850 let parts: Vec<String> = enc.as_bytes().chunks(size).map(|c| String::from_utf8_lossy(c).into_owned()).collect();
1851 let n = parts.len();
1852 parts.into_iter().enumerate().map(|(i, p)| {
1853 let data = if wrap {
1854 // Words of sixteen, as MIME wraps base64.
1855 p.as_bytes().chunks(16).map(|w| String::from_utf8_lossy(w).into_owned()).collect::<Vec<String>>().join(" ")
1856 } else {
1857 p
1858 };
1859 (T0 as i64 + i as i64, fmt!("ds {} {} {}/{}", kind, id, i + 1, n), data)
1860 }).collect()
1861}
1862
1863fn post_rows(sink: &Sink<Or<Shapes, Phrase>>, key: &str, rows: &[(i64, String, String)], at: u64) -> Outcome<usize> {
1864 let borrowed: Vec<(i64, &str, &str)> = rows.iter().map(|(t, g, d)| (*t, g.as_str(), d.as_str())).collect();
1865 Ok(stored(res!(sink.post(&[key], &body("dev", &borrowed), at))))
1866}
1867
1868// The (tag, data) of each line a key has stored, in order.
1869fn kept(sink: &Sink<Or<Shapes, Phrase>>, key: &str) -> Outcome<Vec<(String, String)>> {
1870 let mut out = Vec::new();
1871 for l in res!(lines(&sink.path(&[key]))) {
1872 out.push((text_at(&l, "tag"), text_at(&l, "data")));
1873 }
1874 Ok(out)
1875}
1876
1877fn marked(data: &str) -> bool { data.starts_with("[redacted") }
1878
1879#[test]
1880fn a_base64_snapshot_chunk_is_kept_intact() -> Outcome<()> {
1881 let json = snapshot_json("a quiet moment");
1882 assert!(!chunk_test().hit(&json), "the plain text is innocent, which is what the test is about");
1883 let url = base64::encode_url(json.as_bytes());
1884 let std = base64::encode(json.as_bytes());
1885 let s = Scratch::new("chunk-intact");
1886 let sink = chunk_sink(&s);
1887 // The page's form, Daimond's, and the first Oxegen pages' (words of sixteen).
1888 let forms = [
1889 ("url", rows_of("snapshot", "s1-1", &url, 288, false)),
1890 ("std", rows_of("telemetry", "t1", &std, 360, false)),
1891 ("words", rows_of("snapshot", "s1-2", &url, 288, true)),
1892 ];
1893 for (k, (key, rows)) in forms.iter().enumerate() {
1894 assert!(rows.len() >= 3, "{}: a few chunks", key);
1895 assert_eq!(res!(post_rows(&sink, key, rows, T0 + 10_000 * k as u64)), rows.len());
1896 let got = res!(kept(&sink, key));
1897 assert_eq!(got.len(), rows.len());
1898 for ((tag, data), (_, want_tag, want)) in got.iter().zip(rows.iter()) {
1899 assert_eq!(tag, want_tag);
1900 assert_eq!(data, want, "{}: a chunk of a clean bundle is stored as it came", key);
1901 }
1902 // And what was kept reads back to the bundle.
1903 let whole: String = got.iter().map(|(_, d)| d.replace(' ', "")).collect();
1904 let back = if *key == "std" { res!(base64::decode(&whole)) } else { res!(base64::decode_url(&whole)) };
1905 assert_eq!(String::from_utf8_lossy(&back), json, "{}", key);
1906 }
1907 Ok(())
1908}
1909
1910#[test]
1911fn a_chunk_whose_decoded_text_holds_eight_words_is_covered() -> Outcome<()> {
1912 let json = snapshot_json(&fmt!("the member typed {} by mistake", PHRASE));
1913 let url = base64::encode_url(json.as_bytes());
1914 let s = Scratch::new("chunk-phrase");
1915 let sink = chunk_sink(&s);
1916 let rows = rows_of("snapshot", "s2-1", &url, 288, false);
1917 assert!(rows.len() >= 3);
1918 assert_eq!(res!(post_rows(&sink, "k", &rows, T0)), rows.len());
1919 let got = res!(kept(&sink, "k"));
1920 assert_eq!(got.len(), rows.len());
1921 // The bundle is one: a hit anywhere covers every chunk of it, so a reader finds it broken.
1922 for (i, (tag, data)) in got.iter().enumerate() {
1923 assert_eq!(tag, &rows[i].1);
1924 assert!(marked(data), "chunk {} is covered: {}", i + 1, data);
1925 assert!(!data.contains(&rows[i].2[..16]), "and holds none of the chunk");
1926 }
1927 let bytes = res!(all_bytes(&s.dir()));
1928 assert!(!has(&bytes, "abandon") && !has(&bytes, &url[..40]));
1929 Ok(())
1930}
1931
1932#[test]
1933fn a_chunk_whose_decoded_text_holds_a_key_is_covered() -> Outcome<()> {
1934 let key = cred(1);
1935 let json = snapshot_json(&fmt!("config says {}", key));
1936 let s = Scratch::new("chunk-key");
1937 let sink = chunk_sink(&s);
1938 for (k, (_, (kind, enc, size))) in [
1939 ("url", ("snapshot", base64::encode_url(json.as_bytes()), 288)),
1940 ("std", ("telemetry", base64::encode(json.as_bytes()), 360)),
1941 ].into_iter().enumerate() {
1942 let rows = rows_of(kind, "s3", &enc, size, false);
1943 assert_eq!(res!(post_rows(&sink, &fmt!("k{}", k), &rows, T0 + 10_000 * k as u64)), rows.len());
1944 for (tag, data) in res!(kept(&sink, &fmt!("k{}", k))) {
1945 assert!(marked(&data), "{}: {}", tag, data);
1946 }
1947 }
1948 let bytes = res!(all_bytes(&s.dir()));
1949 assert!(!has(&bytes, &key) && !has(&bytes, &key[..12]));
1950 Ok(())
1951}
1952
1953#[test]
1954fn a_secret_that_straddles_a_cut_is_seen_and_every_chunk_is_covered() -> Outcome<()> {
1955 // The eight words stand across the page's cut at 288 characters (216 bytes), five in the first
1956 // chunk and three in the second.
1957 let json = fmt!("{{\"p\":\"{}typed {} in the box\",\"rest\":{}}}",
1958 "plain words ".repeat(15), PHRASE, snapshot_json("x"));
1959 let a = json.find(PHRASE).unwrap_or(0);
1960 assert!(a < 216 && 216 < a + PHRASE.len(), "the cut falls inside the words: {}", a);
1961 let url = base64::encode_url(json.as_bytes());
1962 let rows = rows_of("snapshot", "s4", &url, 288, false);
1963 assert!(rows.len() >= 2);
1964 // Alone, no chunk holds the eight words: the cut has halved them.
1965 let red = Redact::new().with_head(0).with_test(chunk_test());
1966 for (_, tag, data) in &rows {
1967 let one = vec![(tag.clone(), data.clone())];
1968 assert_eq!(judge(&red, 400, &one), vec![Verdict::Clean], "{} alone is innocent", tag);
1969 }
1970 let s = Scratch::new("chunk-straddle");
1971 let sink = chunk_sink(&s);
1972 assert_eq!(res!(post_rows(&sink, "fwd", &rows, T0)), rows.len());
1973 for (tag, data) in res!(kept(&sink, "fwd")) {
1974 assert!(marked(&data), "{}: {}", tag, data);
1975 }
1976 // The order of the rows in the post makes no difference.
1977 let mut rev = rows.clone();
1978 rev.reverse();
1979 assert_eq!(res!(post_rows(&sink, "rev", &rev, T0 + 10_000)), rev.len());
1980 for (tag, data) in res!(kept(&sink, "rev")) {
1981 assert!(marked(&data), "{}: {}", tag, data);
1982 }
1983 Ok(())
1984}
1985
1986#[test]
1987fn a_payload_that_is_not_the_base64_of_text_is_no_chunk_at_all() -> Outcome<()> {
1988 let s = Scratch::new("chunk-fake");
1989 let sink = chunk_sink(&s);
1990 // Bytes that are not text: an image, say, sent under a snapshot's tag.
1991 let bin: Vec<u8> = (0..300u32).map(|i| (i.wrapping_mul(2_654_435_761) >> 13) as u8 | 0x80).collect();
1992 let image = base64::encode_url(&bin);
1993 assert!(chunk::text(&image).is_none(), "the bytes are not text");
1994 let key = cred(1);
1995 let rows: Vec<(i64, String, String)> = vec![
1996 (1, "ds snapshot f1 1/4".to_string(), PHRASE.to_string()),
1997 (2, "ds snapshot f1 2/4".to_string(), key.clone()),
1998 (3, "ds snapshot f1 3/4".to_string(), image[..288].to_string()),
1999 (4, "ds snapshot f1 4/4".to_string(), fmt!("{}", PHRASE.to_uppercase())),
2000 ];
2001 assert_eq!(res!(post_rows(&sink, "k", &rows, T0)), 4);
2002 for (tag, data) in res!(kept(&sink, "k")) {
2003 assert!(marked(&data), "{} is guarded as any row is: {}", tag, data);
2004 }
2005 let bytes = res!(all_bytes(&s.dir()));
2006 assert!(!has(&bytes, "abandon") && !has(&bytes, "ABANDON") && !has(&bytes, &key) && !has(&bytes, &image[..40]));
2007 Ok(())
2008}
2009
2010#[test]
2011fn only_an_exact_chunk_tag_is_a_chunk() -> Outcome<()> {
2012 let ok = ChunkTag::parse("ds snapshot s1-7 2/5");
2013 assert_eq!(ok, Some(ChunkTag { kind: "snapshot".to_string(), id: "s1-7".to_string(), i: 2, n: 5 }));
2014 assert_eq!(ChunkTag::parse("ds telemetry t9 1/1").map(|t| (t.kind, t.i, t.n)), Some(("telemetry".to_string(), 1, 1)));
2015 let bad = [
2016 "ds snapshot s 0/3", "ds snapshot s 4/3", "ds snapshots s 1/3", "ds snapshot s 1/3 extra", "DS snapshot s 1/3",
2017 "ds snapshot s 1/3", "ds snapshot s 1/3", "ds snapshot s 1/", "ds snapshot s /3", "ds snapshot s a/3",
2018 "ds snapshot s +1/3", "ds snapshot s 1/3 ", " ds snapshot s 1/3", "ds snapshot s", "ds snapshot 1/3", "ds s 1/3",
2019 "ev snapshot s 1/3", "ds snapshot s 1/3/4", "ds snapshot s 1/99999999999", "", "ds",
2020 ];
2021 for t in bad {
2022 assert_eq!(ChunkTag::parse(t), None, "{:?} is not a chunk's tag", t);
2023 }
2024 // Their payloads get no exemption: the same real base64 is covered under each of them.
2025 let enc = base64::encode_url(snapshot_json("x").as_bytes());
2026 let s = Scratch::new("chunk-tags");
2027 let sink = chunk_sink(&s);
2028 let rows: Vec<(i64, String, String)> = bad.iter().enumerate()
2029 .filter(|(_, t)| !t.is_empty())
2030 .map(|(i, t)| (i as i64 + 1, t.to_string(), enc[..288].to_string()))
2031 .collect();
2032 assert_eq!(res!(post_rows(&sink, "k", &rows, T0)), rows.len());
2033 for (tag, data) in res!(kept(&sink, "k")) {
2034 assert!(marked(&data), "{:?} is no chunk, and is guarded as a row: {}", tag, data);
2035 }
2036 // While the exact tag is let through.
2037 let good = vec![(1i64, "ds snapshot s 2/5".to_string(), enc[..288].to_string())];
2038 assert_eq!(res!(post_rows(&sink, "good", &good, T0 + 10_000)), 1);
2039 assert_eq!(res!(kept(&sink, "good")), vec![("ds snapshot s 2/5".to_string(), enc[..288].to_string())]);
2040 Ok(())
2041}
2042
2043#[test]
2044fn a_row_that_is_not_a_chunk_is_guarded_exactly_as_before() -> Outcome<()> {
2045 let enc = base64::encode_url(snapshot_json("x").as_bytes());
2046 let s = Scratch::new("chunk-plain");
2047 let sink = chunk_sink(&s);
2048 let rows = vec![
2049 (1i64, "diag".to_string(), enc[..288].to_string()),
2050 (2, "ds-snapshot s 1/3".to_string(), enc[..288].to_string()),
2051 (3, "diag".to_string(), "an ordinary line of log text".to_string()),
2052 ];
2053 assert_eq!(res!(post_rows(&sink, "k", &rows, T0)), 3);
2054 let got = res!(kept(&sink, "k"));
2055 assert!(marked(&got[0].1) && marked(&got[1].1), "long random-looking runs are covered: {:?}", got);
2056 assert_eq!(got[2].1, "an ordinary line of log text");
2057 Ok(())
2058}
2059
2060#[test]
2061fn a_cut_off_a_quantum_still_reads_as_a_run_and_fails_closed_alone() -> Outcome<()> {
2062 let json = snapshot_json("a quiet moment");
2063 let url = base64::encode_url(json.as_bytes());
2064 let rows = rows_of("snapshot", "s5", &url, 290, false);
2065 assert!(rows.len() >= 3);
2066 // 290 characters is not a whole number of quanta, so no chunk but the first decodes alone.
2067 assert!(chunk::text(&rows[1].2).is_none(), "the second chunk starts inside a quantum");
2068 let s = Scratch::new("chunk-cut");
2069 let sink = chunk_sink(&s);
2070 // Together they are the bundle, and the bundle is innocent.
2071 assert_eq!(res!(post_rows(&sink, "run", &rows, T0)), rows.len());
2072 for (i, (tag, data)) in res!(kept(&sink, "run")).iter().enumerate() {
2073 assert_eq!(data, &rows[i].2, "{}", tag);
2074 }
2075 // One of them posted alone is not provably text, so it meets the whole guard and is covered.
2076 assert_eq!(res!(post_rows(&sink, "alone", &rows[1..2], T0 + 10_000)), 1);
2077 assert!(marked(&res!(kept(&sink, "alone"))[0].1));
2078 Ok(())
2079}
2080
2081#[test]
2082fn the_text_of_a_payload_allows_a_partial_character_at_each_end_and_nothing_else() {
2083 let s = "h\u{e9}llo w\u{f6}rld \u{2603} snowman and a longer tail to be cut at the end \u{2603}";
2084 let b = s.as_bytes();
2085 // Starting inside the first \u{e9}: one stray continuation byte.
2086 let lead = &b[2..];
2087 assert!(std::str::from_utf8(lead).is_err());
2088 assert!(chunk::text(&base64::encode_url(lead)).is_some(), "a cut inside a character at the start");
2089 // Ending inside the last \u{2603}: an unfinished character.
2090 let tail = &b[..b.len() - 1];
2091 assert!(std::str::from_utf8(tail).is_err());
2092 assert!(chunk::text(&base64::encode_url(tail)).is_some(), "a cut inside a character at the end");
2093 assert!(chunk::text(&base64::encode_url(b)).is_some());
2094 // Bytes that are not text anywhere else are not text.
2095 let mut bad = b.to_vec();
2096 bad[10] = 0xff;
2097 assert!(chunk::text(&base64::encode_url(&bad)).is_none());
2098 // Padded, unpadded, white space; the two alphabets mixed are neither.
2099 assert_eq!(chunk::text(&base64::encode(b"foobar!")).as_deref(), Some("foobar!"));
2100 assert_eq!(chunk::text(&base64::encode_url(b"foobar!")).as_deref(), Some("foobar!"));
2101 assert_eq!(chunk::text("Zm9v YmFy\nIQ").as_deref(), Some("foobar!"));
2102 assert_eq!(chunk::text(""), Some(String::new()));
2103 assert!(chunk::text("ab+/cd-_").is_none(), "both alphabets at once");
2104 assert!(chunk::text("Zm9vYmFy!").is_none(), "a stray character");
2105 assert!(chunk::text("Zm9vY").is_none(), "a lone character over a quantum");
2106 // Plain words with their spaces taken out decode to bytes that are not text.
2107 assert!(chunk::text(PHRASE).is_none());
2108}
2109
2110
2111// A bundle that parses as JSON meets the whole redactor (F5, 2026-10-04) ================
2112
2113// A snapshot with one more member in it, which is raw JSON.
2114fn snapshot_with(member: &str) -> String {
2115 let base = snapshot_json("x");
2116 fmt!("{},{}}}", &base[..base.len() - 1], member)
2117}
2118
2119// The eight words of PHRASE spread over a structure's strings in four shapes. The raw text of the
2120// first two is one run, and the letter-only keys and the keys `t` of the last two break it.
2121fn spread(shape: usize) -> String {
2122 let w: Vec<&str> = PHRASE.split(' ').collect();
2123 let join = |v: Vec<String>| v.join(",");
2124 match shape {
2125 0 => fmt!("[{}]", join(w.iter().map(|x| fmt!("\"{}\"", x)).collect())),
2126 1 => fmt!("{{{}}}", join(w.iter().enumerate().map(|(i, x)| fmt!("\"{}\":\"{}\"", i, x)).collect())),
2127 2 => fmt!("{{{}}}", join(w.iter().enumerate().map(|(i, x)| fmt!("\"{}\":\"{}\"", (b'a' + i as u8) as char, x)).collect())),
2128 _ => fmt!("[{}]", join(w.iter().map(|x| fmt!("{{\"t\":\"{}\"}}", x)).collect())),
2129 }
2130}
2131
2132// What a key has stored as a reader reads it: the data of each row joined, decoded as it was sent.
2133fn reread(got: &[(String, String)], std: bool) -> Outcome<String> {
2134 let whole: String = got.iter().map(|(_, d)| d.replace(' ', "")).collect();
2135 let bytes = if std { res!(base64::decode(&whole)) } else { res!(base64::decode_url(&whole)) };
2136 Ok(String::from_utf8_lossy(&bytes).into_owned())
2137}
2138
2139fn parsed(text: &str) -> Outcome<Dat> {
2140 Ok(res!(Dat::decode_json_strict(text, &DecodeLimits::default())))
2141}
2142
2143#[test]
2144fn words_spread_over_a_bundles_strings_are_covered_whatever_stands_between_them() -> Outcome<()> {
2145 let names = ["list", "digits", "letters", "objects"];
2146 let s = Scratch::new("chunk-spread");
2147 let sink = chunk_sink(&s);
2148 for (k, name) in names.iter().enumerate() {
2149 let json = snapshot_with(&fmt!("\"spread\":{}", spread(k)));
2150 let rows = rows_of("snapshot", &fmt!("sp-{}", name), &base64::encode_url(json.as_bytes()), 288, false);
2151 assert!(rows.len() >= 3, "{}: a few chunks", name);
2152 assert_eq!(res!(post_rows(&sink, name, &rows, T0 + 10_000 * k as u64)), rows.len());
2153 let got = res!(kept(&sink, name));
2154 assert_eq!(got.len(), rows.len(), "{}: a row for a row", name);
2155 for ((tag, _), (_, want, _)) in got.iter().zip(rows.iter()) {
2156 assert_eq!(tag, want, "{}: the tags stand", name);
2157 }
2158 let whole = got.iter().all(|(_, d)| marked(d));
2159 assert_eq!(whole, k < 2, "{}: the raw text met the run in the first two shapes alone", name);
2160 if whole {
2161 assert!(!has(&res!(all_bytes(&s.dir())), &rows[0].2[..40]), "{}: no chunk is stored as it came", name);
2162 continue;
2163 }
2164 // The raw text never met the run, so the bundle is covered in place, and is still a bundle.
2165 let text = res!(reread(&got, false));
2166 for w in PHRASE.split(' ') {
2167 assert!(!text.contains(w), "{}: the word {:?} is stored", name, w);
2168 }
2169 let back = res!(parsed(&text));
2170 assert_eq!(text_at(&back, "screen"), "oxegen_home", "{}: the rest of it is kept", name);
2171 assert_eq!(text_at(&back, "app.phase"), "home", "{}", name);
2172 let leaves = fmt!("{:?}", at(&back, "spread"));
2173 assert!(leaves.contains("[redacted"), "{}: the strings of the run are fingerprints: {}", name, leaves);
2174 }
2175 Ok(())
2176}
2177
2178#[test]
2179fn seven_words_spread_over_a_bundle_are_a_bundle_as_it_came() -> Outcome<()> {
2180 let seven: Vec<String> = PHRASE.split(' ').take(7).map(|x| fmt!("{{\"t\":\"{}\"}}", x)).collect();
2181 let json = snapshot_with(&fmt!("\"spread\":[{}]", seven.join(",")));
2182 let rows = rows_of("snapshot", "sv", &base64::encode_url(json.as_bytes()), 288, false);
2183 let s = Scratch::new("chunk-seven");
2184 let sink = chunk_sink(&s);
2185 assert_eq!(res!(post_rows(&sink, "k", &rows, T0)), rows.len());
2186 for ((_, data), (_, _, want)) in res!(kept(&sink, "k")).iter().zip(rows.iter()) {
2187 assert_eq!(data, want, "an innocent bundle is stored as it came");
2188 }
2189 Ok(())
2190}
2191
2192#[test]
2193fn a_secret_named_field_inside_a_bundle_is_covered_in_place() -> Outcome<()> {
2194 let json = snapshot_with("\"priv\":\"hunter2\",\"auth\":{\"seed\":\"s33d\",\"user\":\"ann\"}");
2195 assert!(!chunk_test().hit(&json), "the string test alone finds nothing here, which is the point");
2196 let s = Scratch::new("chunk-named");
2197 let sink = chunk_sink(&s);
2198 let forms = [
2199 ("url", "snapshot", base64::encode_url(json.as_bytes()), 288, false, false),
2200 ("std", "telemetry", base64::encode(json.as_bytes()), 360, false, true),
2201 ("words", "snapshot", base64::encode_url(json.as_bytes()), 288, true, false),
2202 ];
2203 for (k, (key, kind, enc, size, wrap, std)) in forms.iter().enumerate() {
2204 let rows = rows_of(kind, &fmt!("nm-{}", key), enc, *size, *wrap);
2205 assert!(rows.len() >= 3, "{}: a few chunks", key);
2206 assert_eq!(res!(post_rows(&sink, key, &rows, T0 + 10_000 * k as u64)), rows.len());
2207 let got = res!(kept(&sink, key));
2208 assert_eq!(got.len(), rows.len(), "{}: a row for a row", key);
2209 for ((tag, data), (_, want, _)) in got.iter().zip(rows.iter()) {
2210 assert_eq!(tag, want, "{}", key);
2211 assert!(data.len() <= Gates::default().max_data, "{}: a row within the gate", key);
2212 }
2213 // It reads back, in the alphabet it was sent in, as the bundle with two values covered.
2214 let back = res!(parsed(&res!(reread(&got, *std))));
2215 assert_eq!(text_at(&back, "priv"), fingerprint("hunter2", 0), "{}", key);
2216 assert_eq!(text_at(&back, "auth.seed"), fingerprint("s33d", 0), "{}", key);
2217 assert_eq!(text_at(&back, "auth.user"), "ann", "{}: only the secrets go", key);
2218 assert_eq!(text_at(&back, "screen"), "oxegen_home", "{}", key);
2219 }
2220 let bytes = res!(all_bytes(&s.dir()));
2221 assert!(!has(&bytes, "hunter2") && !has(&bytes, "s33d"));
2222 Ok(())
2223}
2224
2225#[test]
2226fn a_bundle_too_big_to_hold_its_covering_is_covered_whole() -> Outcome<()> {
2227 // Each short value becomes a longer fingerprint, so the covered text outgrows the rows it came in.
2228 let members: Vec<String> = (0..60).map(|i| fmt!("\"a_key_{}\":\"{}\"", i, i % 10)).collect();
2229 let json = snapshot_with(&members.join(","));
2230 let rows = rows_of("snapshot", "big", &base64::encode_url(json.as_bytes()), 288, false);
2231 let s = Scratch::new("chunk-big");
2232 let sink = chunk_sink(&s);
2233 assert_eq!(res!(post_rows(&sink, "k", &rows, T0)), rows.len());
2234 let got = res!(kept(&sink, "k"));
2235 assert_eq!(got.len(), rows.len());
2236 for (tag, data) in &got {
2237 assert!(marked(data), "{} is covered whole, since the covering does not fit: {}", tag, data);
2238 }
2239 Ok(())
2240}
2241
2242#[test]
2243fn a_run_that_is_not_the_whole_bundle_is_covered_whole_if_it_needs_covering() -> Outcome<()> {
2244 // One chunk of a set that claims two: whatever the other holds, this one cannot be re-cut.
2245 let json = "{\"priv\":\"hunter2\"}";
2246 let s = Scratch::new("chunk-half");
2247 let sink = chunk_sink(&s);
2248 let rows = vec![(1i64, "ds snapshot h1 1/2".to_string(), base64::encode_url(json.as_bytes()))];
2249 assert_eq!(res!(post_rows(&sink, "half", &rows, T0)), 1);
2250 assert!(marked(&res!(kept(&sink, "half"))[0].1));
2251 // The same text as the whole of a set of one is covered in place.
2252 let one = vec![(1i64, "ds snapshot h2 1/1".to_string(), base64::encode_url(json.as_bytes()))];
2253 assert_eq!(res!(post_rows(&sink, "whole", &one, T0 + 10_000)), 1);
2254 let got = res!(kept(&sink, "whole"));
2255 assert!(!marked(&got[0].1));
2256 let back = res!(parsed(&res!(reread(&got, false))));
2257 assert_eq!(text_at(&back, "priv"), fingerprint("hunter2", 0));
2258 Ok(())
2259}
2260
2261#[test]
2262fn a_test_says_which_of_a_structures_strings_it_reads_only_together() {
2263 let text = "red\ngreen\nblue";
2264 let p = Phrase::new(["red", "green", "blue"], 3);
2265 assert_eq!(p.joined_spans(text), vec![(0, text.len())], "a word run is read across the strings");
2266 assert!(Shapes.joined_spans(text).is_empty(), "a shape is read in one string");
2267 assert!((|s: &str| s.contains("red")).joined_spans(text).is_empty(), "so is a closure's test");
2268 assert_eq!(Or(Shapes, p).joined_spans(text), vec![(0, text.len())], "and `Or` asks both");
2269}
2270
2271#[test]
2272fn a_covering_that_shrinks_a_bundle_still_gives_a_row_for_a_row() -> Outcome<()> {
2273 // A long value that no shape finds, under a name that is a secret's, becomes a short fingerprint.
2274 let long = "w ".repeat(400);
2275 let json = snapshot_with(&fmt!("\"priv\":\"{}\"", long));
2276 assert!(!chunk_test().hit(&json));
2277 let rows = rows_of("snapshot", "sh", &base64::encode_url(json.as_bytes()), 288, false);
2278 let s = Scratch::new("chunk-shrink");
2279 let sink = chunk_sink(&s);
2280 assert_eq!(res!(post_rows(&sink, "k", &rows, T0)), rows.len());
2281 let got = res!(kept(&sink, "k"));
2282 assert_eq!(got.len(), rows.len(), "a row for a row, whatever it holds");
2283 for ((tag, _), (_, want, _)) in got.iter().zip(rows.iter()) {
2284 assert_eq!(tag, want);
2285 }
2286 let (before, after): (usize, usize) = (rows.iter().map(|r| r.2.len()).sum(), got.iter().map(|r| r.1.len()).sum());
2287 assert!(after + 400 < before, "the bundle is shorter for the covering: {} then {}", before, after);
2288 let back = res!(parsed(&res!(reread(&got, false))));
2289 assert_eq!(text_at(&back, "priv"), fingerprint(&long, 0));
2290 assert_eq!(text_at(&back, "screen"), "oxegen_home");
2291 Ok(())
2292}
2293
2294#[test]
2295fn a_bundle_the_strict_reader_refuses_is_stored_covered_whole_and_never_as_sent() -> Outcome<()> {
2296 let deep = fmt!("\"passphrase\":\"hunter2\",\"d\":{}{}", "[".repeat(65), "]".repeat(65));
2297 let cases = [
2298 ("twice", snapshot_with("\"passphrase\":\"hunter2\",\"passphrase\":\"x\"")),
2299 ("surrogate", snapshot_with("\"passphrase\":\"hunter2\",\"t\":\"\\ud83d\"")),
2300 ("deep", snapshot_with(&deep)),
2301 ];
2302 let s = Scratch::new("chunk-refused");
2303 let sink = chunk_sink(&s);
2304 for (k, (name, json)) in cases.iter().enumerate() {
2305 assert!(Dat::decode_json_strict(json, &DecodeLimits::default()).is_err(), "{}: the strict reader refuses it", name);
2306 let rows = rows_of("snapshot", &fmt!("rf-{}", name), &base64::encode_url(json.as_bytes()), 288, false);
2307 assert!(rows.len() >= 3, "{}: a few chunks", name);
2308 assert_eq!(res!(post_rows(&sink, name, &rows, T0 + 10_000 * k as u64)), rows.len());
2309 let got = res!(kept(&sink, name));
2310 assert_eq!(got.len(), rows.len(), "{}: a row for a row", name);
2311 for (tag, data) in &got {
2312 assert!(marked(data), "{}: {} is covered whole: {}", name, tag, data);
2313 }
2314 assert!(!has(&res!(all_bytes(&s.dir())), &rows[0].2[..40]), "{}: no chunk is stored as it came", name);
2315 }
2316 Ok(())
2317}
2318
2319#[test]
2320fn words_that_are_names_or_that_stand_round_a_secret_field_are_covered_in_place() -> Outcome<()> {
2321 let w: Vec<&str> = PHRASE.split(' ').collect();
2322 let names = fmt!("{{{}}}", w.iter().map(|x| fmt!("\"{}\":true", x)).collect::<Vec<String>>().join(","));
2323 // The eight words as the values of eight members, the fourth of them named for a secret: the walk
2324 // covers that one, which used to leave two runs that never reached eight.
2325 let round = fmt!("{{{}}}", w.iter().enumerate()
2326 .map(|(i, x)| fmt!("\"{}\":\"{}\"", if i == 3 { "seed".to_string() } else { ((b'a' + i as u8) as char).to_string() }, x))
2327 .collect::<Vec<String>>().join(","));
2328 let s = Scratch::new("chunk-names");
2329 let sink = chunk_sink(&s);
2330 for (k, (name, member)) in [("names", names), ("round", round)].iter().enumerate() {
2331 let json = snapshot_with(&fmt!("\"spread\":{}", member));
2332 assert!(!chunk_test().hit(&json), "{}: the raw text never meets the run", name);
2333 let rows = rows_of("snapshot", &fmt!("nr-{}", name), &base64::encode_url(json.as_bytes()), 288, false);
2334 assert_eq!(res!(post_rows(&sink, name, &rows, T0 + 10_000 * k as u64)), rows.len());
2335 let got = res!(kept(&sink, name));
2336 assert_eq!(got.len(), rows.len(), "{}: a row for a row", name);
2337 let text = res!(reread(&got, false));
2338 for x in &w {
2339 assert!(!text.contains(&fmt!("\"{}\"", x)), "{}: the word {:?} is stored: {}", name, x, text);
2340 }
2341 let back = res!(parsed(&text));
2342 assert_eq!(text_at(&back, "screen"), "oxegen_home", "{}: the rest of it is kept", name);
2343 }
2344 Ok(())
2345}