Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_net/src/lens/sink.rs

25.8 KiB, 5 runs

created by r1870400018:65646, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! An append-only NDJSON writer with the gates of Daimond's `/api/debug-trace`.
2//!
3//! The sink takes what `gateway/src/handlers/debug_trace.rs` took, a body `{v, device, rows:
4//! [{ts, tag, data}]}` under a key the caller chose, and holds it to the same caps: a body over
5//! `max_body` is refused, more than `max_rows` rows is refused and not truncated, a tag and a
6//! data field are clipped, and one key may post at most once in `min_interval_ms`. It differs
7//! from that handler in four ways, each on purpose.
8//!
9//! * It writes one JSON object a line instead of a text block, which a reader parses with
10//! nothing but `JSON.parse` (see `Entry::line`).
11//! * It is keyed by what the caller passes and never by the body. Daimond names a file for the
12//! session's account and the body's device; here a key is any list of segments, each reduced
13//! to a safe alphabet, and the body's `device` is only a label on each line.
14//! * It redacts. Every row passes `Redact` before it is written, so a caller that forgot to
15//! covers nothing: a secret-named field, a deny-listed place and a string the caller's test
16//! fires on never reach the disk. An `ev` row whose data is not JSON cannot be examined by
17//! name, and is replaced by its fingerprint.
18//! * Its history is bounded by total bytes as well as by age and count, and a file that has gone
19//! quiet can age out, because a session-keyed directory grows by a file for every key.
20//!
21//! Why a rotation keeps its history: rotation once overwrote a single `<name>.log.1`, which held
22//! a device to two files and made the older a sliding window minutes wide. On 2026-09-11 a
23//! device in a live fault filled 2 MiB faster than anyone pulled, and the block holding the
24//! fault had rotated away before the first pull existed. A rotation now closes the file under
25//! the UTC instant it was closed at, and the history is bounded by age and count. Age is read
26//! from the stamp in a file's name and never from its mtime, because a reader mirrors the
27//! directory with `rsync`, which is free to rewrite timestamps.
28
29use super::{
30 chunk::{
31 self,
32 Verdict,
33 },
34 redact::{
35 fingerprint,
36 NoTest,
37 Redact,
38 StrTest,
39 },
40 row::{
41 Entry,
42 Row,
43 },
44};
45use crate::guard::floor::KeyFloor;
46
47use oxedyne_fe2o3_core::prelude::*;
48use oxedyne_fe2o3_jdat::{
49 bdat::limits::DecodeLimits,
50 prelude::*,
51};
52
53use std::{
54 fs,
55 io::Write,
56 path::{
57 Path,
58 PathBuf,
59 },
60 sync::Mutex,
61 time::{
62 SystemTime,
63 UNIX_EPOCH,
64 },
65};
66
67
68const EXT: &str = "ndjson";
69pub(crate) const STAMP_LEN: usize = 16; // `YYYYMMDDTHHMMSSZ`
70const DAY_MS: u64 = 86_400_000;
71const MALFORMED: &str = "(malformed row)";
72
73/// The caps and the retention of a `Sink`.
74#[derive(Clone, Debug)]
75pub struct Gates {
76 pub max_body: usize, // the largest body accepted, in bytes
77 pub max_rows: usize, // the most rows a post may carry
78 pub max_tag: usize, // the longest a tag is kept, in bytes
79 pub max_data: usize, // the longest a data field is kept, in bytes
80 pub min_interval_ms: u64, // the shortest gap between two accepted posts from one key
81 pub max_file: u64, // the size past which a live file is rotated
82 pub retain_days: u64, // how long a rotated file is kept
83 pub max_rotated: usize, // the most rotated files kept for one key
84 pub max_total: u64, // the most bytes the directory may hold
85 pub sweep_idle: bool, // may a live file that has gone quiet age out
86}
87
88impl Gates {
89
90 /// Daimond's own numbers, as `debug_trace.rs` holds them: a device is bounded, the directory
91 /// is not, and nothing that has gone quiet is ever removed.
92 pub fn daimond() -> Self {
93 Self {
94 max_body: 256 * 1024,
95 max_rows: 1_000,
96 max_tag: 64,
97 max_data: 400,
98 min_interval_ms: 2_000,
99 max_file: 2 * 1024 * 1024,
100 retain_days: 14,
101 max_rotated: 200,
102 max_total: u64::MAX,
103 sweep_idle: false,
104 }
105 }
106}
107
108impl Default for Gates {
109 /// Daimond's caps, and the retention the test site plans: 2 MiB files, 7 days or 20 files for
110 /// a key, 200 MiB in all, and a quiet file ages out with the rest.
111 fn default() -> Self {
112 Self {
113 retain_days: 7,
114 max_rotated: 20,
115 max_total: 200 * 1024 * 1024,
116 sweep_idle: true,
117 ..Self::daimond()
118 }
119 }
120}
121
122/// Why a post was turned away. Nothing was written.
123#[derive(Clone, Debug, Eq, PartialEq)]
124pub enum Refusal {
125 TooLarge, // 413
126 Malformed(String), // 400, the body is no JSON
127 NoRows, // 400, there is no `rows` list
128 TooManyRows, // 400
129 TooFast, // 429
130}
131
132impl Refusal {
133
134 /// The HTTP status Daimond answers with.
135 pub fn status(&self) -> u16 {
136 match self {
137 Self::TooLarge => 413,
138 Self::Malformed(_) => 400,
139 Self::NoRows => 400,
140 Self::TooManyRows => 400,
141 Self::TooFast => 429,
142 }
143 }
144
145 /// The sentence Daimond answers with.
146 pub fn message(&self) -> String {
147 match self {
148 Self::TooLarge => "Diagnostic trace too large.".to_string(),
149 Self::Malformed(m) => m.clone(),
150 Self::NoRows => "A trace must carry a `rows` list.".to_string(),
151 Self::TooManyRows => "Too many rows in one trace.".to_string(),
152 Self::TooFast => "Wait a moment before sharing again.".to_string(),
153 }
154 }
155}
156
157/// The result of a post: the rows stored, or the reason none was.
158#[derive(Clone, Debug, Eq, PartialEq)]
159pub enum Posted {
160 Stored(usize),
161 Refused(Refusal),
162}
163
164/// The instant of a rotation as `YYYYMMDDTHHMMSSZ` in UTC. Fixed width, so two stamps sort
165/// chronologically as plain strings and the pruner needs no reverse calendar conversion.
166pub fn stamp(ms: u64) -> String {
167 let secs = (ms / 1_000) as i64;
168 let (y, m, d) = civil_from_days(secs.div_euclid(86_400));
169 let sod = secs.rem_euclid(86_400); // Second of day.
170 fmt!("{:04}{:02}{:02}T{:02}{:02}{:02}Z",
171 y, m, d, sod / 3_600, (sod % 3_600) / 60, sod % 60)
172}
173
174/// Is `s` a rotation stamp?
175pub fn is_stamp(s: &str) -> bool {
176 let b = s.as_bytes();
177 b.len() == STAMP_LEN
178 && b[8] == b'T'
179 && b[15] == b'Z'
180 && b[..8].iter().all(u8::is_ascii_digit)
181 && b[9..15].iter().all(u8::is_ascii_digit)
182}
183
184// Days since 1970-01-01 to a civil date, by Howard Hinnant's algorithm.
185fn civil_from_days(z: i64) -> (i64, u32, u32) {
186 let z = z + 719_468;
187 let era = z.div_euclid(146_097);
188 let doe = z.rem_euclid(146_097);
189 let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365;
190 let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
191 let mp = (5 * doy + 2) / 153;
192 let d = (doy - (153 * mp + 2) / 5 + 1) as u32;
193 let m = (if mp < 10 { mp + 3 } else { mp - 9 }) as u32;
194 (yoe + era * 400 + if m <= 2 { 1 } else { 0 }, m, d)
195}
196
197/// Reduces a segment to a safe filename alphabet, so nothing off the wire can walk out of the
198/// directory or collide with a control character. Kept to 48 characters, since an id is opaque
199/// and a prefix distinguishes it.
200pub fn sanitise(s: &str) -> String {
201 let mut out = String::new();
202 for c in s.chars() {
203 if c.is_ascii_alphanumeric() || c == '-' || c == '_' {
204 out.push(c);
205 } else {
206 out.push('_');
207 }
208 if out.len() >= 48 {
209 break;
210 }
211 }
212 if out.is_empty() {
213 out.push_str("unknown");
214 }
215 out
216}
217
218/// The wall clock in milliseconds since the epoch.
219pub fn now_ms() -> Outcome<u64> {
220 let d = res!(SystemTime::now().duration_since(UNIX_EPOCH));
221 Ok(d.as_millis() as u64)
222}
223
224// Strips control characters and line breaks from a value bound for one line, so a row cannot
225// forge another by carrying its own.
226fn scrub(s: &str) -> String {
227 s.chars().map(|c| if c.is_control() { ' ' } else { c }).collect()
228}
229
230// Caps a string to `n` bytes on a character boundary.
231fn clip(s: &str, n: usize) -> String {
232 if s.len() <= n {
233 return s.to_string();
234 }
235 let mut end = n;
236 while end > 0 && !s.is_char_boundary(end) {
237 end -= 1;
238 }
239 s[..end].to_string()
240}
241
242fn mtime_ms(meta: &fs::Metadata) -> u64 {
243 meta.modified().ok()
244 .and_then(|t| t.duration_since(UNIX_EPOCH).ok())
245 .map_or(0, |d| d.as_millis() as u64)
246}
247
248#[cfg(unix)]
249fn make_dir(dir: &Path) -> Outcome<()> {
250 use std::os::unix::fs::{
251 DirBuilderExt,
252 PermissionsExt,
253 };
254 res!(fs::DirBuilder::new().recursive(true).mode(0o700).create(dir));
255 // A directory that was there already may be open to others. What it holds is debug data, so
256 // it is closed rather than trusted.
257 let meta = res!(fs::metadata(dir));
258 if meta.permissions().mode() & 0o077 != 0 {
259 res!(fs::set_permissions(dir, fs::Permissions::from_mode(0o700)));
260 }
261 Ok(())
262}
263
264#[cfg(not(unix))]
265fn make_dir(dir: &Path) -> Outcome<()> {
266 res!(fs::create_dir_all(dir));
267 Ok(())
268}
269
270#[cfg(unix)]
271fn open_append(path: &Path) -> Outcome<fs::File> {
272 use std::os::unix::fs::OpenOptionsExt;
273 Ok(res!(fs::OpenOptions::new().create(true).append(true).mode(0o600).open(path)))
274}
275
276#[cfg(not(unix))]
277fn open_append(path: &Path) -> Outcome<fs::File> {
278 Ok(res!(fs::OpenOptions::new().create(true).append(true).open(path)))
279}
280
281/// One directory of NDJSON files, a file for each key.
282///
283/// The type parameter is the caller's string test for its `Redact`, which a sink always has: a
284/// plain `Sink::new` redacts by name alone. The directory is made `0700` and the files `0600`.
285/// Nothing here opens a socket.
286pub struct Sink<T: StrTest = NoTest> {
287 dir: PathBuf,
288 gates: Gates,
289 redact: Redact<T>,
290 floor: KeyFloor,
291 lock: Mutex<()>, // one writer at a time, so a rotation cannot meet an append
292}
293
294impl Sink<NoTest> {
295
296 /// A sink under `dir`, which is made when the first row is written.
297 pub fn new<P: Into<PathBuf>>(dir: P, gates: Gates) -> Self {
298 let floor = KeyFloor::new(gates.min_interval_ms);
299 Self {
300 dir: dir.into(),
301 gates,
302 redact: Redact::new(),
303 floor,
304 lock: Mutex::new(()),
305 }
306 }
307}
308
309impl<T: StrTest> Sink<T> {
310
311 pub fn with_redact<U: StrTest>(self, redact: Redact<U>) -> Sink<U> {
312 Sink {
313 dir: self.dir,
314 gates: self.gates,
315 redact,
316 floor: self.floor,
317 lock: self.lock,
318 }
319 }
320
321 pub fn dir(&self) -> &Path { &self.dir }
322
323 pub fn gates(&self) -> &Gates { &self.gates }
324
325 /// The live file for a key. Each segment is reduced to a safe alphabet and the segments are
326 /// joined by a hyphen, so Daimond's `["<account>", "<device>"]` names the file it does.
327 pub fn path(&self, key: &[&str]) -> PathBuf {
328 let stem = if key.is_empty() {
329 sanitise("")
330 } else {
331 key.iter().map(|k| sanitise(k)).collect::<Vec<String>>().join("-")
332 };
333 self.dir.join(fmt!("{}.{}", stem, EXT))
334 }
335
336 /// Takes a client's batch, `{v, device, rows:[{ts, tag, data}]}`, as Daimond's handler does.
337 ///
338 /// The key is the caller's, from a session it trusts and never from the body. The order of
339 /// the checks is Daimond's: size, shape, row count, then the rate floor, so a refused post
340 /// does not use up the key's interval. A refusal is a value and nothing was written.
341 pub fn post(&self, key: &[&str], body: &[u8], now_ms: u64) -> Outcome<Posted> {
342 if body.len() > self.gates.max_body {
343 return Ok(Posted::Refused(Refusal::TooLarge));
344 }
345 let text = match std::str::from_utf8(body) {
346 Ok(t) => t,
347 Err(_) => return Ok(Posted::Refused(Refusal::Malformed(
348 "Request body is not valid UTF-8.".to_string()))),
349 };
350 if text.trim().is_empty() {
351 return Ok(Posted::Refused(Refusal::Malformed(
352 "Request body is empty; expected JSON.".to_string())));
353 }
354 let req = match Dat::decode_json_strict(text, &DecodeLimits::default()) {
355 Ok(d) => d,
356 Err(_) => return Ok(Posted::Refused(Refusal::Malformed(
357 "Request body is not valid JSON.".to_string()))),
358 };
359 let rows = match req.map_get_list(&Dat::Str("rows".to_string())) {
360 Ok(v) => v,
361 Err(_) => return Ok(Posted::Refused(Refusal::NoRows)),
362 };
363 if rows.len() > self.gates.max_rows {
364 return Ok(Posted::Refused(Refusal::TooManyRows));
365 }
366 // The body's device is a label. It names no file and keys no rate.
367 let device = get(&req, "device").and_then(|d| d.get_string()).unwrap_or_default();
368 let device = self.label(&device);
369
370 if !res!(self.floor.admit(&key.join("/"), now_ms)) {
371 return Ok(Posted::Refused(Refusal::TooFast));
372 }
373 let mut posted = Vec::with_capacity(rows.len());
374 for r in rows {
375 let ts = get(r, "ts").and_then(|d| d.get_i64()).unwrap_or(0);
376 let tag = get(r, "tag").and_then(|d| d.get_string()).unwrap_or_default();
377 let data = get(r, "data").and_then(|d| d.get_string()).unwrap_or_default();
378 posted.push((ts, tag, data));
379 }
380 // The chunk rows of a bundle are judged by the text they encode, together (see `chunk`).
381 let seen: Vec<(String, String)> = posted.iter()
382 .map(|(_, tag, data)| (
383 clip(&scrub(tag), self.gates.max_tag),
384 clip(&scrub(data), self.gates.max_data),
385 ))
386 .collect();
387 let verdicts = chunk::judge(&self.redact, self.gates.max_data, &seen);
388 let mut text = String::new();
389 for (((ts, tag, data), (stag, sdata)), verdict) in posted.iter().zip(seen).zip(verdicts) {
390 let entry = match verdict {
391 Verdict::Row => self.entry(*ts, tag, data),
392 Verdict::Clean => Entry { ts: *ts, tag: stag, data: sdata },
393 Verdict::Hit => Entry { ts: *ts, tag: stag, data: self.redact.mark(&sdata) },
394 Verdict::Cover(d) => Entry { ts: *ts, tag: stag, data: d },
395 };
396 text.push_str(&entry.line(&device, now_ms));
397 }
398 res!(self.append(key, &text, now_ms));
399 Ok(Posted::Stored(rows.len()))
400 }
401
402 /// Writes rows built in process, such as a peer's own frames and state, under the same
403 /// redaction, the same field caps and the same rotation as a post. There is no body to cap
404 /// and no rate floor to keep, because the caller is this process.
405 ///
406 /// A row's JSON is fitted to `max_data` by dropping or trimming its largest string, which
407 /// keeps it valid JSON, as `Row::fit` explains.
408 pub fn write(&self, key: &[&str], device: &str, rows: &[Row], now_ms: u64) -> Outcome<usize> {
409 let device = self.label(device);
410 let mut text = String::new();
411 for row in rows {
412 let covered = self.redact.row(row);
413 let mut e = res!(covered.entry(self.gates.max_data));
414 e.tag = clip(&scrub(&e.tag), self.gates.max_tag);
415 text.push_str(&e.line(&device, now_ms));
416 }
417 res!(self.append(key, &text, now_ms));
418 Ok(rows.len())
419 }
420
421 // A device label as it is written beside each line: scrubbed, clipped, and covered like any
422 // other string a client sends, since a label is as free to carry a secret as a row is.
423 fn label(&self, device: &str) -> String {
424 let device = clip(&scrub(device), self.gates.max_tag);
425 match self.redact.text(&device) {
426 Some(covered) => covered,
427 None => device,
428 }
429 }
430
431 // One posted row, as it will be written: scrubbed, covered, then clipped.
432 fn entry(&self, ts: i64, tag: &str, data: &str) -> Entry {
433 // A row that is not the expected shape is written as a marker rather than dropped: a
434 // reader seeing it learns the client sent something unexpected, which is diagnostic.
435 if tag.is_empty() {
436 return Entry { ts, tag: MALFORMED.to_string(), data: String::new() };
437 }
438 let tag = clip(&scrub(tag), self.gates.max_tag);
439 let data = self.cover(&tag, &scrub(data));
440 Entry { ts, tag, data: clip(&data, self.gates.max_data) }
441 }
442
443 // Covers a data field before it is clipped, so a clip cannot leave a secret-named field in a
444 // JSON text too short to parse.
445 fn cover(&self, tag: &str, data: &str) -> String {
446 if tag.starts_with("ev ") {
447 return match Dat::decode_json_strict(data, &DecodeLimits::default()) {
448 Ok(d) => {
449 let (nd, changed) = self.redact.walk(&d);
450 if !changed {
451 // Untouched, so the client's own bytes stand.
452 data.to_string()
453 } else {
454 match super::row::compact(&nd) {
455 Ok(s) => s,
456 Err(_) => fingerprint(data, 0),
457 }
458 }
459 },
460 // An event that is no JSON cannot be examined by name, so it is not kept.
461 Err(_) => fingerprint(data, 0),
462 };
463 }
464 self.redact.text(data).unwrap_or_else(|| data.to_string())
465 }
466
467 // Appends, creating the directory and, where the live file has grown past `max_file`,
468 // closing it under a stamped name first so that the append opens a fresh one.
469 fn append(&self, key: &[&str], text: &str, now_ms: u64) -> Outcome<()> {
470 let _one = lock_mutex!(self.lock);
471 res!(make_dir(&self.dir));
472 let path = self.path(key);
473 let mut sweep = false;
474 match fs::metadata(&path) {
475 Ok(meta) => {
476 if meta.len() > self.gates.max_file {
477 // A failed rotation must not lose the new rows: fall through to the append,
478 // which grows the file past the cap for one more round rather than dropping
479 // the diagnostic.
480 match self.rotate(&path, now_ms) {
481 Ok(_) => sweep = true,
482 Err(e) => warn!("lens: could not rotate {:?}: {}", path, e),
483 }
484 }
485 },
486 // A new file is a new key, and the directory grows by it.
487 Err(_) => sweep = true,
488 }
489 let mut f = res!(open_append(&path));
490 res!(f.write_all(text.as_bytes()));
491 // Sweeping runs only behind a rotation or a new file, so a quiet directory is never walked.
492 if sweep {
493 if let Err(e) = self.sweep(&path, now_ms) {
494 warn!("lens: could not prune {:?} after writing {:?}: {}", self.dir, path, e);
495 }
496 }
497 Ok(())
498 }
499
500 // Closes the live file as `<name>.<stamp>`, giving back the name it now has. An occupied stamp
501 // takes a `-1`, `-2` suffix rather than replacing what is already there.
502 fn rotate(&self, path: &Path, now_ms: u64) -> Outcome<PathBuf> {
503 let name = res!(path.file_name().and_then(|n| n.to_str())
504 .ok_or_else(|| err!("Lens path {:?} has no usable file name.", path; Invalid, Path)));
505 let base = stamp(now_ms);
506 for n in 0..100u32 {
507 let suffix = if n == 0 { base.clone() } else { fmt!("{}-{}", base, n) };
508 let target = self.dir.join(fmt!("{}.{}", name, suffix));
509 if !target.exists() {
510 res!(fs::rename(path, &target));
511 return Ok(target);
512 }
513 }
514 Err(err!("No free rotation name for {:?} at stamp {}.", path, base; Conflict, File))
515 }
516
517 // Prunes the directory. A key's rotated files go by age and by count, then an idle live file
518 // goes by age if the gates say so, then the oldest rotated files (and idle live ones) go until
519 // the directory fits `max_total`. The file being written is never swept.
520 //
521 // Age is read from the stamp in a rotated file's name, and only a live file's age comes from
522 // its mtime, because only the sink writes a live file.
523 fn sweep(&self, keep: &Path, now_ms: u64) -> Outcome<()> {
524 struct Rot { stem: String, st: String, path: PathBuf, len: u64 }
525 struct Live { path: PathBuf, len: u64, at: u64 }
526
527 let mut rots: Vec<Rot> = Vec::new();
528 let mut lives: Vec<Live> = Vec::new();
529 for entry in res!(fs::read_dir(&self.dir)) {
530 let entry = res!(entry);
531 // A name that is not UTF-8 was not written here.
532 let fname = match entry.file_name().into_string() {
533 Ok(s) => s,
534 Err(_) => continue,
535 };
536 let meta = match entry.metadata() {
537 Ok(m) => m,
538 Err(_) => continue,
539 };
540 if !meta.is_file() {
541 continue;
542 }
543 let dot = fmt!(".{}.", EXT);
544 if let Some((stem, tail)) = fname.split_once(dot.as_str()) {
545 // A pre-stamp `<name>.ndjson.1` fails `is_stamp` and is left where it is: it
546 // predates retention, and nothing here should guess at its age.
547 if let Some(st) = tail.get(..STAMP_LEN) {
548 if is_stamp(st) {
549 rots.push(Rot { stem: stem.to_string(), st: st.to_string(), path: entry.path(), len: meta.len() });
550 }
551 }
552 } else if fname.ends_with(&fmt!(".{}", EXT)) {
553 lives.push(Live { path: entry.path(), len: meta.len(), at: mtime_ms(&meta) });
554 }
555 }
556
557 // Age and count, a key at a time. Oldest first within a key, so the first `over` go.
558 let retain = self.gates.retain_days.saturating_mul(DAY_MS);
559 let cutoff = stamp(now_ms.saturating_sub(retain));
560 rots.sort_by(|a, b| a.stem.cmp(&b.stem).then(a.st.cmp(&b.st)).then(a.path.cmp(&b.path)));
561 let mut left: Vec<Rot> = Vec::new();
562 while let Some(stem) = rots.first().map(|r| r.stem.clone()) {
563 let n = rots.iter().take_while(|r| r.stem == stem).count();
564 let over = n.saturating_sub(self.gates.max_rotated);
565 for (k, r) in rots.drain(..n).enumerate() {
566 if k < over || r.st.as_str() < cutoff.as_str() {
567 res!(fs::remove_file(&r.path));
568 } else {
569 left.push(r);
570 }
571 }
572 }
573 let mut rots = left;
574
575 // A live file that has gone quiet.
576 if self.gates.sweep_idle {
577 let mut kept = Vec::new();
578 for l in lives {
579 if l.path != keep && l.at.saturating_add(retain) < now_ms {
580 res!(fs::remove_file(&l.path));
581 } else {
582 kept.push(l);
583 }
584 }
585 lives = kept;
586 }
587
588 // The total. Rotated files first, oldest stamp first across every key, then idle live
589 // ones, oldest first. The file being written is not a candidate.
590 let mut total: u64 = rots.iter().map(|r| r.len).sum::<u64>() + lives.iter().map(|l| l.len).sum::<u64>();
591 if total > self.gates.max_total {
592 rots.sort_by(|a, b| a.st.cmp(&b.st).then(a.path.cmp(&b.path)));
593 for r in &rots {
594 if total <= self.gates.max_total {
595 break;
596 }
597 res!(fs::remove_file(&r.path));
598 total = total.saturating_sub(r.len);
599 }
600 if self.gates.sweep_idle && total > self.gates.max_total {
601 lives.sort_by_key(|l| l.at);
602 for l in &lives {
603 if total <= self.gates.max_total {
604 break;
605 }
606 if l.path != keep {
607 res!(fs::remove_file(&l.path));
608 total = total.saturating_sub(l.len);
609 }
610 }
611 }
612 }
613 Ok(())
614 }
615}
616
617// A member of a map, if it is one and has it.
618fn get<'a>(d: &'a Dat, name: &str) -> Option<&'a Dat> {
619 d.map_get(&Dat::Str(name.to_string())).ok().flatten()
620}
621
622
623#[cfg(test)]
624mod tests {
625 use super::*;
626
627 #[test]
628 fn civil_from_days_is_right_at_the_calendar_edges() {
629 assert_eq!(civil_from_days(0), (1970, 1, 1));
630 assert_eq!(civil_from_days(-1), (1969, 12, 31));
631 assert_eq!(civil_from_days(11_016), (2000, 2, 29)); // A leap year divisible by 400.
632 assert_eq!(civil_from_days(11_017), (2000, 3, 1));
633 assert_eq!(civil_from_days(-25_508), (1900, 3, 1)); // Not a leap year, divisible by 100.
634 assert_eq!(civil_from_days(47_541), (2100, 3, 1));
635 assert_eq!(civil_from_days(19_782), (2024, 2, 29));
636 assert_eq!(civil_from_days(20_343), (2025, 9, 12));
637 }
638
639 #[test]
640 fn clip_cuts_on_a_character_boundary_and_scrub_leaves_no_control() {
641 assert_eq!(clip("abc", 3), "abc");
642 assert_eq!(clip("abcd", 3), "abc");
643 assert_eq!(clip("éé", 3), "é", "a cut through the second é backs up to the boundary");
644 assert_eq!(clip("é", 0), "");
645 assert_eq!(scrub("a\nb\tc\u{7f}d\u{85}"), "a b c d ");
646 assert_eq!(scrub("plain é text"), "plain é text");
647 }
648
649 #[test]
650 fn a_stamp_in_a_name_must_be_the_whole_sixteen() {
651 assert!(is_stamp("20250912T000000Z"));
652 assert!(!is_stamp("20250912T000000"));
653 assert!(!is_stamp("20250912T00000Zx"));
654 assert!(!is_stamp("2025091ZT000000Z"));
655 }
656}