oxedyne/fe2o3/fe2o3_net/src/lens/sink.rs
25.8 KiB, 5 runs
created by r1870400018:65646, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! An append-only NDJSON writer with the gates of Daimond's `/api/debug-trace`. |
| 2 | //! |
| 3 | //! The sink takes what `gateway/src/handlers/debug_trace.rs` took, a body `{v, device, rows: |
| 4 | //! [{ts, tag, data}]}` under a key the caller chose, and holds it to the same caps: a body over |
| 5 | //! `max_body` is refused, more than `max_rows` rows is refused and not truncated, a tag and a |
| 6 | //! data field are clipped, and one key may post at most once in `min_interval_ms`. It differs |
| 7 | //! from that handler in four ways, each on purpose. |
| 8 | //! |
| 9 | //! * It writes one JSON object a line instead of a text block, which a reader parses with |
| 10 | //! nothing but `JSON.parse` (see `Entry::line`). |
| 11 | //! * It is keyed by what the caller passes and never by the body. Daimond names a file for the |
| 12 | //! session's account and the body's device; here a key is any list of segments, each reduced |
| 13 | //! to a safe alphabet, and the body's `device` is only a label on each line. |
| 14 | //! * It redacts. Every row passes `Redact` before it is written, so a caller that forgot to |
| 15 | //! covers nothing: a secret-named field, a deny-listed place and a string the caller's test |
| 16 | //! fires on never reach the disk. An `ev` row whose data is not JSON cannot be examined by |
| 17 | //! name, and is replaced by its fingerprint. |
| 18 | //! * Its history is bounded by total bytes as well as by age and count, and a file that has gone |
| 19 | //! quiet can age out, because a session-keyed directory grows by a file for every key. |
| 20 | //! |
| 21 | //! Why a rotation keeps its history: rotation once overwrote a single `<name>.log.1`, which held |
| 22 | //! a device to two files and made the older a sliding window minutes wide. On 2026-09-11 a |
| 23 | //! device in a live fault filled 2 MiB faster than anyone pulled, and the block holding the |
| 24 | //! fault had rotated away before the first pull existed. A rotation now closes the file under |
| 25 | //! the UTC instant it was closed at, and the history is bounded by age and count. Age is read |
| 26 | //! from the stamp in a file's name and never from its mtime, because a reader mirrors the |
| 27 | //! directory with `rsync`, which is free to rewrite timestamps. |
| 28 | |
| 29 | use super::{ |
| 30 | chunk::{ |
| 31 | self, |
| 32 | Verdict, |
| 33 | }, |
| 34 | redact::{ |
| 35 | fingerprint, |
| 36 | NoTest, |
| 37 | Redact, |
| 38 | StrTest, |
| 39 | }, |
| 40 | row::{ |
| 41 | Entry, |
| 42 | Row, |
| 43 | }, |
| 44 | }; |
| 45 | use crate::guard::floor::KeyFloor; |
| 46 | |
| 47 | use oxedyne_fe2o3_core::prelude::*; |
| 48 | use oxedyne_fe2o3_jdat::{ |
| 49 | bdat::limits::DecodeLimits, |
| 50 | prelude::*, |
| 51 | }; |
| 52 | |
| 53 | use std::{ |
| 54 | fs, |
| 55 | io::Write, |
| 56 | path::{ |
| 57 | Path, |
| 58 | PathBuf, |
| 59 | }, |
| 60 | sync::Mutex, |
| 61 | time::{ |
| 62 | SystemTime, |
| 63 | UNIX_EPOCH, |
| 64 | }, |
| 65 | }; |
| 66 | |
| 67 | |
| 68 | const EXT: &str = "ndjson"; |
| 69 | pub(crate) const STAMP_LEN: usize = 16; // `YYYYMMDDTHHMMSSZ` |
| 70 | const DAY_MS: u64 = 86_400_000; |
| 71 | const MALFORMED: &str = "(malformed row)"; |
| 72 | |
| 73 | /// The caps and the retention of a `Sink`. |
| 74 | #[derive(Clone, Debug)] |
| 75 | pub struct Gates { |
| 76 | pub max_body: usize, // the largest body accepted, in bytes |
| 77 | pub max_rows: usize, // the most rows a post may carry |
| 78 | pub max_tag: usize, // the longest a tag is kept, in bytes |
| 79 | pub max_data: usize, // the longest a data field is kept, in bytes |
| 80 | pub min_interval_ms: u64, // the shortest gap between two accepted posts from one key |
| 81 | pub max_file: u64, // the size past which a live file is rotated |
| 82 | pub retain_days: u64, // how long a rotated file is kept |
| 83 | pub max_rotated: usize, // the most rotated files kept for one key |
| 84 | pub max_total: u64, // the most bytes the directory may hold |
| 85 | pub sweep_idle: bool, // may a live file that has gone quiet age out |
| 86 | } |
| 87 | |
| 88 | impl Gates { |
| 89 | |
| 90 | /// Daimond's own numbers, as `debug_trace.rs` holds them: a device is bounded, the directory |
| 91 | /// is not, and nothing that has gone quiet is ever removed. |
| 92 | pub fn daimond() -> Self { |
| 93 | Self { |
| 94 | max_body: 256 * 1024, |
| 95 | max_rows: 1_000, |
| 96 | max_tag: 64, |
| 97 | max_data: 400, |
| 98 | min_interval_ms: 2_000, |
| 99 | max_file: 2 * 1024 * 1024, |
| 100 | retain_days: 14, |
| 101 | max_rotated: 200, |
| 102 | max_total: u64::MAX, |
| 103 | sweep_idle: false, |
| 104 | } |
| 105 | } |
| 106 | } |
| 107 | |
| 108 | impl Default for Gates { |
| 109 | /// Daimond's caps, and the retention the test site plans: 2 MiB files, 7 days or 20 files for |
| 110 | /// a key, 200 MiB in all, and a quiet file ages out with the rest. |
| 111 | fn default() -> Self { |
| 112 | Self { |
| 113 | retain_days: 7, |
| 114 | max_rotated: 20, |
| 115 | max_total: 200 * 1024 * 1024, |
| 116 | sweep_idle: true, |
| 117 | ..Self::daimond() |
| 118 | } |
| 119 | } |
| 120 | } |
| 121 | |
| 122 | /// Why a post was turned away. Nothing was written. |
| 123 | #[derive(Clone, Debug, Eq, PartialEq)] |
| 124 | pub enum Refusal { |
| 125 | TooLarge, // 413 |
| 126 | Malformed(String), // 400, the body is no JSON |
| 127 | NoRows, // 400, there is no `rows` list |
| 128 | TooManyRows, // 400 |
| 129 | TooFast, // 429 |
| 130 | } |
| 131 | |
| 132 | impl Refusal { |
| 133 | |
| 134 | /// The HTTP status Daimond answers with. |
| 135 | pub fn status(&self) -> u16 { |
| 136 | match self { |
| 137 | Self::TooLarge => 413, |
| 138 | Self::Malformed(_) => 400, |
| 139 | Self::NoRows => 400, |
| 140 | Self::TooManyRows => 400, |
| 141 | Self::TooFast => 429, |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | /// The sentence Daimond answers with. |
| 146 | pub fn message(&self) -> String { |
| 147 | match self { |
| 148 | Self::TooLarge => "Diagnostic trace too large.".to_string(), |
| 149 | Self::Malformed(m) => m.clone(), |
| 150 | Self::NoRows => "A trace must carry a `rows` list.".to_string(), |
| 151 | Self::TooManyRows => "Too many rows in one trace.".to_string(), |
| 152 | Self::TooFast => "Wait a moment before sharing again.".to_string(), |
| 153 | } |
| 154 | } |
| 155 | } |
| 156 | |
| 157 | /// The result of a post: the rows stored, or the reason none was. |
| 158 | #[derive(Clone, Debug, Eq, PartialEq)] |
| 159 | pub enum Posted { |
| 160 | Stored(usize), |
| 161 | Refused(Refusal), |
| 162 | } |
| 163 | |
| 164 | /// The instant of a rotation as `YYYYMMDDTHHMMSSZ` in UTC. Fixed width, so two stamps sort |
| 165 | /// chronologically as plain strings and the pruner needs no reverse calendar conversion. |
| 166 | pub fn stamp(ms: u64) -> String { |
| 167 | let secs = (ms / 1_000) as i64; |
| 168 | let (y, m, d) = civil_from_days(secs.div_euclid(86_400)); |
| 169 | let sod = secs.rem_euclid(86_400); // Second of day. |
| 170 | fmt!("{:04}{:02}{:02}T{:02}{:02}{:02}Z", |
| 171 | y, m, d, sod / 3_600, (sod % 3_600) / 60, sod % 60) |
| 172 | } |
| 173 | |
| 174 | /// Is `s` a rotation stamp? |
| 175 | pub fn is_stamp(s: &str) -> bool { |
| 176 | let b = s.as_bytes(); |
| 177 | b.len() == STAMP_LEN |
| 178 | && b[8] == b'T' |
| 179 | && b[15] == b'Z' |
| 180 | && b[..8].iter().all(u8::is_ascii_digit) |
| 181 | && b[9..15].iter().all(u8::is_ascii_digit) |
| 182 | } |
| 183 | |
| 184 | // Days since 1970-01-01 to a civil date, by Howard Hinnant's algorithm. |
| 185 | fn civil_from_days(z: i64) -> (i64, u32, u32) { |
| 186 | let z = z + 719_468; |
| 187 | let era = z.div_euclid(146_097); |
| 188 | let doe = z.rem_euclid(146_097); |
| 189 | let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365; |
| 190 | let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); |
| 191 | let mp = (5 * doy + 2) / 153; |
| 192 | let d = (doy - (153 * mp + 2) / 5 + 1) as u32; |
| 193 | let m = (if mp < 10 { mp + 3 } else { mp - 9 }) as u32; |
| 194 | (yoe + era * 400 + if m <= 2 { 1 } else { 0 }, m, d) |
| 195 | } |
| 196 | |
| 197 | /// Reduces a segment to a safe filename alphabet, so nothing off the wire can walk out of the |
| 198 | /// directory or collide with a control character. Kept to 48 characters, since an id is opaque |
| 199 | /// and a prefix distinguishes it. |
| 200 | pub fn sanitise(s: &str) -> String { |
| 201 | let mut out = String::new(); |
| 202 | for c in s.chars() { |
| 203 | if c.is_ascii_alphanumeric() || c == '-' || c == '_' { |
| 204 | out.push(c); |
| 205 | } else { |
| 206 | out.push('_'); |
| 207 | } |
| 208 | if out.len() >= 48 { |
| 209 | break; |
| 210 | } |
| 211 | } |
| 212 | if out.is_empty() { |
| 213 | out.push_str("unknown"); |
| 214 | } |
| 215 | out |
| 216 | } |
| 217 | |
| 218 | /// The wall clock in milliseconds since the epoch. |
| 219 | pub fn now_ms() -> Outcome<u64> { |
| 220 | let d = res!(SystemTime::now().duration_since(UNIX_EPOCH)); |
| 221 | Ok(d.as_millis() as u64) |
| 222 | } |
| 223 | |
| 224 | // Strips control characters and line breaks from a value bound for one line, so a row cannot |
| 225 | // forge another by carrying its own. |
| 226 | fn scrub(s: &str) -> String { |
| 227 | s.chars().map(|c| if c.is_control() { ' ' } else { c }).collect() |
| 228 | } |
| 229 | |
| 230 | // Caps a string to `n` bytes on a character boundary. |
| 231 | fn clip(s: &str, n: usize) -> String { |
| 232 | if s.len() <= n { |
| 233 | return s.to_string(); |
| 234 | } |
| 235 | let mut end = n; |
| 236 | while end > 0 && !s.is_char_boundary(end) { |
| 237 | end -= 1; |
| 238 | } |
| 239 | s[..end].to_string() |
| 240 | } |
| 241 | |
| 242 | fn mtime_ms(meta: &fs::Metadata) -> u64 { |
| 243 | meta.modified().ok() |
| 244 | .and_then(|t| t.duration_since(UNIX_EPOCH).ok()) |
| 245 | .map_or(0, |d| d.as_millis() as u64) |
| 246 | } |
| 247 | |
| 248 | #[cfg(unix)] |
| 249 | fn make_dir(dir: &Path) -> Outcome<()> { |
| 250 | use std::os::unix::fs::{ |
| 251 | DirBuilderExt, |
| 252 | PermissionsExt, |
| 253 | }; |
| 254 | res!(fs::DirBuilder::new().recursive(true).mode(0o700).create(dir)); |
| 255 | // A directory that was there already may be open to others. What it holds is debug data, so |
| 256 | // it is closed rather than trusted. |
| 257 | let meta = res!(fs::metadata(dir)); |
| 258 | if meta.permissions().mode() & 0o077 != 0 { |
| 259 | res!(fs::set_permissions(dir, fs::Permissions::from_mode(0o700))); |
| 260 | } |
| 261 | Ok(()) |
| 262 | } |
| 263 | |
| 264 | #[cfg(not(unix))] |
| 265 | fn make_dir(dir: &Path) -> Outcome<()> { |
| 266 | res!(fs::create_dir_all(dir)); |
| 267 | Ok(()) |
| 268 | } |
| 269 | |
| 270 | #[cfg(unix)] |
| 271 | fn open_append(path: &Path) -> Outcome<fs::File> { |
| 272 | use std::os::unix::fs::OpenOptionsExt; |
| 273 | Ok(res!(fs::OpenOptions::new().create(true).append(true).mode(0o600).open(path))) |
| 274 | } |
| 275 | |
| 276 | #[cfg(not(unix))] |
| 277 | fn open_append(path: &Path) -> Outcome<fs::File> { |
| 278 | Ok(res!(fs::OpenOptions::new().create(true).append(true).open(path))) |
| 279 | } |
| 280 | |
| 281 | /// One directory of NDJSON files, a file for each key. |
| 282 | /// |
| 283 | /// The type parameter is the caller's string test for its `Redact`, which a sink always has: a |
| 284 | /// plain `Sink::new` redacts by name alone. The directory is made `0700` and the files `0600`. |
| 285 | /// Nothing here opens a socket. |
| 286 | pub struct Sink<T: StrTest = NoTest> { |
| 287 | dir: PathBuf, |
| 288 | gates: Gates, |
| 289 | redact: Redact<T>, |
| 290 | floor: KeyFloor, |
| 291 | lock: Mutex<()>, // one writer at a time, so a rotation cannot meet an append |
| 292 | } |
| 293 | |
| 294 | impl Sink<NoTest> { |
| 295 | |
| 296 | /// A sink under `dir`, which is made when the first row is written. |
| 297 | pub fn new<P: Into<PathBuf>>(dir: P, gates: Gates) -> Self { |
| 298 | let floor = KeyFloor::new(gates.min_interval_ms); |
| 299 | Self { |
| 300 | dir: dir.into(), |
| 301 | gates, |
| 302 | redact: Redact::new(), |
| 303 | floor, |
| 304 | lock: Mutex::new(()), |
| 305 | } |
| 306 | } |
| 307 | } |
| 308 | |
| 309 | impl<T: StrTest> Sink<T> { |
| 310 | |
| 311 | pub fn with_redact<U: StrTest>(self, redact: Redact<U>) -> Sink<U> { |
| 312 | Sink { |
| 313 | dir: self.dir, |
| 314 | gates: self.gates, |
| 315 | redact, |
| 316 | floor: self.floor, |
| 317 | lock: self.lock, |
| 318 | } |
| 319 | } |
| 320 | |
| 321 | pub fn dir(&self) -> &Path { &self.dir } |
| 322 | |
| 323 | pub fn gates(&self) -> &Gates { &self.gates } |
| 324 | |
| 325 | /// The live file for a key. Each segment is reduced to a safe alphabet and the segments are |
| 326 | /// joined by a hyphen, so Daimond's `["<account>", "<device>"]` names the file it does. |
| 327 | pub fn path(&self, key: &[&str]) -> PathBuf { |
| 328 | let stem = if key.is_empty() { |
| 329 | sanitise("") |
| 330 | } else { |
| 331 | key.iter().map(|k| sanitise(k)).collect::<Vec<String>>().join("-") |
| 332 | }; |
| 333 | self.dir.join(fmt!("{}.{}", stem, EXT)) |
| 334 | } |
| 335 | |
| 336 | /// Takes a client's batch, `{v, device, rows:[{ts, tag, data}]}`, as Daimond's handler does. |
| 337 | /// |
| 338 | /// The key is the caller's, from a session it trusts and never from the body. The order of |
| 339 | /// the checks is Daimond's: size, shape, row count, then the rate floor, so a refused post |
| 340 | /// does not use up the key's interval. A refusal is a value and nothing was written. |
| 341 | pub fn post(&self, key: &[&str], body: &[u8], now_ms: u64) -> Outcome<Posted> { |
| 342 | if body.len() > self.gates.max_body { |
| 343 | return Ok(Posted::Refused(Refusal::TooLarge)); |
| 344 | } |
| 345 | let text = match std::str::from_utf8(body) { |
| 346 | Ok(t) => t, |
| 347 | Err(_) => return Ok(Posted::Refused(Refusal::Malformed( |
| 348 | "Request body is not valid UTF-8.".to_string()))), |
| 349 | }; |
| 350 | if text.trim().is_empty() { |
| 351 | return Ok(Posted::Refused(Refusal::Malformed( |
| 352 | "Request body is empty; expected JSON.".to_string()))); |
| 353 | } |
| 354 | let req = match Dat::decode_json_strict(text, &DecodeLimits::default()) { |
| 355 | Ok(d) => d, |
| 356 | Err(_) => return Ok(Posted::Refused(Refusal::Malformed( |
| 357 | "Request body is not valid JSON.".to_string()))), |
| 358 | }; |
| 359 | let rows = match req.map_get_list(&Dat::Str("rows".to_string())) { |
| 360 | Ok(v) => v, |
| 361 | Err(_) => return Ok(Posted::Refused(Refusal::NoRows)), |
| 362 | }; |
| 363 | if rows.len() > self.gates.max_rows { |
| 364 | return Ok(Posted::Refused(Refusal::TooManyRows)); |
| 365 | } |
| 366 | // The body's device is a label. It names no file and keys no rate. |
| 367 | let device = get(&req, "device").and_then(|d| d.get_string()).unwrap_or_default(); |
| 368 | let device = self.label(&device); |
| 369 | |
| 370 | if !res!(self.floor.admit(&key.join("/"), now_ms)) { |
| 371 | return Ok(Posted::Refused(Refusal::TooFast)); |
| 372 | } |
| 373 | let mut posted = Vec::with_capacity(rows.len()); |
| 374 | for r in rows { |
| 375 | let ts = get(r, "ts").and_then(|d| d.get_i64()).unwrap_or(0); |
| 376 | let tag = get(r, "tag").and_then(|d| d.get_string()).unwrap_or_default(); |
| 377 | let data = get(r, "data").and_then(|d| d.get_string()).unwrap_or_default(); |
| 378 | posted.push((ts, tag, data)); |
| 379 | } |
| 380 | // The chunk rows of a bundle are judged by the text they encode, together (see `chunk`). |
| 381 | let seen: Vec<(String, String)> = posted.iter() |
| 382 | .map(|(_, tag, data)| ( |
| 383 | clip(&scrub(tag), self.gates.max_tag), |
| 384 | clip(&scrub(data), self.gates.max_data), |
| 385 | )) |
| 386 | .collect(); |
| 387 | let verdicts = chunk::judge(&self.redact, self.gates.max_data, &seen); |
| 388 | let mut text = String::new(); |
| 389 | for (((ts, tag, data), (stag, sdata)), verdict) in posted.iter().zip(seen).zip(verdicts) { |
| 390 | let entry = match verdict { |
| 391 | Verdict::Row => self.entry(*ts, tag, data), |
| 392 | Verdict::Clean => Entry { ts: *ts, tag: stag, data: sdata }, |
| 393 | Verdict::Hit => Entry { ts: *ts, tag: stag, data: self.redact.mark(&sdata) }, |
| 394 | Verdict::Cover(d) => Entry { ts: *ts, tag: stag, data: d }, |
| 395 | }; |
| 396 | text.push_str(&entry.line(&device, now_ms)); |
| 397 | } |
| 398 | res!(self.append(key, &text, now_ms)); |
| 399 | Ok(Posted::Stored(rows.len())) |
| 400 | } |
| 401 | |
| 402 | /// Writes rows built in process, such as a peer's own frames and state, under the same |
| 403 | /// redaction, the same field caps and the same rotation as a post. There is no body to cap |
| 404 | /// and no rate floor to keep, because the caller is this process. |
| 405 | /// |
| 406 | /// A row's JSON is fitted to `max_data` by dropping or trimming its largest string, which |
| 407 | /// keeps it valid JSON, as `Row::fit` explains. |
| 408 | pub fn write(&self, key: &[&str], device: &str, rows: &[Row], now_ms: u64) -> Outcome<usize> { |
| 409 | let device = self.label(device); |
| 410 | let mut text = String::new(); |
| 411 | for row in rows { |
| 412 | let covered = self.redact.row(row); |
| 413 | let mut e = res!(covered.entry(self.gates.max_data)); |
| 414 | e.tag = clip(&scrub(&e.tag), self.gates.max_tag); |
| 415 | text.push_str(&e.line(&device, now_ms)); |
| 416 | } |
| 417 | res!(self.append(key, &text, now_ms)); |
| 418 | Ok(rows.len()) |
| 419 | } |
| 420 | |
| 421 | // A device label as it is written beside each line: scrubbed, clipped, and covered like any |
| 422 | // other string a client sends, since a label is as free to carry a secret as a row is. |
| 423 | fn label(&self, device: &str) -> String { |
| 424 | let device = clip(&scrub(device), self.gates.max_tag); |
| 425 | match self.redact.text(&device) { |
| 426 | Some(covered) => covered, |
| 427 | None => device, |
| 428 | } |
| 429 | } |
| 430 | |
| 431 | // One posted row, as it will be written: scrubbed, covered, then clipped. |
| 432 | fn entry(&self, ts: i64, tag: &str, data: &str) -> Entry { |
| 433 | // A row that is not the expected shape is written as a marker rather than dropped: a |
| 434 | // reader seeing it learns the client sent something unexpected, which is diagnostic. |
| 435 | if tag.is_empty() { |
| 436 | return Entry { ts, tag: MALFORMED.to_string(), data: String::new() }; |
| 437 | } |
| 438 | let tag = clip(&scrub(tag), self.gates.max_tag); |
| 439 | let data = self.cover(&tag, &scrub(data)); |
| 440 | Entry { ts, tag, data: clip(&data, self.gates.max_data) } |
| 441 | } |
| 442 | |
| 443 | // Covers a data field before it is clipped, so a clip cannot leave a secret-named field in a |
| 444 | // JSON text too short to parse. |
| 445 | fn cover(&self, tag: &str, data: &str) -> String { |
| 446 | if tag.starts_with("ev ") { |
| 447 | return match Dat::decode_json_strict(data, &DecodeLimits::default()) { |
| 448 | Ok(d) => { |
| 449 | let (nd, changed) = self.redact.walk(&d); |
| 450 | if !changed { |
| 451 | // Untouched, so the client's own bytes stand. |
| 452 | data.to_string() |
| 453 | } else { |
| 454 | match super::row::compact(&nd) { |
| 455 | Ok(s) => s, |
| 456 | Err(_) => fingerprint(data, 0), |
| 457 | } |
| 458 | } |
| 459 | }, |
| 460 | // An event that is no JSON cannot be examined by name, so it is not kept. |
| 461 | Err(_) => fingerprint(data, 0), |
| 462 | }; |
| 463 | } |
| 464 | self.redact.text(data).unwrap_or_else(|| data.to_string()) |
| 465 | } |
| 466 | |
| 467 | // Appends, creating the directory and, where the live file has grown past `max_file`, |
| 468 | // closing it under a stamped name first so that the append opens a fresh one. |
| 469 | fn append(&self, key: &[&str], text: &str, now_ms: u64) -> Outcome<()> { |
| 470 | let _one = lock_mutex!(self.lock); |
| 471 | res!(make_dir(&self.dir)); |
| 472 | let path = self.path(key); |
| 473 | let mut sweep = false; |
| 474 | match fs::metadata(&path) { |
| 475 | Ok(meta) => { |
| 476 | if meta.len() > self.gates.max_file { |
| 477 | // A failed rotation must not lose the new rows: fall through to the append, |
| 478 | // which grows the file past the cap for one more round rather than dropping |
| 479 | // the diagnostic. |
| 480 | match self.rotate(&path, now_ms) { |
| 481 | Ok(_) => sweep = true, |
| 482 | Err(e) => warn!("lens: could not rotate {:?}: {}", path, e), |
| 483 | } |
| 484 | } |
| 485 | }, |
| 486 | // A new file is a new key, and the directory grows by it. |
| 487 | Err(_) => sweep = true, |
| 488 | } |
| 489 | let mut f = res!(open_append(&path)); |
| 490 | res!(f.write_all(text.as_bytes())); |
| 491 | // Sweeping runs only behind a rotation or a new file, so a quiet directory is never walked. |
| 492 | if sweep { |
| 493 | if let Err(e) = self.sweep(&path, now_ms) { |
| 494 | warn!("lens: could not prune {:?} after writing {:?}: {}", self.dir, path, e); |
| 495 | } |
| 496 | } |
| 497 | Ok(()) |
| 498 | } |
| 499 | |
| 500 | // Closes the live file as `<name>.<stamp>`, giving back the name it now has. An occupied stamp |
| 501 | // takes a `-1`, `-2` suffix rather than replacing what is already there. |
| 502 | fn rotate(&self, path: &Path, now_ms: u64) -> Outcome<PathBuf> { |
| 503 | let name = res!(path.file_name().and_then(|n| n.to_str()) |
| 504 | .ok_or_else(|| err!("Lens path {:?} has no usable file name.", path; Invalid, Path))); |
| 505 | let base = stamp(now_ms); |
| 506 | for n in 0..100u32 { |
| 507 | let suffix = if n == 0 { base.clone() } else { fmt!("{}-{}", base, n) }; |
| 508 | let target = self.dir.join(fmt!("{}.{}", name, suffix)); |
| 509 | if !target.exists() { |
| 510 | res!(fs::rename(path, &target)); |
| 511 | return Ok(target); |
| 512 | } |
| 513 | } |
| 514 | Err(err!("No free rotation name for {:?} at stamp {}.", path, base; Conflict, File)) |
| 515 | } |
| 516 | |
| 517 | // Prunes the directory. A key's rotated files go by age and by count, then an idle live file |
| 518 | // goes by age if the gates say so, then the oldest rotated files (and idle live ones) go until |
| 519 | // the directory fits `max_total`. The file being written is never swept. |
| 520 | // |
| 521 | // Age is read from the stamp in a rotated file's name, and only a live file's age comes from |
| 522 | // its mtime, because only the sink writes a live file. |
| 523 | fn sweep(&self, keep: &Path, now_ms: u64) -> Outcome<()> { |
| 524 | struct Rot { stem: String, st: String, path: PathBuf, len: u64 } |
| 525 | struct Live { path: PathBuf, len: u64, at: u64 } |
| 526 | |
| 527 | let mut rots: Vec<Rot> = Vec::new(); |
| 528 | let mut lives: Vec<Live> = Vec::new(); |
| 529 | for entry in res!(fs::read_dir(&self.dir)) { |
| 530 | let entry = res!(entry); |
| 531 | // A name that is not UTF-8 was not written here. |
| 532 | let fname = match entry.file_name().into_string() { |
| 533 | Ok(s) => s, |
| 534 | Err(_) => continue, |
| 535 | }; |
| 536 | let meta = match entry.metadata() { |
| 537 | Ok(m) => m, |
| 538 | Err(_) => continue, |
| 539 | }; |
| 540 | if !meta.is_file() { |
| 541 | continue; |
| 542 | } |
| 543 | let dot = fmt!(".{}.", EXT); |
| 544 | if let Some((stem, tail)) = fname.split_once(dot.as_str()) { |
| 545 | // A pre-stamp `<name>.ndjson.1` fails `is_stamp` and is left where it is: it |
| 546 | // predates retention, and nothing here should guess at its age. |
| 547 | if let Some(st) = tail.get(..STAMP_LEN) { |
| 548 | if is_stamp(st) { |
| 549 | rots.push(Rot { stem: stem.to_string(), st: st.to_string(), path: entry.path(), len: meta.len() }); |
| 550 | } |
| 551 | } |
| 552 | } else if fname.ends_with(&fmt!(".{}", EXT)) { |
| 553 | lives.push(Live { path: entry.path(), len: meta.len(), at: mtime_ms(&meta) }); |
| 554 | } |
| 555 | } |
| 556 | |
| 557 | // Age and count, a key at a time. Oldest first within a key, so the first `over` go. |
| 558 | let retain = self.gates.retain_days.saturating_mul(DAY_MS); |
| 559 | let cutoff = stamp(now_ms.saturating_sub(retain)); |
| 560 | rots.sort_by(|a, b| a.stem.cmp(&b.stem).then(a.st.cmp(&b.st)).then(a.path.cmp(&b.path))); |
| 561 | let mut left: Vec<Rot> = Vec::new(); |
| 562 | while let Some(stem) = rots.first().map(|r| r.stem.clone()) { |
| 563 | let n = rots.iter().take_while(|r| r.stem == stem).count(); |
| 564 | let over = n.saturating_sub(self.gates.max_rotated); |
| 565 | for (k, r) in rots.drain(..n).enumerate() { |
| 566 | if k < over || r.st.as_str() < cutoff.as_str() { |
| 567 | res!(fs::remove_file(&r.path)); |
| 568 | } else { |
| 569 | left.push(r); |
| 570 | } |
| 571 | } |
| 572 | } |
| 573 | let mut rots = left; |
| 574 | |
| 575 | // A live file that has gone quiet. |
| 576 | if self.gates.sweep_idle { |
| 577 | let mut kept = Vec::new(); |
| 578 | for l in lives { |
| 579 | if l.path != keep && l.at.saturating_add(retain) < now_ms { |
| 580 | res!(fs::remove_file(&l.path)); |
| 581 | } else { |
| 582 | kept.push(l); |
| 583 | } |
| 584 | } |
| 585 | lives = kept; |
| 586 | } |
| 587 | |
| 588 | // The total. Rotated files first, oldest stamp first across every key, then idle live |
| 589 | // ones, oldest first. The file being written is not a candidate. |
| 590 | let mut total: u64 = rots.iter().map(|r| r.len).sum::<u64>() + lives.iter().map(|l| l.len).sum::<u64>(); |
| 591 | if total > self.gates.max_total { |
| 592 | rots.sort_by(|a, b| a.st.cmp(&b.st).then(a.path.cmp(&b.path))); |
| 593 | for r in &rots { |
| 594 | if total <= self.gates.max_total { |
| 595 | break; |
| 596 | } |
| 597 | res!(fs::remove_file(&r.path)); |
| 598 | total = total.saturating_sub(r.len); |
| 599 | } |
| 600 | if self.gates.sweep_idle && total > self.gates.max_total { |
| 601 | lives.sort_by_key(|l| l.at); |
| 602 | for l in &lives { |
| 603 | if total <= self.gates.max_total { |
| 604 | break; |
| 605 | } |
| 606 | if l.path != keep { |
| 607 | res!(fs::remove_file(&l.path)); |
| 608 | total = total.saturating_sub(l.len); |
| 609 | } |
| 610 | } |
| 611 | } |
| 612 | } |
| 613 | Ok(()) |
| 614 | } |
| 615 | } |
| 616 | |
| 617 | // A member of a map, if it is one and has it. |
| 618 | fn get<'a>(d: &'a Dat, name: &str) -> Option<&'a Dat> { |
| 619 | d.map_get(&Dat::Str(name.to_string())).ok().flatten() |
| 620 | } |
| 621 | |
| 622 | |
| 623 | #[cfg(test)] |
| 624 | mod tests { |
| 625 | use super::*; |
| 626 | |
| 627 | #[test] |
| 628 | fn civil_from_days_is_right_at_the_calendar_edges() { |
| 629 | assert_eq!(civil_from_days(0), (1970, 1, 1)); |
| 630 | assert_eq!(civil_from_days(-1), (1969, 12, 31)); |
| 631 | assert_eq!(civil_from_days(11_016), (2000, 2, 29)); // A leap year divisible by 400. |
| 632 | assert_eq!(civil_from_days(11_017), (2000, 3, 1)); |
| 633 | assert_eq!(civil_from_days(-25_508), (1900, 3, 1)); // Not a leap year, divisible by 100. |
| 634 | assert_eq!(civil_from_days(47_541), (2100, 3, 1)); |
| 635 | assert_eq!(civil_from_days(19_782), (2024, 2, 29)); |
| 636 | assert_eq!(civil_from_days(20_343), (2025, 9, 12)); |
| 637 | } |
| 638 | |
| 639 | #[test] |
| 640 | fn clip_cuts_on_a_character_boundary_and_scrub_leaves_no_control() { |
| 641 | assert_eq!(clip("abc", 3), "abc"); |
| 642 | assert_eq!(clip("abcd", 3), "abc"); |
| 643 | assert_eq!(clip("éé", 3), "é", "a cut through the second é backs up to the boundary"); |
| 644 | assert_eq!(clip("é", 0), ""); |
| 645 | assert_eq!(scrub("a\nb\tc\u{7f}d\u{85}"), "a b c d "); |
| 646 | assert_eq!(scrub("plain é text"), "plain é text"); |
| 647 | } |
| 648 | |
| 649 | #[test] |
| 650 | fn a_stamp_in_a_name_must_be_the_whole_sixteen() { |
| 651 | assert!(is_stamp("20250912T000000Z")); |
| 652 | assert!(!is_stamp("20250912T000000")); |
| 653 | assert!(!is_stamp("20250912T00000Zx")); |
| 654 | assert!(!is_stamp("2025091ZT000000Z")); |
| 655 | } |
| 656 | } |