oxedyne/fe2o3/fe2o3_o3db_sync/tests/chunk_set.rs
44.7 KiB, 33 runs
created by r1870400018:69929, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! Chunk sets (B1, B2, M1, 2026-10-09): every chunked write has a set of its own, and the |
| 2 | //! store retires the set its key no longer names. The hooks are process-wide, so every test |
| 3 | //! takes one lock. |
| 4 | |
| 5 | mod gc_pair; |
| 6 | |
| 7 | use gc_pair::*; |
| 8 | |
| 9 | use oxedyne_fe2o3_core::prelude::*; |
| 10 | use oxedyne_fe2o3_data::time::Timestamp; |
| 11 | use oxedyne_fe2o3_hash::{ |
| 12 | csum::ChecksumScheme, |
| 13 | hash::HashScheme, |
| 14 | }; |
| 15 | use oxedyne_fe2o3_iop_db::api::{ |
| 16 | Database, |
| 17 | Meta, |
| 18 | }; |
| 19 | use oxedyne_fe2o3_jdat::prelude::*; |
| 20 | use oxedyne_fe2o3_o3db_sync::{ |
| 21 | api::OzoneApi, |
| 22 | base::{ |
| 23 | cfg::OzoneConfig, |
| 24 | constant, |
| 25 | }, |
| 26 | api::CompactReport, |
| 27 | comm::response::Wait, |
| 28 | data::cache::{ |
| 29 | Cache, |
| 30 | CacheEntry, |
| 31 | is_chunk_key, |
| 32 | }, |
| 33 | file::{ |
| 34 | floc::FileLocation, |
| 35 | state::DataState, |
| 36 | stored::RecordDigest, |
| 37 | }, |
| 38 | sweep, |
| 39 | test::{ |
| 40 | hooks, |
| 41 | setup::{ |
| 42 | self, |
| 43 | Uid, |
| 44 | UID_LEN, |
| 45 | }, |
| 46 | }, |
| 47 | verify, |
| 48 | }; |
| 49 | |
| 50 | use std::{ |
| 51 | collections::{ |
| 52 | BTreeMap, |
| 53 | BTreeSet, |
| 54 | }, |
| 55 | path::Path, |
| 56 | sync::{ |
| 57 | Mutex, |
| 58 | MutexGuard, |
| 59 | }, |
| 60 | thread, |
| 61 | time::Duration, |
| 62 | }; |
| 63 | |
| 64 | static HOOKS: Mutex<()> = Mutex::new(()); |
| 65 | |
| 66 | fn lock() -> MutexGuard<'static, ()> { |
| 67 | match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() } |
| 68 | } |
| 69 | |
| 70 | fn cfg() -> Outcome<OzoneConfig> { |
| 71 | let mut c = res!(setup::default_cfg()); |
| 72 | c.num_zones = 1; |
| 73 | c.num_cbots_per_zone = 4; |
| 74 | c.num_fbots_per_zone = 1; |
| 75 | c.num_wbots_per_zone = 2; |
| 76 | c.zone_overrides = BTreeMap::new(); |
| 77 | c.rest_chunk_threshold = 1_000; |
| 78 | c.rest_chunk_bytes = 400; |
| 79 | c.sync_on_write = true; |
| 80 | Ok(c) |
| 81 | } |
| 82 | |
| 83 | // Every byte is `seed ^ j`, so a value assembled from two values' chunks is told by `whole`. |
| 84 | fn val(len: usize, seed: u8) -> Dat { |
| 85 | Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect()) |
| 86 | } |
| 87 | |
| 88 | fn whole(v: &Dat, lens: &[usize]) -> bool { |
| 89 | match v { |
| 90 | Dat::BU32(b) | Dat::BU64(b) | Dat::BU8(b) | Dat::BU16(b) => { |
| 91 | if !lens.contains(&b.len()) || b.is_empty() { return false; } |
| 92 | let seed = b[0]; |
| 93 | b.iter().enumerate().all(|(j, x)| *x == seed ^ (j as u8)) |
| 94 | }, |
| 95 | _ => false, |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | fn start(dir: &str, c: OzoneConfig, wipe: bool) -> Outcome<TestDb> { |
| 100 | log_set_level!("error"); |
| 101 | if wipe { |
| 102 | let _ = std::fs::remove_dir_all(dir); |
| 103 | } |
| 104 | res!(std::fs::create_dir_all(dir)); |
| 105 | let root = res!(Path::new(dir).canonicalize()); |
| 106 | setup::start_db(root, Some(c), schemes(), None, false, wipe) |
| 107 | } |
| 108 | |
| 109 | fn scan_wait() -> Wait { |
| 110 | Wait { |
| 111 | max_wait: Duration::from_secs(60), |
| 112 | check_interval: constant::CHECK_INTERVAL, |
| 113 | } |
| 114 | } |
| 115 | |
| 116 | fn head(db: &TestDb, k: &Dat) -> Outcome<Option<[u64; 5]>> { |
| 117 | match res!(db.api().get_head_wait(k, None)) { |
| 118 | Some((Dat::Tup5u64(t), _)) => Ok(Some(t)), |
| 119 | _ => Ok(None), |
| 120 | } |
| 121 | } |
| 122 | |
| 123 | fn pause(d: Duration) { |
| 124 | thread::sleep(d); |
| 125 | } |
| 126 | |
| 127 | // A value written by a build that derived its set from the key alone reads whole after an |
| 128 | // overwrite of the same length, which moves it to a set of its own and retires the old one. |
| 129 | #[test] |
| 130 | fn legacy_key_derived_value_overwritten_same_length_reads_whole_and_retires() -> Outcome<()> { |
| 131 | let _lock = lock(); |
| 132 | let dir = "./test_db_chunk_set_legacy"; |
| 133 | let db = res!(start(dir, res!(cfg()), true)); |
| 134 | let k = dat!("legacy key"); |
| 135 | let (kbuf, _, _) = res!(db.api().ozone_key_dat(&k, None)); |
| 136 | let legacy = OzoneApi::<{ UID_LEN }, Uid, (), HashScheme, HashScheme, ChecksumScheme>::legacy_chunk_set_id(&kbuf); |
| 137 | let resp = db.api().responder(); |
| 138 | res!(db.api().store_dat_using_responder_forcing_set_id( |
| 139 | k.clone(), val(1_100, 1), Uid::default(), None, resp.clone(), legacy)); |
| 140 | res!(resp.recv_store_ack()); |
| 141 | let old = res!(res!(head(&db, &k)).ok_or_else(|| err!("No legacy head."; Test, Missing))); |
| 142 | assert_eq!(old[0], legacy, "the forced value is not under the legacy set"); |
| 143 | |
| 144 | res!(db.insert(k.clone(), val(1_100, 2), Uid::default(), None)); |
| 145 | let got = res!(db.get(&k, None)); |
| 146 | let new = res!(res!(head(&db, &k)).ok_or_else(|| err!("No new head."; Test, Missing))); |
| 147 | // Retired by the cache bot once the new head is durable, after the put has returned. |
| 148 | let gone = wait_until(Duration::from_secs(10), || db.api().fetch_chunks(&Dat::Tup5u64(old), None).is_err()); |
| 149 | res!(db.close()); |
| 150 | let _ = std::fs::remove_dir_all(dir); |
| 151 | assert!(matches!(&got, Some((v, _)) if *v == val(1_100, 2)), "the overwrite did not read back whole"); |
| 152 | assert_ne!(new[0], legacy, "the overwrite reused the legacy set"); |
| 153 | assert!(gone, "the legacy chunks were not retired"); |
| 154 | Ok(()) |
| 155 | } |
| 156 | |
| 157 | // The orphan sweep, run while a chunked put is between its chunks and its bunch key, leaves the |
| 158 | // put's chunks alone: they are named by no key yet, and only the pending sets say whose they are. |
| 159 | #[test] |
| 160 | fn sweep_spares_chunks_of_a_put_in_flight() -> Outcome<()> { |
| 161 | let _lock = lock(); |
| 162 | let dir = "./test_db_chunk_set_sweep"; |
| 163 | let db = res!(start(dir, res!(cfg()), true)); |
| 164 | let k = dat!("in flight"); |
| 165 | let v = val(3_000, 7); // 8 chunks |
| 166 | hooks::set_chunk_insert_delay(Duration::from_millis(300)); |
| 167 | let put = { |
| 168 | let (db, k, v) = (db.clone(), k.clone(), v.clone()); |
| 169 | thread::spawn(move || db.insert(k, v, Uid::default(), None).map(|_| ()).map_err(|e| fmt!("{}", e))) |
| 170 | }; |
| 171 | pause(Duration::from_millis(400)); |
| 172 | let report = sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait()); |
| 173 | let put = match put.join() { Ok(r) => r, Err(_) => Err(fmt!("panicked")) }; |
| 174 | hooks::set_chunk_insert_delay(Duration::ZERO); |
| 175 | let got = db.get(&k, None); |
| 176 | res!(db.close()); |
| 177 | let _ = std::fs::remove_dir_all(dir); |
| 178 | let report = res!(report); |
| 179 | msg!("sweep mid-put: {}", report.summary().replace('\n', " | ")); |
| 180 | assert!(put.is_ok(), "the put failed: {:?}", put); |
| 181 | assert!(report.skipped_pending > 0, "the sweep saw none of the put's chunks, so it proved nothing"); |
| 182 | assert_eq!(report.orphans_found, 0, "the sweep took the chunks of a put in flight for orphans"); |
| 183 | assert!(matches!(&got, Ok(Some((g, _))) if *g == v), "the put does not read back whole: {:?}", |
| 184 | got.map(|o| o.map(|(d, _)| fmt!("{:?}", d).chars().take(120).collect::<String>()))); |
| 185 | Ok(()) |
| 186 | } |
| 187 | |
| 188 | // A put whose bunch key loses to a newer value already there (its stamp is older) retires its |
| 189 | // own chunks, so the sweep finds nothing, and the newer value survives a restart. |
| 190 | #[test] |
| 191 | fn losing_concurrent_put_retires_its_own_chunks() -> Outcome<()> { |
| 192 | let _lock = lock(); |
| 193 | let dir = "./test_db_chunk_set_losing"; |
| 194 | let k = dat!("losing put"); |
| 195 | let v0 = val(1_100, 3); |
| 196 | let (report, before, after) = { |
| 197 | let db = res!(start(dir, res!(cfg()), true)); |
| 198 | res!(db.insert(k.clone(), v0.clone(), Uid::default(), None)); |
| 199 | let t0 = match res!(db.get(&k, None)) { |
| 200 | Some((_, meta)) => *meta.time, |
| 201 | None => return Err(err!("v0 is missing."; Test, Missing)), |
| 202 | }; |
| 203 | hooks::set_stale_stamp(Some(t0.saturating_sub(Duration::from_secs(1)))); |
| 204 | let lost = db.insert(k.clone(), val(1_500, 4), Uid::default(), None); |
| 205 | hooks::set_stale_stamp(None); |
| 206 | res!(lost); |
| 207 | let before = res!(db.get(&k, None)).map(|(v, _)| v); |
| 208 | let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait())); |
| 209 | res!(db.close()); |
| 210 | let db = res!(start(dir, res!(cfg()), false)); |
| 211 | let after = res!(db.get(&k, None)).map(|(v, _)| v); |
| 212 | res!(db.close()); |
| 213 | (report, before, after) |
| 214 | }; |
| 215 | let _ = std::fs::remove_dir_all(dir); |
| 216 | msg!("losing put: {}", report.summary().replace('\n', " | ")); |
| 217 | assert!(before == Some(v0.clone()), "the older put displaced the newer value"); |
| 218 | assert_eq!(report.orphans_found, 0, "the losing put left its chunks for the sweep"); |
| 219 | assert!(after == Some(v0), "the newer value did not survive a restart"); |
| 220 | Ok(()) |
| 221 | } |
| 222 | |
| 223 | // Two puts of one key at once, of one length, under one clock reading, or one stamp outright. |
| 224 | // Returns, per key, whether it read one value whole live and after a restart, and the failures. |
| 225 | fn two_puts_per_key(dir: &str, stale: bool) -> Outcome<(Vec<bool>, Vec<bool>, Vec<String>)> { |
| 226 | let lens = [1_100usize]; |
| 227 | let keys: Vec<Dat> = (0..6u8).map(|i| dat!(fmt!("tick {}", i))).collect(); |
| 228 | let db = res!(start(dir, res!(cfg()), true)); |
| 229 | let now = *res!(Timestamp::now()); |
| 230 | if stale { hooks::set_stale_stamp(Some(now)); } else { hooks::set_fixed_stamp(Some(now)); } |
| 231 | hooks::set_chunk_insert_delay(Duration::from_millis(5)); |
| 232 | let ws: Vec<_> = keys.iter().enumerate().flat_map(|(i, k)| { |
| 233 | (0..2u8).map(|j| { |
| 234 | let (db, k) = (db.clone(), k.clone()); |
| 235 | let seed = (i as u8).wrapping_mul(16).wrapping_add(j); |
| 236 | thread::spawn(move || db.insert(k, val(1_100, seed), Uid::default(), None) |
| 237 | .map(|_| ()).map_err(|e| fmt!("{}", e))) |
| 238 | }).collect::<Vec<_>>() |
| 239 | }).collect(); |
| 240 | let mut failed = Vec::new(); |
| 241 | for w in ws { |
| 242 | match w.join() { |
| 243 | Ok(Ok(())) => (), |
| 244 | Ok(Err(e)) => failed.push(e.chars().take(200).collect::<String>()), |
| 245 | Err(_) => failed.push(fmt!("panicked")), |
| 246 | } |
| 247 | } |
| 248 | hooks::set_chunk_insert_delay(Duration::ZERO); |
| 249 | hooks::set_fixed_stamp(None); |
| 250 | hooks::set_stale_stamp(None); |
| 251 | let mut live = Vec::new(); |
| 252 | for k in &keys { |
| 253 | live.push(matches!(db.get(k, None), Ok(Some((v, _))) if whole(&v, &lens))); |
| 254 | } |
| 255 | res!(db.close()); |
| 256 | let db = res!(start(dir, res!(cfg()), false)); |
| 257 | let mut restarted = Vec::new(); |
| 258 | for k in &keys { |
| 259 | restarted.push(matches!(db.get(k, None), Ok(Some((v, _))) if whole(&v, &lens))); |
| 260 | } |
| 261 | res!(db.close()); |
| 262 | let _ = std::fs::remove_dir_all(dir); |
| 263 | Ok((live, restarted, failed)) |
| 264 | } |
| 265 | |
| 266 | // Two puts of one key with one stamp get distinct chunk sets, so neither writes over the other's |
| 267 | // chunks and every key reads one value whole. Which value a restart keeps is not asked: two |
| 268 | // records of a key with one stamp have no order, which is why a process never stamps two alike. |
| 269 | #[test] |
| 270 | fn two_puts_one_stamp_get_distinct_sets() -> Outcome<()> { |
| 271 | let _lock = lock(); |
| 272 | let (live, _, failed) = res!(two_puts_per_key("./test_db_chunk_set_stamp", true)); |
| 273 | assert!(failed.is_empty(), "puts failed: {:?}", failed); |
| 274 | assert!(live.iter().all(|b| *b), "a key read torn or failed: {:?}", live); |
| 275 | Ok(()) |
| 276 | } |
| 277 | |
| 278 | // Two puts of one key in one clock tick are stamped apart, so the value a key reads is the one it |
| 279 | // reads after a restart, and its chunks were not retired by the put that lost. |
| 280 | #[test] |
| 281 | fn two_puts_one_clock_tick_settle_alike_across_a_restart() -> Outcome<()> { |
| 282 | let _lock = lock(); |
| 283 | let (live, restarted, failed) = res!(two_puts_per_key("./test_db_chunk_set_tick", false)); |
| 284 | assert!(failed.is_empty(), "puts failed: {:?}", failed); |
| 285 | assert!(live.iter().all(|b| *b), "a key read torn or failed: {:?}", live); |
| 286 | assert!(restarted.iter().all(|b| *b), "after a restart a key read torn or failed: {:?}", restarted); |
| 287 | Ok(()) |
| 288 | } |
| 289 | |
| 290 | // Two records of a key with one stamp: the cache, replay and the scan pick the same one, by place |
| 291 | // (`supersedes`), so a key reads the same whole value live and after a restart, the live set |
| 292 | // verifies, and the put that lost left nothing for the sweep. Each key's two puts are made in |
| 293 | // turn from a thread of its own, so the two writers interleave across the keys. |
| 294 | #[test] |
| 295 | fn equal_stamps_pick_one_record_live_and_after_restart() -> Outcome<()> { |
| 296 | let _lock = lock(); |
| 297 | let dir = "./test_db_chunk_set_equal"; |
| 298 | let lens = [1_100usize]; |
| 299 | let keys: Vec<Dat> = (0..12u8).map(|i| dat!(fmt!("equal {}", i))).collect(); |
| 300 | let mut bad = Vec::new(); |
| 301 | for round in 0..4u8 { |
| 302 | let db = res!(start(dir, res!(cfg()), true)); |
| 303 | hooks::set_stale_stamp(Some(*res!(Timestamp::now()))); |
| 304 | let ws: Vec<_> = keys.iter().enumerate().map(|(i, k)| { |
| 305 | let (db, k) = (db.clone(), k.clone()); |
| 306 | let seed = round.wrapping_mul(64).wrapping_add((i as u8).wrapping_mul(2)); |
| 307 | thread::spawn(move || -> Vec<String> { |
| 308 | let mut errs = Vec::new(); |
| 309 | for j in 0..2u8 { |
| 310 | if let Err(e) = db.insert(k.clone(), val(1_100, seed.wrapping_add(j)), Uid::default(), None) { |
| 311 | errs.push(fmt!("{}", e).chars().take(200).collect::<String>()); |
| 312 | } |
| 313 | } |
| 314 | errs |
| 315 | }) |
| 316 | }).collect(); |
| 317 | for w in ws { |
| 318 | match w.join() { |
| 319 | Ok(errs) => for e in errs { bad.push(fmt!("round {} put: {}", round, e)); }, |
| 320 | Err(_) => bad.push(fmt!("round {} put panicked", round)), |
| 321 | } |
| 322 | } |
| 323 | hooks::set_stale_stamp(None); |
| 324 | let mut live = Vec::new(); |
| 325 | for k in &keys { |
| 326 | let got = res!(db.get(k, None)).map(|(v, _)| v); |
| 327 | if !matches!(&got, Some(v) if whole(v, &lens)) { |
| 328 | bad.push(fmt!("round {} live {:?}: not whole", round, k)); |
| 329 | } |
| 330 | live.push(got); |
| 331 | } |
| 332 | let v = res!(verify::verify_live_set(db.api(), None, scan_wait())); |
| 333 | if !v.clean() { bad.push(fmt!("round {} live verify: {}", round, v.summary(4))); } |
| 334 | res!(db.close()); |
| 335 | let db = res!(start(dir, res!(cfg()), false)); |
| 336 | for (k, want) in keys.iter().zip(&live) { |
| 337 | match db.get(k, None) { |
| 338 | Ok(got) => if got.map(|(v, _)| v) != *want { |
| 339 | bad.push(fmt!("round {} {:?}: the restart reads another value than live", round, k)); |
| 340 | }, |
| 341 | Err(e) => bad.push(fmt!("round {} {:?}: after the restart: {}", round, k, e)), |
| 342 | } |
| 343 | } |
| 344 | let v = res!(verify::verify_live_set(db.api(), None, scan_wait())); |
| 345 | if !v.clean() { bad.push(fmt!("round {} restarted verify: {}", round, v.summary(4))); } |
| 346 | let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait())); |
| 347 | if report.orphans_found != 0 { |
| 348 | bad.push(fmt!("round {} sweep: {}", round, report.summary().replace('\n', " | "))); |
| 349 | } |
| 350 | res!(db.close()); |
| 351 | } |
| 352 | let _ = std::fs::remove_dir_all(dir); |
| 353 | assert!(bad.is_empty(), "equal stamps did not settle on one record:\n{}", bad.join("\n")); |
| 354 | Ok(()) |
| 355 | } |
| 356 | |
| 357 | // With one writer a zone's records lie in write order, so of two puts of a key with one stamp the |
| 358 | // later is the later place and must be the value read, live and after a restart. The interleaved |
| 359 | // test above catches a split between the paths only when replay happens to read in another order; |
| 360 | // this one catches a tie broken by anything but place every time. |
| 361 | #[test] |
| 362 | fn equal_stamps_one_writer_later_put_wins() -> Outcome<()> { |
| 363 | let _lock = lock(); |
| 364 | let dir = "./test_db_chunk_set_equal_one"; |
| 365 | let mut c = res!(cfg()); |
| 366 | c.num_wbots_per_zone = 1; |
| 367 | let keys: Vec<Dat> = (0..6u8).map(|i| dat!(fmt!("equal one {}", i))).collect(); |
| 368 | let db = res!(start(dir, c.clone(), true)); |
| 369 | hooks::set_stale_stamp(Some(*res!(Timestamp::now()))); |
| 370 | for (i, k) in keys.iter().enumerate() { |
| 371 | for j in 0..2u8 { |
| 372 | res!(db.insert(k.clone(), val(1_100, (i as u8) * 2 + j), Uid::default(), None)); |
| 373 | } |
| 374 | } |
| 375 | hooks::set_stale_stamp(None); |
| 376 | let mut bad = Vec::new(); |
| 377 | for (i, k) in keys.iter().enumerate() { |
| 378 | if res!(db.get(k, None)).map(|(v, _)| v) != Some(val(1_100, (i as u8) * 2 + 1)) { |
| 379 | bad.push(fmt!("live {:?}: not the later put", k)); |
| 380 | } |
| 381 | } |
| 382 | res!(db.close()); |
| 383 | let db = res!(start(dir, c, false)); |
| 384 | for (i, k) in keys.iter().enumerate() { |
| 385 | if res!(db.get(k, None)).map(|(v, _)| v) != Some(val(1_100, (i as u8) * 2 + 1)) { |
| 386 | bad.push(fmt!("restarted {:?}: not the later put", k)); |
| 387 | } |
| 388 | } |
| 389 | res!(db.close()); |
| 390 | let _ = std::fs::remove_dir_all(dir); |
| 391 | assert!(bad.is_empty(), "a tie of stamps was not broken by place:\n{}", bad.join("\n")); |
| 392 | Ok(()) |
| 393 | } |
| 394 | |
| 395 | // The riskiest surface: a store reads its key back once its bunch key is durable and retires its |
| 396 | // own set if the key names another. Under many concurrent chunked overwrites of a few keys, with |
| 397 | // slow file reads, no store may retire the set its key ends up naming: every key's current head |
| 398 | // fetches whole, the live set verifies clean, before and after a restart and after a sweep. |
| 399 | #[test] |
| 400 | fn store_never_retires_its_own_live_chunks_under_load() -> Outcome<()> { |
| 401 | let _lock = lock(); |
| 402 | let dir = "./test_db_chunk_set_load"; |
| 403 | let lens = vec![1_100usize, 1_500, 2_300]; |
| 404 | let keys: Vec<Dat> = (0..4u8).map(|i| dat!(fmt!("load {}", i))).collect(); |
| 405 | let mut c = res!(cfg()); |
| 406 | c.cache_size_limit_bytes = 1; |
| 407 | let mut bad = Vec::new(); |
| 408 | let mut failed = Vec::new(); |
| 409 | { |
| 410 | let db = res!(start(dir, c.clone(), true)); |
| 411 | hooks::set_read_delay(Duration::from_millis(3)); |
| 412 | hooks::set_chunk_insert_delay(Duration::from_millis(2)); |
| 413 | let ws: Vec<_> = (0..12usize).map(|t| { |
| 414 | let (db, keys, lens) = (db.clone(), keys.clone(), lens.clone()); |
| 415 | thread::spawn(move || -> Vec<String> { |
| 416 | let mut errs = Vec::new(); |
| 417 | for n in 0..10usize { |
| 418 | let k = keys[(t + n) % keys.len()].clone(); |
| 419 | let seed = (t as u8).wrapping_mul(37).wrapping_add(n as u8); |
| 420 | if let Err(e) = db.insert(k, val(lens[(t * 7 + n) % 3], seed), Uid::default(), None) { |
| 421 | errs.push(fmt!("{}", e).chars().take(200).collect::<String>()); |
| 422 | } |
| 423 | } |
| 424 | errs |
| 425 | }) |
| 426 | }).collect(); |
| 427 | for w in ws { |
| 428 | match w.join() { |
| 429 | Ok(errs) => failed.extend(errs), |
| 430 | Err(_) => failed.push(fmt!("panicked")), |
| 431 | } |
| 432 | } |
| 433 | hooks::set_read_delay(Duration::ZERO); |
| 434 | hooks::set_chunk_insert_delay(Duration::ZERO); |
| 435 | res!(check(&db, &keys, &lens, "live", &mut bad)); |
| 436 | res!(db.close()); |
| 437 | let db = res!(start(dir, c.clone(), false)); |
| 438 | res!(check(&db, &keys, &lens, "restarted", &mut bad)); |
| 439 | let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait())); |
| 440 | msg!("load: {}", report.summary().replace('\n', " | ")); |
| 441 | res!(check(&db, &keys, &lens, "swept", &mut bad)); |
| 442 | res!(db.close()); |
| 443 | } |
| 444 | let _ = std::fs::remove_dir_all(dir); |
| 445 | assert!(failed.is_empty(), "puts failed: {:?}", failed); |
| 446 | assert!(bad.is_empty(), "a store retired chunks its key still names: {:?}", bad); |
| 447 | Ok(()) |
| 448 | } |
| 449 | |
| 450 | fn check(db: &TestDb, keys: &[Dat], lens: &[usize], phase: &str, bad: &mut Vec<String>) -> Outcome<()> { |
| 451 | for k in keys { |
| 452 | match res!(head(db, k)) { |
| 453 | Some(t) => if let Err(e) = db.api().fetch_chunks(&Dat::Tup5u64(t), None) { |
| 454 | bad.push(fmt!("{} {:?}: head {:?}: {}", phase, k, t, e).chars().take(300).collect()); |
| 455 | }, |
| 456 | None => bad.push(fmt!("{} {:?}: no head", phase, k)), |
| 457 | } |
| 458 | match db.get(k, None) { |
| 459 | Ok(Some((v, _))) if whole(&v, lens) => (), |
| 460 | other => bad.push(fmt!("{} {:?}: read {:?}", phase, k, |
| 461 | other.map(|o| o.map(|(d, _)| fmt!("{:?}", d).chars().take(80).collect::<String>())))), |
| 462 | } |
| 463 | } |
| 464 | let report = res!(verify::verify_live_set(db.api(), None, scan_wait())); |
| 465 | if !report.clean() { |
| 466 | bad.push(fmt!("{} verify: {}", phase, report.summary(4))); |
| 467 | } |
| 468 | Ok(()) |
| 469 | } |
| 470 | |
| 471 | // A delete is a store with no chunks: its head tombstone is durable before the chunks of the value |
| 472 | // it deletes are retired, so a read made between the two finds the value whole or gone, never a |
| 473 | // current bunch key missing a chunk (fix A, 2026-10-09). Since the durable mark the cache bot |
| 474 | // retires them after the delete has returned, and `set_chunk_tombstone_delay` holds the chunk |
| 475 | // tombstones so that the read lands between the two. `marker` deletes by storing the deleted marker, as the distributed adapters |
| 476 | // erase. |
| 477 | fn read_during_delete(dir: &str, marker: bool) -> Outcome<()> { |
| 478 | let _lock = lock(); |
| 479 | let db = res!(start(dir, res!(cfg()), true)); |
| 480 | let k = dat!("deleted while read"); |
| 481 | let v = val(1_100, 5); // 3 chunks |
| 482 | let tomb = Dat::Usr(oxedyne_fe2o3_o3db_sync::base::id::usr_kind_id_deleted(), Some(Box::new(Dat::Empty))); |
| 483 | res!(db.insert(k.clone(), v.clone(), Uid::default(), None)); |
| 484 | let old = res!(res!(head(&db, &k)).ok_or_else(|| err!("No head for the value."; Test, Missing))); |
| 485 | hooks::set_chunk_tombstone_delay(Duration::from_millis(300)); |
| 486 | let del = { |
| 487 | let (db, k, tomb) = (db.clone(), k.clone(), tomb.clone()); |
| 488 | thread::spawn(move || { |
| 489 | let r = if marker { |
| 490 | db.api().store(k, tomb, Uid::default()).and_then(|resp| resp.recv_store_ack().map(|_| ())) |
| 491 | } else { |
| 492 | db.delete(&k, Uid::default(), None).map(|_| ()) |
| 493 | }; |
| 494 | r.map_err(|e| fmt!("{}", e)) |
| 495 | }) |
| 496 | }; |
| 497 | pause(Duration::from_millis(100)); |
| 498 | let mid = db.get(&k, None); |
| 499 | let del = match del.join() { Ok(r) => r, Err(_) => Err(fmt!("panicked")) }; |
| 500 | let after = db.get(&k, None); |
| 501 | let gone = wait_until(Duration::from_secs(10), || db.api().fetch_chunks(&Dat::Tup5u64(old), None).is_err()); |
| 502 | hooks::set_chunk_tombstone_delay(Duration::ZERO); |
| 503 | res!(db.close()); |
| 504 | let _ = std::fs::remove_dir_all(dir); |
| 505 | assert!(del.is_ok(), "the delete failed: {:?}", del); |
| 506 | match &mid { |
| 507 | Ok(None) => (), |
| 508 | Ok(Some((g, _))) if *g == v || *g == tomb => (), |
| 509 | other => panic!("a read during the delete found neither the value nor its absence: {}", |
| 510 | fmt!("{:?}", other).chars().take(300).collect::<String>()), |
| 511 | } |
| 512 | match &after { |
| 513 | Ok(None) => (), |
| 514 | Ok(Some((g, _))) if *g == tomb => (), |
| 515 | other => panic!("the key does not read as deleted after the delete: {}", |
| 516 | fmt!("{:?}", other).chars().take(300).collect::<String>()), |
| 517 | } |
| 518 | assert!(gone, "the deleted value's chunks were not retired"); |
| 519 | Ok(()) |
| 520 | } |
| 521 | |
| 522 | #[test] |
| 523 | fn read_during_delete_of_chunked_value_is_whole_or_none() -> Outcome<()> { |
| 524 | read_during_delete("./test_db_chunk_set_read_delete", false) |
| 525 | } |
| 526 | |
| 527 | #[test] |
| 528 | fn read_during_marker_store_delete_is_whole_or_none() -> Outcome<()> { |
| 529 | read_during_delete("./test_db_chunk_set_read_marker", true) |
| 530 | } |
| 531 | |
| 532 | // A delete stamped before the value at its key loses to it, and must leave that value whole: the |
| 533 | // delete retires the set it read only when the key no longer names it. |
| 534 | #[test] |
| 535 | fn stale_delete_keeps_the_newer_value() -> Outcome<()> { |
| 536 | let _lock = lock(); |
| 537 | let dir = "./test_db_chunk_set_stale_delete"; |
| 538 | let db = res!(start(dir, res!(cfg()), true)); |
| 539 | let k = dat!("stale delete"); |
| 540 | let v0 = val(1_100, 6); |
| 541 | res!(db.insert(k.clone(), v0.clone(), Uid::default(), None)); |
| 542 | let t0 = match res!(db.get(&k, None)) { |
| 543 | Some((_, meta)) => *meta.time, |
| 544 | None => return Err(err!("v0 is missing."; Test, Missing)), |
| 545 | }; |
| 546 | hooks::set_stale_stamp(Some(t0.saturating_sub(Duration::from_secs(1)))); |
| 547 | let del = db.delete(&k, Uid::default(), None); |
| 548 | hooks::set_stale_stamp(None); |
| 549 | let got = db.get(&k, None); |
| 550 | res!(db.close()); |
| 551 | let _ = std::fs::remove_dir_all(dir); |
| 552 | res!(del); |
| 553 | assert!(matches!(&got, Ok(Some((g, _))) if *g == v0), "a stale delete took the newer value: {}", |
| 554 | fmt!("{:?}", got).chars().take(300).collect::<String>()); |
| 555 | Ok(()) |
| 556 | } |
| 557 | |
| 558 | // Fix B (2026-10-09): a chunk tombstone is forgotten once every older record of its key is durably |
| 559 | // gone, so a store whose chunked values are overwritten does not grow its caches for ever. |
| 560 | |
| 561 | const VLEN: usize = 1_100; // three chunks of 400 |
| 562 | |
| 563 | fn cfg_b() -> Outcome<OzoneConfig> { |
| 564 | let mut c = res!(cfg()); |
| 565 | c.data_file_max_bytes = 4_000; |
| 566 | c.num_wbots_per_zone = 1; // each writer has a live file of its own, so one puts a value in one file |
| 567 | Ok(c) |
| 568 | } |
| 569 | |
| 570 | fn start_gc(dir: &str, wipe: bool) -> Outcome<TestDb> { |
| 571 | log_set_level!("error"); |
| 572 | if wipe { |
| 573 | let _ = std::fs::remove_dir_all(dir); |
| 574 | } |
| 575 | res!(std::fs::create_dir_all(dir)); |
| 576 | let root = res!(Path::new(dir).canonicalize()); |
| 577 | let db = res!(setup::start_db(root, Some(res!(cfg_b())), schemes(), None, true, wipe)); |
| 578 | // Only `compact_now` collects, so a test places its tombstones and holds before any collection, |
| 579 | // and the reports count every collection and deletion. |
| 580 | res!(db.api().activate_gc(false)); |
| 581 | Ok(db) |
| 582 | } |
| 583 | |
| 584 | #[derive(Debug, Default)] |
| 585 | struct Census { |
| 586 | chunks: usize, // chunk-key cache entries |
| 587 | tombs: BTreeMap<Vec<u8>, FileLocation>, // chunk tombstones, where they lie |
| 588 | files: BTreeSet<u32>, // files of the live chunk entries |
| 589 | entries: usize, // all cache entries |
| 590 | anc: usize, // ancillary bytes of every cache |
| 591 | } |
| 592 | |
| 593 | fn census(db: &TestDb) -> Outcome<Census> { |
| 594 | res!(db.api().settle_for_test(Duration::from_secs(20))); |
| 595 | let mut c = Census::default(); |
| 596 | for (_, cache) in res!(db.api().collect_caches(scan_wait())) { |
| 597 | c.entries += cache.map().len(); |
| 598 | c.anc += cache.get_ancillary_size(); |
| 599 | let tombs = cache.tomb_tracker().tombs(); |
| 600 | for (k, e) in cache.map() { |
| 601 | if !is_chunk_key(k) { continue; } |
| 602 | c.chunks += 1; |
| 603 | let CacheEntry::LocatedValue(mloc, _) = e; |
| 604 | match tombs.contains_key(k) { |
| 605 | true => { c.tombs.insert(k.clone(), *mloc.file_location()); }, |
| 606 | false => { c.files.insert(mloc.file_number()); }, |
| 607 | } |
| 608 | } |
| 609 | } |
| 610 | Ok(c) |
| 611 | } |
| 612 | |
| 613 | // Every cache entry is one current record in the file states, and nothing waits on a record. |
| 614 | fn accounted(db: &TestDb, c: &Census, label: &str) -> Outcome<()> { |
| 615 | let mut cur = 0; |
| 616 | for (_, fmap) in res!(db.api().collect_file_states(scan_wait())) { |
| 617 | for (fnum, fstat) in fmap.map() { |
| 618 | assert!(fstat.pending_old_empty(), "{}: file {} has parked supersessions", label, fnum); |
| 619 | cur += fstat.data_map().values().filter(|d| **d == DataState::Cur).count(); |
| 620 | } |
| 621 | } |
| 622 | assert_eq!(cur, c.entries, "{}: current records against cache entries", label); |
| 623 | Ok(()) |
| 624 | } |
| 625 | |
| 626 | // Compacts until quiet: a dropped tombstone is flagged old only after its shadowed records go. |
| 627 | fn quiet(db: &TestDb) -> Outcome<CompactReport> { |
| 628 | let mut sum = CompactReport::default(); |
| 629 | for _ in 0..3 { |
| 630 | res!(db.api().settle_for_test(Duration::from_secs(20))); |
| 631 | let r = res!(db.api().compact_now(Duration::from_secs(60))); |
| 632 | sum.files_collected += r.files_collected; |
| 633 | sum.files_deleted += r.files_deleted; |
| 634 | pause(Duration::from_millis(300)); |
| 635 | } |
| 636 | Ok(sum) |
| 637 | } |
| 638 | |
| 639 | // As `quiet`, with a file held, which keeps its old bytes and so cannot settle. |
| 640 | fn quiet_held(db: &TestDb) { |
| 641 | for _ in 0..2 { |
| 642 | let _ = db.api().settle_for_test(Duration::from_secs(20)); |
| 643 | let _ = db.api().compact_now(Duration::from_secs(3)); |
| 644 | pause(Duration::from_millis(300)); |
| 645 | } |
| 646 | } |
| 647 | |
| 648 | fn filler(db: &TestDb, tag: &str, n: usize, seed: u8) -> Outcome<()> { |
| 649 | for i in 0..n { |
| 650 | res!(db.insert(dat!(fmt!("filler {} {:03}", tag, i)), val(100, seed), Uid::default(), None)); |
| 651 | } |
| 652 | Ok(()) |
| 653 | } |
| 654 | |
| 655 | fn reads(db: &TestDb, k: &Dat, want: &Dat) -> Outcome<bool> { |
| 656 | Ok(matches!(res!(db.get(k, None)), Some((v, _)) if v == *want)) |
| 657 | } |
| 658 | |
| 659 | fn accumulate(dir: &str, n: usize) -> Outcome<(usize, Census, CompactReport)> { |
| 660 | let db = res!(start_gc(dir, true)); |
| 661 | let k = dat!("accumulate"); |
| 662 | res!(db.insert(k.clone(), val(VLEN, 0), Uid::default(), None)); |
| 663 | let base = res!(census(&db)).chunks; |
| 664 | // Each compaction seals the value just written in a file of its own, which goes all-old once |
| 665 | // the next overwrite lands, so its deletion, and not only a collection, reports the records |
| 666 | // a tombstone shadows gone. |
| 667 | let mut rep = CompactReport::default(); |
| 668 | for i in 1..=n { |
| 669 | res!(db.insert(k.clone(), val(VLEN, i as u8), Uid::default(), None)); |
| 670 | let r = res!(db.api().compact_now(Duration::from_secs(60))); |
| 671 | rep.files_collected += r.files_collected; |
| 672 | rep.files_deleted += r.files_deleted; |
| 673 | } |
| 674 | let r = res!(quiet(&db)); |
| 675 | rep.files_collected += r.files_collected; |
| 676 | rep.files_deleted += r.files_deleted; |
| 677 | let c = res!(census(&db)); |
| 678 | let acc = accounted(&db, &c, "accumulate"); |
| 679 | let ok = res!(reads(&db, &k, &val(VLEN, n as u8))); |
| 680 | res!(db.close()); |
| 681 | let _ = std::fs::remove_dir_all(dir); |
| 682 | res!(acc); |
| 683 | assert!(ok, "the last value does not read back"); |
| 684 | Ok((base, c, rep)) |
| 685 | } |
| 686 | |
| 687 | #[test] |
| 688 | fn chunk_tombstones_do_not_accumulate() -> Outcome<()> { |
| 689 | let _lock = lock(); |
| 690 | let (base1, c1, r1) = res!(accumulate("./test_db_chunk_set_accumulate_a", 140)); |
| 691 | let (base2, c2, r2) = res!(accumulate("./test_db_chunk_set_accumulate_b", 280)); |
| 692 | assert!(base1 >= 2, "the value was not chunked"); |
| 693 | assert_eq!(c1.chunks, base1, "140 overwrites leave {} tombstones", c1.tombs.len()); |
| 694 | assert_eq!(c2.chunks, base2, "280 overwrites leave {} tombstones", c2.tombs.len()); |
| 695 | assert!(c2.anc <= c1.anc + 256, "the ancillary size grows with overwrites: {} then {}", c1.anc, c2.anc); |
| 696 | assert!(r1.files_collected + r2.files_collected > 0, "no file was collected"); |
| 697 | assert!(r1.files_deleted + r2.files_deleted > 0, "no all-old file was deleted"); |
| 698 | Ok(()) |
| 699 | } |
| 700 | |
| 701 | // A value, sealed in a file of its own, overwritten with its tombstones in later files. |
| 702 | fn overwritten(db: &TestDb, k: &Dat) -> Outcome<(usize, BTreeSet<u32>)> { |
| 703 | res!(db.insert(k.clone(), val(VLEN, 1), Uid::default(), None)); |
| 704 | let c0 = res!(census(db)); |
| 705 | res!(filler(db, "a", 40, 1)); |
| 706 | res!(db.insert(k.clone(), val(VLEN, 2), Uid::default(), None)); |
| 707 | res!(filler(db, "b", 40, 1)); |
| 708 | Ok((c0.chunks, c0.files)) |
| 709 | } |
| 710 | |
| 711 | #[test] |
| 712 | fn failed_dir_sync_keeps_tombstone() -> Outcome<()> { |
| 713 | let _lock = lock(); |
| 714 | let dir = "./test_db_chunk_set_dir_sync"; |
| 715 | let db = res!(start_gc(dir, true)); |
| 716 | let k = dat!("dir sync"); |
| 717 | let (base, _) = res!(overwritten(&db, &k)); |
| 718 | hooks::set_dir_sync_failure(true); |
| 719 | quiet_held(&db); |
| 720 | let c = census(&db); |
| 721 | let acc = match &c { Ok(c) => accounted(&db, c, "dir sync"), Err(_) => Ok(()) }; |
| 722 | hooks::set_dir_sync_failure(false); |
| 723 | let ok = reads(&db, &k, &val(VLEN, 2)); |
| 724 | res!(db.close()); |
| 725 | let _ = std::fs::remove_dir_all(dir); |
| 726 | let c = res!(c); |
| 727 | res!(acc); |
| 728 | assert!(res!(ok), "the value does not read back"); |
| 729 | assert!(c.tombs.len() >= base, "{} tombstones kept after a failed directory sync, {} wanted", |
| 730 | c.tombs.len(), base); |
| 731 | Ok(()) |
| 732 | } |
| 733 | |
| 734 | #[test] |
| 735 | fn lone_chunk_tombstones_drop_at_restart() -> Outcome<()> { |
| 736 | let _lock = lock(); |
| 737 | let dir = "./test_db_chunk_set_lone"; |
| 738 | let db = res!(start_gc(dir, true)); |
| 739 | let k = dat!("lone"); |
| 740 | let (base, _) = res!(overwritten(&db, &k)); |
| 741 | hooks::set_dir_sync_failure(true); |
| 742 | quiet_held(&db); |
| 743 | let before = census(&db); |
| 744 | hooks::set_dir_sync_failure(false); |
| 745 | res!(db.close()); |
| 746 | let before = res!(before); |
| 747 | let db = res!(start_gc(dir, false)); |
| 748 | let after = census(&db); |
| 749 | let rep = quiet(&db); |
| 750 | let fin = census(&db); |
| 751 | let acc = match &fin { Ok(c) => accounted(&db, c, "lone"), Err(_) => Ok(()) }; |
| 752 | let ok = reads(&db, &k, &val(VLEN, 2)); |
| 753 | res!(db.close()); |
| 754 | let _ = std::fs::remove_dir_all(dir); |
| 755 | let (after, fin) = (res!(after), res!(fin)); |
| 756 | res!(rep); |
| 757 | res!(acc); |
| 758 | assert!(res!(ok), "the value does not read back"); |
| 759 | assert!(before.tombs.len() >= base, "the setup kept no tombstones"); |
| 760 | assert_eq!(after.chunks, base, "lone tombstones survived the restart: {}", after.tombs.len()); |
| 761 | assert_eq!(fin.chunks, base, "tombstones after compaction: {}", fin.tombs.len()); |
| 762 | Ok(()) |
| 763 | } |
| 764 | |
| 765 | #[test] |
| 766 | fn tombstone_outlives_uncollected_record() -> Outcome<()> { |
| 767 | let _lock = lock(); |
| 768 | let dir = "./test_db_chunk_set_outlives"; |
| 769 | let db = res!(start_gc(dir, true)); |
| 770 | let k = dat!("outlives"); |
| 771 | let (base, files) = res!(overwritten(&db, &k)); |
| 772 | assert_eq!(files.len(), 1, "the first value is not in one file: {:?}", files); |
| 773 | let f = files.iter().next().copied(); |
| 774 | hooks::set_collect_fails_for(f); |
| 775 | quiet_held(&db); |
| 776 | let held = census(&db); |
| 777 | res!(db.close()); |
| 778 | let db = res!(start_gc(dir, false)); |
| 779 | let restarted = census(&db); |
| 780 | hooks::set_collect_fails_for(None); |
| 781 | let rep = quiet(&db); |
| 782 | let fin = census(&db); |
| 783 | let acc = match &fin { Ok(c) => accounted(&db, c, "outlives"), Err(_) => Ok(()) }; |
| 784 | let ok = reads(&db, &k, &val(VLEN, 2)); |
| 785 | res!(db.close()); |
| 786 | let _ = std::fs::remove_dir_all(dir); |
| 787 | let (held, restarted, fin) = (res!(held), res!(restarted), res!(fin)); |
| 788 | res!(rep); |
| 789 | res!(acc); |
| 790 | assert!(res!(ok), "the value does not read back"); |
| 791 | assert!(held.tombs.len() >= base, "a tombstone dropped while its record is on disk: {}", held.tombs.len()); |
| 792 | assert!(restarted.tombs.len() >= base, "a tombstone dropped at a restart while its record is on disk: {}", |
| 793 | restarted.tombs.len()); |
| 794 | assert_eq!(fin.chunks, base, "tombstones after the hold: {}", fin.tombs.len()); |
| 795 | Ok(()) |
| 796 | } |
| 797 | |
| 798 | #[test] |
| 799 | fn reanchored_tombstone_still_drops() -> Outcome<()> { |
| 800 | let _lock = lock(); |
| 801 | let dir = "./test_db_chunk_set_reanchored"; |
| 802 | let db = res!(start_gc(dir, true)); |
| 803 | let k = dat!("reanchored"); |
| 804 | res!(db.insert(k.clone(), val(VLEN, 1), Uid::default(), None)); |
| 805 | let c0 = res!(census(&db)); |
| 806 | assert_eq!(c0.files.len(), 1, "the first value is not in one file: {:?}", c0.files); |
| 807 | res!(filler(&db, "a", 40, 1)); |
| 808 | // Records ahead of the tombstones in their file, then made old, so that collecting that file |
| 809 | // carries the tombstones to new offsets. |
| 810 | res!(filler(&db, "b", 8, 1)); |
| 811 | res!(db.insert(k.clone(), val(VLEN, 2), Uid::default(), None)); |
| 812 | res!(filler(&db, "b", 8, 2)); |
| 813 | res!(filler(&db, "c", 40, 1)); |
| 814 | let placed = res!(census(&db)); |
| 815 | hooks::set_collect_fails_for(c0.files.iter().next().copied()); |
| 816 | quiet_held(&db); |
| 817 | let held = census(&db); |
| 818 | hooks::set_collect_fails_for(None); |
| 819 | let rep = quiet(&db); |
| 820 | let fin = census(&db); |
| 821 | let acc = match &fin { Ok(c) => accounted(&db, c, "reanchored"), Err(_) => Ok(()) }; |
| 822 | let ok = reads(&db, &k, &val(VLEN, 2)); |
| 823 | res!(db.close()); |
| 824 | let _ = std::fs::remove_dir_all(dir); |
| 825 | let (held, fin) = (res!(held), res!(fin)); |
| 826 | res!(rep); |
| 827 | res!(acc); |
| 828 | assert!(res!(ok), "the value does not read back"); |
| 829 | let moved = placed.tombs.iter().filter(|(t, floc)| match held.tombs.get(*t) { |
| 830 | Some(now) => now.start != floc.start, |
| 831 | None => false, |
| 832 | }).count(); |
| 833 | assert!(moved > 0, "no tombstone was carried by a collection: {:?} then {:?}", placed.tombs, held.tombs); |
| 834 | assert_eq!(fin.chunks, c0.chunks, "re-anchored tombstones did not drop: {}", fin.tombs.len()); |
| 835 | Ok(()) |
| 836 | } |
| 837 | |
| 838 | #[test] |
| 839 | fn legacy_tombstone_waits_for_every_older_record() -> Outcome<()> { |
| 840 | let _lock = lock(); |
| 841 | let dir = "./test_db_chunk_set_legacy_wait"; |
| 842 | let db = res!(start_gc(dir, true)); |
| 843 | let k = dat!("legacy wait"); |
| 844 | let (kbuf, _, _) = res!(db.api().ozone_key_dat(&k, None)); |
| 845 | let legacy = OzoneApi::<{ UID_LEN }, Uid, (), HashScheme, HashScheme, ChecksumScheme>::legacy_chunk_set_id(&kbuf); |
| 846 | let mut first = BTreeSet::new(); |
| 847 | for (seed, tag) in [(1u8, "a"), (2u8, "b")] { |
| 848 | let resp = db.api().responder(); |
| 849 | res!(db.api().store_dat_using_responder_forcing_set_id( |
| 850 | k.clone(), val(VLEN, seed), Uid::default(), None, resp.clone(), legacy)); |
| 851 | res!(resp.recv_store_ack()); |
| 852 | if seed == 1 { |
| 853 | first = res!(census(&db)).files; |
| 854 | } |
| 855 | res!(filler(&db, tag, 40, seed)); |
| 856 | } |
| 857 | let old = res!(res!(head(&db, &k)).ok_or_else(|| err!("No legacy head."; Test, Missing))); |
| 858 | let c0 = res!(census(&db)); |
| 859 | assert_eq!(first.len(), 1, "the first legacy value is not in one file: {:?}", first); |
| 860 | assert!(c0.files.is_disjoint(&first), "the two legacy values share a file"); |
| 861 | let f1 = first.iter().next().copied(); |
| 862 | res!(db.insert(k.clone(), val(VLEN, 3), Uid::default(), None)); |
| 863 | res!(filler(&db, "c", 40, 3)); |
| 864 | hooks::set_collect_fails_for(f1); |
| 865 | quiet_held(&db); |
| 866 | let held = census(&db); |
| 867 | res!(db.close()); |
| 868 | let db = res!(start_gc(dir, false)); |
| 869 | let restarted = census(&db); |
| 870 | let report = sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait()); |
| 871 | let gone = db.api().fetch_chunks(&Dat::Tup5u64(old), None).is_err(); |
| 872 | hooks::set_collect_fails_for(None); |
| 873 | let rep = quiet(&db); |
| 874 | let fin = census(&db); |
| 875 | let ok = reads(&db, &k, &val(VLEN, 3)); |
| 876 | res!(db.close()); |
| 877 | let _ = std::fs::remove_dir_all(dir); |
| 878 | let (held, restarted, fin, report) = (res!(held), res!(restarted), res!(fin), res!(report)); |
| 879 | res!(rep); |
| 880 | assert!(res!(ok), "the value does not read back"); |
| 881 | assert!(held.tombs.len() >= c0.chunks, "a legacy tombstone dropped with an older record on disk: {}", |
| 882 | held.tombs.len()); |
| 883 | assert!(restarted.tombs.len() >= c0.chunks, "a legacy tombstone dropped at the restart: {}", |
| 884 | restarted.tombs.len()); |
| 885 | assert_eq!(report.orphans_found, 0, "a retired legacy chunk came back"); |
| 886 | assert!(gone, "the legacy value reads back after its retirement"); |
| 887 | assert_eq!(fin.chunks, c0.chunks, "tombstones after the hold: {}", fin.tombs.len()); |
| 888 | Ok(()) |
| 889 | } |
| 890 | |
| 891 | // A record reported gone before the replay ends drops no tombstone, since an older record of its |
| 892 | // key may lie in a file not yet replayed. Driven on a cache directly: the start sequence lets no |
| 893 | // collection begin before the zone bot sends `ReplayDone`, so no store can reach this order. |
| 894 | #[test] |
| 895 | fn no_tombstone_drops_before_replay_done() -> Outcome<()> { |
| 896 | type C = Cache<{ UID_LEN }, Uid>; |
| 897 | let mut cache = C::new(None); |
| 898 | let k = res!(Dat::Tup5u64([1, 2, 3, 4, 5]).to_bytes(Vec::new())); |
| 899 | assert!(is_chunk_key(&k), "the key is not a chunk key"); |
| 900 | let meta = |s: u64| Meta::<{ UID_LEN }, Uid> { time: Timestamp::new(s, 0), user: Uid::default() }; |
| 901 | let floc = |f: u32| FileLocation { fnum: f, start: 0, klen: k.len() as u64, vlen: 10 }; |
| 902 | let put = |cache: &mut C, cind: Option<usize>, f: u32, s: u64| -> Outcome<()> { |
| 903 | let sup = res!(cache.insert(k.clone(), None, cind, floc(f), meta(s))); |
| 904 | cache.note(&k, cind, &floc(f), &meta(s), &sup); |
| 905 | Ok(()) |
| 906 | }; |
| 907 | let current = |cache: &C| match cache.map().get(&k) { |
| 908 | Some(CacheEntry::LocatedValue(mloc, _)) => Some(mloc.meta().time.secs()), |
| 909 | _ => None, |
| 910 | }; |
| 911 | // The replay meets a chunk record in file 1, then the tombstone in file 3; file 2 waits. |
| 912 | res!(put(&mut cache, Some(0), 1, 1)); |
| 913 | res!(put(&mut cache, None, 3, 3)); |
| 914 | let early = res!(cache.records_gone(1, &[res!(RecordDigest::new(&k, &meta(1)))])); |
| 915 | assert!(early.is_empty(), "a tombstone dropped before the replay ended: {:?}", early); |
| 916 | // File 2's record of the key arrives late, and must not become current. |
| 917 | res!(put(&mut cache, Some(0), 2, 2)); |
| 918 | assert_eq!(current(&cache), Some(3), "a record older than the tombstone became current"); |
| 919 | let at_end = res!(cache.replay_done()); |
| 920 | assert!(at_end.is_empty(), "the tombstone dropped with file 2's record unaccounted for"); |
| 921 | let last = res!(cache.records_gone(2, &[res!(RecordDigest::new(&k, &meta(2)))])); |
| 922 | assert_eq!(last.len(), 1, "the tombstone did not drop once its last older record went"); |
| 923 | assert_eq!(current(&cache), None, "the dropped tombstone is still in the cache"); |
| 924 | Ok(()) |
| 925 | } |
| 926 | |
| 927 | // Run 2 (2026-10-10): a store retires the set of the head it displaced, whatever order |
| 928 | // overlapping puts and deletes land in, so live chunk records are bounded by the current heads |
| 929 | // with no sweep. |
| 930 | |
| 931 | // Live chunk records, tombstones aside. |
| 932 | fn live_chunks(db: &TestDb) -> Outcome<usize> { |
| 933 | res!(db.api().settle_for_test(Duration::from_secs(20))); |
| 934 | let mut n = 0; |
| 935 | for (_, cache) in res!(db.api().collect_caches(scan_wait())) { |
| 936 | let tombs = cache.tomb_tracker().tombs(); |
| 937 | for (k, e) in cache.map() { |
| 938 | if is_chunk_key(k) && matches!(e, CacheEntry::LocatedValue(..)) && !tombs.contains_key(k) { |
| 939 | n += 1; |
| 940 | } |
| 941 | } |
| 942 | } |
| 943 | Ok(n) |
| 944 | } |
| 945 | |
| 946 | // Chunks the current heads name. |
| 947 | fn named(db: &TestDb, keys: &[Dat]) -> Outcome<usize> { |
| 948 | let mut n = 0; |
| 949 | for k in keys { |
| 950 | if let Some(t) = res!(head(db, k)) { |
| 951 | n += t[3] as usize; |
| 952 | } |
| 953 | } |
| 954 | Ok(n) |
| 955 | } |
| 956 | |
| 957 | fn whole_or_none(r: &Outcome<Option<(Dat, Meta<{ UID_LEN }, Uid>)>>) -> bool { |
| 958 | match r { |
| 959 | Ok(None) => true, |
| 960 | Ok(Some((Dat::Usr(..), _))) => true, // a tombstone |
| 961 | Ok(Some((v, _))) => whole(v, &[VLEN]), |
| 962 | Err(_) => false, |
| 963 | } |
| 964 | } |
| 965 | |
| 966 | // Puts B, A and C and a delete of each key, staggered and slowed so that they overlap. |
| 967 | fn overlap_round(db: &TestDb, keys: &[Dat], r: usize, bad: &mut Vec<String>) -> Outcome<()> { |
| 968 | hooks::set_chunk_insert_delay(Duration::from_millis(40)); |
| 969 | let mut ops = Vec::new(); |
| 970 | for (i, k) in keys.iter().enumerate() { |
| 971 | // The delete lands third in even rounds and last in odd ones. |
| 972 | let del_at = if r % 2 == 0 { 30 } else { 45 }; |
| 973 | for (j, (at, put)) in [(0u64, true), (15, true), (del_at, false), (75 - del_at, true)].into_iter().enumerate() { |
| 974 | let (db, k) = (db.clone(), k.clone()); |
| 975 | let seed = ((r * 16 + i * 4 + j) % 251) as u8; |
| 976 | ops.push(thread::spawn(move || { |
| 977 | thread::sleep(Duration::from_millis(at)); |
| 978 | let res = match put { |
| 979 | true => db.insert(k, val(VLEN, seed), Uid::default(), None).map(|_| ()), |
| 980 | false => db.delete(&k, Uid::default(), None).map(|_| ()), |
| 981 | }; |
| 982 | res.map_err(|e| fmt!("{}", e)) |
| 983 | })); |
| 984 | } |
| 985 | } |
| 986 | for _ in 0..4 { |
| 987 | for k in keys { |
| 988 | let got = db.get(k, None); |
| 989 | if !whole_or_none(&got) { |
| 990 | bad.push(fmt!("round {}: a read mid-round was not whole: {:?}", r, |
| 991 | got.map(|o| o.map(|(d, _)| fmt!("{:?}", d).chars().take(80).collect::<String>())))); |
| 992 | } |
| 993 | } |
| 994 | pause(Duration::from_millis(20)); |
| 995 | } |
| 996 | for op in ops { |
| 997 | match op.join() { |
| 998 | Ok(Ok(())) => (), |
| 999 | Ok(Err(e)) => bad.push(fmt!("round {}: {}", r, e.chars().take(200).collect::<String>())), |
| 1000 | Err(_) => bad.push(fmt!("round {}: an op panicked", r)), |
| 1001 | } |
| 1002 | } |
| 1003 | hooks::set_chunk_insert_delay(Duration::ZERO); |
| 1004 | for k in keys { |
| 1005 | let got = db.get(k, None); |
| 1006 | if !whole_or_none(&got) { |
| 1007 | bad.push(fmt!("round {}: {:?} does not read whole after the round", r, k)); |
| 1008 | } |
| 1009 | } |
| 1010 | Ok(()) |
| 1011 | } |
| 1012 | |
| 1013 | #[test] |
| 1014 | fn overlapping_writes_leave_no_live_orphan() -> Outcome<()> { |
| 1015 | let _lock = lock(); |
| 1016 | let dir = "./test_db_chunk_set_overlap_bound"; |
| 1017 | let keys: Vec<Dat> = (0..3).map(|i| dat!(fmt!("overlap bound {}", i))).collect(); |
| 1018 | let mut bad = Vec::new(); |
| 1019 | let db = res!(start_gc(dir, true)); |
| 1020 | for r in 0..20 { |
| 1021 | res!(overlap_round(&db, &keys, r, &mut bad)); |
| 1022 | } |
| 1023 | // A round can end in a delete, so every key is given a value before the restart: each then |
| 1024 | // replays a head whose bytes are not resident, which the first write over it displaces. |
| 1025 | for (i, k) in keys.iter().enumerate() { |
| 1026 | res!(db.insert(k.clone(), val(VLEN, 0xA0 + i as u8), Uid::default(), None)); |
| 1027 | } |
| 1028 | res!(quiet(&db)); |
| 1029 | let (live1, named1) = (res!(live_chunks(&db)), res!(named(&db, &keys))); |
| 1030 | res!(db.close()); |
| 1031 | let db = res!(start_gc(dir, false)); |
| 1032 | for r in 20..22 { |
| 1033 | res!(overlap_round(&db, &keys, r, &mut bad)); |
| 1034 | } |
| 1035 | res!(quiet(&db)); |
| 1036 | let (live2, named2) = (res!(live_chunks(&db)), res!(named(&db, &keys))); |
| 1037 | res!(db.close()); |
| 1038 | let _ = std::fs::remove_dir_all(dir); |
| 1039 | msg!("overlap bound: live {} named {}, after a restart live {} named {}", live1, named1, live2, named2); |
| 1040 | assert!(bad.is_empty(), "{:#?}", bad); |
| 1041 | assert_eq!(live1, named1, "live chunk records against those the current heads name"); |
| 1042 | assert_eq!(live2, named2, "after a restart, live chunk records against those the current heads name"); |
| 1043 | Ok(()) |
| 1044 | } |
| 1045 | |
| 1046 | // Two forced values under one legacy set, of one length: the second overwrites the first's |
| 1047 | // chunks in place, and its head must retire nothing, or it retires its own chunks. |
| 1048 | #[test] |
| 1049 | fn forced_legacy_overwrite_retires_nothing() -> Outcome<()> { |
| 1050 | let _lock = lock(); |
| 1051 | let dir = "./test_db_chunk_set_forced_legacy"; |
| 1052 | let db = res!(start(dir, res!(cfg()), true)); |
| 1053 | let k = dat!("forced legacy"); |
| 1054 | let (kbuf, _, _) = res!(db.api().ozone_key_dat(&k, None)); |
| 1055 | let legacy = OzoneApi::<{ UID_LEN }, Uid, (), HashScheme, HashScheme, ChecksumScheme>::legacy_chunk_set_id(&kbuf); |
| 1056 | for seed in [1u8, 2] { |
| 1057 | let resp = db.api().responder(); |
| 1058 | res!(db.api().store_dat_using_responder_forcing_set_id( |
| 1059 | k.clone(), val(1_100, seed), Uid::default(), None, resp.clone(), legacy)); |
| 1060 | res!(resp.recv_store_ack()); |
| 1061 | } |
| 1062 | let live = res!(db.get(&k, None)).map(|(v, _)| v); |
| 1063 | res!(db.close()); |
| 1064 | let db = res!(start(dir, res!(cfg()), false)); |
| 1065 | let after = res!(db.get(&k, None)).map(|(v, _)| v); |
| 1066 | let v = res!(verify::verify_live_set(db.api(), None, scan_wait())); |
| 1067 | res!(db.close()); |
| 1068 | let _ = std::fs::remove_dir_all(dir); |
| 1069 | assert!(live == Some(val(1_100, 2)), "the second forced value does not read whole live"); |
| 1070 | assert!(after == Some(val(1_100, 2)), "the second forced value does not read whole after a restart"); |
| 1071 | assert!(v.clean(), "verify after restart: {}", v.summary(4)); |
| 1072 | Ok(()) |
| 1073 | } |