oxedyne/fe2o3/fe2o3_o3db_sync/tests/compact_now.rs
29.5 KiB, 7 runs
created by r1870400018:67597, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! `compact_now` erases (A3 R3, 2026-10-08): once it returns `Ok`, no data file holds a record that |
| 2 | //! a write acknowledged before the call superseded or deleted. Every test counts the bytes of the |
| 3 | //! files from outside the store, with a needle that is a whole record's value, and finds it first |
| 4 | //! (the positive control) so that a needle that is never there cannot pass. The hooks used are |
| 5 | //! process-wide, so every test takes one lock and puts each hook back. |
| 6 | |
| 7 | mod gc_pair; |
| 8 | |
| 9 | use gc_pair::*; |
| 10 | |
| 11 | use oxedyne_fe2o3_core::prelude::*; |
| 12 | use oxedyne_fe2o3_iop_db::api::Database; |
| 13 | use oxedyne_fe2o3_jdat::{ |
| 14 | prelude::*, |
| 15 | chunk::PartKey, |
| 16 | }; |
| 17 | use oxedyne_fe2o3_o3db_sync::{ |
| 18 | api::CompactReport, |
| 19 | base::{ |
| 20 | cfg::OzoneConfig, |
| 21 | constant, |
| 22 | }, |
| 23 | comm::response::Wait, |
| 24 | file::core::FileType, |
| 25 | test::{ |
| 26 | hooks, |
| 27 | setup::{ |
| 28 | self, |
| 29 | Uid, |
| 30 | }, |
| 31 | }, |
| 32 | }; |
| 33 | |
| 34 | use std::{ |
| 35 | collections::BTreeMap, |
| 36 | path::{ |
| 37 | Path, |
| 38 | PathBuf, |
| 39 | }, |
| 40 | sync::{ |
| 41 | Arc, |
| 42 | Mutex, |
| 43 | MutexGuard, |
| 44 | atomic::{ |
| 45 | AtomicBool, |
| 46 | AtomicU64, |
| 47 | Ordering, |
| 48 | }, |
| 49 | }, |
| 50 | thread, |
| 51 | time::{ |
| 52 | Duration, |
| 53 | Instant, |
| 54 | }, |
| 55 | }; |
| 56 | |
| 57 | static HOOKS: Mutex<()> = Mutex::new(()); |
| 58 | |
| 59 | fn lock() -> MutexGuard<'static, ()> { |
| 60 | match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() } |
| 61 | } |
| 62 | |
| 63 | const GC_ON: bool = true; |
| 64 | const GC_OFF: bool = false; |
| 65 | |
| 66 | // A store of the plain kind: records of one length, five to a file. |
| 67 | struct Plain { |
| 68 | dir: String, |
| 69 | root: PathBuf, |
| 70 | cfg: OzoneConfig, |
| 71 | probe: Probe, |
| 72 | db: TestDb, |
| 73 | } |
| 74 | |
| 75 | fn plain(name: &str, gc_on: bool, cbots: u16, fbots: u16) -> Outcome<Plain> { |
| 76 | log_set_level!("error"); |
| 77 | let dir = fmt!("./test_db_compact_{}", name); |
| 78 | let _ = std::fs::remove_dir_all(&dir); |
| 79 | res!(std::fs::create_dir_all(&dir)); |
| 80 | let root = res!(Path::new(&dir).canonicalize()); |
| 81 | let probe = res!(probe(&fmt!("./test_db_compact_{}_probe", name))); |
| 82 | let mut cfg = res!(config(5 * probe.len + probe.len / 2)); |
| 83 | cfg.num_cbots_per_zone = cbots; |
| 84 | cfg.num_fbots_per_zone = fbots; |
| 85 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, gc_on, true)); |
| 86 | Ok(Plain { dir, root, cfg, probe, db }) |
| 87 | } |
| 88 | |
| 89 | impl Plain { |
| 90 | // How many data files hold the whole value of key `i` at version `ver`. |
| 91 | fn holds(&self, i: usize, ver: u8) -> usize { |
| 92 | let needle = match value(i, ver).bytes_ref() { |
| 93 | Some(b) => b.clone(), |
| 94 | None => Vec::new(), |
| 95 | }; |
| 96 | assert!(!needle.is_empty(), "the needle of key {} version {} is empty", i, ver); |
| 97 | files_holding(&self.root, &self.cfg, &needle).len() |
| 98 | } |
| 99 | |
| 100 | fn end(self) -> Outcome<()> { |
| 101 | res!(self.db.close()); |
| 102 | let _ = std::fs::remove_dir_all(&self.dir); |
| 103 | Ok(()) |
| 104 | } |
| 105 | } |
| 106 | |
| 107 | /// Reads every key and counts the answers that are not what was last written: a version, or `None` |
| 108 | /// for a key that was deleted. |
| 109 | fn reads(db: &TestDb, want: &[Option<u8>], label: &str) -> usize { |
| 110 | let mut bad = 0; |
| 111 | for (i, w) in want.iter().enumerate() { |
| 112 | match (db.get(&key(i), None), w) { |
| 113 | (Ok(Some((v, _))), Some(ver)) if v == value(i, *ver) => (), |
| 114 | (Ok(None), None) => (), |
| 115 | (got, _) => { |
| 116 | bad += 1; |
| 117 | let s: String = fmt!("{:?}", got.map(|o| o.map(|(d, _)| d))).chars().take(200).collect(); |
| 118 | msg!("{}: key {} wanted {:?} and read {}", label, i, w, s); |
| 119 | }, |
| 120 | } |
| 121 | } |
| 122 | bad |
| 123 | } |
| 124 | |
| 125 | fn small(label: &str, found: usize) -> String { |
| 126 | fmt!("{}: {} data files still hold it", label, found) |
| 127 | } |
| 128 | |
| 129 | #[test] |
| 130 | fn deleted_value_leaves_every_dat() -> Outcome<()> { |
| 131 | let _lock = lock(); |
| 132 | let s = res!(plain("delete", GC_ON, 1, 1)); |
| 133 | let user = Uid::default(); |
| 134 | res!(fill(&s.db)); |
| 135 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 136 | |
| 137 | // Key 2 is in sealed file 1, key 11 in the live file. |
| 138 | for i in [2usize, 11] { |
| 139 | assert!(s.holds(i, 1) >= 1, "positive control: key {} version 1 is in a data file", i); |
| 140 | assert!(res!(s.db.delete(&key(i), user, None)), "the delete of key {} found nothing", i); |
| 141 | } |
| 142 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 143 | thread::sleep(Duration::from_millis(500)); |
| 144 | // One old record is under the trigger, and the live file is never collected, so the delete |
| 145 | // alone has erased nothing. |
| 146 | for i in [2usize, 11] { |
| 147 | assert!(s.holds(i, 1) >= 1, "positive control: the delete of key {} did not erase it by itself", i); |
| 148 | } |
| 149 | |
| 150 | let report = res!(s.db.compact_now(Duration::from_secs(30))); |
| 151 | msg!("{:?}", report); |
| 152 | for i in [2usize, 11] { |
| 153 | assert_eq!(s.holds(i, 1), 0, "{}", small(&fmt!("key {} version 1 after compact_now", i), s.holds(i, 1))); |
| 154 | } |
| 155 | assert!(report.bytes_after < report.bytes_before, "compact_now removed nothing: {:?}", report); |
| 156 | let mut want: Vec<Option<u8>> = vec![Some(1); NKEYS]; |
| 157 | want[2] = None; |
| 158 | want[11] = None; |
| 159 | assert_eq!(reads(&s.db, &want, "After compact_now"), 0); |
| 160 | |
| 161 | // Reopened, so that what the files hold is what is read. |
| 162 | let (root, cfg) = (s.root.clone(), s.cfg.clone()); |
| 163 | res!(s.db.close()); |
| 164 | let db: TestDb = res!(setup::start_db(root, Some(cfg), schemes(), None, GC_ON, false)); |
| 165 | assert_eq!(reads(&db, &want, "After reopening"), 0); |
| 166 | res!(db.close()); |
| 167 | let _ = std::fs::remove_dir_all(&s.dir); |
| 168 | Ok(()) |
| 169 | } |
| 170 | |
| 171 | #[test] |
| 172 | fn overwritten_value_leaves_every_dat() -> Outcome<()> { |
| 173 | let _lock = lock(); |
| 174 | let s = res!(plain("overwrite", GC_ON, 1, 1)); |
| 175 | let user = Uid::default(); |
| 176 | res!(fill(&s.db)); |
| 177 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 178 | |
| 179 | // One record in each sealed file, each under the trigger. |
| 180 | let mut want: Vec<Option<u8>> = vec![Some(1); NKEYS]; |
| 181 | for i in [3usize, 7] { |
| 182 | assert!(s.holds(i, 1) >= 1, "positive control: key {} version 1 is in a data file", i); |
| 183 | res!(s.db.insert(key(i), value(i, 2), user, None)); |
| 184 | want[i] = Some(2); |
| 185 | } |
| 186 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 187 | thread::sleep(Duration::from_millis(500)); |
| 188 | for i in [3usize, 7] { |
| 189 | assert!(s.holds(i, 1) >= 1, "positive control: the overwrite of key {} did not erase it by itself", i); |
| 190 | } |
| 191 | |
| 192 | let report = res!(s.db.compact_now(Duration::from_secs(30))); |
| 193 | msg!("{:?}", report); |
| 194 | for i in [3usize, 7] { |
| 195 | assert_eq!(s.holds(i, 1), 0, "{}", small(&fmt!("key {} version 1 after compact_now", i), s.holds(i, 1))); |
| 196 | assert!(s.holds(i, 2) >= 1, "the new value of key {} is in no data file", i); |
| 197 | } |
| 198 | assert_eq!(reads(&s.db, &want, "After compact_now"), 0); |
| 199 | s.end() |
| 200 | } |
| 201 | |
| 202 | #[test] |
| 203 | fn compact_collects_with_gc_switched_off() -> Outcome<()> { |
| 204 | let _lock = lock(); |
| 205 | let s = res!(plain("gc_off", GC_OFF, 1, 1)); |
| 206 | let user = Uid::default(); |
| 207 | res!(fill(&s.db)); |
| 208 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 209 | assert!(s.holds(3, 1) >= 1, "positive control: key 3 version 1 is in a data file"); |
| 210 | res!(s.db.insert(key(3), value(3, 2), user, None)); |
| 211 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 212 | |
| 213 | // The switch is off and the trigger is not met, so nothing but the order can collect. |
| 214 | thread::sleep(Duration::from_millis(1_000)); |
| 215 | assert!(s.holds(3, 1) >= 1, "positive control: nothing collected with the collector off"); |
| 216 | |
| 217 | let report = res!(s.db.compact_now(Duration::from_secs(30))); |
| 218 | msg!("{:?}", report); |
| 219 | assert_eq!(s.holds(3, 1), 0, "{}", small("key 3 version 1 after compact_now", s.holds(3, 1))); |
| 220 | assert!(report.files_collected + report.files_deleted >= 1, "nothing was collected: {:?}", report); |
| 221 | let mut want: Vec<Option<u8>> = vec![Some(1); NKEYS]; |
| 222 | want[3] = Some(2); |
| 223 | assert_eq!(reads(&s.db, &want, "After compact_now"), 0); |
| 224 | s.end() |
| 225 | } |
| 226 | |
| 227 | // The chunked store: three chunks of 400 bytes in a value of 1,100. |
| 228 | const CHUNKS: u64 = 3; |
| 229 | |
| 230 | fn chunk_cfg() -> Outcome<OzoneConfig> { |
| 231 | let mut c = res!(setup::default_cfg()); |
| 232 | c.num_zones = 1; |
| 233 | c.num_cbots_per_zone = 4; |
| 234 | c.num_fbots_per_zone = 1; |
| 235 | c.num_wbots_per_zone = 2; |
| 236 | c.zone_overrides = BTreeMap::new(); |
| 237 | c.rest_chunk_threshold = 1_000; |
| 238 | c.rest_chunk_bytes = 400; |
| 239 | c.sync_on_write = true; |
| 240 | Ok(c) |
| 241 | } |
| 242 | |
| 243 | fn big(seed: u8) -> Vec<u8> { |
| 244 | (0..1_100usize).map(|j| seed ^ (j as u8)).collect() |
| 245 | } |
| 246 | |
| 247 | // A hundred bytes from the middle of each chunk, so that a chunk is found by its own bytes. |
| 248 | fn windows(seed: u8) -> Vec<Vec<u8>> { |
| 249 | let v = big(seed); |
| 250 | vec![v[100..200].to_vec(), v[500..600].to_vec(), v[900..1_000].to_vec()] |
| 251 | } |
| 252 | |
| 253 | struct Chunked { |
| 254 | dir: String, |
| 255 | root: PathBuf, |
| 256 | cfg: OzoneConfig, |
| 257 | db: TestDb, |
| 258 | } |
| 259 | |
| 260 | fn chunked(name: &str, gc_on: bool) -> Outcome<Chunked> { |
| 261 | log_set_level!("error"); |
| 262 | let dir = fmt!("./test_db_compact_{}", name); |
| 263 | let _ = std::fs::remove_dir_all(&dir); |
| 264 | res!(std::fs::create_dir_all(&dir)); |
| 265 | let root = res!(Path::new(&dir).canonicalize()); |
| 266 | let cfg = res!(chunk_cfg()); |
| 267 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, gc_on, true)); |
| 268 | Ok(Chunked { dir, root, cfg, db }) |
| 269 | } |
| 270 | |
| 271 | impl Chunked { |
| 272 | fn holds(&self, seed: u8) -> Vec<usize> { |
| 273 | windows(seed).iter().map(|w| files_holding(&self.root, &self.cfg, w).len()).collect() |
| 274 | } |
| 275 | |
| 276 | fn num_chunks(&self, k: &Dat) -> Outcome<u64> { |
| 277 | let api = self.db.api(); |
| 278 | let resp = res!(api.fetch_using_schemes(k, None)); |
| 279 | match res!(resp.recv_daticle(api.schemes().encrypter(), None)) { |
| 280 | (Some((Dat::Tup5u64(tup), _)), _) => Ok(PartKey(tup).num_parts()), |
| 281 | other => Err(err!("Key {:?} holds {:?}, not a bunch key.", k, other; Test, Invalid)), |
| 282 | } |
| 283 | } |
| 284 | } |
| 285 | |
| 286 | #[test] |
| 287 | fn chunked_delete_leaves_every_dat() -> Outcome<()> { |
| 288 | let _lock = lock(); |
| 289 | let s = res!(chunked("chunked", GC_OFF)); |
| 290 | let user = Uid::default(); |
| 291 | // A value that stays, in the same file, so that the file is collected and not merely removed. |
| 292 | res!(s.db.insert(key(0), value(0, 9), user, None)); |
| 293 | let k = dat!("chunked erase"); |
| 294 | res!(s.db.insert(k.clone(), Dat::BU32(big(5)), user, None)); |
| 295 | assert_eq!(res!(s.num_chunks(&k)), CHUNKS, "the value did not split into {} chunks", CHUNKS); |
| 296 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 297 | assert!(s.holds(5).iter().all(|n| *n >= 1), "positive control: every chunk is in a data file: {:?}", s.holds(5)); |
| 298 | |
| 299 | hooks::set_chunk_tombstone_delay(Duration::from_millis(300)); |
| 300 | let existed = s.db.delete(&k, user, None); |
| 301 | hooks::set_chunk_tombstone_delay(Duration::ZERO); |
| 302 | assert!(res!(existed), "the delete found nothing"); |
| 303 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 304 | assert!(s.holds(5).iter().all(|n| *n >= 1), "positive control: the delete did not erase the chunks by itself"); |
| 305 | |
| 306 | let report = res!(s.db.compact_now(Duration::from_secs(30))); |
| 307 | msg!("{:?}", report); |
| 308 | assert_eq!(s.holds(5), vec![0, 0, 0], "chunks still in data files after compact_now"); |
| 309 | match res!(s.db.get(&key(0), None)) { |
| 310 | Some((v, _)) => assert_eq!(v, value(0, 9), "the value beside it changed"), |
| 311 | None => return Err(err!("The value that was not deleted is gone."; Test, Missing)), |
| 312 | } |
| 313 | assert!(res!(s.db.get(&k, None)).is_none(), "the deleted value reads back"); |
| 314 | res!(s.db.close()); |
| 315 | let _ = std::fs::remove_dir_all(&s.dir); |
| 316 | Ok(()) |
| 317 | } |
| 318 | |
| 319 | #[test] |
| 320 | fn tombstone_store_leaves_every_dat() -> Outcome<()> { |
| 321 | let _lock = lock(); |
| 322 | let s = res!(chunked("tombstone", GC_OFF)); |
| 323 | let user = Uid::default(); |
| 324 | res!(s.db.insert(key(0), value(0, 9), user, None)); |
| 325 | let k = dat!("tombstone erase"); |
| 326 | res!(s.db.insert(k.clone(), Dat::BU32(big(6)), user, None)); |
| 327 | assert_eq!(res!(s.num_chunks(&k)), CHUNKS, "the value did not split into {} chunks", CHUNKS); |
| 328 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 329 | assert!(s.holds(6).iter().all(|n| *n >= 1), "positive control: every chunk is in a data file: {:?}", s.holds(6)); |
| 330 | |
| 331 | // The erase of the distributed adapters: a store of the deleted marker, not a delete. |
| 332 | let tomb = Dat::Usr(oxedyne_fe2o3_o3db_sync::base::id::usr_kind_id_deleted(), Some(Box::new(Dat::Empty))); |
| 333 | let resp = res!(s.db.api().store(k.clone(), tomb.clone(), user)); |
| 334 | res!(resp.recv_store_ack()); |
| 335 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 336 | assert!(s.holds(6).iter().any(|n| *n >= 1), "positive control: the store alone left no chunk to erase"); |
| 337 | |
| 338 | let report = res!(s.db.compact_now(Duration::from_secs(30))); |
| 339 | msg!("{:?}", report); |
| 340 | assert_eq!(s.holds(6), vec![0, 0, 0], "chunks still in data files after compact_now"); |
| 341 | match res!(s.db.api().get_wait(&k, None)) { |
| 342 | Some((d, _)) => assert_eq!(d, tomb, "the key does not read back as the deleted marker"), |
| 343 | None => (), |
| 344 | } |
| 345 | match res!(s.db.get(&key(0), None)) { |
| 346 | Some((v, _)) => assert_eq!(v, value(0, 9), "the value beside it changed"), |
| 347 | None => return Err(err!("The value that was not deleted is gone."; Test, Missing)), |
| 348 | } |
| 349 | res!(s.db.close()); |
| 350 | let _ = std::fs::remove_dir_all(&s.dir); |
| 351 | Ok(()) |
| 352 | } |
| 353 | |
| 354 | #[test] |
| 355 | fn compact_waits_out_the_barrier() -> Outcome<()> { |
| 356 | let _lock = lock(); |
| 357 | const N: usize = 20; |
| 358 | let s = res!(plain("barrier", GC_OFF, 2, 2)); |
| 359 | let user = Uid::default(); |
| 360 | for i in 0..(N + 2) { |
| 361 | res!(s.db.insert(key(i), value(i, 1), user, None)); |
| 362 | } |
| 363 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 364 | for i in 0..N { |
| 365 | assert!(s.holds(i, 1) >= 1, "positive control: key {} version 1 is in a data file", i); |
| 366 | } |
| 367 | |
| 368 | // Each supersession a file bot passes to another is held, so that the old records are not yet |
| 369 | // counted old when the deletes have all been acknowledged. |
| 370 | hooks::set_forward_delay(Duration::from_millis(200)); |
| 371 | for i in 0..N { |
| 372 | assert!(res!(s.db.delete(&key(i), user, None)), "the delete of key {} found nothing", i); |
| 373 | } |
| 374 | let report = s.db.compact_now(Duration::from_secs(60)); |
| 375 | hooks::set_forward_delay(Duration::ZERO); |
| 376 | let report = res!(report); |
| 377 | msg!("{:?}", report); |
| 378 | let mut left = Vec::new(); |
| 379 | for i in 0..N { |
| 380 | if s.holds(i, 1) > 0 { |
| 381 | left.push(i); |
| 382 | } |
| 383 | } |
| 384 | assert!(left.is_empty(), "keys {:?} are still in data files after compact_now", left); |
| 385 | let mut want: Vec<Option<u8>> = vec![None; N]; |
| 386 | want.push(Some(1)); |
| 387 | want.push(Some(1)); |
| 388 | assert_eq!(reads(&s.db, &want, "After compact_now"), 0); |
| 389 | s.end() |
| 390 | } |
| 391 | |
| 392 | #[test] |
| 393 | fn compact_under_reads() -> Outcome<()> { |
| 394 | let _lock = lock(); |
| 395 | let s = res!(plain("reads", GC_ON, 1, 1)); |
| 396 | let user = Uid::default(); |
| 397 | res!(fill(&s.db)); |
| 398 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 399 | |
| 400 | let issued: Arc<Vec<AtomicU64>> = Arc::new((0..NKEYS).map(|_| AtomicU64::new(1)).collect()); |
| 401 | let acked: Arc<Vec<AtomicU64>> = Arc::new((0..NKEYS).map(|_| AtomicU64::new(1)).collect()); |
| 402 | let stop = Arc::new(AtomicBool::new(false)); |
| 403 | let counts: Arc<[AtomicU64; 5]> = Arc::new([ |
| 404 | AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), |
| 405 | ]); |
| 406 | const READS: usize = 0; const WRONG: usize = 1; const STALE: usize = 2; const PHANTOM: usize = 3; const ERRS: usize = 4; |
| 407 | let mut hs = Vec::new(); |
| 408 | for r in 0..4usize { |
| 409 | let (db, issued, acked, stop, counts) = |
| 410 | (s.db.clone(), issued.clone(), acked.clone(), stop.clone(), counts.clone()); |
| 411 | hs.push(res!(thread::Builder::new().name(fmt!("compact reader {}", r)).spawn(move || { |
| 412 | let mut n = r; |
| 413 | while !stop.load(Ordering::Relaxed) { |
| 414 | n = (n * 7 + 3 + r) % NKEYS; |
| 415 | let lo = acked[n].load(Ordering::Relaxed); |
| 416 | let got = db.get(&key(n), None); |
| 417 | let hi = issued[n].load(Ordering::Relaxed); |
| 418 | counts[READS].fetch_add(1, Ordering::Relaxed); |
| 419 | match got { |
| 420 | Err(e) => { |
| 421 | if counts[ERRS].fetch_add(1, Ordering::Relaxed) < 3 { |
| 422 | msg!("Read error on key {}: {}", n, e); |
| 423 | } |
| 424 | }, |
| 425 | Ok(None) => { counts[WRONG].fetch_add(1, Ordering::Relaxed); }, |
| 426 | Ok(Some((d, _))) => match d.bytes_ref() { |
| 427 | Some(b) if b.len() >= 2 && b[0] as usize == n => { |
| 428 | let ver = b[1] as u64; |
| 429 | if d != value(n, b[1]) || ver > hi { counts[PHANTOM].fetch_add(1, Ordering::Relaxed); } |
| 430 | else if ver < lo { counts[STALE].fetch_add(1, Ordering::Relaxed); } |
| 431 | }, |
| 432 | _ => { counts[WRONG].fetch_add(1, Ordering::Relaxed); }, |
| 433 | }, |
| 434 | } |
| 435 | } |
| 436 | }))); |
| 437 | } |
| 438 | // A thread to keep the cached values away, so that reads go to the files and pin them. |
| 439 | { |
| 440 | let (db, stop) = (s.db.clone(), stop.clone()); |
| 441 | hs.push(res!(thread::Builder::new().name(fmt!("compact clearer")).spawn(move || { |
| 442 | while !stop.load(Ordering::Relaxed) { |
| 443 | let _ = db.api().clear_cache_values(Wait { |
| 444 | max_wait: Duration::from_secs(5), |
| 445 | check_interval: Duration::from_millis(10), |
| 446 | }); |
| 447 | thread::sleep(Duration::from_millis(15)); |
| 448 | } |
| 449 | }))); |
| 450 | } |
| 451 | |
| 452 | let mut failure = None; |
| 453 | for round in 2..12u8 { |
| 454 | for i in 0..NKEYS { |
| 455 | issued[i].store(round as u64, Ordering::Relaxed); |
| 456 | if let Err(e) = s.db.insert(key(i), value(i, round), user, None) { |
| 457 | failure = Some(e); |
| 458 | break; |
| 459 | } |
| 460 | acked[i].store(round as u64, Ordering::Relaxed); |
| 461 | } |
| 462 | if failure.is_some() { break; } |
| 463 | if let Err(e) = s.db.compact_now(Duration::from_secs(60)) { |
| 464 | failure = Some(e); |
| 465 | break; |
| 466 | } |
| 467 | } |
| 468 | stop.store(true, Ordering::Relaxed); |
| 469 | for h in hs { |
| 470 | let _ = h.join(); |
| 471 | } |
| 472 | if let Some(e) = failure { |
| 473 | return Err(e); |
| 474 | } |
| 475 | let c = |i: usize| counts[i].load(Ordering::Relaxed); |
| 476 | msg!("{} reads: {} wrong, {} stale, {} phantom, {} errors.", c(READS), c(WRONG), c(STALE), c(PHANTOM), c(ERRS)); |
| 477 | assert!(c(READS) > 100, "only {} reads ran against the compactions", c(READS)); |
| 478 | assert_eq!(c(WRONG), 0, "reads returned another key's record, or none"); |
| 479 | assert_eq!(c(STALE), 0, "reads returned a version older than one acknowledged before the read"); |
| 480 | assert_eq!(c(PHANTOM), 0, "reads returned a version never written"); |
| 481 | assert_eq!(c(ERRS), 0, "reads failed while compact_now ran"); |
| 482 | let want: Vec<Option<u8>> = vec![Some(11); NKEYS]; |
| 483 | assert_eq!(reads(&s.db, &want, "After the last compact_now"), 0); |
| 484 | for i in 0..NKEYS { |
| 485 | for ver in 1..11u8 { |
| 486 | assert_eq!(s.holds(i, ver), 0, "key {} version {} is in a data file after the last compact_now", i, ver); |
| 487 | } |
| 488 | } |
| 489 | s.end() |
| 490 | } |
| 491 | |
| 492 | #[test] |
| 493 | fn compact_names_what_it_waits_on() -> Outcome<()> { |
| 494 | let _lock = lock(); |
| 495 | let s = res!(plain("names", GC_OFF, 1, 1)); |
| 496 | let user = Uid::default(); |
| 497 | res!(fill(&s.db)); |
| 498 | // Reopened, so that the cache holds locations and no values, and a read of a record in file 1 |
| 499 | // goes to its file bot, which pins the file for the reader. |
| 500 | let (root, cfg, dir) = (s.root.clone(), s.cfg.clone(), s.dir.clone()); |
| 501 | res!(s.db.close()); |
| 502 | let db: TestDb = res!(setup::start_db(root, Some(cfg), schemes(), None, GC_OFF, false)); |
| 503 | res!(db.insert(key(0), value(0, 2), user, None)); |
| 504 | res!(db.api().settle_for_test(Duration::from_secs(10))); |
| 505 | |
| 506 | hooks::set_read_delay(Duration::from_secs(4)); |
| 507 | let reader = { |
| 508 | let db = db.clone(); |
| 509 | thread::spawn(move || db.get(&key(2), None)) |
| 510 | }; |
| 511 | thread::sleep(Duration::from_millis(500)); |
| 512 | let t = Instant::now(); |
| 513 | let result = db.compact_now(Duration::from_secs(1)); |
| 514 | let took = t.elapsed(); |
| 515 | hooks::set_read_delay(Duration::ZERO); |
| 516 | let e = match result { |
| 517 | Err(e) => e, |
| 518 | Ok(r) => return Err(err!("compact_now returned Ok({:?}) with a reader pinning file 1.", r; Test, Invalid)), |
| 519 | }; |
| 520 | let text = fmt!("{}", e); |
| 521 | msg!("compact_now failed after {:?} with: {}", took, text.chars().take(400).collect::<String>()); |
| 522 | assert!(e.tags().contains(&ErrTag::Timeout), "the error is not a timeout: {:?}", e.tags()); |
| 523 | assert!(text.contains("file 1 of zone 1 (readers)"), "the error does not name file 1 and its readers: {}", |
| 524 | text.chars().take(400).collect::<String>()); |
| 525 | assert!(took >= Duration::from_secs(1) && took < Duration::from_secs(3), "the deadline was kept to {:?}", took); |
| 526 | |
| 527 | // Once the reader is done the same call finishes. |
| 528 | match reader.join() { |
| 529 | Ok(got) => match res!(got) { |
| 530 | Some((v, _)) => assert_eq!(v, value(2, 1), "the pinned reader read the wrong value"), |
| 531 | None => return Err(err!("The pinned reader found nothing."; Test, Missing)), |
| 532 | }, |
| 533 | Err(_) => return Err(err!("The reader thread panicked."; Test, Bug)), |
| 534 | } |
| 535 | let report = res!(db.compact_now(Duration::from_secs(30))); |
| 536 | msg!("{:?}", report); |
| 537 | res!(db.close()); |
| 538 | let _ = std::fs::remove_dir_all(&dir); |
| 539 | Ok(()) |
| 540 | } |
| 541 | |
| 542 | #[test] |
| 543 | fn compact_is_idempotent_and_bounded() -> Outcome<()> { |
| 544 | let _lock = lock(); |
| 545 | let s = res!(plain("idem", GC_OFF, 1, 1)); |
| 546 | let user = Uid::default(); |
| 547 | res!(fill(&s.db)); |
| 548 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 549 | for i in 0..NKEYS { |
| 550 | res!(s.db.insert(key(i), value(i, 2), user, None)); |
| 551 | } |
| 552 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 553 | |
| 554 | let files = |s: &Plain| -> usize { |
| 555 | match std::fs::read_dir(zone_dir(&s.root, &s.cfg)) { |
| 556 | Ok(list) => list.flatten().filter(|e| { |
| 557 | e.path().extension().map_or(false, |x| x == constant::DATA_FILE_EXT) |
| 558 | }).count(), |
| 559 | Err(_) => 0, |
| 560 | } |
| 561 | }; |
| 562 | let mut reports: Vec<CompactReport> = Vec::new(); |
| 563 | let mut counts = Vec::new(); |
| 564 | for _ in 0..5 { |
| 565 | reports.push(res!(s.db.compact_now(Duration::from_secs(30)))); |
| 566 | counts.push(files(&s)); |
| 567 | } |
| 568 | msg!("Reports {:?}, data files {:?}.", reports, counts); |
| 569 | assert!(counts[4] <= counts[0] + 1, "data files grew from {} to {} over five calls", counts[0], counts[4]); |
| 570 | for w in reports.windows(2) { |
| 571 | assert!(w[1].bytes_after <= w[0].bytes_after, "size_bytes rose between calls: {:?}", reports); |
| 572 | } |
| 573 | assert_eq!(reports[4].bytes_before, reports[4].bytes_after, "a fifth call on a quiet store still removed bytes"); |
| 574 | for i in 0..NKEYS { |
| 575 | assert_eq!(s.holds(i, 1), 0, "key {} version 1 is in a data file", i); |
| 576 | } |
| 577 | let want: Vec<Option<u8>> = vec![Some(2); NKEYS]; |
| 578 | assert_eq!(reads(&s.db, &want, "After five calls"), 0); |
| 579 | s.end() |
| 580 | } |
| 581 | |
| 582 | #[test] |
| 583 | fn size_falls_by_what_compact_removed() -> Outcome<()> { |
| 584 | let _lock = lock(); |
| 585 | let s = res!(plain("size", GC_OFF, 1, 1)); |
| 586 | let user = Uid::default(); |
| 587 | res!(fill(&s.db)); |
| 588 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 589 | for i in 0..NKEYS { |
| 590 | res!(s.db.insert(key(i), value(i, 2), user, None)); |
| 591 | } |
| 592 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 593 | |
| 594 | let before = settled_len(&s.root, &s.cfg); |
| 595 | let report = res!(s.db.compact_now(Duration::from_secs(30))); |
| 596 | let after = settled_len(&s.root, &s.cfg); |
| 597 | msg!("{:?}; the files held {} bytes before and {} after.", report, before, after); |
| 598 | assert!(before > after, "positive control: compact_now removed nothing from {} bytes", before); |
| 599 | assert_eq!(report.bytes_before, before, "bytes_before is not the length of the files before the call"); |
| 600 | assert_eq!(report.bytes_after, after, "bytes_after is not the length of the files after the call"); |
| 601 | assert_eq!(report.bytes_before - report.bytes_after, before - after, "the report does not give the drop"); |
| 602 | s.end() |
| 603 | } |
| 604 | |
| 605 | #[test] |
| 606 | fn zone_bot_error_ends_compact_now_at_once() -> Outcome<()> { |
| 607 | let _lock = lock(); |
| 608 | let s = res!(plain("error", GC_OFF, 1, 1)); |
| 609 | let user = Uid::default(); |
| 610 | res!(fill(&s.db)); |
| 611 | res!(s.db.insert(key(0), value(0, 2), user, None)); |
| 612 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 613 | |
| 614 | hooks::set_compact_failure(true); |
| 615 | let t = Instant::now(); |
| 616 | let result = s.db.compact_now(Duration::from_secs(30)); |
| 617 | let took = t.elapsed(); |
| 618 | hooks::set_compact_failure(false); |
| 619 | let e = match result { |
| 620 | Err(e) => e, |
| 621 | Ok(r) => return Err(err!("compact_now returned Ok({:?}) though no zone bot passed the order on.", r; Test, Invalid)), |
| 622 | }; |
| 623 | let text = fmt!("{}", e); |
| 624 | msg!("compact_now failed after {:?} with: {}", took, text.chars().take(400).collect::<String>()); |
| 625 | assert!(took < Duration::from_secs(10), "the error took {:?}, so the call waited towards its deadline", took); |
| 626 | assert!(!e.tags().contains(&ErrTag::Timeout), "the error is the deadline's: {:?}", e.tags()); |
| 627 | assert!(text.contains("cannot be passed to the file bots"), "the error lost the zone bot's words: {}", |
| 628 | text.chars().take(400).collect::<String>()); |
| 629 | |
| 630 | // And the store still compacts once the zone bots can pass the order on. |
| 631 | let report = res!(s.db.compact_now(Duration::from_secs(30))); |
| 632 | msg!("{:?}", report); |
| 633 | s.end() |
| 634 | } |
| 635 | |
| 636 | /// The files this process holds open under the root, as the kernel names them: `(deleted)` follows |
| 637 | /// the path of one whose name has been unlinked. |
| 638 | fn open_files_under(root: &Path) -> Vec<String> { |
| 639 | let mut found = Vec::new(); |
| 640 | if let Ok(list) = std::fs::read_dir("/proc/self/fd") { |
| 641 | for entry in list.flatten() { |
| 642 | if let Ok(target) = std::fs::read_link(entry.path()) { |
| 643 | let t = target.to_string_lossy().to_string(); |
| 644 | if t.starts_with(&root.to_string_lossy().to_string()) { |
| 645 | found.push(t); |
| 646 | } |
| 647 | } |
| 648 | } |
| 649 | } |
| 650 | found.sort(); |
| 651 | found |
| 652 | } |
| 653 | |
| 654 | // QA A1-1 (2026-10-08): a file deleted for holding only old records stayed open in the readers' |
| 655 | // file cache, so its bytes stayed allocated on disk after `compact_now` returned. |
| 656 | #[test] |
| 657 | fn deleted_file_is_not_held_open() -> Outcome<()> { |
| 658 | let _lock = lock(); |
| 659 | let s = res!(plain("held_open", GC_OFF, 1, 1)); |
| 660 | let user = Uid::default(); |
| 661 | res!(fill(&s.db)); |
| 662 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 663 | // Reopened, so that the cache holds locations and no values and a read of key 2 goes to a |
| 664 | // reader, which opens file 1 and keeps the handle. |
| 665 | let (root, cfg, dir) = (s.root.clone(), s.cfg.clone(), s.dir.clone()); |
| 666 | res!(s.db.close()); |
| 667 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, GC_OFF, false)); |
| 668 | let mut want: Vec<Option<u8>> = vec![Some(1); NKEYS]; |
| 669 | assert_eq!(reads(&db, &want, "Before the overwrite"), 0); |
| 670 | let dat1 = file(&root, &cfg, FileType::Data, 1).to_string_lossy().to_string(); |
| 671 | let held = open_files_under(&root); |
| 672 | assert!(held.contains(&dat1), "positive control: no reader holds {} open, only {:?}", dat1, held); |
| 673 | |
| 674 | // Every key of file 1 overwritten leaves it holding old records alone. |
| 675 | for i in 0..5usize { |
| 676 | res!(db.insert(key(i), value(i, 2), user, None)); |
| 677 | want[i] = Some(2); |
| 678 | } |
| 679 | res!(db.api().settle_for_test(Duration::from_secs(10))); |
| 680 | let needle = match value(2, 1).bytes_ref() { Some(b) => b.clone(), None => Vec::new() }; |
| 681 | assert!(!files_holding(&root, &cfg, &needle).is_empty(), "positive control: key 2 version 1 is in no file"); |
| 682 | let report = res!(db.compact_now(Duration::from_secs(30))); |
| 683 | msg!("{:?}", report); |
| 684 | assert!(report.files_deleted >= 1, "no file was deleted, so the test proves nothing: {:?}", report); |
| 685 | assert!(!file(&root, &cfg, FileType::Data, 1).is_file(), "file 1 is still on disk"); |
| 686 | assert!(files_holding(&root, &cfg, &needle).is_empty(), "key 2 version 1 is still in a data file"); |
| 687 | |
| 688 | let gone: Vec<String> = open_files_under(&root).into_iter().filter(|t| t.ends_with(" (deleted)")).collect(); |
| 689 | assert!(gone.is_empty(), "compact_now returned with deleted files held open: {:?}", gone); |
| 690 | assert_eq!(reads(&db, &want, "After compact_now"), 0); |
| 691 | res!(db.close()); |
| 692 | let _ = std::fs::remove_dir_all(&dir); |
| 693 | Ok(()) |
| 694 | } |
| 695 | |
| 696 | // QA A1-4 (2026-10-08): the roll to new live files waited out the control deadline of five |
| 697 | // minutes instead of the caller's, so a writer behind a long queue held `compact_now` past its |
| 698 | // deadline before the first check of it. |
| 699 | #[test] |
| 700 | fn deadline_bounds_the_roll_to_new_live_files() -> Outcome<()> { |
| 701 | let _lock = lock(); |
| 702 | const N: usize = 8; |
| 703 | let s = res!(plain("roll", GC_OFF, 2, 2)); |
| 704 | let user = Uid::default(); |
| 705 | for i in 0..(N + 2) { |
| 706 | res!(s.db.insert(key(i), value(i, 1), user, None)); |
| 707 | } |
| 708 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 709 | |
| 710 | // Each supersession a file bot receives from another is held for two seconds, so the writer's |
| 711 | // request to close its old live file waits behind them. |
| 712 | hooks::set_schedule_delay(Duration::from_secs(2)); |
| 713 | for i in 0..N { |
| 714 | assert!(res!(s.db.delete(&key(i), user, None)), "the delete of key {} found nothing", i); |
| 715 | } |
| 716 | let t = Instant::now(); |
| 717 | let result = s.db.compact_now(Duration::from_secs(1)); |
| 718 | let took = t.elapsed(); |
| 719 | hooks::set_schedule_delay(Duration::ZERO); |
| 720 | let e = match result { |
| 721 | Err(e) => e, |
| 722 | Ok(r) => return Err(err!("compact_now returned Ok({:?}) with the file bots held.", r; Test, Invalid)), |
| 723 | }; |
| 724 | let text = fmt!("{}", e); |
| 725 | msg!("compact_now failed after {:?} with: {}", took, text.chars().take(400).collect::<String>()); |
| 726 | assert!(e.tags().contains(&ErrTag::Timeout), "the error is not a timeout: {:?}", e.tags()); |
| 727 | assert!(took < Duration::from_secs(4), "the deadline of one second was kept to {:?}", took); |
| 728 | |
| 729 | // Released, the same call finishes. |
| 730 | let report = res!(s.db.compact_now(Duration::from_secs(60))); |
| 731 | msg!("{:?}", report); |
| 732 | s.end() |
| 733 | } |