Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/compact_now.rs

29.5 KiB, 7 runs

created by r1870400018:67597, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! `compact_now` erases (A3 R3, 2026-10-08): once it returns `Ok`, no data file holds a record that
2//! a write acknowledged before the call superseded or deleted. Every test counts the bytes of the
3//! files from outside the store, with a needle that is a whole record's value, and finds it first
4//! (the positive control) so that a needle that is never there cannot pass. The hooks used are
5//! process-wide, so every test takes one lock and puts each hook back.
6
7mod gc_pair;
8
9use gc_pair::*;
10
11use oxedyne_fe2o3_core::prelude::*;
12use oxedyne_fe2o3_iop_db::api::Database;
13use oxedyne_fe2o3_jdat::{
14 prelude::*,
15 chunk::PartKey,
16};
17use oxedyne_fe2o3_o3db_sync::{
18 api::CompactReport,
19 base::{
20 cfg::OzoneConfig,
21 constant,
22 },
23 comm::response::Wait,
24 file::core::FileType,
25 test::{
26 hooks,
27 setup::{
28 self,
29 Uid,
30 },
31 },
32};
33
34use std::{
35 collections::BTreeMap,
36 path::{
37 Path,
38 PathBuf,
39 },
40 sync::{
41 Arc,
42 Mutex,
43 MutexGuard,
44 atomic::{
45 AtomicBool,
46 AtomicU64,
47 Ordering,
48 },
49 },
50 thread,
51 time::{
52 Duration,
53 Instant,
54 },
55};
56
57static HOOKS: Mutex<()> = Mutex::new(());
58
59fn lock() -> MutexGuard<'static, ()> {
60 match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() }
61}
62
63const GC_ON: bool = true;
64const GC_OFF: bool = false;
65
66// A store of the plain kind: records of one length, five to a file.
67struct Plain {
68 dir: String,
69 root: PathBuf,
70 cfg: OzoneConfig,
71 probe: Probe,
72 db: TestDb,
73}
74
75fn plain(name: &str, gc_on: bool, cbots: u16, fbots: u16) -> Outcome<Plain> {
76 log_set_level!("error");
77 let dir = fmt!("./test_db_compact_{}", name);
78 let _ = std::fs::remove_dir_all(&dir);
79 res!(std::fs::create_dir_all(&dir));
80 let root = res!(Path::new(&dir).canonicalize());
81 let probe = res!(probe(&fmt!("./test_db_compact_{}_probe", name)));
82 let mut cfg = res!(config(5 * probe.len + probe.len / 2));
83 cfg.num_cbots_per_zone = cbots;
84 cfg.num_fbots_per_zone = fbots;
85 let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, gc_on, true));
86 Ok(Plain { dir, root, cfg, probe, db })
87}
88
89impl Plain {
90 // How many data files hold the whole value of key `i` at version `ver`.
91 fn holds(&self, i: usize, ver: u8) -> usize {
92 let needle = match value(i, ver).bytes_ref() {
93 Some(b) => b.clone(),
94 None => Vec::new(),
95 };
96 assert!(!needle.is_empty(), "the needle of key {} version {} is empty", i, ver);
97 files_holding(&self.root, &self.cfg, &needle).len()
98 }
99
100 fn end(self) -> Outcome<()> {
101 res!(self.db.close());
102 let _ = std::fs::remove_dir_all(&self.dir);
103 Ok(())
104 }
105}
106
107/// Reads every key and counts the answers that are not what was last written: a version, or `None`
108/// for a key that was deleted.
109fn reads(db: &TestDb, want: &[Option<u8>], label: &str) -> usize {
110 let mut bad = 0;
111 for (i, w) in want.iter().enumerate() {
112 match (db.get(&key(i), None), w) {
113 (Ok(Some((v, _))), Some(ver)) if v == value(i, *ver) => (),
114 (Ok(None), None) => (),
115 (got, _) => {
116 bad += 1;
117 let s: String = fmt!("{:?}", got.map(|o| o.map(|(d, _)| d))).chars().take(200).collect();
118 msg!("{}: key {} wanted {:?} and read {}", label, i, w, s);
119 },
120 }
121 }
122 bad
123}
124
125fn small(label: &str, found: usize) -> String {
126 fmt!("{}: {} data files still hold it", label, found)
127}
128
129#[test]
130fn deleted_value_leaves_every_dat() -> Outcome<()> {
131 let _lock = lock();
132 let s = res!(plain("delete", GC_ON, 1, 1));
133 let user = Uid::default();
134 res!(fill(&s.db));
135 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
136
137 // Key 2 is in sealed file 1, key 11 in the live file.
138 for i in [2usize, 11] {
139 assert!(s.holds(i, 1) >= 1, "positive control: key {} version 1 is in a data file", i);
140 assert!(res!(s.db.delete(&key(i), user, None)), "the delete of key {} found nothing", i);
141 }
142 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
143 thread::sleep(Duration::from_millis(500));
144 // One old record is under the trigger, and the live file is never collected, so the delete
145 // alone has erased nothing.
146 for i in [2usize, 11] {
147 assert!(s.holds(i, 1) >= 1, "positive control: the delete of key {} did not erase it by itself", i);
148 }
149
150 let report = res!(s.db.compact_now(Duration::from_secs(30)));
151 msg!("{:?}", report);
152 for i in [2usize, 11] {
153 assert_eq!(s.holds(i, 1), 0, "{}", small(&fmt!("key {} version 1 after compact_now", i), s.holds(i, 1)));
154 }
155 assert!(report.bytes_after < report.bytes_before, "compact_now removed nothing: {:?}", report);
156 let mut want: Vec<Option<u8>> = vec![Some(1); NKEYS];
157 want[2] = None;
158 want[11] = None;
159 assert_eq!(reads(&s.db, &want, "After compact_now"), 0);
160
161 // Reopened, so that what the files hold is what is read.
162 let (root, cfg) = (s.root.clone(), s.cfg.clone());
163 res!(s.db.close());
164 let db: TestDb = res!(setup::start_db(root, Some(cfg), schemes(), None, GC_ON, false));
165 assert_eq!(reads(&db, &want, "After reopening"), 0);
166 res!(db.close());
167 let _ = std::fs::remove_dir_all(&s.dir);
168 Ok(())
169}
170
171#[test]
172fn overwritten_value_leaves_every_dat() -> Outcome<()> {
173 let _lock = lock();
174 let s = res!(plain("overwrite", GC_ON, 1, 1));
175 let user = Uid::default();
176 res!(fill(&s.db));
177 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
178
179 // One record in each sealed file, each under the trigger.
180 let mut want: Vec<Option<u8>> = vec![Some(1); NKEYS];
181 for i in [3usize, 7] {
182 assert!(s.holds(i, 1) >= 1, "positive control: key {} version 1 is in a data file", i);
183 res!(s.db.insert(key(i), value(i, 2), user, None));
184 want[i] = Some(2);
185 }
186 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
187 thread::sleep(Duration::from_millis(500));
188 for i in [3usize, 7] {
189 assert!(s.holds(i, 1) >= 1, "positive control: the overwrite of key {} did not erase it by itself", i);
190 }
191
192 let report = res!(s.db.compact_now(Duration::from_secs(30)));
193 msg!("{:?}", report);
194 for i in [3usize, 7] {
195 assert_eq!(s.holds(i, 1), 0, "{}", small(&fmt!("key {} version 1 after compact_now", i), s.holds(i, 1)));
196 assert!(s.holds(i, 2) >= 1, "the new value of key {} is in no data file", i);
197 }
198 assert_eq!(reads(&s.db, &want, "After compact_now"), 0);
199 s.end()
200}
201
202#[test]
203fn compact_collects_with_gc_switched_off() -> Outcome<()> {
204 let _lock = lock();
205 let s = res!(plain("gc_off", GC_OFF, 1, 1));
206 let user = Uid::default();
207 res!(fill(&s.db));
208 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
209 assert!(s.holds(3, 1) >= 1, "positive control: key 3 version 1 is in a data file");
210 res!(s.db.insert(key(3), value(3, 2), user, None));
211 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
212
213 // The switch is off and the trigger is not met, so nothing but the order can collect.
214 thread::sleep(Duration::from_millis(1_000));
215 assert!(s.holds(3, 1) >= 1, "positive control: nothing collected with the collector off");
216
217 let report = res!(s.db.compact_now(Duration::from_secs(30)));
218 msg!("{:?}", report);
219 assert_eq!(s.holds(3, 1), 0, "{}", small("key 3 version 1 after compact_now", s.holds(3, 1)));
220 assert!(report.files_collected + report.files_deleted >= 1, "nothing was collected: {:?}", report);
221 let mut want: Vec<Option<u8>> = vec![Some(1); NKEYS];
222 want[3] = Some(2);
223 assert_eq!(reads(&s.db, &want, "After compact_now"), 0);
224 s.end()
225}
226
227// The chunked store: three chunks of 400 bytes in a value of 1,100.
228const CHUNKS: u64 = 3;
229
230fn chunk_cfg() -> Outcome<OzoneConfig> {
231 let mut c = res!(setup::default_cfg());
232 c.num_zones = 1;
233 c.num_cbots_per_zone = 4;
234 c.num_fbots_per_zone = 1;
235 c.num_wbots_per_zone = 2;
236 c.zone_overrides = BTreeMap::new();
237 c.rest_chunk_threshold = 1_000;
238 c.rest_chunk_bytes = 400;
239 c.sync_on_write = true;
240 Ok(c)
241}
242
243fn big(seed: u8) -> Vec<u8> {
244 (0..1_100usize).map(|j| seed ^ (j as u8)).collect()
245}
246
247// A hundred bytes from the middle of each chunk, so that a chunk is found by its own bytes.
248fn windows(seed: u8) -> Vec<Vec<u8>> {
249 let v = big(seed);
250 vec![v[100..200].to_vec(), v[500..600].to_vec(), v[900..1_000].to_vec()]
251}
252
253struct Chunked {
254 dir: String,
255 root: PathBuf,
256 cfg: OzoneConfig,
257 db: TestDb,
258}
259
260fn chunked(name: &str, gc_on: bool) -> Outcome<Chunked> {
261 log_set_level!("error");
262 let dir = fmt!("./test_db_compact_{}", name);
263 let _ = std::fs::remove_dir_all(&dir);
264 res!(std::fs::create_dir_all(&dir));
265 let root = res!(Path::new(&dir).canonicalize());
266 let cfg = res!(chunk_cfg());
267 let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, gc_on, true));
268 Ok(Chunked { dir, root, cfg, db })
269}
270
271impl Chunked {
272 fn holds(&self, seed: u8) -> Vec<usize> {
273 windows(seed).iter().map(|w| files_holding(&self.root, &self.cfg, w).len()).collect()
274 }
275
276 fn num_chunks(&self, k: &Dat) -> Outcome<u64> {
277 let api = self.db.api();
278 let resp = res!(api.fetch_using_schemes(k, None));
279 match res!(resp.recv_daticle(api.schemes().encrypter(), None)) {
280 (Some((Dat::Tup5u64(tup), _)), _) => Ok(PartKey(tup).num_parts()),
281 other => Err(err!("Key {:?} holds {:?}, not a bunch key.", k, other; Test, Invalid)),
282 }
283 }
284}
285
286#[test]
287fn chunked_delete_leaves_every_dat() -> Outcome<()> {
288 let _lock = lock();
289 let s = res!(chunked("chunked", GC_OFF));
290 let user = Uid::default();
291 // A value that stays, in the same file, so that the file is collected and not merely removed.
292 res!(s.db.insert(key(0), value(0, 9), user, None));
293 let k = dat!("chunked erase");
294 res!(s.db.insert(k.clone(), Dat::BU32(big(5)), user, None));
295 assert_eq!(res!(s.num_chunks(&k)), CHUNKS, "the value did not split into {} chunks", CHUNKS);
296 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
297 assert!(s.holds(5).iter().all(|n| *n >= 1), "positive control: every chunk is in a data file: {:?}", s.holds(5));
298
299 hooks::set_chunk_tombstone_delay(Duration::from_millis(300));
300 let existed = s.db.delete(&k, user, None);
301 hooks::set_chunk_tombstone_delay(Duration::ZERO);
302 assert!(res!(existed), "the delete found nothing");
303 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
304 assert!(s.holds(5).iter().all(|n| *n >= 1), "positive control: the delete did not erase the chunks by itself");
305
306 let report = res!(s.db.compact_now(Duration::from_secs(30)));
307 msg!("{:?}", report);
308 assert_eq!(s.holds(5), vec![0, 0, 0], "chunks still in data files after compact_now");
309 match res!(s.db.get(&key(0), None)) {
310 Some((v, _)) => assert_eq!(v, value(0, 9), "the value beside it changed"),
311 None => return Err(err!("The value that was not deleted is gone."; Test, Missing)),
312 }
313 assert!(res!(s.db.get(&k, None)).is_none(), "the deleted value reads back");
314 res!(s.db.close());
315 let _ = std::fs::remove_dir_all(&s.dir);
316 Ok(())
317}
318
319#[test]
320fn tombstone_store_leaves_every_dat() -> Outcome<()> {
321 let _lock = lock();
322 let s = res!(chunked("tombstone", GC_OFF));
323 let user = Uid::default();
324 res!(s.db.insert(key(0), value(0, 9), user, None));
325 let k = dat!("tombstone erase");
326 res!(s.db.insert(k.clone(), Dat::BU32(big(6)), user, None));
327 assert_eq!(res!(s.num_chunks(&k)), CHUNKS, "the value did not split into {} chunks", CHUNKS);
328 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
329 assert!(s.holds(6).iter().all(|n| *n >= 1), "positive control: every chunk is in a data file: {:?}", s.holds(6));
330
331 // The erase of the distributed adapters: a store of the deleted marker, not a delete.
332 let tomb = Dat::Usr(oxedyne_fe2o3_o3db_sync::base::id::usr_kind_id_deleted(), Some(Box::new(Dat::Empty)));
333 let resp = res!(s.db.api().store(k.clone(), tomb.clone(), user));
334 res!(resp.recv_store_ack());
335 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
336 assert!(s.holds(6).iter().any(|n| *n >= 1), "positive control: the store alone left no chunk to erase");
337
338 let report = res!(s.db.compact_now(Duration::from_secs(30)));
339 msg!("{:?}", report);
340 assert_eq!(s.holds(6), vec![0, 0, 0], "chunks still in data files after compact_now");
341 match res!(s.db.api().get_wait(&k, None)) {
342 Some((d, _)) => assert_eq!(d, tomb, "the key does not read back as the deleted marker"),
343 None => (),
344 }
345 match res!(s.db.get(&key(0), None)) {
346 Some((v, _)) => assert_eq!(v, value(0, 9), "the value beside it changed"),
347 None => return Err(err!("The value that was not deleted is gone."; Test, Missing)),
348 }
349 res!(s.db.close());
350 let _ = std::fs::remove_dir_all(&s.dir);
351 Ok(())
352}
353
354#[test]
355fn compact_waits_out_the_barrier() -> Outcome<()> {
356 let _lock = lock();
357 const N: usize = 20;
358 let s = res!(plain("barrier", GC_OFF, 2, 2));
359 let user = Uid::default();
360 for i in 0..(N + 2) {
361 res!(s.db.insert(key(i), value(i, 1), user, None));
362 }
363 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
364 for i in 0..N {
365 assert!(s.holds(i, 1) >= 1, "positive control: key {} version 1 is in a data file", i);
366 }
367
368 // Each supersession a file bot passes to another is held, so that the old records are not yet
369 // counted old when the deletes have all been acknowledged.
370 hooks::set_forward_delay(Duration::from_millis(200));
371 for i in 0..N {
372 assert!(res!(s.db.delete(&key(i), user, None)), "the delete of key {} found nothing", i);
373 }
374 let report = s.db.compact_now(Duration::from_secs(60));
375 hooks::set_forward_delay(Duration::ZERO);
376 let report = res!(report);
377 msg!("{:?}", report);
378 let mut left = Vec::new();
379 for i in 0..N {
380 if s.holds(i, 1) > 0 {
381 left.push(i);
382 }
383 }
384 assert!(left.is_empty(), "keys {:?} are still in data files after compact_now", left);
385 let mut want: Vec<Option<u8>> = vec![None; N];
386 want.push(Some(1));
387 want.push(Some(1));
388 assert_eq!(reads(&s.db, &want, "After compact_now"), 0);
389 s.end()
390}
391
392#[test]
393fn compact_under_reads() -> Outcome<()> {
394 let _lock = lock();
395 let s = res!(plain("reads", GC_ON, 1, 1));
396 let user = Uid::default();
397 res!(fill(&s.db));
398 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
399
400 let issued: Arc<Vec<AtomicU64>> = Arc::new((0..NKEYS).map(|_| AtomicU64::new(1)).collect());
401 let acked: Arc<Vec<AtomicU64>> = Arc::new((0..NKEYS).map(|_| AtomicU64::new(1)).collect());
402 let stop = Arc::new(AtomicBool::new(false));
403 let counts: Arc<[AtomicU64; 5]> = Arc::new([
404 AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0),
405 ]);
406 const READS: usize = 0; const WRONG: usize = 1; const STALE: usize = 2; const PHANTOM: usize = 3; const ERRS: usize = 4;
407 let mut hs = Vec::new();
408 for r in 0..4usize {
409 let (db, issued, acked, stop, counts) =
410 (s.db.clone(), issued.clone(), acked.clone(), stop.clone(), counts.clone());
411 hs.push(res!(thread::Builder::new().name(fmt!("compact reader {}", r)).spawn(move || {
412 let mut n = r;
413 while !stop.load(Ordering::Relaxed) {
414 n = (n * 7 + 3 + r) % NKEYS;
415 let lo = acked[n].load(Ordering::Relaxed);
416 let got = db.get(&key(n), None);
417 let hi = issued[n].load(Ordering::Relaxed);
418 counts[READS].fetch_add(1, Ordering::Relaxed);
419 match got {
420 Err(e) => {
421 if counts[ERRS].fetch_add(1, Ordering::Relaxed) < 3 {
422 msg!("Read error on key {}: {}", n, e);
423 }
424 },
425 Ok(None) => { counts[WRONG].fetch_add(1, Ordering::Relaxed); },
426 Ok(Some((d, _))) => match d.bytes_ref() {
427 Some(b) if b.len() >= 2 && b[0] as usize == n => {
428 let ver = b[1] as u64;
429 if d != value(n, b[1]) || ver > hi { counts[PHANTOM].fetch_add(1, Ordering::Relaxed); }
430 else if ver < lo { counts[STALE].fetch_add(1, Ordering::Relaxed); }
431 },
432 _ => { counts[WRONG].fetch_add(1, Ordering::Relaxed); },
433 },
434 }
435 }
436 })));
437 }
438 // A thread to keep the cached values away, so that reads go to the files and pin them.
439 {
440 let (db, stop) = (s.db.clone(), stop.clone());
441 hs.push(res!(thread::Builder::new().name(fmt!("compact clearer")).spawn(move || {
442 while !stop.load(Ordering::Relaxed) {
443 let _ = db.api().clear_cache_values(Wait {
444 max_wait: Duration::from_secs(5),
445 check_interval: Duration::from_millis(10),
446 });
447 thread::sleep(Duration::from_millis(15));
448 }
449 })));
450 }
451
452 let mut failure = None;
453 for round in 2..12u8 {
454 for i in 0..NKEYS {
455 issued[i].store(round as u64, Ordering::Relaxed);
456 if let Err(e) = s.db.insert(key(i), value(i, round), user, None) {
457 failure = Some(e);
458 break;
459 }
460 acked[i].store(round as u64, Ordering::Relaxed);
461 }
462 if failure.is_some() { break; }
463 if let Err(e) = s.db.compact_now(Duration::from_secs(60)) {
464 failure = Some(e);
465 break;
466 }
467 }
468 stop.store(true, Ordering::Relaxed);
469 for h in hs {
470 let _ = h.join();
471 }
472 if let Some(e) = failure {
473 return Err(e);
474 }
475 let c = |i: usize| counts[i].load(Ordering::Relaxed);
476 msg!("{} reads: {} wrong, {} stale, {} phantom, {} errors.", c(READS), c(WRONG), c(STALE), c(PHANTOM), c(ERRS));
477 assert!(c(READS) > 100, "only {} reads ran against the compactions", c(READS));
478 assert_eq!(c(WRONG), 0, "reads returned another key's record, or none");
479 assert_eq!(c(STALE), 0, "reads returned a version older than one acknowledged before the read");
480 assert_eq!(c(PHANTOM), 0, "reads returned a version never written");
481 assert_eq!(c(ERRS), 0, "reads failed while compact_now ran");
482 let want: Vec<Option<u8>> = vec![Some(11); NKEYS];
483 assert_eq!(reads(&s.db, &want, "After the last compact_now"), 0);
484 for i in 0..NKEYS {
485 for ver in 1..11u8 {
486 assert_eq!(s.holds(i, ver), 0, "key {} version {} is in a data file after the last compact_now", i, ver);
487 }
488 }
489 s.end()
490}
491
492#[test]
493fn compact_names_what_it_waits_on() -> Outcome<()> {
494 let _lock = lock();
495 let s = res!(plain("names", GC_OFF, 1, 1));
496 let user = Uid::default();
497 res!(fill(&s.db));
498 // Reopened, so that the cache holds locations and no values, and a read of a record in file 1
499 // goes to its file bot, which pins the file for the reader.
500 let (root, cfg, dir) = (s.root.clone(), s.cfg.clone(), s.dir.clone());
501 res!(s.db.close());
502 let db: TestDb = res!(setup::start_db(root, Some(cfg), schemes(), None, GC_OFF, false));
503 res!(db.insert(key(0), value(0, 2), user, None));
504 res!(db.api().settle_for_test(Duration::from_secs(10)));
505
506 hooks::set_read_delay(Duration::from_secs(4));
507 let reader = {
508 let db = db.clone();
509 thread::spawn(move || db.get(&key(2), None))
510 };
511 thread::sleep(Duration::from_millis(500));
512 let t = Instant::now();
513 let result = db.compact_now(Duration::from_secs(1));
514 let took = t.elapsed();
515 hooks::set_read_delay(Duration::ZERO);
516 let e = match result {
517 Err(e) => e,
518 Ok(r) => return Err(err!("compact_now returned Ok({:?}) with a reader pinning file 1.", r; Test, Invalid)),
519 };
520 let text = fmt!("{}", e);
521 msg!("compact_now failed after {:?} with: {}", took, text.chars().take(400).collect::<String>());
522 assert!(e.tags().contains(&ErrTag::Timeout), "the error is not a timeout: {:?}", e.tags());
523 assert!(text.contains("file 1 of zone 1 (readers)"), "the error does not name file 1 and its readers: {}",
524 text.chars().take(400).collect::<String>());
525 assert!(took >= Duration::from_secs(1) && took < Duration::from_secs(3), "the deadline was kept to {:?}", took);
526
527 // Once the reader is done the same call finishes.
528 match reader.join() {
529 Ok(got) => match res!(got) {
530 Some((v, _)) => assert_eq!(v, value(2, 1), "the pinned reader read the wrong value"),
531 None => return Err(err!("The pinned reader found nothing."; Test, Missing)),
532 },
533 Err(_) => return Err(err!("The reader thread panicked."; Test, Bug)),
534 }
535 let report = res!(db.compact_now(Duration::from_secs(30)));
536 msg!("{:?}", report);
537 res!(db.close());
538 let _ = std::fs::remove_dir_all(&dir);
539 Ok(())
540}
541
542#[test]
543fn compact_is_idempotent_and_bounded() -> Outcome<()> {
544 let _lock = lock();
545 let s = res!(plain("idem", GC_OFF, 1, 1));
546 let user = Uid::default();
547 res!(fill(&s.db));
548 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
549 for i in 0..NKEYS {
550 res!(s.db.insert(key(i), value(i, 2), user, None));
551 }
552 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
553
554 let files = |s: &Plain| -> usize {
555 match std::fs::read_dir(zone_dir(&s.root, &s.cfg)) {
556 Ok(list) => list.flatten().filter(|e| {
557 e.path().extension().map_or(false, |x| x == constant::DATA_FILE_EXT)
558 }).count(),
559 Err(_) => 0,
560 }
561 };
562 let mut reports: Vec<CompactReport> = Vec::new();
563 let mut counts = Vec::new();
564 for _ in 0..5 {
565 reports.push(res!(s.db.compact_now(Duration::from_secs(30))));
566 counts.push(files(&s));
567 }
568 msg!("Reports {:?}, data files {:?}.", reports, counts);
569 assert!(counts[4] <= counts[0] + 1, "data files grew from {} to {} over five calls", counts[0], counts[4]);
570 for w in reports.windows(2) {
571 assert!(w[1].bytes_after <= w[0].bytes_after, "size_bytes rose between calls: {:?}", reports);
572 }
573 assert_eq!(reports[4].bytes_before, reports[4].bytes_after, "a fifth call on a quiet store still removed bytes");
574 for i in 0..NKEYS {
575 assert_eq!(s.holds(i, 1), 0, "key {} version 1 is in a data file", i);
576 }
577 let want: Vec<Option<u8>> = vec![Some(2); NKEYS];
578 assert_eq!(reads(&s.db, &want, "After five calls"), 0);
579 s.end()
580}
581
582#[test]
583fn size_falls_by_what_compact_removed() -> Outcome<()> {
584 let _lock = lock();
585 let s = res!(plain("size", GC_OFF, 1, 1));
586 let user = Uid::default();
587 res!(fill(&s.db));
588 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
589 for i in 0..NKEYS {
590 res!(s.db.insert(key(i), value(i, 2), user, None));
591 }
592 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
593
594 let before = settled_len(&s.root, &s.cfg);
595 let report = res!(s.db.compact_now(Duration::from_secs(30)));
596 let after = settled_len(&s.root, &s.cfg);
597 msg!("{:?}; the files held {} bytes before and {} after.", report, before, after);
598 assert!(before > after, "positive control: compact_now removed nothing from {} bytes", before);
599 assert_eq!(report.bytes_before, before, "bytes_before is not the length of the files before the call");
600 assert_eq!(report.bytes_after, after, "bytes_after is not the length of the files after the call");
601 assert_eq!(report.bytes_before - report.bytes_after, before - after, "the report does not give the drop");
602 s.end()
603}
604
605#[test]
606fn zone_bot_error_ends_compact_now_at_once() -> Outcome<()> {
607 let _lock = lock();
608 let s = res!(plain("error", GC_OFF, 1, 1));
609 let user = Uid::default();
610 res!(fill(&s.db));
611 res!(s.db.insert(key(0), value(0, 2), user, None));
612 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
613
614 hooks::set_compact_failure(true);
615 let t = Instant::now();
616 let result = s.db.compact_now(Duration::from_secs(30));
617 let took = t.elapsed();
618 hooks::set_compact_failure(false);
619 let e = match result {
620 Err(e) => e,
621 Ok(r) => return Err(err!("compact_now returned Ok({:?}) though no zone bot passed the order on.", r; Test, Invalid)),
622 };
623 let text = fmt!("{}", e);
624 msg!("compact_now failed after {:?} with: {}", took, text.chars().take(400).collect::<String>());
625 assert!(took < Duration::from_secs(10), "the error took {:?}, so the call waited towards its deadline", took);
626 assert!(!e.tags().contains(&ErrTag::Timeout), "the error is the deadline's: {:?}", e.tags());
627 assert!(text.contains("cannot be passed to the file bots"), "the error lost the zone bot's words: {}",
628 text.chars().take(400).collect::<String>());
629
630 // And the store still compacts once the zone bots can pass the order on.
631 let report = res!(s.db.compact_now(Duration::from_secs(30)));
632 msg!("{:?}", report);
633 s.end()
634}
635
636/// The files this process holds open under the root, as the kernel names them: `(deleted)` follows
637/// the path of one whose name has been unlinked.
638fn open_files_under(root: &Path) -> Vec<String> {
639 let mut found = Vec::new();
640 if let Ok(list) = std::fs::read_dir("/proc/self/fd") {
641 for entry in list.flatten() {
642 if let Ok(target) = std::fs::read_link(entry.path()) {
643 let t = target.to_string_lossy().to_string();
644 if t.starts_with(&root.to_string_lossy().to_string()) {
645 found.push(t);
646 }
647 }
648 }
649 }
650 found.sort();
651 found
652}
653
654// QA A1-1 (2026-10-08): a file deleted for holding only old records stayed open in the readers'
655// file cache, so its bytes stayed allocated on disk after `compact_now` returned.
656#[test]
657fn deleted_file_is_not_held_open() -> Outcome<()> {
658 let _lock = lock();
659 let s = res!(plain("held_open", GC_OFF, 1, 1));
660 let user = Uid::default();
661 res!(fill(&s.db));
662 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
663 // Reopened, so that the cache holds locations and no values and a read of key 2 goes to a
664 // reader, which opens file 1 and keeps the handle.
665 let (root, cfg, dir) = (s.root.clone(), s.cfg.clone(), s.dir.clone());
666 res!(s.db.close());
667 let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, GC_OFF, false));
668 let mut want: Vec<Option<u8>> = vec![Some(1); NKEYS];
669 assert_eq!(reads(&db, &want, "Before the overwrite"), 0);
670 let dat1 = file(&root, &cfg, FileType::Data, 1).to_string_lossy().to_string();
671 let held = open_files_under(&root);
672 assert!(held.contains(&dat1), "positive control: no reader holds {} open, only {:?}", dat1, held);
673
674 // Every key of file 1 overwritten leaves it holding old records alone.
675 for i in 0..5usize {
676 res!(db.insert(key(i), value(i, 2), user, None));
677 want[i] = Some(2);
678 }
679 res!(db.api().settle_for_test(Duration::from_secs(10)));
680 let needle = match value(2, 1).bytes_ref() { Some(b) => b.clone(), None => Vec::new() };
681 assert!(!files_holding(&root, &cfg, &needle).is_empty(), "positive control: key 2 version 1 is in no file");
682 let report = res!(db.compact_now(Duration::from_secs(30)));
683 msg!("{:?}", report);
684 assert!(report.files_deleted >= 1, "no file was deleted, so the test proves nothing: {:?}", report);
685 assert!(!file(&root, &cfg, FileType::Data, 1).is_file(), "file 1 is still on disk");
686 assert!(files_holding(&root, &cfg, &needle).is_empty(), "key 2 version 1 is still in a data file");
687
688 let gone: Vec<String> = open_files_under(&root).into_iter().filter(|t| t.ends_with(" (deleted)")).collect();
689 assert!(gone.is_empty(), "compact_now returned with deleted files held open: {:?}", gone);
690 assert_eq!(reads(&db, &want, "After compact_now"), 0);
691 res!(db.close());
692 let _ = std::fs::remove_dir_all(&dir);
693 Ok(())
694}
695
696// QA A1-4 (2026-10-08): the roll to new live files waited out the control deadline of five
697// minutes instead of the caller's, so a writer behind a long queue held `compact_now` past its
698// deadline before the first check of it.
699#[test]
700fn deadline_bounds_the_roll_to_new_live_files() -> Outcome<()> {
701 let _lock = lock();
702 const N: usize = 8;
703 let s = res!(plain("roll", GC_OFF, 2, 2));
704 let user = Uid::default();
705 for i in 0..(N + 2) {
706 res!(s.db.insert(key(i), value(i, 1), user, None));
707 }
708 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
709
710 // Each supersession a file bot receives from another is held for two seconds, so the writer's
711 // request to close its old live file waits behind them.
712 hooks::set_schedule_delay(Duration::from_secs(2));
713 for i in 0..N {
714 assert!(res!(s.db.delete(&key(i), user, None)), "the delete of key {} found nothing", i);
715 }
716 let t = Instant::now();
717 let result = s.db.compact_now(Duration::from_secs(1));
718 let took = t.elapsed();
719 hooks::set_schedule_delay(Duration::ZERO);
720 let e = match result {
721 Err(e) => e,
722 Ok(r) => return Err(err!("compact_now returned Ok({:?}) with the file bots held.", r; Test, Invalid)),
723 };
724 let text = fmt!("{}", e);
725 msg!("compact_now failed after {:?} with: {}", took, text.chars().take(400).collect::<String>());
726 assert!(e.tags().contains(&ErrTag::Timeout), "the error is not a timeout: {:?}", e.tags());
727 assert!(took < Duration::from_secs(4), "the deadline of one second was kept to {:?}", took);
728
729 // Released, the same call finishes.
730 let report = res!(s.db.compact_now(Duration::from_secs(60)));
731 msg!("{:?}", report);
732 s.end()
733}