Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/qa2_reclaim.rs

17.6 KiB, 32 runs

created by r1870400018:70213, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! QA agent 2 (2026-10-09): fix B reclaim, GC and restart, adversarial.
2mod gc_pair;
3
4use gc_pair::*;
5
6use oxedyne_fe2o3_core::prelude::*;
7use oxedyne_fe2o3_iop_db::api::Database;
8use oxedyne_fe2o3_jdat::prelude::*;
9use oxedyne_fe2o3_o3db_sync::{
10 base::{
11 cfg::OzoneConfig,
12 constant,
13 },
14 comm::response::Wait,
15 data::cache::is_chunk_key,
16 sweep,
17 test::{
18 hooks,
19 setup::{
20 self,
21 Uid,
22 },
23 },
24};
25
26use std::{
27 collections::BTreeMap,
28 path::Path,
29 sync::{
30 Mutex,
31 MutexGuard,
32 atomic::{
33 AtomicBool,
34 AtomicUsize,
35 Ordering,
36 },
37 },
38 thread,
39 time::Duration,
40};
41
42static HOOKS: Mutex<()> = Mutex::new(());
43
44fn lock() -> MutexGuard<'static, ()> {
45 match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() }
46}
47
48fn cfg_b() -> Outcome<OzoneConfig> {
49 let mut c = res!(setup::default_cfg());
50 c.num_zones = 1;
51 c.num_cbots_per_zone = 4;
52 c.num_fbots_per_zone = 1;
53 c.num_wbots_per_zone = 1;
54 c.zone_overrides = BTreeMap::new();
55 c.rest_chunk_threshold = 1_000;
56 c.rest_chunk_bytes = 400;
57 c.sync_on_write = true;
58 c.data_file_max_bytes = 4_000;
59 Ok(c)
60}
61
62const VLEN: usize = 1_100;
63
64fn val(len: usize, seed: u8) -> Dat {
65 Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect())
66}
67
68fn whole(v: &Dat) -> bool {
69 match v {
70 Dat::BU32(b) | Dat::BU64(b) | Dat::BU8(b) | Dat::BU16(b) => {
71 if b.len() != VLEN { return false; }
72 let seed = b[0];
73 b.iter().enumerate().all(|(j, x)| *x == seed ^ (j as u8))
74 },
75 _ => false,
76 }
77}
78
79fn start(dir: &str, c: OzoneConfig, gc_on: bool, wipe: bool) -> Outcome<TestDb> {
80 log_set_level!("error");
81 if wipe {
82 let _ = std::fs::remove_dir_all(dir);
83 }
84 res!(std::fs::create_dir_all(dir));
85 let root = res!(Path::new(dir).canonicalize());
86 setup::start_db(root, Some(c), schemes(), None, gc_on, wipe)
87}
88
89fn scan_wait() -> Wait {
90 Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL }
91}
92
93fn head(db: &TestDb, k: &Dat) -> Outcome<Option<[u64; 5]>> {
94 match res!(db.api().get_head_wait(k, None)) {
95 Some((Dat::Tup5u64(t), _)) => Ok(Some(t)),
96 _ => Ok(None),
97 }
98}
99
100#[derive(Debug, Default, Clone)]
101struct Census {
102 chunks: usize,
103 tombs: usize,
104 live: usize,
105 anc: usize,
106 older: usize,
107 disk: u64,
108}
109
110fn census(db: &TestDb, dir: &str) -> Outcome<Census> {
111 let _ = db.api().settle_for_test(Duration::from_secs(20));
112 let mut c = Census::default();
113 for (_, cache) in res!(db.api().collect_caches(scan_wait())) {
114 c.anc += cache.get_ancillary_size();
115 let tt = cache.tomb_tracker();
116 c.older += tt.older().values().map(|s| s.len()).sum::<usize>();
117 for (k, _) in cache.map() {
118 if !is_chunk_key(k) { continue; }
119 c.chunks += 1;
120 match tt.tombs().contains_key(k) {
121 true => c.tombs += 1,
122 false => c.live += 1,
123 }
124 }
125 }
126 c.disk = dir_len(&res!(Path::new(dir).canonicalize()));
127 Ok(c)
128}
129
130// Background GC on throughout, several keys overwritten from one thread while another reads.
131// The footprint, the tombstones held and the ancillary size must plateau, not grow with writes.
132#[test]
133fn qa2_bg_gc_overwrite_bounded() -> Outcome<()> {
134 let _lock = lock();
135 let dir = "./test_db_qa2_bg_bounded";
136 let db = res!(start(dir, res!(cfg_b()), true, true));
137 let keys: Vec<Dat> = (0..3).map(|i| dat!(fmt!("qa2 bounded {}", i))).collect();
138 for k in &keys {
139 res!(db.insert(k.clone(), val(VLEN, 0), Uid::default(), None));
140 }
141 let stop = AtomicBool::new(false);
142 let torn = AtomicUsize::new(0);
143 let missing = AtomicUsize::new(0);
144 let mut series = Vec::new();
145 let rounds = 5;
146 let per = 120;
147 let result: Outcome<()> = thread::scope(|s| {
148 s.spawn(|| {
149 while !stop.load(Ordering::Relaxed) {
150 for k in &keys {
151 match db.get(k, None) {
152 Ok(Some((v, _))) => if !whole(&v) { torn.fetch_add(1, Ordering::Relaxed); },
153 Ok(None) => { missing.fetch_add(1, Ordering::Relaxed); },
154 Err(_) => { torn.fetch_add(1, Ordering::Relaxed); },
155 }
156 }
157 }
158 });
159 let mut n = 1u32;
160 for _ in 0..rounds {
161 for _ in 0..per {
162 for k in &keys {
163 if let Err(e) = db.insert(k.clone(), val(VLEN, (n % 251) as u8 + 1), Uid::default(), None) {
164 stop.store(true, Ordering::Relaxed);
165 return Err(e);
166 }
167 }
168 n += 1;
169 }
170 thread::sleep(Duration::from_secs(2));
171 match census(&db, dir) {
172 Ok(c) => series.push(c),
173 Err(e) => { stop.store(true, Ordering::Relaxed); return Err(e); },
174 }
175 }
176 stop.store(true, Ordering::Relaxed);
177 Ok(())
178 });
179 let mut fin = Vec::new();
180 for k in &keys {
181 fin.push(res!(db.get(k, None)).map(|(v, _)| whole(&v)).unwrap_or(false));
182 }
183 res!(db.close());
184 // Restart: replay drops the lone ones; the store must be the same three values.
185 let db = res!(start(dir, res!(cfg_b()), true, false));
186 let after = res!(census(&db, dir));
187 let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait()));
188 let mut fin2 = Vec::new();
189 for k in &keys {
190 fin2.push(res!(db.get(k, None)).map(|(v, _)| whole(&v)).unwrap_or(false));
191 }
192 res!(db.close());
193 let _ = std::fs::remove_dir_all(dir);
194 res!(result);
195 for (i, c) in series.iter().enumerate() {
196 println!("QA2 bounded round {} ({} overwrites): {:?}", i + 1, (i + 1) * per * keys.len(), c);
197 }
198 println!("QA2 bounded after restart: {:?}, sweep {:?}", after, report.orphans_found);
199 assert_eq!(torn.load(Ordering::Relaxed), 0, "torn or failed reads");
200 assert_eq!(missing.load(Ordering::Relaxed), 0, "reads found no value");
201 assert!(fin.iter().all(|b| *b) && fin2.iter().all(|b| *b), "final values {:?} {:?}", fin, fin2);
202 assert_eq!(report.orphans_found, 0, "orphans after restart");
203 let first = &series[1];
204 let last = &series[rounds - 1];
205 assert!(last.tombs <= first.tombs * 2 + 40, "tombstones grow: {} then {}", first.tombs, last.tombs);
206 assert!(last.anc <= first.anc * 2 + 4_096, "ancillary grows: {} then {}", first.anc, last.anc);
207 assert!(last.disk <= first.disk * 2 + 40_000, "disk grows: {} then {}", first.disk, last.disk);
208 Ok(())
209}
210
211// The chunk_leak shrink geometry with background GC, sampled every ten overwrites over two
212// hundred, to tell a linear leak from sampling jitter around a plateau. Run under each sync
213// policy: a discard held for a barrier can be flagged old after a collection has carried its
214// record into a small rewritten file, and that file must still collect.
215#[test]
216fn qa2_shrink_footprint_series() -> Outcome<()> {
217 shrink_series("./test_db_qa2_shrink", true, 0)
218}
219
220#[test]
221fn qa2_shrink_footprint_series_interval() -> Outcome<()> {
222 shrink_series("./test_db_qa2_shrink_interval", false, 200)
223}
224
225#[test]
226fn qa2_shrink_footprint_series_never() -> Outcome<()> {
227 shrink_series("./test_db_qa2_shrink_never", false, 0)
228}
229
230// Sealed files still holding more old bytes than the collector lets stand once it is quiet:
231// past the trigger's share of the maximum file length, or at the high share of the file's own
232// length that catches a shrunk file flagged old late. Polled for ten seconds, so that
233// collections under way can finish.
234fn stranded(db: &TestDb, max: u64) -> Outcome<Vec<(u32, u64, usize)>> {
235 let mut found = Vec::new();
236 for _ in 0..40 {
237 found.clear();
238 for (_, fmap) in res!(db.api().collect_file_states(scan_wait())) {
239 for (fnum, fstat) in fmap.map() {
240 let old = fstat.get_old_sum() as f64;
241 let size = fstat.get_data_file_size();
242 let of_max = 100.0 * old / max as f64;
243 let of_own = if size == 0 { 0.0 } else { 100.0 * old / size as f64 };
244 if !fstat.is_live() && (of_max > constant::OLD_DATA_PERCENT_GC_TRIGGER
245 || of_own >= constant::OLD_SHARE_GC_TRIGGER)
246 {
247 found.push((*fnum, fstat.get_old_sum(), size));
248 }
249 }
250 }
251 if found.is_empty() {
252 break;
253 }
254 thread::sleep(Duration::from_millis(250));
255 }
256 Ok(found)
257}
258
259// `sync_on_write`, else `interval_ms`, else no periodic barrier at all.
260fn shrink_series(dir: &str, sync_on_write: bool, interval_ms: u64) -> Outcome<()> {
261 let _lock = lock();
262 let mut c = res!(setup::default_cfg());
263 c.data_file_max_bytes = 16_000;
264 c.rest_chunk_threshold = 3_000;
265 c.rest_chunk_bytes = 1_000;
266 c.sync_on_write = sync_on_write;
267 c.sync_every_n_writes = 0;
268 c.sync_interval_ms = interval_ms;
269 c.zone_overrides = BTreeMap::new();
270 let db = res!(start(dir, c, true, true));
271 let k = dat!("chunked:shrinking");
272 let big = Dat::BU32(vec![7u8; 6_000]);
273 res!(db.insert(k.clone(), big, Uid::default(), None));
274 let mut samples = Vec::new();
275 for i in 1..=200u32 {
276 let v = Dat::BU32((0..3_200).map(|j| (i as u8).wrapping_add((j % 251) as u8)).collect());
277 res!(db.insert(k.clone(), v, Uid::default(), None));
278 if i % 10 == 0 {
279 thread::sleep(Duration::from_secs(3));
280 let root = res!(Path::new(dir).canonicalize());
281 let mut tombs = 0;
282 for (_, cache) in res!(db.api().collect_caches(scan_wait())) {
283 tombs += cache.tomb_tracker().tombs().len();
284 }
285 samples.push((i, dir_len(&root), tombs));
286 }
287 }
288 let left = res!(stranded(&db, 16_000));
289 let rep = res!(db.api().compact_now(Duration::from_secs(60)));
290 let root = res!(Path::new(dir).canonicalize());
291 let compacted = dir_len(&root);
292 res!(db.close());
293 let _ = std::fs::remove_dir_all(dir);
294 for s in &samples {
295 println!("QA2 shrink {} after {:3}: dir {:6} bytes, tombstones held {}", dir, s.0, s.1, s.2);
296 }
297 println!("QA2 shrink compacted: {} bytes, {:?}", compacted, rep);
298 assert!(left.is_empty(), "{}: sealed files left over the trigger (file, old, size): {:?}", dir, left);
299 let w10 = samples[0].1;
300 let mx = samples.iter().map(|s| s.1).max().unwrap_or(0);
301 let late: Vec<u64> = samples[10..].iter().map(|s| s.1).collect();
302 let late_avg = late.iter().sum::<u64>() / late.len() as u64;
303 let early_avg = samples[1..10].iter().map(|s| s.1).sum::<u64>() / 9;
304 println!("QA2 shrink {}: first {} max {} early avg {} late avg {}", dir, w10, mx, early_avg, late_avg);
305 assert!(late_avg <= early_avg * 3 / 2, "linear growth: early avg {} late avg {}", early_avg, late_avg);
306 Ok(())
307}
308
309// Cycles of overwrite, delete, held files and failed directory syncs, each closed and reopened
310// with GC on. A set once retired must never read again, the sweep must find no orphan after a
311// clean close, and every live value must read whole.
312#[test]
313fn qa2_restart_cycles_no_resurrection() -> Outcome<()> {
314 let _lock = lock();
315 let dir = "./test_db_qa2_cycles";
316 let keys: Vec<Dat> = (0..4).map(|i| dat!(fmt!("qa2 cycle {}", i))).collect();
317 let mut retired: Vec<[u64; 5]> = Vec::new();
318 let mut want: BTreeMap<usize, Option<u8>> = BTreeMap::new();
319 let mut bad: Vec<String> = Vec::new();
320 let mut seed = 1u8;
321 for cycle in 0..10usize {
322 let db = res!(start(dir, res!(cfg_b()), true, cycle == 0));
323 // After the replay: retired sets stay gone, no orphan, live values whole.
324 for h in &retired {
325 if db.api().fetch_chunks(&Dat::Tup5u64(*h), None).is_ok() {
326 bad.push(fmt!("cycle {}: retired set {:?} reads again", cycle, h));
327 }
328 }
329 let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait()));
330 if report.orphans_found != 0 {
331 bad.push(fmt!("cycle {}: {} orphans after restart", cycle, report.orphans_found));
332 }
333 for (i, w) in &want {
334 let got = res!(db.get(&keys[*i], None));
335 match (w, got) {
336 (Some(s), Some((v, _))) => if v != val(VLEN, *s) {
337 bad.push(fmt!("cycle {}: key {} wrong value", cycle, i));
338 },
339 (None, None) => (),
340 (w, g) => bad.push(fmt!("cycle {}: key {} want {:?} got some={}", cycle, i, w, g.is_some())),
341 }
342 }
343 let c = res!(census(&db, dir));
344 println!("QA2 cycle {} start: {:?}", cycle, c);
345 // A hold on some existing file, or failed directory syncs, for part of the cycle.
346 match cycle % 3 {
347 0 => hooks::set_collect_fails_for(Some((cycle as u32 % 5) + 1)),
348 1 => hooks::set_dir_sync_failure(true),
349 _ => (),
350 }
351 for round in 0..6 {
352 for (i, k) in keys.iter().enumerate() {
353 if let Some(h) = res!(head(&db, k)) {
354 if (round + i + cycle) % 7 == 3 {
355 res!(db.delete(k, Uid::default(), None));
356 retired.push(h);
357 want.insert(i, None);
358 continue;
359 }
360 retired.push(h);
361 }
362 seed = seed.wrapping_add(1).max(1);
363 res!(db.insert(k.clone(), val(VLEN, seed), Uid::default(), None));
364 want.insert(i, Some(seed));
365 }
366 if round == 3 {
367 hooks::set_collect_fails_for(None);
368 hooks::set_dir_sync_failure(false);
369 }
370 }
371 if cycle % 2 == 0 {
372 let _ = db.api().compact_now(Duration::from_secs(30));
373 }
374 thread::sleep(Duration::from_millis(500));
375 res!(db.close());
376 }
377 let db = res!(start(dir, res!(cfg_b()), true, false));
378 for h in &retired {
379 if db.api().fetch_chunks(&Dat::Tup5u64(*h), None).is_ok() {
380 bad.push(fmt!("final: retired set {:?} reads again", h));
381 }
382 }
383 let fin = res!(census(&db, dir));
384 res!(db.close());
385 let _ = std::fs::remove_dir_all(dir);
386 println!("QA2 cycles final: {:?}", fin);
387 assert!(bad.is_empty(), "{:#?}", bad);
388 Ok(())
389}
390
391// The chunk-form key refusal: a small user value under a Tup5u64 key is refused, a delete of
392// one is not, and an ordinary key that merely starts with the Tup5u64 code byte is not refused.
393#[test]
394fn qa2_chunk_form_key_refusal() -> Outcome<()> {
395 let _lock = lock();
396 let dir = "./test_db_qa2_refusal";
397 let db = res!(start(dir, res!(cfg_b()), false, true));
398 let tk = Dat::Tup5u64([1, 2, 3, 4, 5]);
399 let small = db.insert(tk.clone(), dat!("small"), Uid::default(), None);
400 let del = db.delete(&tk, Uid::default(), None);
401 // Byte strings whose raw form starts with 0x93 are framed by their own Dat code, so never
402 // decode as a Tup5u64.
403 let mut raw = vec![0x93u8];
404 raw.extend_from_slice(&[0u8; 40]);
405 let bk = Dat::BU8(raw);
406 let bytes_key = db.insert(bk.clone(), dat!("v"), Uid::default(), None);
407 let read_back = db.get(&bk, None);
408 res!(db.close());
409 let _ = std::fs::remove_dir_all(dir);
410 assert!(small.is_err(), "a small value under a Tup5u64 key was accepted");
411 assert!(del.is_ok(), "a delete under a Tup5u64 key was refused: {:?}", del.err());
412 assert!(bytes_key.is_ok(), "a byte key starting 0x93 was refused");
413 assert!(matches!(res!(read_back), Some(_)), "the byte key does not read back");
414 Ok(())
415}
416
417// A store's config.jdat keeps `init_load_caches: true`, because a rolled-back binary needs the
418// field to read it, and an unchanged open and close leaves the file byte for byte as it was. A
419// stored `false` asks for an open without replay, which no longer exists, and is refused by name.
420#[test]
421fn qa2_stored_init_load_caches_kept_and_false_refused() -> Outcome<()> {
422 let _lock = lock();
423 let dir = "./test_db_qa2_ilc";
424 let db = res!(start(dir, res!(cfg_b()), false, true));
425 res!(db.close());
426 let path = OzoneConfig::config_path(&res!(Path::new(dir).canonicalize()));
427 let before = res!(std::fs::read_to_string(&path));
428 if !before.lines().any(|l| l.contains("\"init_load_caches\"") && l.contains("true")) {
429 return Err(err!("A new store's config.jdat lacks init_load_caches: true:\n{}", before;
430 Test, Missing));
431 }
432 for _ in 0..2 {
433 let db = res!(start(dir, res!(cfg_b()), false, false));
434 res!(db.close());
435 let after = res!(std::fs::read_to_string(&path));
436 if after != before {
437 return Err(err!("An unchanged open and close rewrote config.jdat:\nwas\n{}\nnow\n{}",
438 before, after; Test, Mismatch));
439 }
440 }
441 let falsed = before.lines()
442 .map(|l| if l.contains("\"init_load_caches\"") { l.replace("true", "false") } else { l.to_string() })
443 .collect::<Vec<_>>().join("\n") + "\n";
444 if falsed == before {
445 return Err(err!("The test could not set init_load_caches to false."; Test, Bug));
446 }
447 res!(std::fs::write(&path, &falsed));
448 match start(dir, res!(cfg_b()), false, false) {
449 Ok(db) => {
450 let _ = db.close();
451 return Err(err!("A store whose config.jdat sets init_load_caches to false opened.";
452 Test, Mismatch));
453 },
454 Err(e) => if !fmt!("{}", e).contains("init_load_caches") {
455 return Err(err!("The refusal does not name init_load_caches: {}", e; Test, Mismatch));
456 },
457 }
458 let _ = std::fs::remove_dir_all(dir);
459 Ok(())
460}