Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/qa1_chunkset.rs

29.7 KiB, 36 runs

created by r1870400018:70211, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// 2026-10-09 ore-a: describe() now labels the known literals ("small", "small one", "small two") and every deleted-kind Usr value as tomb; TORN only for anything else.
2//! QA agent 1 (2026-10-09), chunk sets and fix A: adversarial probes of the write, delete and read
3//! paths. The hooks are process-wide, so every test takes one lock. Cross-process cases run the
4//! phases in child processes of this binary (`qa1_child`, env QA1_MODE / QA1_DIR / QA1_T), so a
5//! phase starts with a fresh process stamp, as a real restart does.
6
7mod gc_pair;
8
9use gc_pair::*;
10
11use oxedyne_fe2o3_core::prelude::*;
12use oxedyne_fe2o3_data::time::Timestamp;
13use oxedyne_fe2o3_hash::{
14 csum::ChecksumScheme,
15 hash::HashScheme,
16};
17use oxedyne_fe2o3_iop_db::api::Database;
18use oxedyne_fe2o3_jdat::prelude::*;
19use oxedyne_fe2o3_o3db_sync::{
20 api::OzoneApi,
21 base::{
22 cfg::OzoneConfig,
23 constant,
24 },
25 comm::response::Wait,
26 data::cache::{
27 CacheEntry,
28 is_chunk_key,
29 },
30 file::refusal::Refusal,
31 sweep,
32 test::{
33 hooks,
34 setup::{
35 self,
36 Uid,
37 UID_LEN,
38 },
39 },
40 verify,
41};
42
43use std::{
44 collections::BTreeMap,
45 path::Path,
46 process::Command,
47 sync::{
48 Arc,
49 Mutex,
50 MutexGuard,
51 atomic::{
52 AtomicBool,
53 AtomicUsize,
54 Ordering,
55 },
56 },
57 thread,
58 time::Duration,
59};
60
61static HOOKS: Mutex<()> = Mutex::new(());
62
63fn lock() -> MutexGuard<'static, ()> {
64 match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() }
65}
66
67fn cfg() -> Outcome<OzoneConfig> {
68 let mut c = res!(setup::default_cfg());
69 c.num_zones = 1;
70 c.num_cbots_per_zone = 4;
71 c.num_fbots_per_zone = 1;
72 c.num_wbots_per_zone = 2;
73 c.zone_overrides = BTreeMap::new();
74 c.rest_chunk_threshold = 1_000;
75 c.rest_chunk_bytes = 400;
76 c.sync_on_write = true;
77 Ok(c)
78}
79
80fn val(len: usize, seed: u8) -> Dat {
81 Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect())
82}
83
84fn seed_of(v: &Dat) -> Option<(usize, u8)> {
85 match v {
86 Dat::BU32(b) | Dat::BU64(b) | Dat::BU8(b) | Dat::BU16(b) if !b.is_empty() => {
87 let s = b[0];
88 if b.iter().enumerate().all(|(j, x)| *x == s ^ (j as u8)) { Some((b.len(), s)) } else { None }
89 },
90 _ => None,
91 }
92}
93
94fn tomb() -> Dat {
95 Dat::Usr(oxedyne_fe2o3_o3db_sync::base::id::usr_kind_id_deleted(), Some(Box::new(Dat::Empty)))
96}
97
98fn start(dir: &str, c: OzoneConfig, wipe: bool) -> Outcome<TestDb> {
99 log_set_level!("error");
100 if wipe {
101 let _ = std::fs::remove_dir_all(dir);
102 }
103 res!(std::fs::create_dir_all(dir));
104 let root = res!(Path::new(dir).canonicalize());
105 setup::start_db(root, Some(c), schemes(), None, false, wipe)
106}
107
108fn scan_wait() -> Wait {
109 Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL }
110}
111
112fn head(db: &TestDb, k: &Dat) -> Outcome<Option<[u64; 5]>> {
113 match res!(db.api().get_head_wait(k, None)) {
114 Some((Dat::Tup5u64(t), _)) => Ok(Some(t)),
115 _ => Ok(None),
116 }
117}
118
119// What a read found: "none", "tomb", "L/S" for a whole value of length L and seed S, "TORN", or
120// "ERR:<text>".
121fn describe(r: &Outcome<Option<(Dat, oxedyne_fe2o3_iop_db::api::Meta<UID_LEN, Uid>)>>) -> String {
122 match r {
123 Ok(None) => fmt!("none"),
124 Ok(Some((Dat::Usr(kind, _), _))) if *kind == oxedyne_fe2o3_o3db_sync::base::id::usr_kind_id_deleted() => fmt!("tomb"),
125 Ok(Some((d, _))) if *d == dat!("small") => fmt!("small"),
126 Ok(Some((d, _))) if *d == dat!("small one") => fmt!("small one"),
127 Ok(Some((d, _))) if *d == dat!("small two") => fmt!("small two"),
128 Ok(Some((d, _))) => match seed_of(d) {
129 Some((l, s)) => fmt!("{}/{}", l, s),
130 None => fmt!("TORN:{:?}", d).chars().take(120).collect(),
131 },
132 Err(e) => fmt!("ERR:{}", fmt!("{}", e).chars().take(160).collect::<String>()),
133 }
134}
135
136fn stamp_of(db: &TestDb, k: &Dat) -> String {
137 match db.api().get_head_wait(k, None) {
138 Ok(Some((_, m))) => fmt!("{:?}", *m.time),
139 other => fmt!("{:?}", other.map(|o| o.is_some())),
140 }
141}
142
143// ---------------------------------------------------------------------------------------------
144// Child-process phases.
145
146fn child(mode: &str, dir: &str, t: Option<u128>) -> Outcome<String> {
147 let exe = res!(std::env::current_exe());
148 let mut cmd = Command::new(exe);
149 cmd.args(["qa1_child", "--exact", "--ignored", "--nocapture", "--test-threads=1"])
150 .env("QA1_MODE", mode)
151 .env("QA1_DIR", dir);
152 if let Some(t) = t { cmd.env("QA1_T", fmt!("{}", t)); }
153 let out = res!(cmd.output());
154 let so = String::from_utf8_lossy(&out.stdout).to_string();
155 // The harness prints `test qa1_child ... ` with no newline, so the child's first line
156 // follows it on the same line.
157 let lines: Vec<&str> = so.lines().filter_map(|l| l.find("QA1 ").map(|i| &l[i..])).collect();
158 Ok(fmt!("status={:?}\n{}", out.status.code(), lines.join("\n")))
159}
160
161#[test]
162#[ignore]
163fn qa1_child() -> Outcome<()> {
164 let mode = match std::env::var("QA1_MODE") { Ok(m) => m, Err(_) => return Ok(()) };
165 let dir = res!(std::env::var("QA1_DIR"));
166 let t: Option<u128> = std::env::var("QA1_T").ok().and_then(|s| s.parse().ok());
167 let dur = |n: u128| Duration::new((n / 1_000_000_000) as u64, (n % 1_000_000_000) as u32);
168 match mode.as_str() {
169 // F3: a store written while the clock read a minute ahead.
170 "f3_ahead" => {
171 let db = res!(start(&dir, res!(cfg()), true));
172 if let Some(t) = t { hooks::set_fixed_stamp(Some(dur(t))); }
173 res!(db.insert(dat!("k1"), val(1_100, 1), Uid::default(), None));
174 res!(db.insert(dat!("k2"), dat!("small one"), Uid::default(), None));
175 res!(db.insert(dat!("k3"), val(1_100, 3), Uid::default(), None));
176 hooks::set_fixed_stamp(None);
177 println!("QA1 ahead k1={} stamp={}", describe(&db.get(&dat!("k1"), None)), stamp_of(&db, &dat!("k1")));
178 res!(db.close());
179 },
180 // F3: the next process, with the clock back where it should be.
181 "f3_behind" => {
182 let db = res!(start(&dir, res!(cfg()), false));
183 let p1 = db.insert(dat!("k1"), val(1_500, 2), Uid::default(), None).map(|_| ());
184 let p2 = db.insert(dat!("k2"), dat!("small two"), Uid::default(), None).map(|_| ());
185 let d3 = db.delete(&dat!("k3"), Uid::default(), None).map(|_| ());
186 let p4 = db.insert(dat!("k4"), val(1_100, 4), Uid::default(), None).map(|_| ());
187 println!("QA1 calls put_k1={:?} put_k2={:?} del_k3={:?} put_k4={:?}",
188 p1.is_ok(), p2.is_ok(), d3.is_ok(), p4.is_ok());
189 println!("QA1 live k1={} k2={:?} k3={} k4={} k1stamp={}",
190 describe(&db.get(&dat!("k1"), None)),
191 db.get(&dat!("k2"), None).map(|o| o.map(|(d, _)| d)).ok(),
192 describe(&db.get(&dat!("k3"), None)),
193 describe(&db.get(&dat!("k4"), None)),
194 stamp_of(&db, &dat!("k1")));
195 let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait()));
196 println!("QA1 sweep orphans_found={} skipped_pending={}", report.orphans_found, report.skipped_pending);
197 res!(db.close());
198 let db = res!(start(&dir, res!(cfg()), false));
199 println!("QA1 restarted k1={} k2={:?} k3={}",
200 describe(&db.get(&dat!("k1"), None)),
201 db.get(&dat!("k2"), None).map(|o| o.map(|(d, _)| d)).ok(),
202 describe(&db.get(&dat!("k3"), None)));
203 res!(db.close());
204 },
205 // Equal stamps across processes: phase a and b write the same keys in the same order
206 // with the clock fixed at T, so each key's two records carry one stamp.
207 "tie_a" | "tie_b" => {
208 let wipe = mode == "tie_a";
209 let seed = if wipe { 1u8 } else { 2u8 };
210 let db = res!(start(&dir, res!(cfg()), wipe));
211 if let Some(t) = t { hooks::set_fixed_stamp(Some(dur(t))); }
212 let mut stamps = Vec::new();
213 for i in 0..12u8 {
214 let k = dat!(fmt!("tie {}", i));
215 let r = db.insert(k.clone(), val(1_100, seed.wrapping_add(i * 16)), Uid::default(), None);
216 stamps.push(fmt!("{}:{}", i, if r.is_ok() { stamp_of(&db, &k) } else { fmt!("ERR") }));
217 }
218 hooks::set_fixed_stamp(None);
219 let reads: Vec<String> = (0..12u8).map(|i| describe(&db.get(&dat!(fmt!("tie {}", i)), None))).collect();
220 println!("QA1 {} reads {:?}", mode, reads);
221 println!("QA1 {} stamps {:?}", mode, stamps);
222 res!(db.close());
223 },
224 "tie_c" => {
225 let db = res!(start(&dir, res!(cfg()), false));
226 let reads: Vec<String> = (0..12u8).map(|i| describe(&db.get(&dat!(fmt!("tie {}", i)), None))).collect();
227 let stamps: Vec<String> = (0..12u8).map(|i| stamp_of(&db, &dat!(fmt!("tie {}", i)))).collect();
228 let v = res!(verify::verify_live_set(db.api(), None, scan_wait()));
229 println!("QA1 tie_c reads {:?}", reads);
230 println!("QA1 tie_c stamps {:?}", stamps);
231 println!("QA1 tie_c verify clean={} {}", v.clean(), v.summary(4).replace('\n', " | "));
232 res!(db.close());
233 },
234 // A legacy value, then an overwrite of one length killed between its durable head and its
235 // retire, or before its head.
236 "legacy_after_head" | "legacy_before_head" => {
237 let db = res!(start(&dir, res!(cfg()), true));
238 let k = dat!("legacy crash");
239 let (kbuf, _, _) = res!(db.api().ozone_key_dat(&k, None));
240 let legacy = OzoneApi::<{ UID_LEN }, Uid, (), HashScheme, HashScheme, ChecksumScheme>::legacy_chunk_set_id(&kbuf);
241 let resp = db.api().responder();
242 res!(db.api().store_dat_using_responder_forcing_set_id(
243 k.clone(), val(1_100, 1), Uid::default(), None, resp.clone(), legacy));
244 res!(resp.recv_store_ack());
245 println!("QA1 legacy head={:?}", res!(head(&db, &k)));
246 if mode == "legacy_after_head" {
247 hooks::set_retire_drop(true);
248 } else {
249 hooks::set_chunk_insert_delay(Duration::from_secs(20));
250 }
251 let (db2, k2) = (db.clone(), k.clone());
252 let _put = thread::spawn(move || db2.insert(k2, val(1_100, 2), Uid::default(), None).map(|_| ()));
253 thread::sleep(Duration::from_millis(1_500));
254 println!("QA1 aborting");
255 std::process::abort();
256 },
257 // Overwritten chunked values, then a collection killed between its new data file and
258 // its new index file.
259 "collect_crash" => {
260 let db = res!(start_collecting(&dir, true));
261 for round in 0..6u8 {
262 for i in 0..4u8 {
263 res!(db.insert(dat!(fmt!("crash {}", i)), val(1_100, round * 16 + i), Uid::default(), None));
264 }
265 }
266 res!(db.delete(&dat!("crash 3"), Uid::default(), None));
267 res!(db.api().settle_for_test(Duration::from_secs(20)));
268 hooks::set_commit_delay(Duration::from_secs(30));
269 let db2 = db.clone();
270 let _gc = thread::spawn(move || db2.api().compact_now(Duration::from_secs(60)).map(|_| ()));
271 thread::sleep(Duration::from_millis(2_500));
272 println!("QA1 aborting");
273 std::process::abort();
274 },
275 _ => return Err(err!("Unknown QA1_MODE {}.", mode; Test, Invalid)),
276 }
277 Ok(())
278}
279
280fn start_collecting(dir: &str, wipe: bool) -> Outcome<TestDb> {
281 let mut c = res!(cfg());
282 c.data_file_max_bytes = 4_000;
283 c.num_wbots_per_zone = 1;
284 log_set_level!("error");
285 if wipe {
286 let _ = std::fs::remove_dir_all(dir);
287 }
288 res!(std::fs::create_dir_all(dir));
289 let root = res!(Path::new(dir).canonicalize());
290 let db = res!(setup::start_db(root, Some(c), schemes(), None, true, wipe));
291 res!(db.api().activate_gc(false));
292 Ok(db)
293}
294
295// Live chunk records, tombstones aside, and the chunks the current heads name.
296fn chunk_bound(db: &TestDb, keys: &[Dat]) -> Outcome<(usize, usize)> {
297 res!(db.api().settle_for_test(Duration::from_secs(20)));
298 let mut live = 0;
299 for (_, cache) in res!(db.api().collect_caches(scan_wait())) {
300 let tombs = cache.tomb_tracker().tombs();
301 for (k, e) in cache.map() {
302 if is_chunk_key(k) && matches!(e, CacheEntry::LocatedValue(..)) && !tombs.contains_key(k) {
303 live += 1;
304 }
305 }
306 }
307 let mut named = 0;
308 for k in keys {
309 if let Some(t) = res!(head(db, k)) {
310 named += t[3] as usize;
311 }
312 }
313 Ok((live, named))
314}
315
316// A process killed mid-collection, between its new data file and its new index file: every key
317// reads its last value whole after a restart, before and after the sweep, verify is clean, and
318// the next overwrites leave no more live chunks than the heads name.
319#[test]
320fn qa1_kill_mid_collection() -> Outcome<()> {
321 let _lock = lock();
322 let dir = "./test_db_qa1_collect_crash";
323 let out = res!(child("collect_crash", dir, None));
324 let mut bad = Vec::new();
325 if !out.contains("QA1 aborting") { bad.push(fmt!("child did not reach the crash: {}", out)); }
326 let keys: Vec<Dat> = (0..4).map(|i| dat!(fmt!("crash {}", i))).collect();
327 let want = |i: usize| if i == 3 { fmt!("none") } else { fmt!("1100/{}", 5 * 16 + i) };
328 let reads = |db: &TestDb| -> Vec<String> { keys.iter()
329 .map(|k| match describe(&db.get(k, None)) { d if d == "tomb" => fmt!("none"), d => d }).collect() };
330 let wanted: Vec<String> = (0..4).map(want).collect();
331 let db = res!(start_collecting(dir, false));
332 let got = reads(&db);
333 if got != wanted { bad.push(fmt!("restart read {:?} want {:?}", got, wanted)); }
334 let v = res!(verify::verify_live_set(db.api(), None, scan_wait()));
335 if !v.clean() { bad.push(fmt!("verify {}", v.summary(4))); }
336 let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait()));
337 let swept = reads(&db);
338 if swept != wanted { bad.push(fmt!("after sweep read {:?}", swept)); }
339 for (i, k) in keys.iter().enumerate() {
340 res!(db.insert(k.clone(), val(1_100, 0xC0 + i as u8), Uid::default(), None));
341 }
342 for _ in 0..3 {
343 res!(db.api().settle_for_test(Duration::from_secs(20)));
344 res!(db.api().compact_now(Duration::from_secs(60)));
345 }
346 let (live, named) = res!(chunk_bound(&db, &keys));
347 if live != named { bad.push(fmt!("live chunks {} against {} named", live, named)); }
348 res!(db.close());
349 let db = res!(start_collecting(dir, false));
350 let last = reads(&db);
351 let lasts: Vec<String> = (0..4).map(|i| fmt!("1100/{}", 0xC0 + i)).collect();
352 if last != lasts { bad.push(fmt!("final restart read {:?}", last)); }
353 let v = res!(verify::verify_live_set(db.api(), None, scan_wait()));
354 if !v.clean() { bad.push(fmt!("verify end {}", v.summary(4))); }
355 res!(db.close());
356 let _ = std::fs::remove_dir_all(dir);
357 println!("QA1 collect crash sweep_orphans={} {}", report.orphans_found, out.replace('\n', " | "));
358 assert!(bad.is_empty(), "{:#?}", bad);
359 Ok(())
360}
361
362// ---------------------------------------------------------------------------------------------
363// Q: two overlapping puts where the earlier-stamped publishes and reads back first and the later
364// one wins: is the middle set left for the sweep? Counts rounds with orphans after both settle.
365
366#[test]
367fn qa1_overlap_orphans() -> Outcome<()> {
368 let _lock = lock();
369 let dir = "./test_db_qa1_overlap";
370 let db = res!(start(dir, res!(cfg()), true));
371 let mut rounds_with_orphans = 0usize;
372 let mut orphan_total = 0usize;
373 let mut bad = Vec::new();
374 let rounds = 10usize;
375 for r in 0..rounds {
376 let k = dat!(fmt!("overlap {}", r));
377 res!(db.insert(k.clone(), val(1_100, 0), Uid::default(), None));
378 hooks::set_chunk_insert_delay(Duration::from_millis(40));
379 let b = { let (db, k) = (db.clone(), k.clone());
380 thread::spawn(move || db.insert(k, val(1_100, 0xB0), Uid::default(), None).map(|_| ()).map_err(|e| fmt!("{}", e))) };
381 thread::sleep(Duration::from_millis(15));
382 let a = { let (db, k) = (db.clone(), k.clone());
383 thread::spawn(move || db.insert(k, val(1_100, 0xA0), Uid::default(), None).map(|_| ()).map_err(|e| fmt!("{}", e))) };
384 let rb = match b.join() { Ok(x) => x, Err(_) => Err(fmt!("panicked")) };
385 let ra = match a.join() { Ok(x) => x, Err(_) => Err(fmt!("panicked")) };
386 hooks::set_chunk_insert_delay(Duration::ZERO);
387 if rb.is_err() || ra.is_err() { bad.push(fmt!("round {}: puts {:?} {:?}", r, rb, ra)); }
388 let got = describe(&db.get(&k, None));
389 if !(got == "1100/160" || got == "1100/176") { bad.push(fmt!("round {}: read {}", r, got)); }
390 let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait()));
391 if report.orphans_found > 0 { rounds_with_orphans += 1; orphan_total += report.orphans_found; }
392 let after = describe(&db.get(&k, None));
393 if after != got { bad.push(fmt!("round {}: after sweep read {} (was {})", r, after, got)); }
394 println!("QA1 overlap round {} final={} orphans_found={}", r, got, report.orphans_found);
395 }
396 res!(db.close());
397 let _ = std::fs::remove_dir_all(dir);
398 println!("QA1 overlap rounds_with_orphans={}/{} orphan_chunks={}", rounds_with_orphans, rounds, orphan_total);
399 assert!(bad.is_empty(), "{:?}", bad);
400 assert_eq!(rounds_with_orphans, 0, "overlapping puts left chunk sets for the sweep: {} chunks over {} rounds",
401 orphan_total, rounds_with_orphans);
402 Ok(())
403}
404
405// ---------------------------------------------------------------------------------------------
406// Q: concurrent puts and deletes (both roads) on one key, with readers: every read whole-or-none,
407// the final value whole, the same across a restart, verify clean, and the sweep takes nothing live.
408
409fn puts_deletes(dir: &str, cache_lim: Option<usize>, iters: usize) -> Outcome<(Vec<String>, usize)> {
410 let mut c = res!(cfg());
411 if let Some(l) = cache_lim { c.cache_size_limit_bytes = l as u64; }
412 let mut bad = Vec::new();
413 let mut orphans = 0usize;
414 let lens = [1_100usize, 1_500, 2_300];
415 for it in 0..iters {
416 let db = res!(start(dir, c.clone(), true));
417 let k = dat!("contended");
418 hooks::set_read_delay(Duration::from_millis(2));
419 hooks::set_chunk_insert_delay(Duration::from_millis(2));
420 let stop = Arc::new(AtomicBool::new(false));
421 let torn = Arc::new(Mutex::new(Vec::<String>::new()));
422 let nreads = Arc::new(AtomicUsize::new(0));
423 let readers: Vec<_> = (0..3).map(|_| {
424 let (db, k, stop, torn, nreads) = (db.clone(), k.clone(), stop.clone(), torn.clone(), nreads.clone());
425 thread::spawn(move || {
426 while !stop.load(Ordering::Relaxed) {
427 let d = describe(&db.get(&k, None));
428 nreads.fetch_add(1, Ordering::Relaxed);
429 if d.starts_with("TORN") || d.starts_with("ERR") {
430 if let Ok(mut t) = torn.lock() { t.push(d); }
431 }
432 }
433 })
434 }).collect();
435 let writers: Vec<_> = (0..6usize).map(|t| {
436 let (db, k) = (db.clone(), k.clone());
437 thread::spawn(move || -> Vec<String> {
438 let mut errs = Vec::new();
439 for n in 0..15usize {
440 let r = match (t + n) % 5 {
441 0 => db.delete(&k, Uid::default(), None).map(|_| ()),
442 1 => db.api().store(k.clone(), tomb(), Uid::default())
443 .and_then(|resp| resp.recv_store_ack().map(|_| ())),
444 2 => db.insert(k.clone(), dat!("small"), Uid::default(), None).map(|_| ()),
445 _ => db.insert(k.clone(), val(lens[(t + n) % 3], (t as u8) * 16 + n as u8), Uid::default(), None).map(|_| ()),
446 };
447 if let Err(e) = r { errs.push(fmt!("{}", e).chars().take(200).collect()); }
448 }
449 errs
450 })
451 }).collect();
452 for w in writers {
453 match w.join() { Ok(e) => for x in e { bad.push(fmt!("it {} write: {}", it, x)); }, Err(_) => bad.push(fmt!("panic")) }
454 }
455 stop.store(true, Ordering::Relaxed);
456 for r in readers { let _ = r.join(); }
457 hooks::set_read_delay(Duration::ZERO);
458 hooks::set_chunk_insert_delay(Duration::ZERO);
459 if let Ok(t) = torn.lock() { for x in t.iter().take(5) { bad.push(fmt!("it {} mid-read: {}", it, x)); } }
460 let live = describe(&db.get(&k, None));
461 if live.starts_with("TORN") || live.starts_with("ERR") { bad.push(fmt!("it {} live final {}", it, live)); }
462 if let Some(t) = res!(head(&db, &k)) {
463 if let Err(e) = db.api().fetch_chunks(&Dat::Tup5u64(t), None) {
464 bad.push(fmt!("it {} live head {:?} missing chunks: {}", it, t, fmt!("{}", e).chars().take(160).collect::<String>()));
465 }
466 }
467 let v = res!(verify::verify_live_set(db.api(), None, scan_wait()));
468 if !v.clean() { bad.push(fmt!("it {} verify live: {}", it, v.summary(4))); }
469 res!(db.close());
470 let db = res!(start(dir, c.clone(), false));
471 let rest = describe(&db.get(&k, None));
472 if rest != live { bad.push(fmt!("it {} restart read {} but live read {}", it, rest, live)); }
473 let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait()));
474 orphans += report.orphans_found;
475 let swept = describe(&db.get(&k, None));
476 if swept != live { bad.push(fmt!("it {} after sweep read {} but live read {}", it, swept, live)); }
477 let v = res!(verify::verify_live_set(db.api(), None, scan_wait()));
478 if !v.clean() { bad.push(fmt!("it {} verify swept: {}", it, v.summary(4))); }
479 res!(db.close());
480 println!("QA1 contended it {} final={} reads={} orphans={}", it, live, nreads.load(Ordering::Relaxed), report.orphans_found);
481 }
482 let _ = std::fs::remove_dir_all(dir);
483 Ok((bad, orphans))
484}
485
486#[test]
487fn qa1_puts_deletes_one_key_default_cache() -> Outcome<()> {
488 let _lock = lock();
489 let (bad, orphans) = res!(puts_deletes("./test_db_qa1_pd_def", None, 4));
490 println!("QA1 contended default orphans_total={}", orphans);
491 assert!(bad.is_empty(), "{:#?}", bad);
492 Ok(())
493}
494
495#[test]
496fn qa1_puts_deletes_one_key_tiny_cache() -> Outcome<()> {
497 let _lock = lock();
498 let (bad, orphans) = res!(puts_deletes("./test_db_qa1_pd_tiny", Some(1), 4));
499 println!("QA1 contended tiny orphans_total={}", orphans);
500 assert!(bad.is_empty(), "{:#?}", bad);
501 Ok(())
502}
503
504// ---------------------------------------------------------------------------------------------
505// F3 characterised: a process stamps a minute ahead, the next process stamps by the true clock.
506
507#[test]
508fn qa1_f3_clock_back_across_restart() -> Outcome<()> {
509 let _lock = lock();
510 let dir = "./test_db_qa1_f3";
511 let _ = std::fs::remove_dir_all(dir);
512 let ahead = (*res!(Timestamp::now()) + Duration::from_secs(60)).as_nanos();
513 let a = res!(child("f3_ahead", dir, Some(ahead)));
514 let b = res!(child("f3_behind", dir, None));
515 let _ = std::fs::remove_dir_all(dir);
516 println!("{}\n{}", a, b);
517 assert!(b.contains("live k1=1500/2"), "a put after a clock step back is lost although it returned Ok:\n{}", b);
518 assert!(b.contains("k3=none") || b.contains("k3=tomb"), "a delete after a clock step back is lost:\n{}", b);
519 Ok(())
520}
521
522// Equal stamps across processes: does a restart keep the value the cache kept, whole?
523#[test]
524fn qa1_equal_stamp_across_processes() -> Outcome<()> {
525 let _lock = lock();
526 let dir = "./test_db_qa1_tie";
527 let mut outs = Vec::new();
528 let mut bad = Vec::new();
529 for round in 0..4 {
530 let _ = std::fs::remove_dir_all(dir);
531 let t = (*res!(Timestamp::now())).as_nanos();
532 let a = res!(child("tie_a", dir, Some(t)));
533 let b = res!(child("tie_b", dir, Some(t)));
534 let c = res!(child("tie_c", dir, None));
535 let reads = |s: &str, tag: &str| s.lines().find(|l| l.starts_with(&fmt!("QA1 {} reads", tag))).unwrap_or("").to_string();
536 let (rb, rc) = (reads(&b, "tie_b"), reads(&c, "tie_c"));
537 let rb = rb.replace("tie_b", "");
538 let rc = rc.replace("tie_c", "");
539 if rb != rc || c.contains("TORN") || c.contains("ERR") || c.contains("clean=false") {
540 bad.push(fmt!("round {}: live {} vs restart {}\n{}\n{}", round, rb, rc, b, c));
541 }
542 outs.push(fmt!("round {}\n{}\n{}\n{}", round, a, b, c));
543 }
544 let _ = std::fs::remove_dir_all(dir);
545 println!("{}", outs.join("\n"));
546 assert!(bad.is_empty(), "equal stamps across processes diverge at restart:\n{}", bad.join("\n"));
547 Ok(())
548}
549
550// ---------------------------------------------------------------------------------------------
551// Legacy (key-derived) set: an overwrite of one length killed after its durable head and before
552// its retire, or before its head. After restart: whole, verify clean, sweep spares the live set,
553// and overwrite, delete and restart behave.
554
555fn legacy_crash(mode: &str, dir: &str) -> Outcome<Vec<String>> {
556 let out = res!(child(mode, dir, None));
557 let mut bad = Vec::new();
558 if !out.contains("QA1 aborting") { bad.push(fmt!("child did not reach the crash: {}", out)); }
559 let k = dat!("legacy crash");
560 let db = res!(start(dir, res!(cfg()), false));
561 let got = describe(&db.get(&k, None));
562 let want = if mode == "legacy_after_head" { "1100/2" } else { "1100/1" };
563 if got != want { bad.push(fmt!("{}: restart read {} want {}", mode, got, want)); }
564 let v = res!(verify::verify_live_set(db.api(), None, scan_wait()));
565 if !v.clean() { bad.push(fmt!("{}: verify {}", mode, v.summary(4))); }
566 let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait()));
567 let swept = describe(&db.get(&k, None));
568 if swept != want { bad.push(fmt!("{}: after sweep read {}", mode, swept)); }
569 res!(db.insert(k.clone(), val(1_100, 3), Uid::default(), None));
570 let over = describe(&db.get(&k, None));
571 if over != "1100/3" { bad.push(fmt!("{}: overwrite read {}", mode, over)); }
572 res!(db.close());
573 let db = res!(start(dir, res!(cfg()), false));
574 let over2 = describe(&db.get(&k, None));
575 if over2 != "1100/3" { bad.push(fmt!("{}: overwrite after restart read {}", mode, over2)); }
576 res!(db.delete(&k, Uid::default(), None));
577 let del = describe(&db.get(&k, None));
578 if !(del == "none" || del == "tomb") { bad.push(fmt!("{}: delete read {}", mode, del)); }
579 res!(db.close());
580 let db = res!(start(dir, res!(cfg()), false));
581 let del2 = describe(&db.get(&k, None));
582 if !(del2 == "none" || del2 == "tomb") { bad.push(fmt!("{}: delete after restart read {}", mode, del2)); }
583 let v = res!(verify::verify_live_set(db.api(), None, scan_wait()));
584 if !v.clean() { bad.push(fmt!("{}: verify end {}", mode, v.summary(4))); }
585 res!(db.close());
586 println!("QA1 {} restart={} sweep_orphans={} {}", mode, got, report.orphans_found, out.replace('\n', " | "));
587 let _ = std::fs::remove_dir_all(dir);
588 Ok(bad)
589}
590
591#[test]
592fn qa1_legacy_crash_mid_overwrite() -> Outcome<()> {
593 let _lock = lock();
594 let mut bad = res!(legacy_crash("legacy_after_head", "./test_db_qa1_legacy_a"));
595 bad.extend(res!(legacy_crash("legacy_before_head", "./test_db_qa1_legacy_b")));
596 assert!(bad.is_empty(), "{:#?}", bad);
597 Ok(())
598}
599
600// ---------------------------------------------------------------------------------------------
601// A bunch key whose barrier fails after its chunks were written. Since 2026-10-10 (F2) the
602// failure stops the store: the head is past its file's mark, so it is not served, the sweep is
603// refused, and the restart cuts it, leaving the value a barrier proved. It used to be served
604// and kept as an unconfirmed head that landed.
605
606#[test]
607fn qa1_unconfirmed_head_that_lands() -> Outcome<()> {
608 let _lock = lock();
609 let dir = "./test_db_qa1_unconf";
610 let db = res!(start(dir, res!(cfg()), true));
611 let k = dat!("unconfirmed head");
612 res!(db.insert(k.clone(), val(1_100, 1), Uid::default(), None));
613 res!(db.api().sync_now(scan_wait()));
614 hooks::set_chunk_insert_delay(Duration::from_millis(400));
615 let put = { let (db, k) = (db.clone(), k.clone());
616 thread::spawn(move || db.insert(k, val(1_100, 2), Uid::default(), None).map(|_| ()).map_err(|e| fmt!("{}", e))) };
617 thread::sleep(Duration::from_millis(200));
618 hooks::set_barrier_failure(true);
619 let put = match put.join() { Ok(x) => x, Err(_) => Err(fmt!("panicked")) };
620 hooks::set_barrier_failure(false);
621 hooks::set_chunk_insert_delay(Duration::ZERO);
622 let faulted = db.api().fault().is_some();
623 let live = describe(&db.get(&k, None));
624 let report = sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait());
625 let _ = db.close();
626 res!(Refusal::clear(Path::new(dir))); // as the operator would
627 let db = res!(start(dir, res!(cfg()), false));
628 let rest = describe(&db.get(&k, None));
629 let v = res!(verify::verify_live_set(db.api(), None, scan_wait()));
630 res!(db.close());
631 let _ = std::fs::remove_dir_all(dir);
632 println!("QA1 unconfirmed put={:?} faulted={} live={} sweep={:?} restart={} verify={}",
633 put.as_ref().map_err(|e| e.chars().take(200).collect::<String>()), faulted, live,
634 report.as_ref().map(|r| r.orphans_found), rest, v.clean());
635 assert!(put.is_err(), "a put whose barrier failed was reported stored");
636 assert!(faulted, "a failed barrier did not stop the store");
637 assert!(live == "1100/1" || live.starts_with("ERR:"), "live read {}", live);
638 assert!(report.is_err(), "the sweep ran on a store stopped by a failed sync");
639 assert_eq!(rest, "1100/1", "restart read");
640 assert!(v.clean(), "verify after restart: {}", v.summary(4));
641 Ok(())
642}