oxedyne/fe2o3/fe2o3_o3db_sync/tests/qa1_chunkset.rs
29.7 KiB, 36 runs
created by r1870400018:70211, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | // 2026-10-09 ore-a: describe() now labels the known literals ("small", "small one", "small two") and every deleted-kind Usr value as tomb; TORN only for anything else. |
| 2 | //! QA agent 1 (2026-10-09), chunk sets and fix A: adversarial probes of the write, delete and read |
| 3 | //! paths. The hooks are process-wide, so every test takes one lock. Cross-process cases run the |
| 4 | //! phases in child processes of this binary (`qa1_child`, env QA1_MODE / QA1_DIR / QA1_T), so a |
| 5 | //! phase starts with a fresh process stamp, as a real restart does. |
| 6 | |
| 7 | mod gc_pair; |
| 8 | |
| 9 | use gc_pair::*; |
| 10 | |
| 11 | use oxedyne_fe2o3_core::prelude::*; |
| 12 | use oxedyne_fe2o3_data::time::Timestamp; |
| 13 | use oxedyne_fe2o3_hash::{ |
| 14 | csum::ChecksumScheme, |
| 15 | hash::HashScheme, |
| 16 | }; |
| 17 | use oxedyne_fe2o3_iop_db::api::Database; |
| 18 | use oxedyne_fe2o3_jdat::prelude::*; |
| 19 | use oxedyne_fe2o3_o3db_sync::{ |
| 20 | api::OzoneApi, |
| 21 | base::{ |
| 22 | cfg::OzoneConfig, |
| 23 | constant, |
| 24 | }, |
| 25 | comm::response::Wait, |
| 26 | data::cache::{ |
| 27 | CacheEntry, |
| 28 | is_chunk_key, |
| 29 | }, |
| 30 | file::refusal::Refusal, |
| 31 | sweep, |
| 32 | test::{ |
| 33 | hooks, |
| 34 | setup::{ |
| 35 | self, |
| 36 | Uid, |
| 37 | UID_LEN, |
| 38 | }, |
| 39 | }, |
| 40 | verify, |
| 41 | }; |
| 42 | |
| 43 | use std::{ |
| 44 | collections::BTreeMap, |
| 45 | path::Path, |
| 46 | process::Command, |
| 47 | sync::{ |
| 48 | Arc, |
| 49 | Mutex, |
| 50 | MutexGuard, |
| 51 | atomic::{ |
| 52 | AtomicBool, |
| 53 | AtomicUsize, |
| 54 | Ordering, |
| 55 | }, |
| 56 | }, |
| 57 | thread, |
| 58 | time::Duration, |
| 59 | }; |
| 60 | |
| 61 | static HOOKS: Mutex<()> = Mutex::new(()); |
| 62 | |
| 63 | fn lock() -> MutexGuard<'static, ()> { |
| 64 | match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() } |
| 65 | } |
| 66 | |
| 67 | fn cfg() -> Outcome<OzoneConfig> { |
| 68 | let mut c = res!(setup::default_cfg()); |
| 69 | c.num_zones = 1; |
| 70 | c.num_cbots_per_zone = 4; |
| 71 | c.num_fbots_per_zone = 1; |
| 72 | c.num_wbots_per_zone = 2; |
| 73 | c.zone_overrides = BTreeMap::new(); |
| 74 | c.rest_chunk_threshold = 1_000; |
| 75 | c.rest_chunk_bytes = 400; |
| 76 | c.sync_on_write = true; |
| 77 | Ok(c) |
| 78 | } |
| 79 | |
| 80 | fn val(len: usize, seed: u8) -> Dat { |
| 81 | Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect()) |
| 82 | } |
| 83 | |
| 84 | fn seed_of(v: &Dat) -> Option<(usize, u8)> { |
| 85 | match v { |
| 86 | Dat::BU32(b) | Dat::BU64(b) | Dat::BU8(b) | Dat::BU16(b) if !b.is_empty() => { |
| 87 | let s = b[0]; |
| 88 | if b.iter().enumerate().all(|(j, x)| *x == s ^ (j as u8)) { Some((b.len(), s)) } else { None } |
| 89 | }, |
| 90 | _ => None, |
| 91 | } |
| 92 | } |
| 93 | |
| 94 | fn tomb() -> Dat { |
| 95 | Dat::Usr(oxedyne_fe2o3_o3db_sync::base::id::usr_kind_id_deleted(), Some(Box::new(Dat::Empty))) |
| 96 | } |
| 97 | |
| 98 | fn start(dir: &str, c: OzoneConfig, wipe: bool) -> Outcome<TestDb> { |
| 99 | log_set_level!("error"); |
| 100 | if wipe { |
| 101 | let _ = std::fs::remove_dir_all(dir); |
| 102 | } |
| 103 | res!(std::fs::create_dir_all(dir)); |
| 104 | let root = res!(Path::new(dir).canonicalize()); |
| 105 | setup::start_db(root, Some(c), schemes(), None, false, wipe) |
| 106 | } |
| 107 | |
| 108 | fn scan_wait() -> Wait { |
| 109 | Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL } |
| 110 | } |
| 111 | |
| 112 | fn head(db: &TestDb, k: &Dat) -> Outcome<Option<[u64; 5]>> { |
| 113 | match res!(db.api().get_head_wait(k, None)) { |
| 114 | Some((Dat::Tup5u64(t), _)) => Ok(Some(t)), |
| 115 | _ => Ok(None), |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | // What a read found: "none", "tomb", "L/S" for a whole value of length L and seed S, "TORN", or |
| 120 | // "ERR:<text>". |
| 121 | fn describe(r: &Outcome<Option<(Dat, oxedyne_fe2o3_iop_db::api::Meta<UID_LEN, Uid>)>>) -> String { |
| 122 | match r { |
| 123 | Ok(None) => fmt!("none"), |
| 124 | Ok(Some((Dat::Usr(kind, _), _))) if *kind == oxedyne_fe2o3_o3db_sync::base::id::usr_kind_id_deleted() => fmt!("tomb"), |
| 125 | Ok(Some((d, _))) if *d == dat!("small") => fmt!("small"), |
| 126 | Ok(Some((d, _))) if *d == dat!("small one") => fmt!("small one"), |
| 127 | Ok(Some((d, _))) if *d == dat!("small two") => fmt!("small two"), |
| 128 | Ok(Some((d, _))) => match seed_of(d) { |
| 129 | Some((l, s)) => fmt!("{}/{}", l, s), |
| 130 | None => fmt!("TORN:{:?}", d).chars().take(120).collect(), |
| 131 | }, |
| 132 | Err(e) => fmt!("ERR:{}", fmt!("{}", e).chars().take(160).collect::<String>()), |
| 133 | } |
| 134 | } |
| 135 | |
| 136 | fn stamp_of(db: &TestDb, k: &Dat) -> String { |
| 137 | match db.api().get_head_wait(k, None) { |
| 138 | Ok(Some((_, m))) => fmt!("{:?}", *m.time), |
| 139 | other => fmt!("{:?}", other.map(|o| o.is_some())), |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | // --------------------------------------------------------------------------------------------- |
| 144 | // Child-process phases. |
| 145 | |
| 146 | fn child(mode: &str, dir: &str, t: Option<u128>) -> Outcome<String> { |
| 147 | let exe = res!(std::env::current_exe()); |
| 148 | let mut cmd = Command::new(exe); |
| 149 | cmd.args(["qa1_child", "--exact", "--ignored", "--nocapture", "--test-threads=1"]) |
| 150 | .env("QA1_MODE", mode) |
| 151 | .env("QA1_DIR", dir); |
| 152 | if let Some(t) = t { cmd.env("QA1_T", fmt!("{}", t)); } |
| 153 | let out = res!(cmd.output()); |
| 154 | let so = String::from_utf8_lossy(&out.stdout).to_string(); |
| 155 | // The harness prints `test qa1_child ... ` with no newline, so the child's first line |
| 156 | // follows it on the same line. |
| 157 | let lines: Vec<&str> = so.lines().filter_map(|l| l.find("QA1 ").map(|i| &l[i..])).collect(); |
| 158 | Ok(fmt!("status={:?}\n{}", out.status.code(), lines.join("\n"))) |
| 159 | } |
| 160 | |
| 161 | #[test] |
| 162 | #[ignore] |
| 163 | fn qa1_child() -> Outcome<()> { |
| 164 | let mode = match std::env::var("QA1_MODE") { Ok(m) => m, Err(_) => return Ok(()) }; |
| 165 | let dir = res!(std::env::var("QA1_DIR")); |
| 166 | let t: Option<u128> = std::env::var("QA1_T").ok().and_then(|s| s.parse().ok()); |
| 167 | let dur = |n: u128| Duration::new((n / 1_000_000_000) as u64, (n % 1_000_000_000) as u32); |
| 168 | match mode.as_str() { |
| 169 | // F3: a store written while the clock read a minute ahead. |
| 170 | "f3_ahead" => { |
| 171 | let db = res!(start(&dir, res!(cfg()), true)); |
| 172 | if let Some(t) = t { hooks::set_fixed_stamp(Some(dur(t))); } |
| 173 | res!(db.insert(dat!("k1"), val(1_100, 1), Uid::default(), None)); |
| 174 | res!(db.insert(dat!("k2"), dat!("small one"), Uid::default(), None)); |
| 175 | res!(db.insert(dat!("k3"), val(1_100, 3), Uid::default(), None)); |
| 176 | hooks::set_fixed_stamp(None); |
| 177 | println!("QA1 ahead k1={} stamp={}", describe(&db.get(&dat!("k1"), None)), stamp_of(&db, &dat!("k1"))); |
| 178 | res!(db.close()); |
| 179 | }, |
| 180 | // F3: the next process, with the clock back where it should be. |
| 181 | "f3_behind" => { |
| 182 | let db = res!(start(&dir, res!(cfg()), false)); |
| 183 | let p1 = db.insert(dat!("k1"), val(1_500, 2), Uid::default(), None).map(|_| ()); |
| 184 | let p2 = db.insert(dat!("k2"), dat!("small two"), Uid::default(), None).map(|_| ()); |
| 185 | let d3 = db.delete(&dat!("k3"), Uid::default(), None).map(|_| ()); |
| 186 | let p4 = db.insert(dat!("k4"), val(1_100, 4), Uid::default(), None).map(|_| ()); |
| 187 | println!("QA1 calls put_k1={:?} put_k2={:?} del_k3={:?} put_k4={:?}", |
| 188 | p1.is_ok(), p2.is_ok(), d3.is_ok(), p4.is_ok()); |
| 189 | println!("QA1 live k1={} k2={:?} k3={} k4={} k1stamp={}", |
| 190 | describe(&db.get(&dat!("k1"), None)), |
| 191 | db.get(&dat!("k2"), None).map(|o| o.map(|(d, _)| d)).ok(), |
| 192 | describe(&db.get(&dat!("k3"), None)), |
| 193 | describe(&db.get(&dat!("k4"), None)), |
| 194 | stamp_of(&db, &dat!("k1"))); |
| 195 | let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait())); |
| 196 | println!("QA1 sweep orphans_found={} skipped_pending={}", report.orphans_found, report.skipped_pending); |
| 197 | res!(db.close()); |
| 198 | let db = res!(start(&dir, res!(cfg()), false)); |
| 199 | println!("QA1 restarted k1={} k2={:?} k3={}", |
| 200 | describe(&db.get(&dat!("k1"), None)), |
| 201 | db.get(&dat!("k2"), None).map(|o| o.map(|(d, _)| d)).ok(), |
| 202 | describe(&db.get(&dat!("k3"), None))); |
| 203 | res!(db.close()); |
| 204 | }, |
| 205 | // Equal stamps across processes: phase a and b write the same keys in the same order |
| 206 | // with the clock fixed at T, so each key's two records carry one stamp. |
| 207 | "tie_a" | "tie_b" => { |
| 208 | let wipe = mode == "tie_a"; |
| 209 | let seed = if wipe { 1u8 } else { 2u8 }; |
| 210 | let db = res!(start(&dir, res!(cfg()), wipe)); |
| 211 | if let Some(t) = t { hooks::set_fixed_stamp(Some(dur(t))); } |
| 212 | let mut stamps = Vec::new(); |
| 213 | for i in 0..12u8 { |
| 214 | let k = dat!(fmt!("tie {}", i)); |
| 215 | let r = db.insert(k.clone(), val(1_100, seed.wrapping_add(i * 16)), Uid::default(), None); |
| 216 | stamps.push(fmt!("{}:{}", i, if r.is_ok() { stamp_of(&db, &k) } else { fmt!("ERR") })); |
| 217 | } |
| 218 | hooks::set_fixed_stamp(None); |
| 219 | let reads: Vec<String> = (0..12u8).map(|i| describe(&db.get(&dat!(fmt!("tie {}", i)), None))).collect(); |
| 220 | println!("QA1 {} reads {:?}", mode, reads); |
| 221 | println!("QA1 {} stamps {:?}", mode, stamps); |
| 222 | res!(db.close()); |
| 223 | }, |
| 224 | "tie_c" => { |
| 225 | let db = res!(start(&dir, res!(cfg()), false)); |
| 226 | let reads: Vec<String> = (0..12u8).map(|i| describe(&db.get(&dat!(fmt!("tie {}", i)), None))).collect(); |
| 227 | let stamps: Vec<String> = (0..12u8).map(|i| stamp_of(&db, &dat!(fmt!("tie {}", i)))).collect(); |
| 228 | let v = res!(verify::verify_live_set(db.api(), None, scan_wait())); |
| 229 | println!("QA1 tie_c reads {:?}", reads); |
| 230 | println!("QA1 tie_c stamps {:?}", stamps); |
| 231 | println!("QA1 tie_c verify clean={} {}", v.clean(), v.summary(4).replace('\n', " | ")); |
| 232 | res!(db.close()); |
| 233 | }, |
| 234 | // A legacy value, then an overwrite of one length killed between its durable head and its |
| 235 | // retire, or before its head. |
| 236 | "legacy_after_head" | "legacy_before_head" => { |
| 237 | let db = res!(start(&dir, res!(cfg()), true)); |
| 238 | let k = dat!("legacy crash"); |
| 239 | let (kbuf, _, _) = res!(db.api().ozone_key_dat(&k, None)); |
| 240 | let legacy = OzoneApi::<{ UID_LEN }, Uid, (), HashScheme, HashScheme, ChecksumScheme>::legacy_chunk_set_id(&kbuf); |
| 241 | let resp = db.api().responder(); |
| 242 | res!(db.api().store_dat_using_responder_forcing_set_id( |
| 243 | k.clone(), val(1_100, 1), Uid::default(), None, resp.clone(), legacy)); |
| 244 | res!(resp.recv_store_ack()); |
| 245 | println!("QA1 legacy head={:?}", res!(head(&db, &k))); |
| 246 | if mode == "legacy_after_head" { |
| 247 | hooks::set_retire_drop(true); |
| 248 | } else { |
| 249 | hooks::set_chunk_insert_delay(Duration::from_secs(20)); |
| 250 | } |
| 251 | let (db2, k2) = (db.clone(), k.clone()); |
| 252 | let _put = thread::spawn(move || db2.insert(k2, val(1_100, 2), Uid::default(), None).map(|_| ())); |
| 253 | thread::sleep(Duration::from_millis(1_500)); |
| 254 | println!("QA1 aborting"); |
| 255 | std::process::abort(); |
| 256 | }, |
| 257 | // Overwritten chunked values, then a collection killed between its new data file and |
| 258 | // its new index file. |
| 259 | "collect_crash" => { |
| 260 | let db = res!(start_collecting(&dir, true)); |
| 261 | for round in 0..6u8 { |
| 262 | for i in 0..4u8 { |
| 263 | res!(db.insert(dat!(fmt!("crash {}", i)), val(1_100, round * 16 + i), Uid::default(), None)); |
| 264 | } |
| 265 | } |
| 266 | res!(db.delete(&dat!("crash 3"), Uid::default(), None)); |
| 267 | res!(db.api().settle_for_test(Duration::from_secs(20))); |
| 268 | hooks::set_commit_delay(Duration::from_secs(30)); |
| 269 | let db2 = db.clone(); |
| 270 | let _gc = thread::spawn(move || db2.api().compact_now(Duration::from_secs(60)).map(|_| ())); |
| 271 | thread::sleep(Duration::from_millis(2_500)); |
| 272 | println!("QA1 aborting"); |
| 273 | std::process::abort(); |
| 274 | }, |
| 275 | _ => return Err(err!("Unknown QA1_MODE {}.", mode; Test, Invalid)), |
| 276 | } |
| 277 | Ok(()) |
| 278 | } |
| 279 | |
| 280 | fn start_collecting(dir: &str, wipe: bool) -> Outcome<TestDb> { |
| 281 | let mut c = res!(cfg()); |
| 282 | c.data_file_max_bytes = 4_000; |
| 283 | c.num_wbots_per_zone = 1; |
| 284 | log_set_level!("error"); |
| 285 | if wipe { |
| 286 | let _ = std::fs::remove_dir_all(dir); |
| 287 | } |
| 288 | res!(std::fs::create_dir_all(dir)); |
| 289 | let root = res!(Path::new(dir).canonicalize()); |
| 290 | let db = res!(setup::start_db(root, Some(c), schemes(), None, true, wipe)); |
| 291 | res!(db.api().activate_gc(false)); |
| 292 | Ok(db) |
| 293 | } |
| 294 | |
| 295 | // Live chunk records, tombstones aside, and the chunks the current heads name. |
| 296 | fn chunk_bound(db: &TestDb, keys: &[Dat]) -> Outcome<(usize, usize)> { |
| 297 | res!(db.api().settle_for_test(Duration::from_secs(20))); |
| 298 | let mut live = 0; |
| 299 | for (_, cache) in res!(db.api().collect_caches(scan_wait())) { |
| 300 | let tombs = cache.tomb_tracker().tombs(); |
| 301 | for (k, e) in cache.map() { |
| 302 | if is_chunk_key(k) && matches!(e, CacheEntry::LocatedValue(..)) && !tombs.contains_key(k) { |
| 303 | live += 1; |
| 304 | } |
| 305 | } |
| 306 | } |
| 307 | let mut named = 0; |
| 308 | for k in keys { |
| 309 | if let Some(t) = res!(head(db, k)) { |
| 310 | named += t[3] as usize; |
| 311 | } |
| 312 | } |
| 313 | Ok((live, named)) |
| 314 | } |
| 315 | |
| 316 | // A process killed mid-collection, between its new data file and its new index file: every key |
| 317 | // reads its last value whole after a restart, before and after the sweep, verify is clean, and |
| 318 | // the next overwrites leave no more live chunks than the heads name. |
| 319 | #[test] |
| 320 | fn qa1_kill_mid_collection() -> Outcome<()> { |
| 321 | let _lock = lock(); |
| 322 | let dir = "./test_db_qa1_collect_crash"; |
| 323 | let out = res!(child("collect_crash", dir, None)); |
| 324 | let mut bad = Vec::new(); |
| 325 | if !out.contains("QA1 aborting") { bad.push(fmt!("child did not reach the crash: {}", out)); } |
| 326 | let keys: Vec<Dat> = (0..4).map(|i| dat!(fmt!("crash {}", i))).collect(); |
| 327 | let want = |i: usize| if i == 3 { fmt!("none") } else { fmt!("1100/{}", 5 * 16 + i) }; |
| 328 | let reads = |db: &TestDb| -> Vec<String> { keys.iter() |
| 329 | .map(|k| match describe(&db.get(k, None)) { d if d == "tomb" => fmt!("none"), d => d }).collect() }; |
| 330 | let wanted: Vec<String> = (0..4).map(want).collect(); |
| 331 | let db = res!(start_collecting(dir, false)); |
| 332 | let got = reads(&db); |
| 333 | if got != wanted { bad.push(fmt!("restart read {:?} want {:?}", got, wanted)); } |
| 334 | let v = res!(verify::verify_live_set(db.api(), None, scan_wait())); |
| 335 | if !v.clean() { bad.push(fmt!("verify {}", v.summary(4))); } |
| 336 | let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait())); |
| 337 | let swept = reads(&db); |
| 338 | if swept != wanted { bad.push(fmt!("after sweep read {:?}", swept)); } |
| 339 | for (i, k) in keys.iter().enumerate() { |
| 340 | res!(db.insert(k.clone(), val(1_100, 0xC0 + i as u8), Uid::default(), None)); |
| 341 | } |
| 342 | for _ in 0..3 { |
| 343 | res!(db.api().settle_for_test(Duration::from_secs(20))); |
| 344 | res!(db.api().compact_now(Duration::from_secs(60))); |
| 345 | } |
| 346 | let (live, named) = res!(chunk_bound(&db, &keys)); |
| 347 | if live != named { bad.push(fmt!("live chunks {} against {} named", live, named)); } |
| 348 | res!(db.close()); |
| 349 | let db = res!(start_collecting(dir, false)); |
| 350 | let last = reads(&db); |
| 351 | let lasts: Vec<String> = (0..4).map(|i| fmt!("1100/{}", 0xC0 + i)).collect(); |
| 352 | if last != lasts { bad.push(fmt!("final restart read {:?}", last)); } |
| 353 | let v = res!(verify::verify_live_set(db.api(), None, scan_wait())); |
| 354 | if !v.clean() { bad.push(fmt!("verify end {}", v.summary(4))); } |
| 355 | res!(db.close()); |
| 356 | let _ = std::fs::remove_dir_all(dir); |
| 357 | println!("QA1 collect crash sweep_orphans={} {}", report.orphans_found, out.replace('\n', " | ")); |
| 358 | assert!(bad.is_empty(), "{:#?}", bad); |
| 359 | Ok(()) |
| 360 | } |
| 361 | |
| 362 | // --------------------------------------------------------------------------------------------- |
| 363 | // Q: two overlapping puts where the earlier-stamped publishes and reads back first and the later |
| 364 | // one wins: is the middle set left for the sweep? Counts rounds with orphans after both settle. |
| 365 | |
| 366 | #[test] |
| 367 | fn qa1_overlap_orphans() -> Outcome<()> { |
| 368 | let _lock = lock(); |
| 369 | let dir = "./test_db_qa1_overlap"; |
| 370 | let db = res!(start(dir, res!(cfg()), true)); |
| 371 | let mut rounds_with_orphans = 0usize; |
| 372 | let mut orphan_total = 0usize; |
| 373 | let mut bad = Vec::new(); |
| 374 | let rounds = 10usize; |
| 375 | for r in 0..rounds { |
| 376 | let k = dat!(fmt!("overlap {}", r)); |
| 377 | res!(db.insert(k.clone(), val(1_100, 0), Uid::default(), None)); |
| 378 | hooks::set_chunk_insert_delay(Duration::from_millis(40)); |
| 379 | let b = { let (db, k) = (db.clone(), k.clone()); |
| 380 | thread::spawn(move || db.insert(k, val(1_100, 0xB0), Uid::default(), None).map(|_| ()).map_err(|e| fmt!("{}", e))) }; |
| 381 | thread::sleep(Duration::from_millis(15)); |
| 382 | let a = { let (db, k) = (db.clone(), k.clone()); |
| 383 | thread::spawn(move || db.insert(k, val(1_100, 0xA0), Uid::default(), None).map(|_| ()).map_err(|e| fmt!("{}", e))) }; |
| 384 | let rb = match b.join() { Ok(x) => x, Err(_) => Err(fmt!("panicked")) }; |
| 385 | let ra = match a.join() { Ok(x) => x, Err(_) => Err(fmt!("panicked")) }; |
| 386 | hooks::set_chunk_insert_delay(Duration::ZERO); |
| 387 | if rb.is_err() || ra.is_err() { bad.push(fmt!("round {}: puts {:?} {:?}", r, rb, ra)); } |
| 388 | let got = describe(&db.get(&k, None)); |
| 389 | if !(got == "1100/160" || got == "1100/176") { bad.push(fmt!("round {}: read {}", r, got)); } |
| 390 | let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait())); |
| 391 | if report.orphans_found > 0 { rounds_with_orphans += 1; orphan_total += report.orphans_found; } |
| 392 | let after = describe(&db.get(&k, None)); |
| 393 | if after != got { bad.push(fmt!("round {}: after sweep read {} (was {})", r, after, got)); } |
| 394 | println!("QA1 overlap round {} final={} orphans_found={}", r, got, report.orphans_found); |
| 395 | } |
| 396 | res!(db.close()); |
| 397 | let _ = std::fs::remove_dir_all(dir); |
| 398 | println!("QA1 overlap rounds_with_orphans={}/{} orphan_chunks={}", rounds_with_orphans, rounds, orphan_total); |
| 399 | assert!(bad.is_empty(), "{:?}", bad); |
| 400 | assert_eq!(rounds_with_orphans, 0, "overlapping puts left chunk sets for the sweep: {} chunks over {} rounds", |
| 401 | orphan_total, rounds_with_orphans); |
| 402 | Ok(()) |
| 403 | } |
| 404 | |
| 405 | // --------------------------------------------------------------------------------------------- |
| 406 | // Q: concurrent puts and deletes (both roads) on one key, with readers: every read whole-or-none, |
| 407 | // the final value whole, the same across a restart, verify clean, and the sweep takes nothing live. |
| 408 | |
| 409 | fn puts_deletes(dir: &str, cache_lim: Option<usize>, iters: usize) -> Outcome<(Vec<String>, usize)> { |
| 410 | let mut c = res!(cfg()); |
| 411 | if let Some(l) = cache_lim { c.cache_size_limit_bytes = l as u64; } |
| 412 | let mut bad = Vec::new(); |
| 413 | let mut orphans = 0usize; |
| 414 | let lens = [1_100usize, 1_500, 2_300]; |
| 415 | for it in 0..iters { |
| 416 | let db = res!(start(dir, c.clone(), true)); |
| 417 | let k = dat!("contended"); |
| 418 | hooks::set_read_delay(Duration::from_millis(2)); |
| 419 | hooks::set_chunk_insert_delay(Duration::from_millis(2)); |
| 420 | let stop = Arc::new(AtomicBool::new(false)); |
| 421 | let torn = Arc::new(Mutex::new(Vec::<String>::new())); |
| 422 | let nreads = Arc::new(AtomicUsize::new(0)); |
| 423 | let readers: Vec<_> = (0..3).map(|_| { |
| 424 | let (db, k, stop, torn, nreads) = (db.clone(), k.clone(), stop.clone(), torn.clone(), nreads.clone()); |
| 425 | thread::spawn(move || { |
| 426 | while !stop.load(Ordering::Relaxed) { |
| 427 | let d = describe(&db.get(&k, None)); |
| 428 | nreads.fetch_add(1, Ordering::Relaxed); |
| 429 | if d.starts_with("TORN") || d.starts_with("ERR") { |
| 430 | if let Ok(mut t) = torn.lock() { t.push(d); } |
| 431 | } |
| 432 | } |
| 433 | }) |
| 434 | }).collect(); |
| 435 | let writers: Vec<_> = (0..6usize).map(|t| { |
| 436 | let (db, k) = (db.clone(), k.clone()); |
| 437 | thread::spawn(move || -> Vec<String> { |
| 438 | let mut errs = Vec::new(); |
| 439 | for n in 0..15usize { |
| 440 | let r = match (t + n) % 5 { |
| 441 | 0 => db.delete(&k, Uid::default(), None).map(|_| ()), |
| 442 | 1 => db.api().store(k.clone(), tomb(), Uid::default()) |
| 443 | .and_then(|resp| resp.recv_store_ack().map(|_| ())), |
| 444 | 2 => db.insert(k.clone(), dat!("small"), Uid::default(), None).map(|_| ()), |
| 445 | _ => db.insert(k.clone(), val(lens[(t + n) % 3], (t as u8) * 16 + n as u8), Uid::default(), None).map(|_| ()), |
| 446 | }; |
| 447 | if let Err(e) = r { errs.push(fmt!("{}", e).chars().take(200).collect()); } |
| 448 | } |
| 449 | errs |
| 450 | }) |
| 451 | }).collect(); |
| 452 | for w in writers { |
| 453 | match w.join() { Ok(e) => for x in e { bad.push(fmt!("it {} write: {}", it, x)); }, Err(_) => bad.push(fmt!("panic")) } |
| 454 | } |
| 455 | stop.store(true, Ordering::Relaxed); |
| 456 | for r in readers { let _ = r.join(); } |
| 457 | hooks::set_read_delay(Duration::ZERO); |
| 458 | hooks::set_chunk_insert_delay(Duration::ZERO); |
| 459 | if let Ok(t) = torn.lock() { for x in t.iter().take(5) { bad.push(fmt!("it {} mid-read: {}", it, x)); } } |
| 460 | let live = describe(&db.get(&k, None)); |
| 461 | if live.starts_with("TORN") || live.starts_with("ERR") { bad.push(fmt!("it {} live final {}", it, live)); } |
| 462 | if let Some(t) = res!(head(&db, &k)) { |
| 463 | if let Err(e) = db.api().fetch_chunks(&Dat::Tup5u64(t), None) { |
| 464 | bad.push(fmt!("it {} live head {:?} missing chunks: {}", it, t, fmt!("{}", e).chars().take(160).collect::<String>())); |
| 465 | } |
| 466 | } |
| 467 | let v = res!(verify::verify_live_set(db.api(), None, scan_wait())); |
| 468 | if !v.clean() { bad.push(fmt!("it {} verify live: {}", it, v.summary(4))); } |
| 469 | res!(db.close()); |
| 470 | let db = res!(start(dir, c.clone(), false)); |
| 471 | let rest = describe(&db.get(&k, None)); |
| 472 | if rest != live { bad.push(fmt!("it {} restart read {} but live read {}", it, rest, live)); } |
| 473 | let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait())); |
| 474 | orphans += report.orphans_found; |
| 475 | let swept = describe(&db.get(&k, None)); |
| 476 | if swept != live { bad.push(fmt!("it {} after sweep read {} but live read {}", it, swept, live)); } |
| 477 | let v = res!(verify::verify_live_set(db.api(), None, scan_wait())); |
| 478 | if !v.clean() { bad.push(fmt!("it {} verify swept: {}", it, v.summary(4))); } |
| 479 | res!(db.close()); |
| 480 | println!("QA1 contended it {} final={} reads={} orphans={}", it, live, nreads.load(Ordering::Relaxed), report.orphans_found); |
| 481 | } |
| 482 | let _ = std::fs::remove_dir_all(dir); |
| 483 | Ok((bad, orphans)) |
| 484 | } |
| 485 | |
| 486 | #[test] |
| 487 | fn qa1_puts_deletes_one_key_default_cache() -> Outcome<()> { |
| 488 | let _lock = lock(); |
| 489 | let (bad, orphans) = res!(puts_deletes("./test_db_qa1_pd_def", None, 4)); |
| 490 | println!("QA1 contended default orphans_total={}", orphans); |
| 491 | assert!(bad.is_empty(), "{:#?}", bad); |
| 492 | Ok(()) |
| 493 | } |
| 494 | |
| 495 | #[test] |
| 496 | fn qa1_puts_deletes_one_key_tiny_cache() -> Outcome<()> { |
| 497 | let _lock = lock(); |
| 498 | let (bad, orphans) = res!(puts_deletes("./test_db_qa1_pd_tiny", Some(1), 4)); |
| 499 | println!("QA1 contended tiny orphans_total={}", orphans); |
| 500 | assert!(bad.is_empty(), "{:#?}", bad); |
| 501 | Ok(()) |
| 502 | } |
| 503 | |
| 504 | // --------------------------------------------------------------------------------------------- |
| 505 | // F3 characterised: a process stamps a minute ahead, the next process stamps by the true clock. |
| 506 | |
| 507 | #[test] |
| 508 | fn qa1_f3_clock_back_across_restart() -> Outcome<()> { |
| 509 | let _lock = lock(); |
| 510 | let dir = "./test_db_qa1_f3"; |
| 511 | let _ = std::fs::remove_dir_all(dir); |
| 512 | let ahead = (*res!(Timestamp::now()) + Duration::from_secs(60)).as_nanos(); |
| 513 | let a = res!(child("f3_ahead", dir, Some(ahead))); |
| 514 | let b = res!(child("f3_behind", dir, None)); |
| 515 | let _ = std::fs::remove_dir_all(dir); |
| 516 | println!("{}\n{}", a, b); |
| 517 | assert!(b.contains("live k1=1500/2"), "a put after a clock step back is lost although it returned Ok:\n{}", b); |
| 518 | assert!(b.contains("k3=none") || b.contains("k3=tomb"), "a delete after a clock step back is lost:\n{}", b); |
| 519 | Ok(()) |
| 520 | } |
| 521 | |
| 522 | // Equal stamps across processes: does a restart keep the value the cache kept, whole? |
| 523 | #[test] |
| 524 | fn qa1_equal_stamp_across_processes() -> Outcome<()> { |
| 525 | let _lock = lock(); |
| 526 | let dir = "./test_db_qa1_tie"; |
| 527 | let mut outs = Vec::new(); |
| 528 | let mut bad = Vec::new(); |
| 529 | for round in 0..4 { |
| 530 | let _ = std::fs::remove_dir_all(dir); |
| 531 | let t = (*res!(Timestamp::now())).as_nanos(); |
| 532 | let a = res!(child("tie_a", dir, Some(t))); |
| 533 | let b = res!(child("tie_b", dir, Some(t))); |
| 534 | let c = res!(child("tie_c", dir, None)); |
| 535 | let reads = |s: &str, tag: &str| s.lines().find(|l| l.starts_with(&fmt!("QA1 {} reads", tag))).unwrap_or("").to_string(); |
| 536 | let (rb, rc) = (reads(&b, "tie_b"), reads(&c, "tie_c")); |
| 537 | let rb = rb.replace("tie_b", ""); |
| 538 | let rc = rc.replace("tie_c", ""); |
| 539 | if rb != rc || c.contains("TORN") || c.contains("ERR") || c.contains("clean=false") { |
| 540 | bad.push(fmt!("round {}: live {} vs restart {}\n{}\n{}", round, rb, rc, b, c)); |
| 541 | } |
| 542 | outs.push(fmt!("round {}\n{}\n{}\n{}", round, a, b, c)); |
| 543 | } |
| 544 | let _ = std::fs::remove_dir_all(dir); |
| 545 | println!("{}", outs.join("\n")); |
| 546 | assert!(bad.is_empty(), "equal stamps across processes diverge at restart:\n{}", bad.join("\n")); |
| 547 | Ok(()) |
| 548 | } |
| 549 | |
| 550 | // --------------------------------------------------------------------------------------------- |
| 551 | // Legacy (key-derived) set: an overwrite of one length killed after its durable head and before |
| 552 | // its retire, or before its head. After restart: whole, verify clean, sweep spares the live set, |
| 553 | // and overwrite, delete and restart behave. |
| 554 | |
| 555 | fn legacy_crash(mode: &str, dir: &str) -> Outcome<Vec<String>> { |
| 556 | let out = res!(child(mode, dir, None)); |
| 557 | let mut bad = Vec::new(); |
| 558 | if !out.contains("QA1 aborting") { bad.push(fmt!("child did not reach the crash: {}", out)); } |
| 559 | let k = dat!("legacy crash"); |
| 560 | let db = res!(start(dir, res!(cfg()), false)); |
| 561 | let got = describe(&db.get(&k, None)); |
| 562 | let want = if mode == "legacy_after_head" { "1100/2" } else { "1100/1" }; |
| 563 | if got != want { bad.push(fmt!("{}: restart read {} want {}", mode, got, want)); } |
| 564 | let v = res!(verify::verify_live_set(db.api(), None, scan_wait())); |
| 565 | if !v.clean() { bad.push(fmt!("{}: verify {}", mode, v.summary(4))); } |
| 566 | let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait())); |
| 567 | let swept = describe(&db.get(&k, None)); |
| 568 | if swept != want { bad.push(fmt!("{}: after sweep read {}", mode, swept)); } |
| 569 | res!(db.insert(k.clone(), val(1_100, 3), Uid::default(), None)); |
| 570 | let over = describe(&db.get(&k, None)); |
| 571 | if over != "1100/3" { bad.push(fmt!("{}: overwrite read {}", mode, over)); } |
| 572 | res!(db.close()); |
| 573 | let db = res!(start(dir, res!(cfg()), false)); |
| 574 | let over2 = describe(&db.get(&k, None)); |
| 575 | if over2 != "1100/3" { bad.push(fmt!("{}: overwrite after restart read {}", mode, over2)); } |
| 576 | res!(db.delete(&k, Uid::default(), None)); |
| 577 | let del = describe(&db.get(&k, None)); |
| 578 | if !(del == "none" || del == "tomb") { bad.push(fmt!("{}: delete read {}", mode, del)); } |
| 579 | res!(db.close()); |
| 580 | let db = res!(start(dir, res!(cfg()), false)); |
| 581 | let del2 = describe(&db.get(&k, None)); |
| 582 | if !(del2 == "none" || del2 == "tomb") { bad.push(fmt!("{}: delete after restart read {}", mode, del2)); } |
| 583 | let v = res!(verify::verify_live_set(db.api(), None, scan_wait())); |
| 584 | if !v.clean() { bad.push(fmt!("{}: verify end {}", mode, v.summary(4))); } |
| 585 | res!(db.close()); |
| 586 | println!("QA1 {} restart={} sweep_orphans={} {}", mode, got, report.orphans_found, out.replace('\n', " | ")); |
| 587 | let _ = std::fs::remove_dir_all(dir); |
| 588 | Ok(bad) |
| 589 | } |
| 590 | |
| 591 | #[test] |
| 592 | fn qa1_legacy_crash_mid_overwrite() -> Outcome<()> { |
| 593 | let _lock = lock(); |
| 594 | let mut bad = res!(legacy_crash("legacy_after_head", "./test_db_qa1_legacy_a")); |
| 595 | bad.extend(res!(legacy_crash("legacy_before_head", "./test_db_qa1_legacy_b"))); |
| 596 | assert!(bad.is_empty(), "{:#?}", bad); |
| 597 | Ok(()) |
| 598 | } |
| 599 | |
| 600 | // --------------------------------------------------------------------------------------------- |
| 601 | // A bunch key whose barrier fails after its chunks were written. Since 2026-10-10 (F2) the |
| 602 | // failure stops the store: the head is past its file's mark, so it is not served, the sweep is |
| 603 | // refused, and the restart cuts it, leaving the value a barrier proved. It used to be served |
| 604 | // and kept as an unconfirmed head that landed. |
| 605 | |
| 606 | #[test] |
| 607 | fn qa1_unconfirmed_head_that_lands() -> Outcome<()> { |
| 608 | let _lock = lock(); |
| 609 | let dir = "./test_db_qa1_unconf"; |
| 610 | let db = res!(start(dir, res!(cfg()), true)); |
| 611 | let k = dat!("unconfirmed head"); |
| 612 | res!(db.insert(k.clone(), val(1_100, 1), Uid::default(), None)); |
| 613 | res!(db.api().sync_now(scan_wait())); |
| 614 | hooks::set_chunk_insert_delay(Duration::from_millis(400)); |
| 615 | let put = { let (db, k) = (db.clone(), k.clone()); |
| 616 | thread::spawn(move || db.insert(k, val(1_100, 2), Uid::default(), None).map(|_| ()).map_err(|e| fmt!("{}", e))) }; |
| 617 | thread::sleep(Duration::from_millis(200)); |
| 618 | hooks::set_barrier_failure(true); |
| 619 | let put = match put.join() { Ok(x) => x, Err(_) => Err(fmt!("panicked")) }; |
| 620 | hooks::set_barrier_failure(false); |
| 621 | hooks::set_chunk_insert_delay(Duration::ZERO); |
| 622 | let faulted = db.api().fault().is_some(); |
| 623 | let live = describe(&db.get(&k, None)); |
| 624 | let report = sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait()); |
| 625 | let _ = db.close(); |
| 626 | res!(Refusal::clear(Path::new(dir))); // as the operator would |
| 627 | let db = res!(start(dir, res!(cfg()), false)); |
| 628 | let rest = describe(&db.get(&k, None)); |
| 629 | let v = res!(verify::verify_live_set(db.api(), None, scan_wait())); |
| 630 | res!(db.close()); |
| 631 | let _ = std::fs::remove_dir_all(dir); |
| 632 | println!("QA1 unconfirmed put={:?} faulted={} live={} sweep={:?} restart={} verify={}", |
| 633 | put.as_ref().map_err(|e| e.chars().take(200).collect::<String>()), faulted, live, |
| 634 | report.as_ref().map(|r| r.orphans_found), rest, v.clean()); |
| 635 | assert!(put.is_err(), "a put whose barrier failed was reported stored"); |
| 636 | assert!(faulted, "a failed barrier did not stop the store"); |
| 637 | assert!(live == "1100/1" || live.starts_with("ERR:"), "live read {}", live); |
| 638 | assert!(report.is_err(), "the sweep ran on a store stopped by a failed sync"); |
| 639 | assert_eq!(rest, "1100/1", "restart read"); |
| 640 | assert!(v.clean(), "verify after restart: {}", v.summary(4)); |
| 641 | Ok(()) |
| 642 | } |