oxedyne/fe2o3/fe2o3_o3db_sync/tests/durable_mark_races.rs
15.6 KiB, 8 runs
created by r1870400018:70850, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! Races of the durable mark (2026-10-10, from QA agent 2's probes P1, P5 and P6). A chunked |
| 2 | //! put's answer waits for the barrier over its chunks, a power cut taken at random through a run |
| 3 | //! with automatic collection on never reads a torn, lost or stale value, and a close racing a |
| 4 | //! writer leaves every live file covered by its mark. The hooks are process-wide, so the tests |
| 5 | //! run one at a time. |
| 6 | |
| 7 | mod gc_pair; |
| 8 | |
| 9 | use gc_pair::*; |
| 10 | |
| 11 | use oxedyne_fe2o3_core::prelude::*; |
| 12 | use oxedyne_fe2o3_iop_db::api::{ |
| 13 | Database, |
| 14 | Meta, |
| 15 | }; |
| 16 | use oxedyne_fe2o3_jdat::prelude::*; |
| 17 | use oxedyne_fe2o3_o3db_sync::{ |
| 18 | base::{ |
| 19 | cfg::OzoneConfig, |
| 20 | constant, |
| 21 | }, |
| 22 | comm::response::Wait, |
| 23 | file::{ |
| 24 | core::FileType, |
| 25 | zdir::ZoneDir, |
| 26 | }, |
| 27 | sweep, |
| 28 | test::{ |
| 29 | hooks, |
| 30 | setup::{ |
| 31 | self, |
| 32 | Uid, |
| 33 | UID_LEN, |
| 34 | }, |
| 35 | }, |
| 36 | verify, |
| 37 | }; |
| 38 | |
| 39 | use std::{ |
| 40 | collections::BTreeMap, |
| 41 | fs, |
| 42 | path::{ |
| 43 | Path, |
| 44 | PathBuf, |
| 45 | }, |
| 46 | sync::{ |
| 47 | Arc, |
| 48 | Mutex, |
| 49 | atomic::{ |
| 50 | AtomicBool, |
| 51 | Ordering, |
| 52 | }, |
| 53 | }, |
| 54 | time::{ |
| 55 | Duration, |
| 56 | Instant, |
| 57 | }, |
| 58 | }; |
| 59 | |
| 60 | static SERIAL: Mutex<()> = Mutex::new(()); |
| 61 | |
| 62 | #[derive(Clone, Copy, Debug)] |
| 63 | enum Policy { |
| 64 | EveryWrite, |
| 65 | EveryN(u32), |
| 66 | Interval(u64), |
| 67 | Never, |
| 68 | } |
| 69 | |
| 70 | const POLICIES: [Policy; 4] = [ |
| 71 | Policy::EveryWrite, |
| 72 | Policy::EveryN(100_000), |
| 73 | Policy::Interval(3_600_000), |
| 74 | Policy::Never, |
| 75 | ]; |
| 76 | |
| 77 | fn cfg(p: Policy, zones: u16, wbots: u16) -> Outcome<OzoneConfig> { |
| 78 | let mut c = res!(setup::default_cfg()); |
| 79 | c.num_zones = zones; |
| 80 | c.num_cbots_per_zone = 4; |
| 81 | c.num_fbots_per_zone = 1; |
| 82 | c.num_wbots_per_zone = wbots; |
| 83 | c.zone_overrides = BTreeMap::new(); |
| 84 | c.rest_chunk_threshold = 1_000; |
| 85 | c.rest_chunk_bytes = 400; |
| 86 | c.sync_on_write = false; |
| 87 | c.sync_every_n_writes = 0; |
| 88 | c.sync_interval_ms = 0; |
| 89 | match p { |
| 90 | Policy::EveryWrite => c.sync_on_write = true, |
| 91 | Policy::EveryN(n) => c.sync_every_n_writes = n, |
| 92 | Policy::Interval(t) => c.sync_interval_ms = t, |
| 93 | Policy::Never => (), |
| 94 | } |
| 95 | Ok(c) |
| 96 | } |
| 97 | |
| 98 | fn lock() -> std::sync::MutexGuard<'static, ()> { |
| 99 | let g = match SERIAL.lock() { Ok(g) => g, Err(p) => p.into_inner() }; |
| 100 | hooks::set_barrier_failure(false); |
| 101 | hooks::set_barrier_failure_for(None); |
| 102 | hooks::set_fault_record_fails(false); |
| 103 | hooks::set_owed_drop(false); |
| 104 | hooks::set_barrier_delay(Duration::ZERO); |
| 105 | g |
| 106 | } |
| 107 | |
| 108 | fn val(len: usize, seed: u8) -> Dat { |
| 109 | Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect()) |
| 110 | } |
| 111 | |
| 112 | fn fresh(name: &str) -> Outcome<PathBuf> { |
| 113 | let dir = fmt!("./test_db_races_{}", name); |
| 114 | let _ = fs::remove_dir_all(&dir); |
| 115 | res!(fs::create_dir_all(&dir)); |
| 116 | Ok(res!(Path::new(&dir).canonicalize())) |
| 117 | } |
| 118 | |
| 119 | fn open(root: &Path, c: &OzoneConfig, gc: bool, wipe: bool) -> Outcome<TestDb> { |
| 120 | log_set_level!("error"); |
| 121 | setup::start_db(root.to_path_buf(), Some(c.clone()), schemes(), None, gc, wipe) |
| 122 | } |
| 123 | |
| 124 | fn wait() -> Wait { |
| 125 | Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL } |
| 126 | } |
| 127 | |
| 128 | fn put(db: &TestDb, k: &Dat, v: Dat) -> Outcome<()> { |
| 129 | db.insert(k.clone(), v, Uid::default(), None).map(|_| ()) |
| 130 | } |
| 131 | |
| 132 | fn describe(r: &Outcome<Option<(Dat, Meta<{ UID_LEN }, Uid>)>>) -> String { |
| 133 | match r { |
| 134 | Ok(None) => fmt!("none"), |
| 135 | Ok(Some((Dat::BU32(b), _))) | Ok(Some((Dat::BU8(b), _))) | Ok(Some((Dat::BU16(b), _))) | |
| 136 | Ok(Some((Dat::BU64(b), _))) if !b.is_empty() |
| 137 | && b.iter().enumerate().all(|(j, x)| *x == b[0] ^ (j as u8)) => fmt!("{}/{}", b.len(), b[0]), |
| 138 | Ok(Some((d, _))) => fmt!("TORN:{:?}", d).chars().take(100).collect(), |
| 139 | Err(e) => fmt!("ERR:{}", fmt!("{}", e).chars().take(200).collect::<String>()), |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | fn read(db: &TestDb, k: &Dat) -> String { |
| 144 | describe(&db.get(k, None)) |
| 145 | } |
| 146 | |
| 147 | fn zdir(root: &Path, c: &OzoneConfig, z: usize) -> PathBuf { |
| 148 | c.zone_root(root).join(fmt!("zone_{:03}", z + 1)) |
| 149 | } |
| 150 | |
| 151 | /// The power cut of `durable_mark.rs`, at one instant (`gc_pair::instant_cut`). |
| 152 | fn power_cut(db: &TestDb, root: &Path, c: &OzoneConfig, to: &str) -> Outcome<PathBuf> { |
| 153 | let dst = res!(fresh(to)); |
| 154 | res!(instant_cut(db, root, c, &dst)); |
| 155 | Ok(dst) |
| 156 | } |
| 157 | |
| 158 | fn read_after_cut(db: &TestDb, root: &Path, c: &OzoneConfig, to: &str, keys: &[Dat]) -> Outcome<Vec<String>> { |
| 159 | let cut = res!(power_cut(db, root, c, to)); |
| 160 | let db2 = res!(open(&cut, c, false, false)); |
| 161 | let got = keys.iter().map(|k| read(&db2, k)).collect(); |
| 162 | res!(db2.close()); |
| 163 | let _ = fs::remove_dir_all(&cut); |
| 164 | Ok(got) |
| 165 | } |
| 166 | |
| 167 | // --------------------------------------------------------------------------------------------- |
| 168 | // R1 (P1, M8). A chunked put's caller waits on the barrier over its chunks (M8 closer), and a power |
| 169 | // cut while that barrier is in progress gives the old value, never the new head without chunks. |
| 170 | #[test] |
| 171 | fn a_chunk_answer_waits_for_its_barrier() -> Outcome<()> { |
| 172 | let _g = lock(); |
| 173 | for p in [Policy::Interval(3_600_000), Policy::Interval(200), Policy::Never, Policy::EveryN(100_000)] { |
| 174 | for (z, w) in [(1u16, 1u16), (2, 2)] { |
| 175 | let label = fmt!("{:?} {}x{}", p, z, w); |
| 176 | let c = res!(cfg(p, z, w)); |
| 177 | let root = res!(fresh("p1")); |
| 178 | let db = Arc::new(res!(open(&root, &c, false, true))); |
| 179 | let k = dat!("races p1 chunked"); |
| 180 | res!(put(&db, &k, val(3_000, 1))); |
| 181 | res!(db.api().sync_now(wait())); |
| 182 | let delay = Duration::from_millis(900); |
| 183 | hooks::set_barrier_delay(delay); |
| 184 | let db2 = db.clone(); |
| 185 | let k2 = k.clone(); |
| 186 | let t0 = Instant::now(); |
| 187 | let h = std::thread::spawn(move || { |
| 188 | let r = put(&db2, &k2, val(3_000, 2)); |
| 189 | (r, Instant::now()) |
| 190 | }); |
| 191 | std::thread::sleep(Duration::from_millis(300)); |
| 192 | let mid = res!(read_after_cut(&db, &root, &c, "p1cut", &[k.clone()])); |
| 193 | let (r, t1) = match h.join() { |
| 194 | Ok(x) => x, |
| 195 | Err(_) => return Err(err!("put thread panicked"; Test)), |
| 196 | }; |
| 197 | hooks::set_barrier_delay(Duration::ZERO); |
| 198 | res!(r); |
| 199 | let took = t1.duration_since(t0); |
| 200 | println!("{}: chunked put took {:?}; cut mid-barrier read {}", label, took, mid[0]); |
| 201 | assert!(took >= delay - Duration::from_millis(50), |
| 202 | "{}: the chunked put returned in {:?}, before the barrier over its chunks ({:?})", label, took, delay); |
| 203 | assert_eq!(mid[0], "3000/1", "{}: a cut during the chunk barrier did not give the old value", label); |
| 204 | let after = res!(read_after_cut(&db, &root, &c, "p1cut", &[k.clone()])); |
| 205 | // The head is past its mark under these policies unless an owed barrier covered it. |
| 206 | assert!(after[0] == "3000/1" || after[0] == "3000/2", "{}: after: {}", label, after[0]); |
| 207 | assert_eq!(read(&db, &k), "3000/2", "{}", label); |
| 208 | match Arc::try_unwrap(db) { |
| 209 | Ok(db) => res!(db.close()), |
| 210 | Err(_) => return Err(err!("db still shared"; Test)), |
| 211 | } |
| 212 | let _ = fs::remove_dir_all(&root); |
| 213 | } |
| 214 | } |
| 215 | Ok(()) |
| 216 | } |
| 217 | |
| 218 | // M8. A writer finishes its barrier before it takes the next record, so a head sharing a writer |
| 219 | // with a chunk waits on the chunk's barrier whether or not the chunk's answer was held. With one |
| 220 | // writer in each of sixteen zones a head usually lands in a zone that took none of its three |
| 221 | // chunks; only the held chunk answers then keep the put from returning, and the head from being |
| 222 | // written, before its chunks are durable. |
| 223 | #[test] |
| 224 | fn a_head_with_its_own_writer_waits_for_its_chunks() -> Outcome<()> { |
| 225 | let _g = lock(); |
| 226 | let delay = Duration::from_millis(600); |
| 227 | for p in [Policy::Never, Policy::Interval(3_600_000)] { |
| 228 | let c = res!(cfg(p, 16, 1)); |
| 229 | let root = res!(fresh("m8")); |
| 230 | let db = res!(open(&root, &c, false, true)); |
| 231 | for i in 0..8 { |
| 232 | let label = fmt!("{:?} key {}", p, i); |
| 233 | let k = dat!(fmt!("races m8 chunked {}", i)); |
| 234 | res!(put(&db, &k, val(1_200, 1))); |
| 235 | res!(db.api().sync_now(wait())); |
| 236 | hooks::set_barrier_delay(delay); |
| 237 | let t0 = Instant::now(); |
| 238 | let r = put(&db, &k, val(1_200, 2)); |
| 239 | let took = t0.elapsed(); |
| 240 | hooks::set_barrier_delay(Duration::ZERO); |
| 241 | res!(r); |
| 242 | assert!(took >= delay - Duration::from_millis(50), |
| 243 | "{}: the chunked put returned in {:?}, before the barrier over its chunks ({:?})", |
| 244 | label, took, delay); |
| 245 | assert_eq!(read(&db, &k), "1200/2", "{}", label); |
| 246 | } |
| 247 | res!(db.close()); |
| 248 | let _ = fs::remove_dir_all(&root); |
| 249 | } |
| 250 | Ok(()) |
| 251 | } |
| 252 | |
| 253 | // P5: the power-cut oracle with automatic collection on, small files and more ops, every policy. |
| 254 | fn model(seed: u64, p: Policy, zones: u16, wbots: u16) -> Outcome<()> { |
| 255 | let label = fmt!("seed {} {:?} {}x{}", seed, p, zones, wbots); |
| 256 | let mut c = res!(cfg(p, zones, wbots)); |
| 257 | c.data_file_max_bytes = 5_000; |
| 258 | let root = res!(fresh("p5")); |
| 259 | let db = res!(open(&root, &c, true, true)); |
| 260 | let keys: Vec<Dat> = (0..8).map(|i| dat!(fmt!("races model {}", i))).collect(); |
| 261 | let mut hist: Vec<Vec<String>> = vec![vec![fmt!("none")]; keys.len()]; |
| 262 | let mut covered: Vec<usize> = vec![1; keys.len()]; |
| 263 | let mut rng = seed | 1; |
| 264 | let mut next = |n: u64| { rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17; rng % n }; |
| 265 | let mut wseed = 0u8; |
| 266 | let mut cut_n = 0; |
| 267 | for step in 0..120 { |
| 268 | let i = next(keys.len() as u64) as usize; |
| 269 | match next(24) { |
| 270 | 0..=6 => { |
| 271 | wseed = wseed.wrapping_add(1).max(1); |
| 272 | let len = 1_100 + next(500) as usize; |
| 273 | res!(put(&db, &keys[i], val(len, wseed))); |
| 274 | hist[i].push(fmt!("{}/{}", len, wseed)); |
| 275 | }, |
| 276 | 7..=15 => { |
| 277 | wseed = wseed.wrapping_add(1).max(1); |
| 278 | let len = 50 + next(250) as usize; |
| 279 | res!(put(&db, &keys[i], val(len, wseed))); |
| 280 | hist[i].push(fmt!("{}/{}", len, wseed)); |
| 281 | }, |
| 282 | 16..=17 => { |
| 283 | res!(db.delete(&keys[i], Uid::default(), None)); |
| 284 | hist[i].push(fmt!("none")); |
| 285 | }, |
| 286 | 18 => { |
| 287 | res!(db.api().sync_now(wait())); |
| 288 | for (h, n) in hist.iter().zip(covered.iter_mut()) { *n = h.len(); } |
| 289 | }, |
| 290 | 19 => { res!(db.api().new_live_files()); }, |
| 291 | 20 => { res!(sweep::sweep_orphans(db.api(), Uid::default(), None, wait())); }, |
| 292 | 21 => { |
| 293 | // Let owed barriers and collections run, then cut mid-run. |
| 294 | std::thread::sleep(Duration::from_millis(next(400))); |
| 295 | let got = res!(read_after_cut(&db, &root, &c, "p5cut", &keys)); |
| 296 | for (j, g) in got.iter().enumerate() { |
| 297 | let ok = &hist[j][covered[j] - 1..]; |
| 298 | assert!(ok.contains(g), "{} step {}: key {} reads {} after a mid-run cut; allowed {:?}", label, step, j, g, ok); |
| 299 | } |
| 300 | cut_n += 1; |
| 301 | }, |
| 302 | _ => { |
| 303 | // A cut right now, with no pause. |
| 304 | let got = res!(read_after_cut(&db, &root, &c, "p5cut", &keys)); |
| 305 | for (j, g) in got.iter().enumerate() { |
| 306 | let ok = &hist[j][covered[j] - 1..]; |
| 307 | assert!(ok.contains(g), "{} step {}: key {} reads {} after an immediate cut; allowed {:?}", label, step, j, g, ok); |
| 308 | } |
| 309 | cut_n += 1; |
| 310 | }, |
| 311 | } |
| 312 | if let Policy::EveryWrite = p { |
| 313 | for (h, n) in hist.iter().zip(covered.iter_mut()) { *n = h.len(); } |
| 314 | } |
| 315 | } |
| 316 | res!(db.api().settle_for_test(Duration::from_secs(20))); |
| 317 | let got = res!(read_after_cut(&db, &root, &c, "p5cut", &keys)); |
| 318 | for (i, g) in got.iter().enumerate() { |
| 319 | let ok = &hist[i][covered[i] - 1..]; |
| 320 | assert!(ok.contains(g), "{}: key {} reads {} after the final cut; allowed {:?}", label, i, g, ok); |
| 321 | } |
| 322 | for (i, k) in keys.iter().enumerate() { |
| 323 | let want = match hist[i].last() { Some(w) => w.clone(), None => fmt!("none") }; |
| 324 | assert_eq!(read(&db, k), want, "{}: live read of key {}", label, i); |
| 325 | } |
| 326 | let v = res!(verify::verify_live_set(db.api(), None, wait())); |
| 327 | assert!(v.clean(), "{}: {}", label, v.summary(4)); |
| 328 | res!(db.close()); |
| 329 | let db = res!(open(&root, &c, true, false)); |
| 330 | for (i, k) in keys.iter().enumerate() { |
| 331 | let want = match hist[i].last() { Some(w) => w.clone(), None => fmt!("none") }; |
| 332 | assert_eq!(read(&db, k), want, "{}: read of key {} after a clean restart", label, i); |
| 333 | } |
| 334 | res!(db.close()); |
| 335 | let _ = fs::remove_dir_all(&root); |
| 336 | println!("{}: {} mid-run cuts passed", label, cut_n); |
| 337 | Ok(()) |
| 338 | } |
| 339 | |
| 340 | #[test] |
| 341 | fn the_power_cut_oracle_with_collection_on() -> Outcome<()> { |
| 342 | let _g = lock(); |
| 343 | let base: u64 = std::env::var("RACES_SEED").ok().and_then(|s| s.parse().ok()).unwrap_or(0xa2a2_0001); |
| 344 | let reps: u64 = std::env::var("RACES_REPS").ok().and_then(|s| s.parse().ok()).unwrap_or(1); |
| 345 | for n in 0..reps { |
| 346 | for p in POLICIES { |
| 347 | for (z, w) in [(1u16, 1u16), (2, 2)] { |
| 348 | res!(model(base + n * 7_919, p, z, w)); |
| 349 | } |
| 350 | } |
| 351 | } |
| 352 | Ok(()) |
| 353 | } |
| 354 | |
| 355 | // --------------------------------------------------------------------------------------------- |
| 356 | // M5 (P6): close with writes racing it under Never (M5). After close, every live file's mark must |
| 357 | // cover the file's length on disk: the announced close barriered what was written after the |
| 358 | // close's own sync_now. |
| 359 | #[test] |
| 360 | fn a_close_covers_racing_writes() -> Outcome<()> { |
| 361 | let _g = lock(); |
| 362 | for (z, w) in [(1u16, 1u16), (2, 2)] { |
| 363 | for round in 0..5 { |
| 364 | let label = fmt!("Never {}x{} round {}", z, w, round); |
| 365 | let mut c = res!(cfg(Policy::Never, z, w)); |
| 366 | c.data_file_max_bytes = 10_000_000; |
| 367 | let root = res!(fresh("p6")); |
| 368 | let db = Arc::new(res!(open(&root, &c, false, true))); |
| 369 | let stop = Arc::new(AtomicBool::new(false)); |
| 370 | let (db2, stop2) = (db.clone(), stop.clone()); |
| 371 | let h = std::thread::spawn(move || { |
| 372 | let mut i = 0usize; |
| 373 | let mut ok = 0usize; |
| 374 | while !stop2.load(Ordering::SeqCst) && i < 200_000 { |
| 375 | if put(&db2, &dat!(fmt!("races p6 {}", i % 50)), val(60, (i % 200) as u8 + 1)).is_ok() { |
| 376 | ok += 1; |
| 377 | } else { |
| 378 | break; |
| 379 | } |
| 380 | i += 1; |
| 381 | } |
| 382 | ok |
| 383 | }); |
| 384 | std::thread::sleep(Duration::from_millis(300)); |
| 385 | let marks = db.api().chans().marks().clone(); |
| 386 | let r = db.close(); |
| 387 | stop.store(true, Ordering::SeqCst); |
| 388 | let ok = h.join().unwrap_or(0); |
| 389 | let mut bad = Vec::new(); |
| 390 | for zi in 0..z as usize { |
| 391 | for (f, (d, i)) in res!(marks.files(zi)) { |
| 392 | let dl = fs::metadata(zdir(&root, &c, zi).join(ZoneDir::relative_file_path(&FileType::Data, f))).map(|m| m.len()).unwrap_or(0); |
| 393 | let il = fs::metadata(zdir(&root, &c, zi).join(ZoneDir::relative_file_path(&FileType::Index, f))).map(|m| m.len()).unwrap_or(0); |
| 394 | if d < dl || i < il { |
| 395 | bad.push(fmt!("zone {} file {}: mark ({}, {}) disk ({}, {})", zi, f, d, i, dl, il)); |
| 396 | } |
| 397 | } |
| 398 | } |
| 399 | println!("{}: close {:?}, {} writes ok, uncovered: {:?}", label, r.is_ok(), ok, bad); |
| 400 | assert!(bad.is_empty(), "{}: close left written bytes past the mark: {:?}", label, bad); |
| 401 | drop(db); |
| 402 | let _ = fs::remove_dir_all(&root); |
| 403 | } |
| 404 | } |
| 405 | Ok(()) |
| 406 | } |