oxedyne/fe2o3/fe2o3_o3db_sync/tests/qa_r2b.rs
20.7 KiB, 1 run
created by r1870400018:70860, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! QA round 2, agent B (2026-10-11): zone+fnum cut on a restored store and across zones whose |
| 2 | //! file numbers collide; the F3 rollover cut twice in one window, under a collection, and at the |
| 3 | //! seal. Every test compares what was acknowledged against what reads back after the reopen. |
| 4 | mod gc_pair; |
| 5 | |
| 6 | use gc_pair::*; |
| 7 | |
| 8 | use oxedyne_fe2o3_core::prelude::*; |
| 9 | use oxedyne_fe2o3_iop_db::api::Database; |
| 10 | use oxedyne_fe2o3_jdat::prelude::*; |
| 11 | use oxedyne_fe2o3_o3db_sync::{ |
| 12 | base::{ |
| 13 | cfg::OzoneConfig, |
| 14 | constant, |
| 15 | }, |
| 16 | comm::response::Wait, |
| 17 | file::{ |
| 18 | core::FileType, |
| 19 | refusal::Refusal, |
| 20 | zdir::ZoneDir, |
| 21 | }, |
| 22 | test::{ |
| 23 | hooks, |
| 24 | setup::{ |
| 25 | self, |
| 26 | Uid, |
| 27 | }, |
| 28 | }, |
| 29 | verify, |
| 30 | }; |
| 31 | |
| 32 | use std::{ |
| 33 | collections::BTreeMap, |
| 34 | fs, |
| 35 | path::{ |
| 36 | Path, |
| 37 | PathBuf, |
| 38 | }, |
| 39 | sync::{ |
| 40 | Arc, |
| 41 | Mutex, |
| 42 | }, |
| 43 | time::Duration, |
| 44 | }; |
| 45 | |
| 46 | static SERIAL: Mutex<()> = Mutex::new(()); |
| 47 | |
| 48 | fn lock() -> std::sync::MutexGuard<'static, ()> { |
| 49 | let g = match SERIAL.lock() { Ok(g) => g, Err(p) => p.into_inner() }; |
| 50 | reset(); |
| 51 | g |
| 52 | } |
| 53 | |
| 54 | fn reset() { |
| 55 | hooks::set_barrier_failure(false); |
| 56 | hooks::set_barrier_failure_for(None); |
| 57 | hooks::set_fault_record_fails(false); |
| 58 | hooks::set_barrier_delay(Duration::ZERO); |
| 59 | hooks::set_collect_delay(Duration::ZERO); |
| 60 | } |
| 61 | |
| 62 | #[derive(Clone, Copy)] |
| 63 | enum Pol { Every, N(u32), Never } |
| 64 | |
| 65 | fn cfg(zones: u16, p: Pol, max: u64) -> Outcome<OzoneConfig> { |
| 66 | let mut c = res!(setup::default_cfg()); |
| 67 | c.num_zones = zones; |
| 68 | c.num_wbots_per_zone = 1; |
| 69 | c.zone_overrides = BTreeMap::new(); |
| 70 | c.sync_on_write = matches!(p, Pol::Every); |
| 71 | c.sync_every_n_writes = match p { Pol::N(n) => n, _ => 0 }; |
| 72 | c.sync_interval_ms = 0; |
| 73 | c.data_file_max_bytes = max; |
| 74 | Ok(c) |
| 75 | } |
| 76 | |
| 77 | fn fresh(dir: &str) -> Outcome<PathBuf> { |
| 78 | let _ = fs::remove_dir_all(dir); |
| 79 | res!(fs::create_dir_all(dir)); |
| 80 | Ok(res!(Path::new(dir).canonicalize())) |
| 81 | } |
| 82 | |
| 83 | fn open(root: &Path, c: &OzoneConfig, wipe: bool) -> Outcome<TestDb> { |
| 84 | log_set_level!("error"); |
| 85 | setup::start_db(root.to_path_buf(), Some(c.clone()), schemes(), None, false, wipe) |
| 86 | } |
| 87 | |
| 88 | fn wait() -> Wait { |
| 89 | Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL } |
| 90 | } |
| 91 | |
| 92 | fn val(seed: u8) -> Dat { |
| 93 | Dat::BU32((0..100).map(|j| seed ^ (j as u8)).collect()) |
| 94 | } |
| 95 | |
| 96 | fn put(db: &TestDb, k: &str, seed: u8) -> Outcome<()> { |
| 97 | db.insert(dat!(k), val(seed), Uid::default(), None).map(|_| ()) |
| 98 | } |
| 99 | |
| 100 | fn reads(db: &TestDb, k: &str) -> Outcome<Option<Dat>> { |
| 101 | Ok(res!(db.get(&dat!(k), None)).map(|(v, _)| v)) |
| 102 | } |
| 103 | |
| 104 | fn len(p: &Path) -> u64 { |
| 105 | fs::metadata(p).map(|m| m.len()).unwrap_or(0) |
| 106 | } |
| 107 | |
| 108 | fn zdir(root: &Path, c: &OzoneConfig, z: usize) -> PathBuf { |
| 109 | c.zone_root(root).join(fmt!("zone_{:03}", z)) |
| 110 | } |
| 111 | |
| 112 | fn dat(zd: &Path, f: u32) -> PathBuf { zd.join(ZoneDir::relative_file_path(&FileType::Data, f)) } |
| 113 | |
| 114 | // Every ozone file of a zone directory with its length. |
| 115 | fn lens(zd: &Path) -> Outcome<BTreeMap<String, u64>> { |
| 116 | let mut m = BTreeMap::new(); |
| 117 | for e in res!(fs::read_dir(zd)) { |
| 118 | let p = res!(e).path(); |
| 119 | if p.is_file() { |
| 120 | m.insert(p.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(), len(&p)); |
| 121 | } |
| 122 | } |
| 123 | Ok(m) |
| 124 | } |
| 125 | |
| 126 | fn data_total(zd: &Path) -> u64 { |
| 127 | match lens(zd) { |
| 128 | Ok(m) => m.iter().filter(|(n, _)| n.ends_with(".dat")).map(|(_, l)| *l).sum(), |
| 129 | Err(_) => 0, |
| 130 | } |
| 131 | } |
| 132 | |
| 133 | fn last_fnum(zd: &Path) -> Option<u32> { |
| 134 | (1..10_000u32).rev().find(|f| dat(zd, *f).exists()) |
| 135 | } |
| 136 | |
| 137 | fn copy_tree(from: &Path, to: &Path) -> Outcome<()> { |
| 138 | res!(fs::create_dir_all(to)); |
| 139 | for e in res!(fs::read_dir(from)) { |
| 140 | let p = res!(e).path(); |
| 141 | let name = res!(p.file_name().ok_or_else(|| err!("{:?} has no name.", p; Test))); |
| 142 | match p.is_dir() { |
| 143 | true => res!(copy_tree(&p, &to.join(name))), |
| 144 | false => { res!(fs::copy(&p, to.join(name))); }, |
| 145 | } |
| 146 | } |
| 147 | Ok(()) |
| 148 | } |
| 149 | |
| 150 | // What each key was told: durable (proven by a barrier), acknowledged but not proven (under a |
| 151 | // lazy policy), and refused. After the reopen a key reads its durable value or an acknowledged |
| 152 | // one (under EveryWrite, an acknowledged one if it has any), never only a refused one. |
| 153 | #[derive(Default)] |
| 154 | struct Book { |
| 155 | every: bool, |
| 156 | durable: BTreeMap<String, u8>, |
| 157 | acked: BTreeMap<String, Vec<u8>>, |
| 158 | failed: BTreeMap<String, Vec<u8>>, |
| 159 | } |
| 160 | |
| 161 | impl Book { |
| 162 | fn durable(&mut self, k: &str, s: u8) { self.durable.insert(k.to_string(), s); self.acked.remove(k); } |
| 163 | fn acked(&mut self, k: &str, s: u8) { |
| 164 | if self.every { self.durable(k, s); } else { self.acked.entry(k.to_string()).or_default().push(s); } |
| 165 | } |
| 166 | fn failed(&mut self, k: &str, s: u8) { self.failed.entry(k.to_string()).or_default().push(s); } |
| 167 | // Everything acknowledged so far is now durable (a sync_now that succeeded). |
| 168 | fn synced(&mut self) { |
| 169 | let acked = std::mem::take(&mut self.acked); |
| 170 | for (k, v) in acked { if let Some(s) = v.last() { self.durable.insert(k, *s); } } |
| 171 | } |
| 172 | } |
| 173 | |
| 174 | // (lost, resurrected, torn) |
| 175 | fn check(db: &TestDb, label: &str, b: &Book) -> Outcome<(usize, usize, usize)> { |
| 176 | let mut keys: Vec<&String> = b.durable.keys().chain(b.acked.keys()).chain(b.failed.keys()).collect(); |
| 177 | keys.sort(); |
| 178 | keys.dedup(); |
| 179 | let (mut lost, mut risen, mut torn) = (0, 0, 0); |
| 180 | for k in keys { |
| 181 | let got = res!(reads(db, k)); |
| 182 | let acked = b.acked.get(k).cloned().unwrap_or_default(); |
| 183 | let allowed: Vec<u8> = b.durable.get(k).cloned().into_iter().chain(acked.iter().cloned()).collect(); |
| 184 | match got { |
| 185 | None => if b.durable.contains_key(k) { |
| 186 | msg!("{}: LOST {} (durable seed {:?})", label, k, b.durable.get(k)); |
| 187 | lost += 1; |
| 188 | }, |
| 189 | Some(g) => if allowed.iter().any(|s| val(*s) == g) { |
| 190 | // fine |
| 191 | } else if b.failed.get(k).map(|f| f.iter().any(|s| val(*s) == g)).unwrap_or(false) { |
| 192 | msg!("{}: RESURRECTED {} (refused value served; allowed {:?})", label, k, allowed); |
| 193 | risen += 1; |
| 194 | } else { |
| 195 | msg!("{}: TORN {} (allowed {:?})", label, k, allowed); |
| 196 | torn += 1; |
| 197 | }, |
| 198 | } |
| 199 | } |
| 200 | Ok((lost, risen, torn)) |
| 201 | } |
| 202 | |
| 203 | // --- Surface 1 --------------------------------------------------------------------------------- |
| 204 | |
| 205 | /// Two zones each hold a file 1. The fault is in zone 2 only; zone 1's file 1, longer than zone |
| 206 | /// 2's mark, must come through the open untouched. |
| 207 | #[test] |
| 208 | fn colliding_file_numbers_cut_only_the_named_zone() -> Outcome<()> { |
| 209 | let _g = lock(); |
| 210 | let c = res!(cfg(2, Pol::Every, 1_000_000)); |
| 211 | // Which zone each key lands in, learned from a throwaway store. |
| 212 | let probe = res!(fresh("./test_db_qa_r2b_probe")); |
| 213 | let mut z1 = Vec::new(); |
| 214 | let mut z2 = Vec::new(); |
| 215 | { |
| 216 | let db = res!(open(&probe, &c, true)); |
| 217 | for i in 0..80 { |
| 218 | let k = fmt!("collide {}", i); |
| 219 | let (a, b) = (data_total(&zdir(&probe, &c, 1)), data_total(&zdir(&probe, &c, 2))); |
| 220 | res!(put(&db, &k, 1)); |
| 221 | let (a2, b2) = (data_total(&zdir(&probe, &c, 1)), data_total(&zdir(&probe, &c, 2))); |
| 222 | if a2 > a { z1.push(k); } else if b2 > b { z2.push(k); } |
| 223 | } |
| 224 | let _ = db.close(); |
| 225 | } |
| 226 | let _ = fs::remove_dir_all(&probe); |
| 227 | msg!("collide: {} keys in zone 1, {} in zone 2", z1.len(), z2.len()); |
| 228 | assert!(z1.len() > 10 && z2.len() > 3, "the probe did not split the keys"); |
| 229 | |
| 230 | let root = res!(fresh("./test_db_qa_r2b_collide")); |
| 231 | let db = res!(open(&root, &c, true)); |
| 232 | let mut bk = Book { every: true, ..Default::default() }; |
| 233 | for (i, k) in z1.iter().enumerate() { |
| 234 | res!(put(&db, k, 10 + i as u8)); |
| 235 | bk.durable(k, 10 + i as u8); |
| 236 | } |
| 237 | for k in &z2[0..2] { |
| 238 | res!(put(&db, k, 3)); |
| 239 | bk.durable(k, 3); |
| 240 | } |
| 241 | hooks::set_barrier_failure(true); |
| 242 | let lost = put(&db, &z2[2], 4); |
| 243 | hooks::set_barrier_failure(false); |
| 244 | assert!(lost.is_err(), "the failed write was confirmed"); |
| 245 | bk.failed(&z2[2], 4); |
| 246 | let report = res!(db.api().fault().ok_or_else(|| err!("No fault."; Test))); |
| 247 | let _ = db.close(); |
| 248 | msg!("collide cuts: {:?}", report.cuts.iter().map(|c| (c.zone, c.fnum, c.data_upto, c.index_upto)).collect::<Vec<_>>()); |
| 249 | assert!(report.cuts.iter().all(|c| c.zone == 1), "a cut names zone 1: {:?}", report.cuts); |
| 250 | let z1dir = zdir(&root, &c, 1); |
| 251 | for cut in &report.cuts { |
| 252 | assert!(len(&dat(&z1dir, cut.fnum)) > cut.data_upto, |
| 253 | "precondition: zone 1's file {} is not longer than zone 2's mark", cut.fnum); |
| 254 | } |
| 255 | let before = res!(lens(&z1dir)); |
| 256 | assert!(res!(Refusal::clear(&root))); |
| 257 | let db = res!(open(&root, &c, false)); |
| 258 | let t = res!(check(&db, "collide", &bk)); |
| 259 | res!(db.close()); |
| 260 | assert_eq!(t, (0, 0, 0), "collide: (lost, resurrected, torn) {:?}", t); |
| 261 | let after = res!(lens(&z1dir)); |
| 262 | for (n, b) in &before { |
| 263 | if n.ends_with(".dat") || n.ends_with(".ind") { |
| 264 | assert_eq!(after.get(n), Some(b), "zone 1's {} changed in the open: {} to {:?}", n, b, after.get(n)); |
| 265 | } |
| 266 | } |
| 267 | let z2dir = zdir(&root, &c, 2); |
| 268 | for cut in &report.cuts { |
| 269 | assert!(len(&dat(&z2dir, cut.fnum)) <= cut.data_upto, "zone 2's file {} was not cut", cut.fnum); |
| 270 | } |
| 271 | let _ = fs::remove_dir_all(&root); |
| 272 | Ok(()) |
| 273 | } |
| 274 | |
| 275 | /// A store restored from an older backup under its newer record: (a) the file in doubt is in |
| 276 | /// the backup, shorter than its mark, and the open is a no-op cut; (b) the backup predates the |
| 277 | /// file in doubt, and the open refuses. Then the operator restores the whole backup over the |
| 278 | /// root: does the store open? |
| 279 | #[test] |
| 280 | fn a_restore_from_an_older_backup() -> Outcome<()> { |
| 281 | let _g = lock(); |
| 282 | let c = res!(cfg(1, Pol::Every, 1_000_000)); |
| 283 | let root = res!(fresh("./test_db_qa_r2b_restore")); |
| 284 | let early = res!(fresh("./test_db_qa_r2b_restore_early")); |
| 285 | let late = res!(fresh("./test_db_qa_r2b_restore_late")); |
| 286 | let zd = zdir(&root, &c, 1); |
| 287 | let mut db = res!(open(&root, &c, true)); |
| 288 | let mut bk = Book { every: true, ..Default::default() }; |
| 289 | for i in 0..5 { res!(put(&db, &fmt!("a {}", i), i + 1)); bk.durable(&fmt!("a {}", i), i + 1); } |
| 290 | let _ = db.close(); |
| 291 | res!(copy_tree(&root, &early)); // before the file in doubt existed |
| 292 | db = res!(open(&root, &c, false)); |
| 293 | res!(db.api().new_live_files()); |
| 294 | for i in 0..5 { res!(put(&db, &fmt!("b {}", i), i + 20)); bk.durable(&fmt!("b {}", i), i + 20); } |
| 295 | let _ = db.close(); |
| 296 | res!(copy_tree(&root, &late)); // the file in doubt there, shorter than its mark |
| 297 | db = res!(open(&root, &c, false)); |
| 298 | for i in 0..5 { res!(put(&db, &fmt!("c {}", i), i + 40)); } |
| 299 | hooks::set_barrier_failure(true); |
| 300 | assert!(put(&db, "lost", 99).is_err()); |
| 301 | hooks::set_barrier_failure(false); |
| 302 | let report = res!(db.api().fault().ok_or_else(|| err!("No fault."; Test))); |
| 303 | let _ = db.close(); |
| 304 | msg!("restore cuts {:?}, files now {:?}", report.cuts.iter().map(|c| (c.fnum, c.data_upto)).collect::<Vec<_>>(), res!(lens(&zd))); |
| 305 | let record = res!(fs::read_to_string(root.join(constant::FAULT_FILENAME))); |
| 306 | |
| 307 | // (a) late backup under the record. |
| 308 | let a = res!(fresh("./test_db_qa_r2b_restore_a")); |
| 309 | res!(copy_tree(&late, &a)); |
| 310 | res!(fs::write(a.join(constant::FAULT_FILENAME), &record)); |
| 311 | let alens = res!(lens(&zdir(&a, &c, 1))); |
| 312 | let db = res!(open(&a, &c, false)); |
| 313 | bk.failed("lost", 99); |
| 314 | let t = res!(check(&db, "restore a", &bk)); |
| 315 | for i in 0..5 { assert_eq!(res!(reads(&db, &fmt!("c {}", i))), None, "a key newer than the backup appeared"); } |
| 316 | res!(db.close()); |
| 317 | assert_eq!(t, (0, 0, 0), "restore a: (lost, resurrected, torn) {:?}", t); |
| 318 | let alens2 = res!(lens(&zdir(&a, &c, 1))); |
| 319 | for (n, b) in &alens { |
| 320 | if n.ends_with(".dat") || n.ends_with(".ind") { |
| 321 | assert_eq!(alens2.get(n), Some(b), "restore a: {} changed", n); |
| 322 | } |
| 323 | } |
| 324 | |
| 325 | // (b) early backup under the record and the marker. |
| 326 | let b = res!(fresh("./test_db_qa_r2b_restore_b")); |
| 327 | res!(copy_tree(&early, &b)); |
| 328 | res!(fs::write(b.join(constant::FAULT_FILENAME), &record)); |
| 329 | match open(&b, &c, false) { |
| 330 | Ok(db) => { let _ = db.close(); msg!("restore b: OPENED over a record naming a file it lacks"); }, |
| 331 | Err(e) => msg!("restore b: refused: {}", fmt!("{}", e).chars().take(600).collect::<String>()), |
| 332 | } |
| 333 | let _ = fs::remove_dir_all(&root); |
| 334 | let _ = fs::remove_dir_all(&early); |
| 335 | let _ = fs::remove_dir_all(&late); |
| 336 | let _ = fs::remove_dir_all(&a); |
| 337 | let _ = fs::remove_dir_all(&b); |
| 338 | Ok(()) |
| 339 | } |
| 340 | |
| 341 | // --- Surface 2 --------------------------------------------------------------------------------- |
| 342 | |
| 343 | // Fills the live file during a held, failing barrier, so the writer rolls over, possibly more |
| 344 | // than once. Returns the store's root, its config, the first and last live files, the ack map, |
| 345 | // the failed map and the fault. |
| 346 | fn rollover_window(dir: &str, p: Pol, max: u64, nputs: usize, overwrite: bool, collect: bool) |
| 347 | -> Outcome<()> |
| 348 | { |
| 349 | let label = dir.to_string(); |
| 350 | let root = res!(fresh(dir)); |
| 351 | let c = res!(cfg(1, p, max)); |
| 352 | let zd = zdir(&root, &c, 1); |
| 353 | let db = Arc::new(res!(open(&root, &c, true))); |
| 354 | let mut bk = Book { every: matches!(p, Pol::Every), ..Default::default() }; |
| 355 | // Several files, with old versions for a collection to compact. |
| 356 | for round in 0..3u8 { |
| 357 | for i in 0..10 { |
| 358 | res!(put(&db, &fmt!("k {}", i), round * 10 + i as u8 + 1)); |
| 359 | bk.acked(&fmt!("k {}", i), round * 10 + i as u8 + 1); |
| 360 | } |
| 361 | res!(db.api().sync_now(wait())); |
| 362 | bk.synced(); |
| 363 | } |
| 364 | let first = res!(last_fnum(&zd).ok_or_else(|| err!("No file."; Test))); |
| 365 | res!(put(&db, "dirty", 9)); |
| 366 | bk.acked("dirty", 9); |
| 367 | let compactor = if collect { |
| 368 | hooks::set_collect_delay(Duration::from_millis(600)); |
| 369 | let dbc = db.clone(); |
| 370 | let h = std::thread::spawn(move || dbc.compact_now(Duration::from_secs(30)).map(|r| fmt!("{:?}", r))); |
| 371 | std::thread::sleep(Duration::from_millis(50)); |
| 372 | Some(h) |
| 373 | } else { None }; |
| 374 | hooks::set_barrier_delay(Duration::from_millis(800)); |
| 375 | hooks::set_barrier_failure(true); |
| 376 | let db2 = db.clone(); |
| 377 | let sync = std::thread::spawn(move || db2.api().sync_now(wait())); |
| 378 | std::thread::sleep(Duration::from_millis(100)); |
| 379 | let puts: Vec<_> = (0..nputs).map(|i| { |
| 380 | let db3 = db.clone(); |
| 381 | let k = if overwrite && i % 2 == 0 { fmt!("k {}", (i / 2) % 10) } else { fmt!("during {}", i) }; |
| 382 | let s = 100 + i as u8; |
| 383 | std::thread::spawn(move || (k.clone(), s, put(&db3, &k, s))) |
| 384 | }).collect(); |
| 385 | let synced = sync.join(); |
| 386 | let mut acked_during = 0; |
| 387 | let mut nfailed = 0; |
| 388 | for p in puts { |
| 389 | if let Ok((k, s, r)) = p.join() { |
| 390 | match r { |
| 391 | Ok(()) => { bk.acked(&k, s); acked_during += 1; }, |
| 392 | Err(_) => { bk.failed(&k, s); nfailed += 1; }, |
| 393 | } |
| 394 | } |
| 395 | } |
| 396 | reset(); |
| 397 | let gc = compactor.map(|h| h.join()); |
| 398 | msg!("{}: sync {:?}, {} during acked, {} failed, compaction {:?}", label, |
| 399 | synced.as_ref().map(|r| r.is_ok()), acked_during, nfailed, gc.map(|g| g.map(|r| r.is_ok()))); |
| 400 | let last = res!(last_fnum(&zd).ok_or_else(|| err!("No file."; Test))); |
| 401 | let report = res!(db.api().fault().ok_or_else(|| err!("No fault."; Test))); |
| 402 | // Let any late raise land. |
| 403 | let _ = wait_until(Duration::from_secs(5), || db.api().fault() |
| 404 | .map(|r| (first + 1..=last).all(|f| r.cuts.iter().any(|c| c.fnum == f))).unwrap_or(false)); |
| 405 | let report2 = res!(db.api().fault().ok_or_else(|| err!("No fault."; Test))); |
| 406 | msg!("{}: files {}..{}, cuts {:?} (first look {})", label, first, last, |
| 407 | report2.cuts.iter().map(|c| (c.fnum, c.data_upto, c.index_upto)).collect::<Vec<_>>(), report.cuts.len()); |
| 408 | match Arc::try_unwrap(db) { |
| 409 | Ok(db) => { let _ = db.close(); }, |
| 410 | Err(_) => return Err(err!("{}: the store is still shared.", label; Test)), |
| 411 | } |
| 412 | let record = res!(fs::read_to_string(root.join(constant::FAULT_FILENAME))); |
| 413 | let pre = res!(lens(&zd)); |
| 414 | msg!("{}: record:\n{}files before reopen {:?}", label, record, pre); |
| 415 | let mut missing = Vec::new(); |
| 416 | for f in (first + 1)..=last { |
| 417 | if dat(&zd, f).exists() && !record.lines().any(|l| l.starts_with(&fmt!("cut 0 {} ", f))) { |
| 418 | missing.push(f); |
| 419 | } |
| 420 | } |
| 421 | assert!(res!(Refusal::clear(&root))); |
| 422 | let db = res!(open(&root, &c, false)); |
| 423 | let t1 = res!(check(&db, &label, &bk)); |
| 424 | let v = res!(verify::verify_live_set(db.api(), None, wait())); |
| 425 | res!(db.close()); |
| 426 | for cut in &report2.cuts { |
| 427 | assert!(len(&cut.data) <= cut.data_upto && len(&cut.index) <= cut.index_upto, |
| 428 | "{}: file {} past its mark ({}, {}): ({}, {})", label, cut.fnum, cut.data_upto, |
| 429 | cut.index_upto, len(&cut.data), len(&cut.index)); |
| 430 | } |
| 431 | // Round trip: a write after the cut, a second reopen. |
| 432 | let db = res!(open(&root, &c, false)); |
| 433 | res!(put(&db, "after", 77)); |
| 434 | res!(db.close()); |
| 435 | let db = res!(open(&root, &c, false)); |
| 436 | let t2 = res!(check(&db, &label, &bk)); |
| 437 | let after = res!(reads(&db, "after")); |
| 438 | res!(db.close()); |
| 439 | assert!(missing.is_empty(), "{}: files rolled over to but not in the record: {:?}", label, missing); |
| 440 | assert!(v.clean(), "{}: {}", label, v.summary(4)); |
| 441 | assert_eq!((t1, t2), ((0, 0, 0), (0, 0, 0)), "{}: (lost, resurrected, torn) first reopen {:?}, second {:?}", label, t1, t2); |
| 442 | assert_eq!(after, Some(val(77)), "{}: the write after the cut was lost", label); |
| 443 | let _ = fs::remove_dir_all(&root); |
| 444 | Ok(()) |
| 445 | } |
| 446 | |
| 447 | #[test] |
| 448 | fn rollover_twice_never() -> Outcome<()> { |
| 449 | let _g = lock(); |
| 450 | let r = rollover_window("./test_db_qa_r2b_roll2_never", Pol::Never, 2_000, 60, false, false); |
| 451 | reset(); |
| 452 | r |
| 453 | } |
| 454 | |
| 455 | #[test] |
| 456 | fn rollover_twice_every_write_overwrites() -> Outcome<()> { |
| 457 | let _g = lock(); |
| 458 | let r = rollover_window("./test_db_qa_r2b_roll2_every", Pol::Every, 2_000, 60, true, false); |
| 459 | reset(); |
| 460 | r |
| 461 | } |
| 462 | |
| 463 | #[test] |
| 464 | fn rollover_under_collection_every_write() -> Outcome<()> { |
| 465 | let _g = lock(); |
| 466 | let r = rollover_window("./test_db_qa_r2b_rollgc_every", Pol::Every, 2_000, 60, true, true); |
| 467 | reset(); |
| 468 | r |
| 469 | } |
| 470 | |
| 471 | #[test] |
| 472 | fn rollover_under_collection_every_n() -> Outcome<()> { |
| 473 | let _g = lock(); |
| 474 | let r = rollover_window("./test_db_qa_r2b_rollgc_n", Pol::N(1000), 2_000, 60, true, true); |
| 475 | reset(); |
| 476 | r |
| 477 | } |
| 478 | |
| 479 | /// The first barrier to fail is the seal's: EveryN with a large N runs none per record, so the |
| 480 | /// rollover's own barrier over the old file is the one that fails. |
| 481 | #[test] |
| 482 | fn a_fault_exactly_at_the_seal() -> Outcome<()> { |
| 483 | let _g = lock(); |
| 484 | let root = res!(fresh("./test_db_qa_r2b_seal")); |
| 485 | let c = res!(cfg(1, Pol::N(100_000), 2_000)); |
| 486 | let zd = zdir(&root, &c, 1); |
| 487 | let db = res!(open(&root, &c, true)); |
| 488 | let mut bk = Book::default(); |
| 489 | for i in 0..5 { res!(put(&db, &fmt!("s {}", i), i + 1)); bk.acked(&fmt!("s {}", i), i + 1); } |
| 490 | res!(db.api().sync_now(wait())); |
| 491 | bk.synced(); |
| 492 | let n = res!(last_fnum(&zd).ok_or_else(|| err!("No file."; Test))); |
| 493 | let mark = len(&dat(&zd, n)); |
| 494 | hooks::set_barrier_failure_for(Some(n)); |
| 495 | let mut acked_unsynced = BTreeMap::new(); |
| 496 | let mut nfailed = 0; |
| 497 | for i in 0..60u8 { |
| 498 | let k = fmt!("t {}", i); |
| 499 | match put(&db, &k, 50 + i) { |
| 500 | Ok(()) => { acked_unsynced.insert(k.clone(), 50 + i); bk.acked(&k, 50 + i); }, |
| 501 | Err(_) => { bk.failed(&k, 50 + i); nfailed += 1; }, |
| 502 | } |
| 503 | } |
| 504 | hooks::set_barrier_failure_for(None); |
| 505 | let report = res!(db.api().fault().ok_or_else(|| err!("The seal's failed barrier raised no fault."; Test))); |
| 506 | let last = res!(last_fnum(&zd).ok_or_else(|| err!("No file."; Test))); |
| 507 | let _ = db.close(); |
| 508 | let record = res!(fs::read_to_string(root.join(constant::FAULT_FILENAME))); |
| 509 | msg!("seal: file {} mark {}, last {}, {} acked unsynced, {} failed, cuts {:?}\nrecord:\n{}", n, mark, last, |
| 510 | acked_unsynced.len(), nfailed, |
| 511 | report.cuts.iter().map(|c| (c.fnum, c.data_upto, c.index_upto)).collect::<Vec<_>>(), record); |
| 512 | assert!(last > n, "no rollover happened"); |
| 513 | assert!(record.lines().any(|l| l.starts_with(&fmt!("cut 0 {} ", n))), "the sealed file is not in the record"); |
| 514 | for f in (n + 1)..=last { |
| 515 | assert!(record.lines().any(|l| l.starts_with(&fmt!("cut 0 {} ", f))), "file {} is not in the record", f); |
| 516 | } |
| 517 | assert!(res!(Refusal::clear(&root))); |
| 518 | let db = res!(open(&root, &c, false)); |
| 519 | let t = res!(check(&db, "seal", &bk)); |
| 520 | let mut survived = 0; |
| 521 | for (k, s) in &acked_unsynced { |
| 522 | if res!(reads(&db, k)) == Some(val(*s)) { survived += 1; } |
| 523 | } |
| 524 | res!(db.close()); |
| 525 | msg!("seal: {} of {} acked-but-unsynced (EveryN) survived the cut", survived, acked_unsynced.len()); |
| 526 | assert_eq!(t, (0, 0, 0), "seal: (lost, resurrected, torn) {:?}", t); |
| 527 | assert!(len(&dat(&zd, n)) <= mark.max(report.cuts.iter().find(|c| c.fnum == n).map(|c| c.data_upto).unwrap_or(0)), |
| 528 | "file {} is past its mark", n); |
| 529 | let _ = fs::remove_dir_all(&root); |
| 530 | Ok(()) |
| 531 | } |