oxedyne/fe2o3/fe2o3_o3db_sync/tests/durable_mark.rs
36.5 KiB, 25 runs
created by r1870400018:70848, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! The durable mark (Ozone, 2026-10-10). A discard waits for its winner's barrier, a failed |
| 2 | //! barrier stops the store for good, and the next open cuts every file in doubt back to its mark. |
| 3 | //! Each test that can runs under every sync policy, and under two zones of two writers. The |
| 4 | //! power cut is modelled in process: the store is copied as its files stand, every live file of |
| 5 | //! the copy is cut to its published mark, and the copy is opened. The hooks are process-wide, |
| 6 | //! so the tests run one at a time. |
| 7 | |
| 8 | mod gc_pair; |
| 9 | |
| 10 | use gc_pair::*; |
| 11 | |
| 12 | use oxedyne_fe2o3_core::prelude::*; |
| 13 | use oxedyne_fe2o3_iop_db::api::{ |
| 14 | Database, |
| 15 | Meta, |
| 16 | }; |
| 17 | use oxedyne_fe2o3_jdat::prelude::*; |
| 18 | use oxedyne_fe2o3_o3db_sync::{ |
| 19 | base::{ |
| 20 | cfg::OzoneConfig, |
| 21 | constant, |
| 22 | }, |
| 23 | comm::response::Wait, |
| 24 | data::cache::{ |
| 25 | CacheEntry, |
| 26 | is_chunk_key, |
| 27 | }, |
| 28 | file::{ |
| 29 | core::FileType, |
| 30 | refusal::Refusal, |
| 31 | zdir::ZoneDir, |
| 32 | }, |
| 33 | file::state::DataState, |
| 34 | sweep, |
| 35 | test::{ |
| 36 | hooks, |
| 37 | setup::{ |
| 38 | self, |
| 39 | Uid, |
| 40 | UID_LEN, |
| 41 | }, |
| 42 | }, |
| 43 | verify, |
| 44 | }; |
| 45 | |
| 46 | use std::{ |
| 47 | collections::{ |
| 48 | BTreeMap, |
| 49 | BTreeSet, |
| 50 | }, |
| 51 | fs, |
| 52 | path::{ |
| 53 | Path, |
| 54 | PathBuf, |
| 55 | }, |
| 56 | sync::Mutex, |
| 57 | thread, |
| 58 | time::{ |
| 59 | Duration, |
| 60 | Instant, |
| 61 | }, |
| 62 | }; |
| 63 | |
| 64 | static SERIAL: Mutex<()> = Mutex::new(()); |
| 65 | |
| 66 | #[derive(Clone, Copy, Debug)] |
| 67 | enum Policy { |
| 68 | EveryWrite, |
| 69 | EveryN(u32), |
| 70 | Interval(u64), |
| 71 | Never, |
| 72 | } |
| 73 | |
| 74 | const POLICIES: [Policy; 4] = [ |
| 75 | Policy::EveryWrite, |
| 76 | Policy::EveryN(1_000), |
| 77 | Policy::Interval(3_600_000), |
| 78 | Policy::Never, |
| 79 | ]; |
| 80 | |
| 81 | const SHAPES: [(u16, u16); 2] = [(1, 1), (2, 2)]; // zones, writers per zone |
| 82 | |
| 83 | fn cfg(p: Policy, zones: u16, wbots: u16) -> Outcome<OzoneConfig> { |
| 84 | let mut c = res!(setup::default_cfg()); |
| 85 | c.num_zones = zones; |
| 86 | c.num_cbots_per_zone = 4; |
| 87 | c.num_fbots_per_zone = 1; |
| 88 | c.num_wbots_per_zone = wbots; |
| 89 | c.zone_overrides = BTreeMap::new(); |
| 90 | c.rest_chunk_threshold = 1_000; |
| 91 | c.rest_chunk_bytes = 400; |
| 92 | c.sync_on_write = false; |
| 93 | c.sync_every_n_writes = 0; |
| 94 | c.sync_interval_ms = 0; |
| 95 | match p { |
| 96 | Policy::EveryWrite => c.sync_on_write = true, |
| 97 | Policy::EveryN(n) => c.sync_every_n_writes = n, |
| 98 | Policy::Interval(t) => c.sync_interval_ms = t, |
| 99 | Policy::Never => (), |
| 100 | } |
| 101 | Ok(c) |
| 102 | } |
| 103 | |
| 104 | fn lock() -> std::sync::MutexGuard<'static, ()> { |
| 105 | let g = match SERIAL.lock() { Ok(g) => g, Err(p) => p.into_inner() }; |
| 106 | hooks::set_barrier_failure(false); |
| 107 | hooks::set_barrier_failure_for(None); |
| 108 | hooks::set_fault_record_fails(false); |
| 109 | hooks::set_owed_drop(false); |
| 110 | hooks::set_barrier_delay(Duration::ZERO); |
| 111 | hooks::set_head_hold(false); |
| 112 | g |
| 113 | } |
| 114 | |
| 115 | fn val(len: usize, seed: u8) -> Dat { |
| 116 | Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect()) |
| 117 | } |
| 118 | |
| 119 | fn fresh(dir: &str) -> Outcome<PathBuf> { |
| 120 | let _ = fs::remove_dir_all(dir); |
| 121 | res!(fs::create_dir_all(dir)); |
| 122 | Ok(res!(Path::new(dir).canonicalize())) |
| 123 | } |
| 124 | |
| 125 | fn open(root: &Path, c: &OzoneConfig, wipe: bool) -> Outcome<TestDb> { |
| 126 | log_set_level!("error"); |
| 127 | setup::start_db(root.to_path_buf(), Some(c.clone()), schemes(), None, false, wipe) |
| 128 | } |
| 129 | |
| 130 | fn wait() -> Wait { |
| 131 | Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL } |
| 132 | } |
| 133 | |
| 134 | fn put(db: &TestDb, k: &Dat, v: Dat) -> Outcome<()> { |
| 135 | db.insert(k.clone(), v, Uid::default(), None).map(|_| ()) |
| 136 | } |
| 137 | |
| 138 | // "L/S" for a whole value of length L and seed S as `val` writes it, else what was read. |
| 139 | fn describe(r: &Outcome<Option<(Dat, Meta<{ UID_LEN }, Uid>)>>) -> String { |
| 140 | match r { |
| 141 | Ok(None) => fmt!("none"), |
| 142 | Ok(Some((Dat::BU32(b), _))) | Ok(Some((Dat::BU8(b), _))) | Ok(Some((Dat::BU16(b), _))) | |
| 143 | Ok(Some((Dat::BU64(b), _))) if !b.is_empty() |
| 144 | && b.iter().enumerate().all(|(j, x)| *x == b[0] ^ (j as u8)) => fmt!("{}/{}", b.len(), b[0]), |
| 145 | Ok(Some((d, _))) => fmt!("TORN:{:?}", d).chars().take(100).collect(), |
| 146 | Err(e) => fmt!("ERR:{}", fmt!("{}", e).chars().take(800).collect::<String>()), |
| 147 | } |
| 148 | } |
| 149 | |
| 150 | fn read(db: &TestDb, k: &Dat) -> String { |
| 151 | describe(&db.get(k, None)) |
| 152 | } |
| 153 | |
| 154 | fn head(db: &TestDb, k: &Dat) -> Outcome<Option<[u64; 5]>> { |
| 155 | match res!(db.api().get_head_wait(k, None)) { |
| 156 | Some((Dat::Tup5u64(t), _)) => Ok(Some(t)), |
| 157 | _ => Ok(None), |
| 158 | } |
| 159 | } |
| 160 | |
| 161 | // Are all the chunks of the set `t` still readable? |
| 162 | fn chunks_live(db: &TestDb, t: [u64; 5]) -> bool { |
| 163 | db.api().fetch_chunks(&Dat::Tup5u64(t), None).is_ok() |
| 164 | } |
| 165 | |
| 166 | fn zdir(root: &Path, c: &OzoneConfig, z: usize) -> PathBuf { |
| 167 | c.zone_root(root).join(fmt!("zone_{:03}", z + 1)) |
| 168 | } |
| 169 | |
| 170 | /// A power cut now: the store is imaged at one instant (`gc_pair::instant_cut`), and each |
| 171 | /// live file of the image is cut to the mark its last barrier published. |
| 172 | fn power_cut(db: &TestDb, root: &Path, c: &OzoneConfig, to: &str) -> Outcome<PathBuf> { |
| 173 | let dst = res!(fresh(to)); |
| 174 | res!(instant_cut(db, root, c, &dst)); |
| 175 | Ok(dst) |
| 176 | } |
| 177 | |
| 178 | // Reads `keys` from a store cut at a power cut, and closes it. |
| 179 | fn read_after_cut(db: &TestDb, root: &Path, c: &OzoneConfig, to: &str, keys: &[Dat]) -> Outcome<Vec<String>> { |
| 180 | let cut = res!(power_cut(db, root, c, to)); |
| 181 | let db2 = res!(open(&cut, c, false)); |
| 182 | let got = keys.iter().map(|k| read(&db2, k)).collect(); |
| 183 | res!(db2.close()); |
| 184 | let _ = fs::remove_dir_all(&cut); |
| 185 | Ok(got) |
| 186 | } |
| 187 | |
| 188 | /// Live chunk-key cache entries, those that are not tombstones. |
| 189 | // The cache bot answers before it enters the hold, so a positive control waits for it. |
| 190 | fn held_soon(db: &TestDb) -> bool { |
| 191 | wait_until(Duration::from_secs(1), || db.api().held_effects() > 0) |
| 192 | } |
| 193 | |
| 194 | fn live_chunks(db: &TestDb) -> Outcome<usize> { |
| 195 | res!(db.api().settle_for_test(Duration::from_secs(20))); |
| 196 | let mut n = 0; |
| 197 | for (_, cache) in res!(db.api().collect_caches(wait())) { |
| 198 | let tombs = cache.tomb_tracker().tombs(); |
| 199 | for (k, e) in cache.map() { |
| 200 | let CacheEntry::LocatedValue(_, _) = e; |
| 201 | if is_chunk_key(k) && !tombs.contains_key(k) { |
| 202 | n += 1; |
| 203 | } |
| 204 | } |
| 205 | } |
| 206 | Ok(n) |
| 207 | } |
| 208 | |
| 209 | // The live chunk keys, those that are not tombstones. |
| 210 | fn chunk_keys(db: &TestDb) -> Outcome<BTreeSet<Vec<u8>>> { |
| 211 | res!(db.api().settle_for_test(Duration::from_secs(20))); |
| 212 | let mut keys = BTreeSet::new(); |
| 213 | for (_, cache) in res!(db.api().collect_caches(wait())) { |
| 214 | let tombs = cache.tomb_tracker().tombs(); |
| 215 | for k in cache.map().keys() { |
| 216 | if is_chunk_key(k) && !tombs.contains_key(k) { |
| 217 | keys.insert(k.clone()); |
| 218 | } |
| 219 | } |
| 220 | } |
| 221 | Ok(keys) |
| 222 | } |
| 223 | |
| 224 | // Fails every barrier from now, makes one over a new record, and lets the switch go. The store |
| 225 | // is stopped after. |
| 226 | fn fail_a_barrier(db: &TestDb) -> Outcome<()> { |
| 227 | res!(put(db, &dat!("durable mark uncovered"), val(50, 7))); |
| 228 | hooks::set_barrier_failure(true); |
| 229 | let r = db.api().sync_now(wait()); |
| 230 | hooks::set_barrier_failure(false); |
| 231 | match r { |
| 232 | Err(_) => Ok(()), |
| 233 | Ok(()) => Err(err!("A barrier armed to fail succeeded."; Test, Unexpected)), |
| 234 | } |
| 235 | } |
| 236 | |
| 237 | /// P, a durable chunked value, is displaced by U, whose barrier fails. Nothing P owns is |
| 238 | /// retired or collected, the stopped store refuses compaction and the sweep, and the restart |
| 239 | /// cuts U, so P is the key's value again. `small` makes P unchunked. |
| 240 | fn displaced_by_a_failed_write(p: Policy, zones: u16, wbots: u16, small: bool, collect: bool) -> Outcome<()> { |
| 241 | let label = fmt!("{:?} {}x{} small={} collect={}", p, zones, wbots, small, collect); |
| 242 | let c = res!(cfg(p, zones, wbots)); |
| 243 | let root = res!(fresh("./test_db_durable_mark_p")); |
| 244 | let db = res!(open(&root, &c, true)); |
| 245 | let k = dat!("durable mark p"); |
| 246 | let pv = if small { val(200, 2) } else { val(1_500, 2) }; |
| 247 | let want = if small { "200/2" } else { "1500/2" }; |
| 248 | res!(put(&db, &k, pv)); |
| 249 | res!(db.api().sync_now(wait())); |
| 250 | let set = res!(head(&db, &k)); |
| 251 | // U's own barrier fails under `EveryWrite`; otherwise the forced one does. |
| 252 | hooks::set_barrier_failure(true); |
| 253 | let u = put(&db, &k, val(50, 3)); |
| 254 | let _ = db.api().sync_now(wait()); |
| 255 | hooks::set_barrier_failure(false); |
| 256 | assert!(db.api().fault().is_some(), "{}: a failed barrier did not stop the store", label); |
| 257 | if let Policy::EveryWrite = p { |
| 258 | assert!(u.is_err(), "{}: U was acknowledged although its barrier failed", label); |
| 259 | } |
| 260 | if collect { |
| 261 | assert!(db.compact_now(Duration::from_secs(60)).is_err(), "{}: a stopped store compacted", label); |
| 262 | assert!(sweep::sweep_orphans(db.api(), Uid::default(), None, wait()).is_err(), |
| 263 | "{}: a stopped store swept", label); |
| 264 | } |
| 265 | if let Some(t) = set { |
| 266 | assert!(chunks_live(&db, t), "{}: P's chunks were retired behind a failed barrier", label); |
| 267 | } |
| 268 | let _ = db.close(); |
| 269 | res!(Refusal::clear(&root)); // as the operator would |
| 270 | let db = res!(open(&root, &c, false)); |
| 271 | assert_eq!(read(&db, &k), want, "{}: the restart did not give P back", label); |
| 272 | assert!(db.api().fault().is_none(), "{}: the restart left the fault", label); |
| 273 | let v = res!(verify::verify_live_set(db.api(), None, wait())); |
| 274 | assert!(v.clean(), "{}: {}", label, v.summary(4)); |
| 275 | res!(db.close()); |
| 276 | let _ = fs::remove_dir_all(&root); |
| 277 | Ok(()) |
| 278 | } |
| 279 | |
| 280 | #[test] |
| 281 | fn a_head_displaced_by_a_failed_write_keeps_its_value() -> Outcome<()> { |
| 282 | let _g = lock(); |
| 283 | for p in POLICIES { |
| 284 | for (z, w) in SHAPES { |
| 285 | res!(displaced_by_a_failed_write(p, z, w, false, false)); |
| 286 | res!(displaced_by_a_failed_write(p, z, w, false, true)); |
| 287 | } |
| 288 | } |
| 289 | res!(displaced_by_a_failed_write(Policy::Interval(3_600_000), 1, 1, true, true)); |
| 290 | res!(displaced_by_a_failed_write(Policy::Interval(3_600_000), 2, 2, true, true)); |
| 291 | Ok(()) |
| 292 | } |
| 293 | |
| 294 | /// With no fault, P's discard waits for Q's barrier: compaction does not drop P while Q may yet |
| 295 | /// be lost, so a power cut leaves P. Once Q is durable the late discard is applied through the |
| 296 | /// collection that carried P, and marks nothing else old. |
| 297 | #[test] |
| 298 | fn collection_waits_for_the_winners_barrier() -> Outcome<()> { |
| 299 | let _g = lock(); |
| 300 | hooks::set_owed_drop(true); // the hold's window stays open until the barrier under test |
| 301 | for (z, w) in SHAPES { |
| 302 | let label = fmt!("{}x{}", z, w); |
| 303 | let mut c = res!(cfg(Policy::Interval(3_600_000), z, w)); |
| 304 | c.data_file_max_bytes = 4_000; |
| 305 | let root = res!(fresh("./test_db_durable_mark_collect")); |
| 306 | let db = res!(open(&root, &c, true)); |
| 307 | let k = dat!("durable mark collect"); |
| 308 | let others: Vec<Dat> = (0..8).map(|i| dat!(fmt!("durable mark other {}", i))).collect(); |
| 309 | res!(put(&db, &k, val(300, 2))); |
| 310 | for (i, o) in others.iter().enumerate() { |
| 311 | res!(put(&db, o, val(300, 10 + i as u8))); |
| 312 | } |
| 313 | res!(db.api().sync_now(wait())); |
| 314 | res!(db.api().new_live_files()); |
| 315 | res!(put(&db, &k, val(300, 4))); |
| 316 | assert!(held_soon(&db), "{}: P's discard was not held behind Q's barrier", label); |
| 317 | let got = res!(read_after_cut(&db, &root, &c, "./test_db_durable_mark_collect_cut", &[k.clone()])); |
| 318 | assert_eq!(got[0], "300/2", "{}: a power cut before Q's barrier did not give P back", label); |
| 319 | // Compaction seals the live file first, which makes Q durable; P may go only after that. |
| 320 | let _ = db.compact_now(Duration::from_secs(60)); |
| 321 | let got = res!(read_after_cut(&db, &root, &c, "./test_db_durable_mark_collect_cut", &[k.clone()])); |
| 322 | assert!(got[0] == "300/2" || got[0] == "300/4", |
| 323 | "{}: a power cut after compaction lost P under Q: {}", label, got[0]); |
| 324 | res!(db.api().sync_now(wait())); |
| 325 | assert!(wait_until(Duration::from_secs(10), || db.api().held_effects() == 0), |
| 326 | "{}: the hold outlived Q's barrier", label); |
| 327 | res!(db.compact_now(Duration::from_secs(60))); |
| 328 | assert_eq!(read(&db, &k), "300/4", "{}", label); |
| 329 | for (i, o) in others.iter().enumerate() { |
| 330 | assert_eq!(read(&db, o), fmt!("300/{}", 10 + i), "{}: the late discard hit another record", label); |
| 331 | } |
| 332 | let v = res!(verify::verify_live_set(db.api(), None, wait())); |
| 333 | assert!(v.clean(), "{}: {}", label, v.summary(4)); |
| 334 | res!(db.close()); |
| 335 | let db = res!(open(&root, &c, false)); |
| 336 | for (i, o) in others.iter().enumerate() { |
| 337 | assert_eq!(read(&db, o), fmt!("300/{}", 10 + i), "{}: after the restart", label); |
| 338 | } |
| 339 | res!(db.close()); |
| 340 | let _ = fs::remove_dir_all(&root); |
| 341 | } |
| 342 | Ok(()) |
| 343 | } |
| 344 | |
| 345 | /// A hold lasts until the barrier the policy makes, and no longer. |
| 346 | #[test] |
| 347 | fn holds_release_at_each_policys_barrier() -> Outcome<()> { |
| 348 | let _g = lock(); |
| 349 | let k = dat!("durable mark hold"); |
| 350 | let quiet = |db: &TestDb| wait_until(Duration::from_secs(10), || db.api().held_effects() == 0); |
| 351 | |
| 352 | let c = res!(cfg(Policy::EveryWrite, 1, 1)); |
| 353 | let root = res!(fresh("./test_db_durable_mark_hold")); |
| 354 | let db = res!(open(&root, &c, true)); |
| 355 | res!(put(&db, &k, val(50, 1))); |
| 356 | res!(put(&db, &k, val(50, 2))); |
| 357 | assert!(quiet(&db), "EveryWrite: a hold outlived its write's own barrier"); |
| 358 | res!(db.close()); |
| 359 | |
| 360 | let c = res!(cfg(Policy::EveryN(5), 1, 1)); |
| 361 | let db = res!(open(&root, &c, true)); |
| 362 | res!(put(&db, &k, val(50, 1))); |
| 363 | res!(db.api().sync_now(wait())); |
| 364 | res!(put(&db, &k, val(50, 2))); |
| 365 | assert!(held_soon(&db), "EveryN(5): released before the fifth write"); |
| 366 | for i in 0..4 { |
| 367 | res!(put(&db, &dat!(fmt!("durable mark filler {}", i)), val(50, 9))); |
| 368 | } |
| 369 | assert!(quiet(&db), "EveryN(5): neither the fifth write's barrier nor the owed one released the hold"); |
| 370 | res!(db.close()); |
| 371 | |
| 372 | let c = res!(cfg(Policy::Never, 1, 1)); |
| 373 | let db = res!(open(&root, &c, true)); |
| 374 | res!(put(&db, &k, val(50, 1))); |
| 375 | res!(db.api().sync_now(wait())); |
| 376 | res!(put(&db, &k, val(50, 2))); |
| 377 | assert!(held_soon(&db), "Never: released with no barrier"); |
| 378 | res!(db.api().new_live_files()); |
| 379 | assert!(quiet(&db), "Never: sealing the live file did not release the hold"); |
| 380 | res!(put(&db, &k, val(50, 3))); |
| 381 | assert!(held_soon(&db), "Never: released with no barrier"); |
| 382 | res!(db.api().sync_now(wait())); |
| 383 | assert!(quiet(&db), "Never: sync_now did not release the hold"); |
| 384 | res!(db.close()); |
| 385 | let _ = fs::remove_dir_all(&root); |
| 386 | Ok(()) |
| 387 | } |
| 388 | |
| 389 | /// A failed barrier is never retried into success (fsyncgate): the marks stay where they were, |
| 390 | /// the holds stay held, and nothing more is written, whatever the disk does after. |
| 391 | #[test] |
| 392 | fn a_failed_barrier_gets_no_second_chance() -> Outcome<()> { |
| 393 | let _g = lock(); |
| 394 | hooks::set_owed_drop(true); // the hold's window stays open until the barrier under test |
| 395 | let c = res!(cfg(Policy::Interval(3_600_000), 1, 1)); |
| 396 | let root = res!(fresh("./test_db_durable_mark_fsyncgate")); |
| 397 | let db = res!(open(&root, &c, true)); |
| 398 | let k = dat!("durable mark fsyncgate"); |
| 399 | res!(put(&db, &k, val(1_500, 2))); |
| 400 | res!(db.api().sync_now(wait())); |
| 401 | let _ = put(&db, &k, val(1_500, 4)); |
| 402 | let _ = held_soon(&db); |
| 403 | let held = db.api().held_effects(); |
| 404 | assert!(held > 0, "positive control: the overwrite's discard is held"); |
| 405 | res!(fail_a_barrier(&db)); |
| 406 | let marks = res!(db.api().chans().marks().files(0)); |
| 407 | assert!(put(&db, &dat!("durable mark after"), val(50, 1)).is_err(), "a write after the fault was taken"); |
| 408 | assert!(db.api().sync_now(wait()).is_err(), "a barrier after the fault succeeded"); |
| 409 | let _ = db.api().new_live_files(); |
| 410 | std::thread::sleep(Duration::from_millis(200)); |
| 411 | assert_eq!(res!(db.api().chans().marks().files(0)), marks, "a mark moved after the fault"); |
| 412 | assert_eq!(db.api().held_effects(), held, "a hold was released after the fault"); |
| 413 | let _ = db.close(); |
| 414 | let _ = fs::remove_dir_all(&root); |
| 415 | Ok(()) |
| 416 | } |
| 417 | |
| 418 | /// After a fault, writes, compaction and the sweep are refused with the advice to restart, a |
| 419 | /// read of what a barrier covered is served, and the fault is recorded for the next open. |
| 420 | #[test] |
| 421 | fn a_fault_stops_writes() -> Outcome<()> { |
| 422 | let _g = lock(); |
| 423 | let c = res!(cfg(Policy::Interval(3_600_000), 1, 1)); |
| 424 | let root = res!(fresh("./test_db_durable_mark_stops")); |
| 425 | let db = res!(open(&root, &c, true)); |
| 426 | let k = dat!("durable mark stops"); |
| 427 | res!(put(&db, &k, val(1_500, 2))); |
| 428 | res!(db.api().sync_now(wait())); |
| 429 | res!(fail_a_barrier(&db)); |
| 430 | let advice = |r: Outcome<()>, what: &str| match r { |
| 431 | Ok(()) => panic!("{} was taken by a stopped store", what), |
| 432 | Err(e) => assert!(fmt!("{}", e).contains("Restart the store"), "{}: {}", what, e), |
| 433 | }; |
| 434 | advice(put(&db, &dat!("durable mark new"), val(50, 1)), "a put"); |
| 435 | advice(db.delete(&k, Uid::default(), None).map(|_| ()), "a delete"); |
| 436 | advice(sweep::sweep_orphans(db.api(), Uid::default(), None, wait()).map(|_| ()), "the sweep"); |
| 437 | assert!(db.compact_now(Duration::from_secs(60)).is_err(), "compaction ran on a stopped store"); |
| 438 | assert_eq!(read(&db, &k), "1500/2", "a covered read was refused"); |
| 439 | assert!(db.api().fault().is_some()); |
| 440 | assert!(root.join(constant::FAULT_FILENAME).exists(), "no fault record was written"); |
| 441 | let _ = db.close(); |
| 442 | let _ = fs::remove_dir_all(&root); |
| 443 | Ok(()) |
| 444 | } |
| 445 | |
| 446 | /// The next open cuts every file in doubt to its mark, keeps the tail in quarantine and clears |
| 447 | /// the fault record. A second open over the same record changes no length. |
| 448 | #[test] |
| 449 | fn the_open_cuts_to_the_marks() -> Outcome<()> { |
| 450 | let _g = lock(); |
| 451 | hooks::set_owed_drop(true); // the hold's window stays open until the barrier under test |
| 452 | let c = res!(cfg(Policy::Interval(3_600_000), 1, 1)); |
| 453 | let root = res!(fresh("./test_db_durable_mark_cut")); |
| 454 | let db = res!(open(&root, &c, true)); |
| 455 | let k = dat!("durable mark cut"); |
| 456 | res!(put(&db, &k, val(300, 2))); |
| 457 | res!(db.api().sync_now(wait())); |
| 458 | let _ = put(&db, &k, val(300, 4)); |
| 459 | let marks = res!(db.api().chans().marks().files(0)); |
| 460 | res!(fail_a_barrier(&db)); |
| 461 | let _ = db.close(); |
| 462 | let fault = res!(fs::read(root.join(constant::FAULT_FILENAME))); |
| 463 | let lens = |root: &Path| -> Vec<(u64, u64)> { |
| 464 | marks.iter().map(|(f, _)| ( |
| 465 | size(&zdir(root, &c, 0).join(ZoneDir::relative_file_path(&FileType::Data, *f))), |
| 466 | size(&zdir(root, &c, 0).join(ZoneDir::relative_file_path(&FileType::Index, *f))), |
| 467 | )).collect() |
| 468 | }; |
| 469 | // The fault cut the tails in process; a crash before that cut would leave them, so one is |
| 470 | // put back for the open to find. |
| 471 | for (f, _) in &marks { |
| 472 | let p = zdir(&root, &c, 0).join(ZoneDir::relative_file_path(&FileType::Data, *f)); |
| 473 | let mut b = res!(fs::read(&p)); |
| 474 | b.extend_from_slice(&[0xab; 64]); |
| 475 | res!(fs::write(&p, &b)); |
| 476 | } |
| 477 | let before = lens(&root); |
| 478 | assert!(marks.iter().zip(&before).any(|((_, m), l)| l.0 > m.0), "positive control: nothing past a mark"); |
| 479 | res!(Refusal::clear(&root)); // as the operator would |
| 480 | let db = res!(open(&root, &c, false)); |
| 481 | assert_eq!(read(&db, &k), "300/2"); |
| 482 | res!(db.close()); |
| 483 | let after = lens(&root); |
| 484 | for ((f, m), l) in marks.iter().zip(&after) { |
| 485 | assert_eq!(*l, *m, "file {} was not cut to its mark", f); |
| 486 | } |
| 487 | assert!(!root.join(constant::FAULT_FILENAME).exists(), "the fault record outlived the cut"); |
| 488 | let q = zdir(&root, &c, 0).join(constant::QUARANTINE_DIR); |
| 489 | assert!(res!(fs::read_dir(&q)).count() > 0, "the cut tail was not quarantined"); |
| 490 | let held = |q: &Path| -> Outcome<Vec<(PathBuf, u64)>> { |
| 491 | let mut v = Vec::new(); |
| 492 | for e in res!(fs::read_dir(q)) { |
| 493 | let p = res!(e).path(); |
| 494 | v.push((p.clone(), size(&p))); |
| 495 | } |
| 496 | v.sort(); |
| 497 | Ok(v) |
| 498 | }; |
| 499 | let evidence = res!(held(&q)); |
| 500 | res!(fs::write(root.join(constant::FAULT_FILENAME), &fault)); |
| 501 | let db = res!(open(&root, &c, false)); |
| 502 | assert_eq!(read(&db, &k), "300/2"); |
| 503 | res!(db.close()); |
| 504 | assert_eq!(lens(&root), after, "a second cut over a replanted fault record changed a length"); |
| 505 | // A file already at its mark has nothing to cut, so the first cut's evidence stands. |
| 506 | assert_eq!(res!(held(&q)), evidence, "a second cut rewrote the quarantined tails"); |
| 507 | let _ = fs::remove_dir_all(&root); |
| 508 | Ok(()) |
| 509 | } |
| 510 | |
| 511 | /// A fault record that cannot be written still stops the store in process. |
| 512 | #[test] |
| 513 | fn a_fault_stops_the_store_without_its_record() -> Outcome<()> { |
| 514 | let _g = lock(); |
| 515 | let c = res!(cfg(Policy::Interval(3_600_000), 1, 1)); |
| 516 | let root = res!(fresh("./test_db_durable_mark_norecord")); |
| 517 | let db = res!(open(&root, &c, true)); |
| 518 | res!(put(&db, &dat!("durable mark norecord"), val(50, 1))); |
| 519 | hooks::set_fault_record_fails(true); |
| 520 | let r = fail_a_barrier(&db); |
| 521 | hooks::set_fault_record_fails(false); |
| 522 | res!(r); |
| 523 | assert!(db.api().fault().is_some(), "the store did not stop"); |
| 524 | assert!(put(&db, &dat!("durable mark norecord 2"), val(50, 1)).is_err(), "a write was taken"); |
| 525 | let _ = db.close(); |
| 526 | let _ = fs::remove_dir_all(&root); |
| 527 | Ok(()) |
| 528 | } |
| 529 | |
| 530 | /// A chunked put's chunks are answered only once a barrier covers them, so a power cut taken |
| 531 | /// while its head is held back keeps every chunk of the put and no head (R1). The data files are |
| 532 | /// large enough that no rollover seal makes the chunks durable by the way, and each barrier is |
| 533 | /// slowed, so a chunk answered before its barrier is cut short in the image. One store is held |
| 534 | /// at a time, since the hold blocks the server bot driving it, and its chunk set is counted |
| 535 | /// exactly: the chunk keys the store added, from the live store once the put has returned. |
| 536 | #[test] |
| 537 | fn a_head_never_outlives_its_chunks() -> Outcome<()> { |
| 538 | let _g = lock(); |
| 539 | let mut bad = Vec::new(); |
| 540 | for p in POLICIES { |
| 541 | for (z, w) in [(1u16, 1u16), (1, 4), (2, 2)] { |
| 542 | let label = fmt!("{:?} {}x{}", p, z, w); |
| 543 | let mut c = res!(cfg(p, z, w)); |
| 544 | c.data_file_max_bytes = 1_048_576; |
| 545 | let root = res!(fresh("./test_db_durable_mark_r1")); |
| 546 | let db = res!(open(&root, &c, true)); |
| 547 | res!(put(&db, &dat!("durable mark r1 base"), val(1_500, 2))); |
| 548 | res!(db.api().sync_now(wait())); |
| 549 | let base = res!(chunk_keys(&db)); |
| 550 | let k = dat!("durable mark r1"); |
| 551 | let held = hooks::heads_held(); |
| 552 | hooks::set_barrier_delay(Duration::from_millis(900)); |
| 553 | hooks::set_head_hold(true); |
| 554 | let putter = { |
| 555 | let db = db.clone(); |
| 556 | let k = k.clone(); |
| 557 | thread::spawn(move || put(&db, &k, val(3_000, 4))) |
| 558 | }; |
| 559 | let reached = wait_until(Duration::from_secs(30), || hooks::heads_held() > held); |
| 560 | hooks::set_barrier_delay(Duration::ZERO); |
| 561 | let cut = power_cut(&db, &root, &c, "./test_db_durable_mark_r1_cut"); |
| 562 | hooks::set_head_hold(false); |
| 563 | let done = match putter.join() { Ok(r) => r, Err(_) => Err(err!("{}: the put panicked.", label; Test)) }; |
| 564 | res!(done); |
| 565 | let cut = res!(cut); |
| 566 | let new: BTreeSet<Vec<u8>> = res!(chunk_keys(&db)).difference(&base).cloned().collect(); |
| 567 | let db2 = res!(open(&cut, &c, false)); |
| 568 | let image = res!(chunk_keys(&db2)); |
| 569 | let got = read(&db2, &k); |
| 570 | res!(db2.close()); |
| 571 | let _ = fs::remove_dir_all(&cut); |
| 572 | let after = res!(read_after_cut(&db, &root, &c, "./test_db_durable_mark_r1_cut", &[k.clone()])); |
| 573 | res!(db.close()); |
| 574 | let _ = fs::remove_dir_all(&root); |
| 575 | let missing = new.difference(&image).count(); |
| 576 | if !reached { |
| 577 | bad.push(fmt!("{}: the put never reached the head hold", label)); |
| 578 | } |
| 579 | if new.len() < 2 { |
| 580 | bad.push(fmt!("{}: the put added {} chunks, so it was not chunked", label, new.len())); |
| 581 | } |
| 582 | if missing > 0 { |
| 583 | bad.push(fmt!("{}: a cut at the head hold lost {} of the put's {} answered chunks", |
| 584 | label, missing, new.len())); |
| 585 | } |
| 586 | if got != "none" { |
| 587 | bad.push(fmt!("{}: a cut at the head hold read {}, but no head had been sent", label, got)); |
| 588 | } |
| 589 | let whole = after.iter().all(|g| g == "3000/4" || (!matches!(p, Policy::EveryWrite) && g == "none")); |
| 590 | if !whole { |
| 591 | bad.push(fmt!("{}: a cut after the put returned read {:?}", label, after)); |
| 592 | } |
| 593 | println!("R1 {}: {} chunks, {} missing at the hold, then {:?}", label, new.len(), missing, after); |
| 594 | } |
| 595 | } |
| 596 | if !bad.is_empty() { |
| 597 | return Err(err!("{:?}", bad; Test)); |
| 598 | } |
| 599 | Ok(()) |
| 600 | } |
| 601 | |
| 602 | /// The sweep makes its scan durable before it retires, which releases what the scan saw; a |
| 603 | /// barrier that fails there stops the sweep with nothing retired. |
| 604 | #[test] |
| 605 | fn the_sweep_syncs_after_its_scan() -> Outcome<()> { |
| 606 | let _g = lock(); |
| 607 | hooks::set_owed_drop(true); // the hold's window stays open until the barrier under test |
| 608 | let c = res!(cfg(Policy::Interval(3_600_000), 1, 1)); |
| 609 | let root = res!(fresh("./test_db_durable_mark_sweep")); |
| 610 | let db = res!(open(&root, &c, true)); |
| 611 | let k = dat!("durable mark sweep"); |
| 612 | res!(put(&db, &k, val(1_500, 2))); |
| 613 | res!(db.api().sync_now(wait())); |
| 614 | let first = res!(res!(head(&db, &k)).ok_or_else(|| err!("No head."; Test, Missing))); |
| 615 | res!(put(&db, &k, val(1_500, 4))); |
| 616 | assert!(held_soon(&db), "positive control: the discard is held"); |
| 617 | assert!(chunks_live(&db, first), "positive control: P's chunks are live before the sweep"); |
| 618 | res!(sweep::sweep_orphans(db.api(), Uid::default(), None, wait())); |
| 619 | // The retires the sweep's barrier releases are themselves written after it, so they are |
| 620 | // held in turn: what is asserted is that P's displaced chunk set was retired. |
| 621 | assert!(wait_until(Duration::from_secs(10), || !chunks_live(&db, first)), |
| 622 | "the sweep's barrier did not release the retire of P's chunks"); |
| 623 | res!(db.close()); |
| 624 | |
| 625 | let db = res!(open(&root, &c, true)); |
| 626 | res!(put(&db, &k, val(1_500, 2))); |
| 627 | res!(db.api().sync_now(wait())); |
| 628 | let old = res!(res!(head(&db, &k)).ok_or_else(|| err!("No head."; Test, Missing))); |
| 629 | res!(put(&db, &k, val(1_500, 4))); |
| 630 | hooks::set_barrier_failure(true); |
| 631 | let swept = sweep::sweep_orphans(db.api(), Uid::default(), None, wait()); |
| 632 | hooks::set_barrier_failure(false); |
| 633 | assert!(swept.is_err(), "the sweep ran past a failed barrier: {:?}", swept); |
| 634 | assert!(db.api().fault().is_some(), "the failed barrier did not stop the store"); |
| 635 | assert!(chunks_live(&db, old), "P's chunks were retired"); |
| 636 | let _ = db.close(); |
| 637 | res!(Refusal::clear(&root)); // as the operator would |
| 638 | let db = res!(open(&root, &c, false)); |
| 639 | assert_eq!(read(&db, &k), "1500/2", "the restart did not give P back"); |
| 640 | res!(db.close()); |
| 641 | let _ = fs::remove_dir_all(&root); |
| 642 | Ok(()) |
| 643 | } |
| 644 | |
| 645 | /// Under `Never`, a clean close makes everything durable, so every displaced chunk set is retired |
| 646 | /// and the reopened store holds the chunks of the last value only. |
| 647 | #[test] |
| 648 | fn a_clean_close_under_never_retires() -> Outcome<()> { |
| 649 | let _g = lock(); |
| 650 | let c = res!(cfg(Policy::Never, 1, 1)); |
| 651 | let root = res!(fresh("./test_db_durable_mark_never")); |
| 652 | let db = res!(open(&root, &c, true)); |
| 653 | let k = dat!("durable mark never"); |
| 654 | for s in 1..6u8 { |
| 655 | res!(put(&db, &k, val(1_500, s))); |
| 656 | } |
| 657 | res!(db.close()); |
| 658 | let db = res!(open(&root, &c, false)); |
| 659 | assert_eq!(read(&db, &k), "1500/5"); |
| 660 | assert_eq!(res!(live_chunks(&db)), 4, "displaced chunk sets outlived a clean close"); |
| 661 | res!(db.close()); |
| 662 | let _ = fs::remove_dir_all(&root); |
| 663 | Ok(()) |
| 664 | } |
| 665 | |
| 666 | // The records the file states flag current, and those they flag old. |
| 667 | fn flagged(db: &TestDb) -> Outcome<(usize, usize)> { |
| 668 | let mut n = (0, 0); |
| 669 | for (_, fstates) in res!(db.api().collect_file_states(constant::USER_REQUEST_WAIT)) { |
| 670 | for (_, fstat) in fstates.map() { |
| 671 | for (_, d) in fstat.data_map() { |
| 672 | match d { |
| 673 | DataState::Cur => n.0 += 1, |
| 674 | DataState::Old(_) => n.1 += 1, |
| 675 | } |
| 676 | } |
| 677 | } |
| 678 | } |
| 679 | Ok(n) |
| 680 | } |
| 681 | |
| 682 | /// `Never` and `EveryN` owe a held discard a barrier within `HELD_BARRIER`: after overwrites |
| 683 | /// into a live file that then stops growing, with no seal, `sync_now` or close, the losers are |
| 684 | /// flagged old and the displaced chunk set retired. Two writers per zone check that the |
| 685 | /// request reaches the writer that owns the winner's file. |
| 686 | #[test] |
| 687 | fn an_idle_store_is_owed_its_barrier() -> Outcome<()> { |
| 688 | let _g = lock(); |
| 689 | const N: usize = 12; |
| 690 | let bound = Duration::from_millis(1_500); |
| 691 | for p in [Policy::Never, Policy::EveryN(1_000)] { |
| 692 | for (zones, wbots) in SHAPES { |
| 693 | let label = fmt!("{:?} {}x{}", p, zones, wbots); |
| 694 | let mut c = res!(cfg(p, zones, wbots)); |
| 695 | c.data_file_max_bytes = 1_000_000; |
| 696 | let root = res!(fresh("./test_db_durable_mark_owed")); |
| 697 | let db = res!(open(&root, &c, true)); |
| 698 | let big = dat!("durable mark owed chunked"); |
| 699 | for i in 0..N { |
| 700 | res!(put(&db, &dat!(fmt!("durable mark owed {}", i)), val(50, 1))); |
| 701 | } |
| 702 | res!(put(&db, &big, val(1_500, 1))); |
| 703 | res!(db.api().sync_now(wait())); |
| 704 | res!(put(&db, &big, val(1_500, 2))); |
| 705 | assert!(held_soon(&db), "{}: an overwrite was released with no barrier", label); |
| 706 | for i in 0..N { |
| 707 | res!(put(&db, &dat!(fmt!("durable mark owed {}", i)), val(50, 2))); |
| 708 | } |
| 709 | let begun = Instant::now(); |
| 710 | assert!(wait_until(bound, || db.api().held_effects() == 0), |
| 711 | "{}: {} discards still held {:?} after the writes stopped", |
| 712 | label, db.api().held_effects(), bound); |
| 713 | let took = begun.elapsed(); |
| 714 | // The losers: the small values, and the chunked value's head and four chunks. |
| 715 | assert_eq!(res!(flagged(&db)).1, N + 5, "{}: losers not flagged old", label); |
| 716 | assert_eq!(res!(live_chunks(&db)), 4, "{}: the displaced chunk set was not retired", label); |
| 717 | assert_eq!(read(&db, &big), "1500/2"); |
| 718 | println!("{}: holds released {:?} after the last write.", label, took); |
| 719 | res!(db.close()); |
| 720 | let _ = fs::remove_dir_all(&root); |
| 721 | } |
| 722 | } |
| 723 | Ok(()) |
| 724 | } |
| 725 | |
| 726 | /// A burst of inserts that displace nothing holds nothing, so under `Never` it costs no barrier |
| 727 | /// at all: one would fail, and stop the store. |
| 728 | #[test] |
| 729 | fn an_insert_only_burst_costs_no_barrier() -> Outcome<()> { |
| 730 | let _g = lock(); |
| 731 | let mut c = res!(cfg(Policy::Never, 1, 1)); |
| 732 | c.data_file_max_bytes = 1_000_000; |
| 733 | let root = res!(fresh("./test_db_durable_mark_inserts")); |
| 734 | let db = res!(open(&root, &c, true)); |
| 735 | let failed = hooks::barriers_failed(); |
| 736 | hooks::set_barrier_failure(true); |
| 737 | for i in 0..200 { |
| 738 | res!(put(&db, &dat!(fmt!("durable mark insert {}", i)), val(50, 1))); |
| 739 | } |
| 740 | // Absence is what is tested: three periods in which an owed barrier would have run. |
| 741 | std::thread::sleep(constant::HELD_BARRIER * 3); |
| 742 | hooks::set_barrier_failure(false); |
| 743 | assert_eq!(hooks::barriers_failed(), failed, "an insert-only burst under Never ran a barrier"); |
| 744 | assert!(db.api().fault().is_none(), "an insert-only burst under Never faulted the store"); |
| 745 | res!(db.close()); |
| 746 | let _ = fs::remove_dir_all(&root); |
| 747 | Ok(()) |
| 748 | } |
| 749 | |
| 750 | /// The two gates the owed barrier closed, under `Never` with no write after the overwrites: |
| 751 | /// the superseded records of a sealed file start its collection, and the rollover's file |
| 752 | /// states settle to one current record per key, each within a bounded time. |
| 753 | #[test] |
| 754 | fn an_idle_store_under_never_collects_and_settles() -> Outcome<()> { |
| 755 | let _g = lock(); |
| 756 | const NK: usize = 10; // most of the first file |
| 757 | const NF: usize = 8; // seals it part way through |
| 758 | let bound = Duration::from_secs(2); |
| 759 | let mut c = res!(cfg(Policy::Never, 1, 1)); |
| 760 | c.num_cbots_per_zone = 1; |
| 761 | c.data_file_max_bytes = 16_000; // collection triggers at 4,800 superseded bytes |
| 762 | c.rest_chunk_threshold = 3_000; |
| 763 | let root = res!(fresh("./test_db_durable_mark_collect")); |
| 764 | log_set_level!("error"); |
| 765 | let db = res!(setup::start_db(root.clone(), Some(c.clone()), schemes(), None, true, true)); |
| 766 | for i in 0..NK { |
| 767 | res!(put(&db, &dat!(fmt!("durable mark key {}", i)), val(1_000, 1))); |
| 768 | } |
| 769 | for i in 0..NF { |
| 770 | res!(put(&db, &dat!(fmt!("durable mark filler {}", i)), val(1_000, 1))); |
| 771 | } |
| 772 | assert!(zdir(&root, &c, 0).join(ZoneDir::relative_file_path(&FileType::Data, 2)).is_file(), |
| 773 | "the fillers did not seal data file 1"); |
| 774 | // Held in collection long enough to be seen there. |
| 775 | hooks::set_collect_delay(Duration::from_millis(1_500)); |
| 776 | for i in 0..(NK / 2) { |
| 777 | res!(put(&db, &dat!(fmt!("durable mark key {}", i)), val(1_000, 2))); |
| 778 | } |
| 779 | let begun = Instant::now(); |
| 780 | let settled = wait_until(bound, || matches!(flagged(&db), Ok((n, _)) if n == NK + NF)); |
| 781 | let collecting = wait_until(bound, || match db.api().collect_file_states(wait()) { |
| 782 | Ok(states) => states.values().any(|s| s.map().get(&1).map_or(false, |f| f.gc_active())), |
| 783 | Err(_) => false, |
| 784 | }); |
| 785 | let took = begun.elapsed(); |
| 786 | hooks::set_collect_delay(Duration::ZERO); |
| 787 | assert!(settled, "rollover under Never: {:?} current and old, not {} current, {:?} after the \ |
| 788 | last write", flagged(&db), NK + NF, took); |
| 789 | assert!(collecting, "data file 1 did not start its collection within {:?} of the last write", took); |
| 790 | println!("Never, idle: settled and collecting {:?} after the last write.", took); |
| 791 | res!(db.close()); |
| 792 | let _ = fs::remove_dir_all(&root); |
| 793 | Ok(()) |
| 794 | } |
| 795 | |
| 796 | // A small deterministic generator, so that a failing seed can be run again. |
| 797 | struct Rng(u64); |
| 798 | |
| 799 | impl Rng { |
| 800 | fn next(&mut self) -> u64 { |
| 801 | self.0 ^= self.0 << 13; |
| 802 | self.0 ^= self.0 >> 7; |
| 803 | self.0 ^= self.0 << 17; |
| 804 | self.0 |
| 805 | } |
| 806 | fn below(&mut self, n: u64) -> u64 { self.next() % n } |
| 807 | } |
| 808 | |
| 809 | /// The power-cut oracle (R5). Random chunked and small puts, overwrites, deletes, barriers, |
| 810 | /// compactions and sweeps, then a cut at the marks: no read is torn, and no key comes back |
| 811 | /// missing or older than its newest write a barrier covered. |
| 812 | fn model(seed: u64, p: Policy, zones: u16, wbots: u16) -> Outcome<()> { |
| 813 | let label = fmt!("seed {} {:?} {}x{}", seed, p, zones, wbots); |
| 814 | let mut c = res!(cfg(p, zones, wbots)); |
| 815 | c.data_file_max_bytes = 6_000; |
| 816 | let root = res!(fresh("./test_db_durable_mark_model")); |
| 817 | let db = res!(open(&root, &c, true)); |
| 818 | let keys: Vec<Dat> = (0..6).map(|i| dat!(fmt!("durable mark model {}", i))).collect(); |
| 819 | // Per key, what it held after each acknowledged write, and how many of them are covered. |
| 820 | let mut hist: Vec<Vec<String>> = vec![vec![fmt!("none")]; keys.len()]; |
| 821 | let mut covered: Vec<usize> = vec![1; keys.len()]; |
| 822 | let mut rng = Rng(seed | 1); |
| 823 | let mut wseed = 0u8; |
| 824 | for _ in 0..60 { |
| 825 | let i = rng.below(keys.len() as u64) as usize; |
| 826 | match rng.below(20) { |
| 827 | 0..=6 => { |
| 828 | wseed = wseed.wrapping_add(1).max(1); |
| 829 | let len = 1_100 + rng.below(500) as usize; |
| 830 | res!(put(&db, &keys[i], val(len, wseed))); |
| 831 | hist[i].push(fmt!("{}/{}", len, wseed)); |
| 832 | }, |
| 833 | 7..=12 => { |
| 834 | wseed = wseed.wrapping_add(1).max(1); |
| 835 | let len = 50 + rng.below(250) as usize; |
| 836 | res!(put(&db, &keys[i], val(len, wseed))); |
| 837 | hist[i].push(fmt!("{}/{}", len, wseed)); |
| 838 | }, |
| 839 | 13..=14 => { |
| 840 | res!(db.delete(&keys[i], Uid::default(), None)); |
| 841 | hist[i].push(fmt!("none")); |
| 842 | }, |
| 843 | 15..=16 => { |
| 844 | res!(db.api().sync_now(wait())); |
| 845 | for (h, n) in hist.iter().zip(covered.iter_mut()) { |
| 846 | *n = h.len(); |
| 847 | } |
| 848 | }, |
| 849 | 17..=18 => { res!(db.compact_now(Duration::from_secs(60))); }, |
| 850 | _ => { res!(sweep::sweep_orphans(db.api(), Uid::default(), None, wait())); }, |
| 851 | } |
| 852 | if let Policy::EveryWrite = p { |
| 853 | for (h, n) in hist.iter().zip(covered.iter_mut()) { |
| 854 | *n = h.len(); |
| 855 | } |
| 856 | } |
| 857 | } |
| 858 | res!(db.api().settle_for_test(Duration::from_secs(20))); |
| 859 | let got = res!(read_after_cut(&db, &root, &c, "./test_db_durable_mark_model_cut", &keys)); |
| 860 | for (i, g) in got.iter().enumerate() { |
| 861 | let ok = &hist[i][covered[i] - 1..]; |
| 862 | assert!(ok.contains(g), "{}: key {} reads {} after the cut; allowed {:?}", label, i, g, ok); |
| 863 | } |
| 864 | res!(db.close()); |
| 865 | let _ = fs::remove_dir_all(&root); |
| 866 | Ok(()) |
| 867 | } |
| 868 | |
| 869 | #[test] |
| 870 | fn power_cut_oracle() -> Outcome<()> { |
| 871 | let _g = lock(); |
| 872 | let base = match std::env::var("DURABLE_MARK_SEED") { |
| 873 | Ok(s) => res!(s.parse::<u64>().map_err(|e| err!(e, "DURABLE_MARK_SEED"; Test, Invalid))), |
| 874 | Err(_) => 0x5eed_0000, |
| 875 | }; |
| 876 | let t0 = Instant::now(); |
| 877 | for n in 0..3u64 { |
| 878 | for p in POLICIES { |
| 879 | for (z, w) in SHAPES { |
| 880 | res!(model(base + n * 7_919, p, z, w)); |
| 881 | } |
| 882 | } |
| 883 | } |
| 884 | msg!("The oracle ran 24 stores in {:?}.", t0.elapsed()); |
| 885 | Ok(()) |
| 886 | } |