Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/durable_mark.rs

36.5 KiB, 25 runs

created by r1870400018:70848, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! The durable mark (Ozone, 2026-10-10). A discard waits for its winner's barrier, a failed
2//! barrier stops the store for good, and the next open cuts every file in doubt back to its mark.
3//! Each test that can runs under every sync policy, and under two zones of two writers. The
4//! power cut is modelled in process: the store is copied as its files stand, every live file of
5//! the copy is cut to its published mark, and the copy is opened. The hooks are process-wide,
6//! so the tests run one at a time.
7
8mod gc_pair;
9
10use gc_pair::*;
11
12use oxedyne_fe2o3_core::prelude::*;
13use oxedyne_fe2o3_iop_db::api::{
14 Database,
15 Meta,
16};
17use oxedyne_fe2o3_jdat::prelude::*;
18use oxedyne_fe2o3_o3db_sync::{
19 base::{
20 cfg::OzoneConfig,
21 constant,
22 },
23 comm::response::Wait,
24 data::cache::{
25 CacheEntry,
26 is_chunk_key,
27 },
28 file::{
29 core::FileType,
30 refusal::Refusal,
31 zdir::ZoneDir,
32 },
33 file::state::DataState,
34 sweep,
35 test::{
36 hooks,
37 setup::{
38 self,
39 Uid,
40 UID_LEN,
41 },
42 },
43 verify,
44};
45
46use std::{
47 collections::{
48 BTreeMap,
49 BTreeSet,
50 },
51 fs,
52 path::{
53 Path,
54 PathBuf,
55 },
56 sync::Mutex,
57 thread,
58 time::{
59 Duration,
60 Instant,
61 },
62};
63
64static SERIAL: Mutex<()> = Mutex::new(());
65
66#[derive(Clone, Copy, Debug)]
67enum Policy {
68 EveryWrite,
69 EveryN(u32),
70 Interval(u64),
71 Never,
72}
73
74const POLICIES: [Policy; 4] = [
75 Policy::EveryWrite,
76 Policy::EveryN(1_000),
77 Policy::Interval(3_600_000),
78 Policy::Never,
79];
80
81const SHAPES: [(u16, u16); 2] = [(1, 1), (2, 2)]; // zones, writers per zone
82
83fn cfg(p: Policy, zones: u16, wbots: u16) -> Outcome<OzoneConfig> {
84 let mut c = res!(setup::default_cfg());
85 c.num_zones = zones;
86 c.num_cbots_per_zone = 4;
87 c.num_fbots_per_zone = 1;
88 c.num_wbots_per_zone = wbots;
89 c.zone_overrides = BTreeMap::new();
90 c.rest_chunk_threshold = 1_000;
91 c.rest_chunk_bytes = 400;
92 c.sync_on_write = false;
93 c.sync_every_n_writes = 0;
94 c.sync_interval_ms = 0;
95 match p {
96 Policy::EveryWrite => c.sync_on_write = true,
97 Policy::EveryN(n) => c.sync_every_n_writes = n,
98 Policy::Interval(t) => c.sync_interval_ms = t,
99 Policy::Never => (),
100 }
101 Ok(c)
102}
103
104fn lock() -> std::sync::MutexGuard<'static, ()> {
105 let g = match SERIAL.lock() { Ok(g) => g, Err(p) => p.into_inner() };
106 hooks::set_barrier_failure(false);
107 hooks::set_barrier_failure_for(None);
108 hooks::set_fault_record_fails(false);
109 hooks::set_owed_drop(false);
110 hooks::set_barrier_delay(Duration::ZERO);
111 hooks::set_head_hold(false);
112 g
113}
114
115fn val(len: usize, seed: u8) -> Dat {
116 Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect())
117}
118
119fn fresh(dir: &str) -> Outcome<PathBuf> {
120 let _ = fs::remove_dir_all(dir);
121 res!(fs::create_dir_all(dir));
122 Ok(res!(Path::new(dir).canonicalize()))
123}
124
125fn open(root: &Path, c: &OzoneConfig, wipe: bool) -> Outcome<TestDb> {
126 log_set_level!("error");
127 setup::start_db(root.to_path_buf(), Some(c.clone()), schemes(), None, false, wipe)
128}
129
130fn wait() -> Wait {
131 Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL }
132}
133
134fn put(db: &TestDb, k: &Dat, v: Dat) -> Outcome<()> {
135 db.insert(k.clone(), v, Uid::default(), None).map(|_| ())
136}
137
138// "L/S" for a whole value of length L and seed S as `val` writes it, else what was read.
139fn describe(r: &Outcome<Option<(Dat, Meta<{ UID_LEN }, Uid>)>>) -> String {
140 match r {
141 Ok(None) => fmt!("none"),
142 Ok(Some((Dat::BU32(b), _))) | Ok(Some((Dat::BU8(b), _))) | Ok(Some((Dat::BU16(b), _))) |
143 Ok(Some((Dat::BU64(b), _))) if !b.is_empty()
144 && b.iter().enumerate().all(|(j, x)| *x == b[0] ^ (j as u8)) => fmt!("{}/{}", b.len(), b[0]),
145 Ok(Some((d, _))) => fmt!("TORN:{:?}", d).chars().take(100).collect(),
146 Err(e) => fmt!("ERR:{}", fmt!("{}", e).chars().take(800).collect::<String>()),
147 }
148}
149
150fn read(db: &TestDb, k: &Dat) -> String {
151 describe(&db.get(k, None))
152}
153
154fn head(db: &TestDb, k: &Dat) -> Outcome<Option<[u64; 5]>> {
155 match res!(db.api().get_head_wait(k, None)) {
156 Some((Dat::Tup5u64(t), _)) => Ok(Some(t)),
157 _ => Ok(None),
158 }
159}
160
161// Are all the chunks of the set `t` still readable?
162fn chunks_live(db: &TestDb, t: [u64; 5]) -> bool {
163 db.api().fetch_chunks(&Dat::Tup5u64(t), None).is_ok()
164}
165
166fn zdir(root: &Path, c: &OzoneConfig, z: usize) -> PathBuf {
167 c.zone_root(root).join(fmt!("zone_{:03}", z + 1))
168}
169
170/// A power cut now: the store is imaged at one instant (`gc_pair::instant_cut`), and each
171/// live file of the image is cut to the mark its last barrier published.
172fn power_cut(db: &TestDb, root: &Path, c: &OzoneConfig, to: &str) -> Outcome<PathBuf> {
173 let dst = res!(fresh(to));
174 res!(instant_cut(db, root, c, &dst));
175 Ok(dst)
176}
177
178// Reads `keys` from a store cut at a power cut, and closes it.
179fn read_after_cut(db: &TestDb, root: &Path, c: &OzoneConfig, to: &str, keys: &[Dat]) -> Outcome<Vec<String>> {
180 let cut = res!(power_cut(db, root, c, to));
181 let db2 = res!(open(&cut, c, false));
182 let got = keys.iter().map(|k| read(&db2, k)).collect();
183 res!(db2.close());
184 let _ = fs::remove_dir_all(&cut);
185 Ok(got)
186}
187
188/// Live chunk-key cache entries, those that are not tombstones.
189// The cache bot answers before it enters the hold, so a positive control waits for it.
190fn held_soon(db: &TestDb) -> bool {
191 wait_until(Duration::from_secs(1), || db.api().held_effects() > 0)
192}
193
194fn live_chunks(db: &TestDb) -> Outcome<usize> {
195 res!(db.api().settle_for_test(Duration::from_secs(20)));
196 let mut n = 0;
197 for (_, cache) in res!(db.api().collect_caches(wait())) {
198 let tombs = cache.tomb_tracker().tombs();
199 for (k, e) in cache.map() {
200 let CacheEntry::LocatedValue(_, _) = e;
201 if is_chunk_key(k) && !tombs.contains_key(k) {
202 n += 1;
203 }
204 }
205 }
206 Ok(n)
207}
208
209// The live chunk keys, those that are not tombstones.
210fn chunk_keys(db: &TestDb) -> Outcome<BTreeSet<Vec<u8>>> {
211 res!(db.api().settle_for_test(Duration::from_secs(20)));
212 let mut keys = BTreeSet::new();
213 for (_, cache) in res!(db.api().collect_caches(wait())) {
214 let tombs = cache.tomb_tracker().tombs();
215 for k in cache.map().keys() {
216 if is_chunk_key(k) && !tombs.contains_key(k) {
217 keys.insert(k.clone());
218 }
219 }
220 }
221 Ok(keys)
222}
223
224// Fails every barrier from now, makes one over a new record, and lets the switch go. The store
225// is stopped after.
226fn fail_a_barrier(db: &TestDb) -> Outcome<()> {
227 res!(put(db, &dat!("durable mark uncovered"), val(50, 7)));
228 hooks::set_barrier_failure(true);
229 let r = db.api().sync_now(wait());
230 hooks::set_barrier_failure(false);
231 match r {
232 Err(_) => Ok(()),
233 Ok(()) => Err(err!("A barrier armed to fail succeeded."; Test, Unexpected)),
234 }
235}
236
237/// P, a durable chunked value, is displaced by U, whose barrier fails. Nothing P owns is
238/// retired or collected, the stopped store refuses compaction and the sweep, and the restart
239/// cuts U, so P is the key's value again. `small` makes P unchunked.
240fn displaced_by_a_failed_write(p: Policy, zones: u16, wbots: u16, small: bool, collect: bool) -> Outcome<()> {
241 let label = fmt!("{:?} {}x{} small={} collect={}", p, zones, wbots, small, collect);
242 let c = res!(cfg(p, zones, wbots));
243 let root = res!(fresh("./test_db_durable_mark_p"));
244 let db = res!(open(&root, &c, true));
245 let k = dat!("durable mark p");
246 let pv = if small { val(200, 2) } else { val(1_500, 2) };
247 let want = if small { "200/2" } else { "1500/2" };
248 res!(put(&db, &k, pv));
249 res!(db.api().sync_now(wait()));
250 let set = res!(head(&db, &k));
251 // U's own barrier fails under `EveryWrite`; otherwise the forced one does.
252 hooks::set_barrier_failure(true);
253 let u = put(&db, &k, val(50, 3));
254 let _ = db.api().sync_now(wait());
255 hooks::set_barrier_failure(false);
256 assert!(db.api().fault().is_some(), "{}: a failed barrier did not stop the store", label);
257 if let Policy::EveryWrite = p {
258 assert!(u.is_err(), "{}: U was acknowledged although its barrier failed", label);
259 }
260 if collect {
261 assert!(db.compact_now(Duration::from_secs(60)).is_err(), "{}: a stopped store compacted", label);
262 assert!(sweep::sweep_orphans(db.api(), Uid::default(), None, wait()).is_err(),
263 "{}: a stopped store swept", label);
264 }
265 if let Some(t) = set {
266 assert!(chunks_live(&db, t), "{}: P's chunks were retired behind a failed barrier", label);
267 }
268 let _ = db.close();
269 res!(Refusal::clear(&root)); // as the operator would
270 let db = res!(open(&root, &c, false));
271 assert_eq!(read(&db, &k), want, "{}: the restart did not give P back", label);
272 assert!(db.api().fault().is_none(), "{}: the restart left the fault", label);
273 let v = res!(verify::verify_live_set(db.api(), None, wait()));
274 assert!(v.clean(), "{}: {}", label, v.summary(4));
275 res!(db.close());
276 let _ = fs::remove_dir_all(&root);
277 Ok(())
278}
279
280#[test]
281fn a_head_displaced_by_a_failed_write_keeps_its_value() -> Outcome<()> {
282 let _g = lock();
283 for p in POLICIES {
284 for (z, w) in SHAPES {
285 res!(displaced_by_a_failed_write(p, z, w, false, false));
286 res!(displaced_by_a_failed_write(p, z, w, false, true));
287 }
288 }
289 res!(displaced_by_a_failed_write(Policy::Interval(3_600_000), 1, 1, true, true));
290 res!(displaced_by_a_failed_write(Policy::Interval(3_600_000), 2, 2, true, true));
291 Ok(())
292}
293
294/// With no fault, P's discard waits for Q's barrier: compaction does not drop P while Q may yet
295/// be lost, so a power cut leaves P. Once Q is durable the late discard is applied through the
296/// collection that carried P, and marks nothing else old.
297#[test]
298fn collection_waits_for_the_winners_barrier() -> Outcome<()> {
299 let _g = lock();
300 hooks::set_owed_drop(true); // the hold's window stays open until the barrier under test
301 for (z, w) in SHAPES {
302 let label = fmt!("{}x{}", z, w);
303 let mut c = res!(cfg(Policy::Interval(3_600_000), z, w));
304 c.data_file_max_bytes = 4_000;
305 let root = res!(fresh("./test_db_durable_mark_collect"));
306 let db = res!(open(&root, &c, true));
307 let k = dat!("durable mark collect");
308 let others: Vec<Dat> = (0..8).map(|i| dat!(fmt!("durable mark other {}", i))).collect();
309 res!(put(&db, &k, val(300, 2)));
310 for (i, o) in others.iter().enumerate() {
311 res!(put(&db, o, val(300, 10 + i as u8)));
312 }
313 res!(db.api().sync_now(wait()));
314 res!(db.api().new_live_files());
315 res!(put(&db, &k, val(300, 4)));
316 assert!(held_soon(&db), "{}: P's discard was not held behind Q's barrier", label);
317 let got = res!(read_after_cut(&db, &root, &c, "./test_db_durable_mark_collect_cut", &[k.clone()]));
318 assert_eq!(got[0], "300/2", "{}: a power cut before Q's barrier did not give P back", label);
319 // Compaction seals the live file first, which makes Q durable; P may go only after that.
320 let _ = db.compact_now(Duration::from_secs(60));
321 let got = res!(read_after_cut(&db, &root, &c, "./test_db_durable_mark_collect_cut", &[k.clone()]));
322 assert!(got[0] == "300/2" || got[0] == "300/4",
323 "{}: a power cut after compaction lost P under Q: {}", label, got[0]);
324 res!(db.api().sync_now(wait()));
325 assert!(wait_until(Duration::from_secs(10), || db.api().held_effects() == 0),
326 "{}: the hold outlived Q's barrier", label);
327 res!(db.compact_now(Duration::from_secs(60)));
328 assert_eq!(read(&db, &k), "300/4", "{}", label);
329 for (i, o) in others.iter().enumerate() {
330 assert_eq!(read(&db, o), fmt!("300/{}", 10 + i), "{}: the late discard hit another record", label);
331 }
332 let v = res!(verify::verify_live_set(db.api(), None, wait()));
333 assert!(v.clean(), "{}: {}", label, v.summary(4));
334 res!(db.close());
335 let db = res!(open(&root, &c, false));
336 for (i, o) in others.iter().enumerate() {
337 assert_eq!(read(&db, o), fmt!("300/{}", 10 + i), "{}: after the restart", label);
338 }
339 res!(db.close());
340 let _ = fs::remove_dir_all(&root);
341 }
342 Ok(())
343}
344
345/// A hold lasts until the barrier the policy makes, and no longer.
346#[test]
347fn holds_release_at_each_policys_barrier() -> Outcome<()> {
348 let _g = lock();
349 let k = dat!("durable mark hold");
350 let quiet = |db: &TestDb| wait_until(Duration::from_secs(10), || db.api().held_effects() == 0);
351
352 let c = res!(cfg(Policy::EveryWrite, 1, 1));
353 let root = res!(fresh("./test_db_durable_mark_hold"));
354 let db = res!(open(&root, &c, true));
355 res!(put(&db, &k, val(50, 1)));
356 res!(put(&db, &k, val(50, 2)));
357 assert!(quiet(&db), "EveryWrite: a hold outlived its write's own barrier");
358 res!(db.close());
359
360 let c = res!(cfg(Policy::EveryN(5), 1, 1));
361 let db = res!(open(&root, &c, true));
362 res!(put(&db, &k, val(50, 1)));
363 res!(db.api().sync_now(wait()));
364 res!(put(&db, &k, val(50, 2)));
365 assert!(held_soon(&db), "EveryN(5): released before the fifth write");
366 for i in 0..4 {
367 res!(put(&db, &dat!(fmt!("durable mark filler {}", i)), val(50, 9)));
368 }
369 assert!(quiet(&db), "EveryN(5): neither the fifth write's barrier nor the owed one released the hold");
370 res!(db.close());
371
372 let c = res!(cfg(Policy::Never, 1, 1));
373 let db = res!(open(&root, &c, true));
374 res!(put(&db, &k, val(50, 1)));
375 res!(db.api().sync_now(wait()));
376 res!(put(&db, &k, val(50, 2)));
377 assert!(held_soon(&db), "Never: released with no barrier");
378 res!(db.api().new_live_files());
379 assert!(quiet(&db), "Never: sealing the live file did not release the hold");
380 res!(put(&db, &k, val(50, 3)));
381 assert!(held_soon(&db), "Never: released with no barrier");
382 res!(db.api().sync_now(wait()));
383 assert!(quiet(&db), "Never: sync_now did not release the hold");
384 res!(db.close());
385 let _ = fs::remove_dir_all(&root);
386 Ok(())
387}
388
389/// A failed barrier is never retried into success (fsyncgate): the marks stay where they were,
390/// the holds stay held, and nothing more is written, whatever the disk does after.
391#[test]
392fn a_failed_barrier_gets_no_second_chance() -> Outcome<()> {
393 let _g = lock();
394 hooks::set_owed_drop(true); // the hold's window stays open until the barrier under test
395 let c = res!(cfg(Policy::Interval(3_600_000), 1, 1));
396 let root = res!(fresh("./test_db_durable_mark_fsyncgate"));
397 let db = res!(open(&root, &c, true));
398 let k = dat!("durable mark fsyncgate");
399 res!(put(&db, &k, val(1_500, 2)));
400 res!(db.api().sync_now(wait()));
401 let _ = put(&db, &k, val(1_500, 4));
402 let _ = held_soon(&db);
403 let held = db.api().held_effects();
404 assert!(held > 0, "positive control: the overwrite's discard is held");
405 res!(fail_a_barrier(&db));
406 let marks = res!(db.api().chans().marks().files(0));
407 assert!(put(&db, &dat!("durable mark after"), val(50, 1)).is_err(), "a write after the fault was taken");
408 assert!(db.api().sync_now(wait()).is_err(), "a barrier after the fault succeeded");
409 let _ = db.api().new_live_files();
410 std::thread::sleep(Duration::from_millis(200));
411 assert_eq!(res!(db.api().chans().marks().files(0)), marks, "a mark moved after the fault");
412 assert_eq!(db.api().held_effects(), held, "a hold was released after the fault");
413 let _ = db.close();
414 let _ = fs::remove_dir_all(&root);
415 Ok(())
416}
417
418/// After a fault, writes, compaction and the sweep are refused with the advice to restart, a
419/// read of what a barrier covered is served, and the fault is recorded for the next open.
420#[test]
421fn a_fault_stops_writes() -> Outcome<()> {
422 let _g = lock();
423 let c = res!(cfg(Policy::Interval(3_600_000), 1, 1));
424 let root = res!(fresh("./test_db_durable_mark_stops"));
425 let db = res!(open(&root, &c, true));
426 let k = dat!("durable mark stops");
427 res!(put(&db, &k, val(1_500, 2)));
428 res!(db.api().sync_now(wait()));
429 res!(fail_a_barrier(&db));
430 let advice = |r: Outcome<()>, what: &str| match r {
431 Ok(()) => panic!("{} was taken by a stopped store", what),
432 Err(e) => assert!(fmt!("{}", e).contains("Restart the store"), "{}: {}", what, e),
433 };
434 advice(put(&db, &dat!("durable mark new"), val(50, 1)), "a put");
435 advice(db.delete(&k, Uid::default(), None).map(|_| ()), "a delete");
436 advice(sweep::sweep_orphans(db.api(), Uid::default(), None, wait()).map(|_| ()), "the sweep");
437 assert!(db.compact_now(Duration::from_secs(60)).is_err(), "compaction ran on a stopped store");
438 assert_eq!(read(&db, &k), "1500/2", "a covered read was refused");
439 assert!(db.api().fault().is_some());
440 assert!(root.join(constant::FAULT_FILENAME).exists(), "no fault record was written");
441 let _ = db.close();
442 let _ = fs::remove_dir_all(&root);
443 Ok(())
444}
445
446/// The next open cuts every file in doubt to its mark, keeps the tail in quarantine and clears
447/// the fault record. A second open over the same record changes no length.
448#[test]
449fn the_open_cuts_to_the_marks() -> Outcome<()> {
450 let _g = lock();
451 hooks::set_owed_drop(true); // the hold's window stays open until the barrier under test
452 let c = res!(cfg(Policy::Interval(3_600_000), 1, 1));
453 let root = res!(fresh("./test_db_durable_mark_cut"));
454 let db = res!(open(&root, &c, true));
455 let k = dat!("durable mark cut");
456 res!(put(&db, &k, val(300, 2)));
457 res!(db.api().sync_now(wait()));
458 let _ = put(&db, &k, val(300, 4));
459 let marks = res!(db.api().chans().marks().files(0));
460 res!(fail_a_barrier(&db));
461 let _ = db.close();
462 let fault = res!(fs::read(root.join(constant::FAULT_FILENAME)));
463 let lens = |root: &Path| -> Vec<(u64, u64)> {
464 marks.iter().map(|(f, _)| (
465 size(&zdir(root, &c, 0).join(ZoneDir::relative_file_path(&FileType::Data, *f))),
466 size(&zdir(root, &c, 0).join(ZoneDir::relative_file_path(&FileType::Index, *f))),
467 )).collect()
468 };
469 // The fault cut the tails in process; a crash before that cut would leave them, so one is
470 // put back for the open to find.
471 for (f, _) in &marks {
472 let p = zdir(&root, &c, 0).join(ZoneDir::relative_file_path(&FileType::Data, *f));
473 let mut b = res!(fs::read(&p));
474 b.extend_from_slice(&[0xab; 64]);
475 res!(fs::write(&p, &b));
476 }
477 let before = lens(&root);
478 assert!(marks.iter().zip(&before).any(|((_, m), l)| l.0 > m.0), "positive control: nothing past a mark");
479 res!(Refusal::clear(&root)); // as the operator would
480 let db = res!(open(&root, &c, false));
481 assert_eq!(read(&db, &k), "300/2");
482 res!(db.close());
483 let after = lens(&root);
484 for ((f, m), l) in marks.iter().zip(&after) {
485 assert_eq!(*l, *m, "file {} was not cut to its mark", f);
486 }
487 assert!(!root.join(constant::FAULT_FILENAME).exists(), "the fault record outlived the cut");
488 let q = zdir(&root, &c, 0).join(constant::QUARANTINE_DIR);
489 assert!(res!(fs::read_dir(&q)).count() > 0, "the cut tail was not quarantined");
490 let held = |q: &Path| -> Outcome<Vec<(PathBuf, u64)>> {
491 let mut v = Vec::new();
492 for e in res!(fs::read_dir(q)) {
493 let p = res!(e).path();
494 v.push((p.clone(), size(&p)));
495 }
496 v.sort();
497 Ok(v)
498 };
499 let evidence = res!(held(&q));
500 res!(fs::write(root.join(constant::FAULT_FILENAME), &fault));
501 let db = res!(open(&root, &c, false));
502 assert_eq!(read(&db, &k), "300/2");
503 res!(db.close());
504 assert_eq!(lens(&root), after, "a second cut over a replanted fault record changed a length");
505 // A file already at its mark has nothing to cut, so the first cut's evidence stands.
506 assert_eq!(res!(held(&q)), evidence, "a second cut rewrote the quarantined tails");
507 let _ = fs::remove_dir_all(&root);
508 Ok(())
509}
510
511/// A fault record that cannot be written still stops the store in process.
512#[test]
513fn a_fault_stops_the_store_without_its_record() -> Outcome<()> {
514 let _g = lock();
515 let c = res!(cfg(Policy::Interval(3_600_000), 1, 1));
516 let root = res!(fresh("./test_db_durable_mark_norecord"));
517 let db = res!(open(&root, &c, true));
518 res!(put(&db, &dat!("durable mark norecord"), val(50, 1)));
519 hooks::set_fault_record_fails(true);
520 let r = fail_a_barrier(&db);
521 hooks::set_fault_record_fails(false);
522 res!(r);
523 assert!(db.api().fault().is_some(), "the store did not stop");
524 assert!(put(&db, &dat!("durable mark norecord 2"), val(50, 1)).is_err(), "a write was taken");
525 let _ = db.close();
526 let _ = fs::remove_dir_all(&root);
527 Ok(())
528}
529
530/// A chunked put's chunks are answered only once a barrier covers them, so a power cut taken
531/// while its head is held back keeps every chunk of the put and no head (R1). The data files are
532/// large enough that no rollover seal makes the chunks durable by the way, and each barrier is
533/// slowed, so a chunk answered before its barrier is cut short in the image. One store is held
534/// at a time, since the hold blocks the server bot driving it, and its chunk set is counted
535/// exactly: the chunk keys the store added, from the live store once the put has returned.
536#[test]
537fn a_head_never_outlives_its_chunks() -> Outcome<()> {
538 let _g = lock();
539 let mut bad = Vec::new();
540 for p in POLICIES {
541 for (z, w) in [(1u16, 1u16), (1, 4), (2, 2)] {
542 let label = fmt!("{:?} {}x{}", p, z, w);
543 let mut c = res!(cfg(p, z, w));
544 c.data_file_max_bytes = 1_048_576;
545 let root = res!(fresh("./test_db_durable_mark_r1"));
546 let db = res!(open(&root, &c, true));
547 res!(put(&db, &dat!("durable mark r1 base"), val(1_500, 2)));
548 res!(db.api().sync_now(wait()));
549 let base = res!(chunk_keys(&db));
550 let k = dat!("durable mark r1");
551 let held = hooks::heads_held();
552 hooks::set_barrier_delay(Duration::from_millis(900));
553 hooks::set_head_hold(true);
554 let putter = {
555 let db = db.clone();
556 let k = k.clone();
557 thread::spawn(move || put(&db, &k, val(3_000, 4)))
558 };
559 let reached = wait_until(Duration::from_secs(30), || hooks::heads_held() > held);
560 hooks::set_barrier_delay(Duration::ZERO);
561 let cut = power_cut(&db, &root, &c, "./test_db_durable_mark_r1_cut");
562 hooks::set_head_hold(false);
563 let done = match putter.join() { Ok(r) => r, Err(_) => Err(err!("{}: the put panicked.", label; Test)) };
564 res!(done);
565 let cut = res!(cut);
566 let new: BTreeSet<Vec<u8>> = res!(chunk_keys(&db)).difference(&base).cloned().collect();
567 let db2 = res!(open(&cut, &c, false));
568 let image = res!(chunk_keys(&db2));
569 let got = read(&db2, &k);
570 res!(db2.close());
571 let _ = fs::remove_dir_all(&cut);
572 let after = res!(read_after_cut(&db, &root, &c, "./test_db_durable_mark_r1_cut", &[k.clone()]));
573 res!(db.close());
574 let _ = fs::remove_dir_all(&root);
575 let missing = new.difference(&image).count();
576 if !reached {
577 bad.push(fmt!("{}: the put never reached the head hold", label));
578 }
579 if new.len() < 2 {
580 bad.push(fmt!("{}: the put added {} chunks, so it was not chunked", label, new.len()));
581 }
582 if missing > 0 {
583 bad.push(fmt!("{}: a cut at the head hold lost {} of the put's {} answered chunks",
584 label, missing, new.len()));
585 }
586 if got != "none" {
587 bad.push(fmt!("{}: a cut at the head hold read {}, but no head had been sent", label, got));
588 }
589 let whole = after.iter().all(|g| g == "3000/4" || (!matches!(p, Policy::EveryWrite) && g == "none"));
590 if !whole {
591 bad.push(fmt!("{}: a cut after the put returned read {:?}", label, after));
592 }
593 println!("R1 {}: {} chunks, {} missing at the hold, then {:?}", label, new.len(), missing, after);
594 }
595 }
596 if !bad.is_empty() {
597 return Err(err!("{:?}", bad; Test));
598 }
599 Ok(())
600}
601
602/// The sweep makes its scan durable before it retires, which releases what the scan saw; a
603/// barrier that fails there stops the sweep with nothing retired.
604#[test]
605fn the_sweep_syncs_after_its_scan() -> Outcome<()> {
606 let _g = lock();
607 hooks::set_owed_drop(true); // the hold's window stays open until the barrier under test
608 let c = res!(cfg(Policy::Interval(3_600_000), 1, 1));
609 let root = res!(fresh("./test_db_durable_mark_sweep"));
610 let db = res!(open(&root, &c, true));
611 let k = dat!("durable mark sweep");
612 res!(put(&db, &k, val(1_500, 2)));
613 res!(db.api().sync_now(wait()));
614 let first = res!(res!(head(&db, &k)).ok_or_else(|| err!("No head."; Test, Missing)));
615 res!(put(&db, &k, val(1_500, 4)));
616 assert!(held_soon(&db), "positive control: the discard is held");
617 assert!(chunks_live(&db, first), "positive control: P's chunks are live before the sweep");
618 res!(sweep::sweep_orphans(db.api(), Uid::default(), None, wait()));
619 // The retires the sweep's barrier releases are themselves written after it, so they are
620 // held in turn: what is asserted is that P's displaced chunk set was retired.
621 assert!(wait_until(Duration::from_secs(10), || !chunks_live(&db, first)),
622 "the sweep's barrier did not release the retire of P's chunks");
623 res!(db.close());
624
625 let db = res!(open(&root, &c, true));
626 res!(put(&db, &k, val(1_500, 2)));
627 res!(db.api().sync_now(wait()));
628 let old = res!(res!(head(&db, &k)).ok_or_else(|| err!("No head."; Test, Missing)));
629 res!(put(&db, &k, val(1_500, 4)));
630 hooks::set_barrier_failure(true);
631 let swept = sweep::sweep_orphans(db.api(), Uid::default(), None, wait());
632 hooks::set_barrier_failure(false);
633 assert!(swept.is_err(), "the sweep ran past a failed barrier: {:?}", swept);
634 assert!(db.api().fault().is_some(), "the failed barrier did not stop the store");
635 assert!(chunks_live(&db, old), "P's chunks were retired");
636 let _ = db.close();
637 res!(Refusal::clear(&root)); // as the operator would
638 let db = res!(open(&root, &c, false));
639 assert_eq!(read(&db, &k), "1500/2", "the restart did not give P back");
640 res!(db.close());
641 let _ = fs::remove_dir_all(&root);
642 Ok(())
643}
644
645/// Under `Never`, a clean close makes everything durable, so every displaced chunk set is retired
646/// and the reopened store holds the chunks of the last value only.
647#[test]
648fn a_clean_close_under_never_retires() -> Outcome<()> {
649 let _g = lock();
650 let c = res!(cfg(Policy::Never, 1, 1));
651 let root = res!(fresh("./test_db_durable_mark_never"));
652 let db = res!(open(&root, &c, true));
653 let k = dat!("durable mark never");
654 for s in 1..6u8 {
655 res!(put(&db, &k, val(1_500, s)));
656 }
657 res!(db.close());
658 let db = res!(open(&root, &c, false));
659 assert_eq!(read(&db, &k), "1500/5");
660 assert_eq!(res!(live_chunks(&db)), 4, "displaced chunk sets outlived a clean close");
661 res!(db.close());
662 let _ = fs::remove_dir_all(&root);
663 Ok(())
664}
665
666// The records the file states flag current, and those they flag old.
667fn flagged(db: &TestDb) -> Outcome<(usize, usize)> {
668 let mut n = (0, 0);
669 for (_, fstates) in res!(db.api().collect_file_states(constant::USER_REQUEST_WAIT)) {
670 for (_, fstat) in fstates.map() {
671 for (_, d) in fstat.data_map() {
672 match d {
673 DataState::Cur => n.0 += 1,
674 DataState::Old(_) => n.1 += 1,
675 }
676 }
677 }
678 }
679 Ok(n)
680}
681
682/// `Never` and `EveryN` owe a held discard a barrier within `HELD_BARRIER`: after overwrites
683/// into a live file that then stops growing, with no seal, `sync_now` or close, the losers are
684/// flagged old and the displaced chunk set retired. Two writers per zone check that the
685/// request reaches the writer that owns the winner's file.
686#[test]
687fn an_idle_store_is_owed_its_barrier() -> Outcome<()> {
688 let _g = lock();
689 const N: usize = 12;
690 let bound = Duration::from_millis(1_500);
691 for p in [Policy::Never, Policy::EveryN(1_000)] {
692 for (zones, wbots) in SHAPES {
693 let label = fmt!("{:?} {}x{}", p, zones, wbots);
694 let mut c = res!(cfg(p, zones, wbots));
695 c.data_file_max_bytes = 1_000_000;
696 let root = res!(fresh("./test_db_durable_mark_owed"));
697 let db = res!(open(&root, &c, true));
698 let big = dat!("durable mark owed chunked");
699 for i in 0..N {
700 res!(put(&db, &dat!(fmt!("durable mark owed {}", i)), val(50, 1)));
701 }
702 res!(put(&db, &big, val(1_500, 1)));
703 res!(db.api().sync_now(wait()));
704 res!(put(&db, &big, val(1_500, 2)));
705 assert!(held_soon(&db), "{}: an overwrite was released with no barrier", label);
706 for i in 0..N {
707 res!(put(&db, &dat!(fmt!("durable mark owed {}", i)), val(50, 2)));
708 }
709 let begun = Instant::now();
710 assert!(wait_until(bound, || db.api().held_effects() == 0),
711 "{}: {} discards still held {:?} after the writes stopped",
712 label, db.api().held_effects(), bound);
713 let took = begun.elapsed();
714 // The losers: the small values, and the chunked value's head and four chunks.
715 assert_eq!(res!(flagged(&db)).1, N + 5, "{}: losers not flagged old", label);
716 assert_eq!(res!(live_chunks(&db)), 4, "{}: the displaced chunk set was not retired", label);
717 assert_eq!(read(&db, &big), "1500/2");
718 println!("{}: holds released {:?} after the last write.", label, took);
719 res!(db.close());
720 let _ = fs::remove_dir_all(&root);
721 }
722 }
723 Ok(())
724}
725
726/// A burst of inserts that displace nothing holds nothing, so under `Never` it costs no barrier
727/// at all: one would fail, and stop the store.
728#[test]
729fn an_insert_only_burst_costs_no_barrier() -> Outcome<()> {
730 let _g = lock();
731 let mut c = res!(cfg(Policy::Never, 1, 1));
732 c.data_file_max_bytes = 1_000_000;
733 let root = res!(fresh("./test_db_durable_mark_inserts"));
734 let db = res!(open(&root, &c, true));
735 let failed = hooks::barriers_failed();
736 hooks::set_barrier_failure(true);
737 for i in 0..200 {
738 res!(put(&db, &dat!(fmt!("durable mark insert {}", i)), val(50, 1)));
739 }
740 // Absence is what is tested: three periods in which an owed barrier would have run.
741 std::thread::sleep(constant::HELD_BARRIER * 3);
742 hooks::set_barrier_failure(false);
743 assert_eq!(hooks::barriers_failed(), failed, "an insert-only burst under Never ran a barrier");
744 assert!(db.api().fault().is_none(), "an insert-only burst under Never faulted the store");
745 res!(db.close());
746 let _ = fs::remove_dir_all(&root);
747 Ok(())
748}
749
750/// The two gates the owed barrier closed, under `Never` with no write after the overwrites:
751/// the superseded records of a sealed file start its collection, and the rollover's file
752/// states settle to one current record per key, each within a bounded time.
753#[test]
754fn an_idle_store_under_never_collects_and_settles() -> Outcome<()> {
755 let _g = lock();
756 const NK: usize = 10; // most of the first file
757 const NF: usize = 8; // seals it part way through
758 let bound = Duration::from_secs(2);
759 let mut c = res!(cfg(Policy::Never, 1, 1));
760 c.num_cbots_per_zone = 1;
761 c.data_file_max_bytes = 16_000; // collection triggers at 4,800 superseded bytes
762 c.rest_chunk_threshold = 3_000;
763 let root = res!(fresh("./test_db_durable_mark_collect"));
764 log_set_level!("error");
765 let db = res!(setup::start_db(root.clone(), Some(c.clone()), schemes(), None, true, true));
766 for i in 0..NK {
767 res!(put(&db, &dat!(fmt!("durable mark key {}", i)), val(1_000, 1)));
768 }
769 for i in 0..NF {
770 res!(put(&db, &dat!(fmt!("durable mark filler {}", i)), val(1_000, 1)));
771 }
772 assert!(zdir(&root, &c, 0).join(ZoneDir::relative_file_path(&FileType::Data, 2)).is_file(),
773 "the fillers did not seal data file 1");
774 // Held in collection long enough to be seen there.
775 hooks::set_collect_delay(Duration::from_millis(1_500));
776 for i in 0..(NK / 2) {
777 res!(put(&db, &dat!(fmt!("durable mark key {}", i)), val(1_000, 2)));
778 }
779 let begun = Instant::now();
780 let settled = wait_until(bound, || matches!(flagged(&db), Ok((n, _)) if n == NK + NF));
781 let collecting = wait_until(bound, || match db.api().collect_file_states(wait()) {
782 Ok(states) => states.values().any(|s| s.map().get(&1).map_or(false, |f| f.gc_active())),
783 Err(_) => false,
784 });
785 let took = begun.elapsed();
786 hooks::set_collect_delay(Duration::ZERO);
787 assert!(settled, "rollover under Never: {:?} current and old, not {} current, {:?} after the \
788 last write", flagged(&db), NK + NF, took);
789 assert!(collecting, "data file 1 did not start its collection within {:?} of the last write", took);
790 println!("Never, idle: settled and collecting {:?} after the last write.", took);
791 res!(db.close());
792 let _ = fs::remove_dir_all(&root);
793 Ok(())
794}
795
796// A small deterministic generator, so that a failing seed can be run again.
797struct Rng(u64);
798
799impl Rng {
800 fn next(&mut self) -> u64 {
801 self.0 ^= self.0 << 13;
802 self.0 ^= self.0 >> 7;
803 self.0 ^= self.0 << 17;
804 self.0
805 }
806 fn below(&mut self, n: u64) -> u64 { self.next() % n }
807}
808
809/// The power-cut oracle (R5). Random chunked and small puts, overwrites, deletes, barriers,
810/// compactions and sweeps, then a cut at the marks: no read is torn, and no key comes back
811/// missing or older than its newest write a barrier covered.
812fn model(seed: u64, p: Policy, zones: u16, wbots: u16) -> Outcome<()> {
813 let label = fmt!("seed {} {:?} {}x{}", seed, p, zones, wbots);
814 let mut c = res!(cfg(p, zones, wbots));
815 c.data_file_max_bytes = 6_000;
816 let root = res!(fresh("./test_db_durable_mark_model"));
817 let db = res!(open(&root, &c, true));
818 let keys: Vec<Dat> = (0..6).map(|i| dat!(fmt!("durable mark model {}", i))).collect();
819 // Per key, what it held after each acknowledged write, and how many of them are covered.
820 let mut hist: Vec<Vec<String>> = vec![vec![fmt!("none")]; keys.len()];
821 let mut covered: Vec<usize> = vec![1; keys.len()];
822 let mut rng = Rng(seed | 1);
823 let mut wseed = 0u8;
824 for _ in 0..60 {
825 let i = rng.below(keys.len() as u64) as usize;
826 match rng.below(20) {
827 0..=6 => {
828 wseed = wseed.wrapping_add(1).max(1);
829 let len = 1_100 + rng.below(500) as usize;
830 res!(put(&db, &keys[i], val(len, wseed)));
831 hist[i].push(fmt!("{}/{}", len, wseed));
832 },
833 7..=12 => {
834 wseed = wseed.wrapping_add(1).max(1);
835 let len = 50 + rng.below(250) as usize;
836 res!(put(&db, &keys[i], val(len, wseed)));
837 hist[i].push(fmt!("{}/{}", len, wseed));
838 },
839 13..=14 => {
840 res!(db.delete(&keys[i], Uid::default(), None));
841 hist[i].push(fmt!("none"));
842 },
843 15..=16 => {
844 res!(db.api().sync_now(wait()));
845 for (h, n) in hist.iter().zip(covered.iter_mut()) {
846 *n = h.len();
847 }
848 },
849 17..=18 => { res!(db.compact_now(Duration::from_secs(60))); },
850 _ => { res!(sweep::sweep_orphans(db.api(), Uid::default(), None, wait())); },
851 }
852 if let Policy::EveryWrite = p {
853 for (h, n) in hist.iter().zip(covered.iter_mut()) {
854 *n = h.len();
855 }
856 }
857 }
858 res!(db.api().settle_for_test(Duration::from_secs(20)));
859 let got = res!(read_after_cut(&db, &root, &c, "./test_db_durable_mark_model_cut", &keys));
860 for (i, g) in got.iter().enumerate() {
861 let ok = &hist[i][covered[i] - 1..];
862 assert!(ok.contains(g), "{}: key {} reads {} after the cut; allowed {:?}", label, i, g, ok);
863 }
864 res!(db.close());
865 let _ = fs::remove_dir_all(&root);
866 Ok(())
867}
868
869#[test]
870fn power_cut_oracle() -> Outcome<()> {
871 let _g = lock();
872 let base = match std::env::var("DURABLE_MARK_SEED") {
873 Ok(s) => res!(s.parse::<u64>().map_err(|e| err!(e, "DURABLE_MARK_SEED"; Test, Invalid))),
874 Err(_) => 0x5eed_0000,
875 };
876 let t0 = Instant::now();
877 for n in 0..3u64 {
878 for p in POLICIES {
879 for (z, w) in SHAPES {
880 res!(model(base + n * 7_919, p, z, w));
881 }
882 }
883 }
884 msg!("The oracle ran 24 stores in {:?}.", t0.elapsed());
885 Ok(())
886}