oxedyne/fe2o3/fe2o3_o3db_sync/tests/chunk_period.rs
5.2 KiB, 1 run
created by r1870400018:74445, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! A plain record's period under a long put's chunks (Ozone chunk barriers, QA B A1, 2026-10-12). |
| 2 | //! A store's chunks owe no timed barrier of their own, so under `Interval` a plain record that |
| 3 | //! lands when the period is not yet due is covered by the syncer's timer alone. A syncer that |
| 4 | //! lags its writer through a long put always finds a chunk waiting, so the timer never runs out, |
| 5 | //! and the plain record waited for the put's own `SyncNow`, seconds later. The syncer now checks |
| 6 | //! what is owed after each item it takes in. First seen by QA B as a code reading; the probe |
| 7 | //! became this test. The hooks are process-wide, so the tests run one at a time. |
| 8 | |
| 9 | mod gc_pair; |
| 10 | |
| 11 | use gc_pair::*; |
| 12 | |
| 13 | use oxedyne_fe2o3_core::prelude::*; |
| 14 | use oxedyne_fe2o3_iop_db::api::Database; |
| 15 | use oxedyne_fe2o3_jdat::prelude::*; |
| 16 | use oxedyne_fe2o3_o3db_sync::{ |
| 17 | base::{ |
| 18 | cfg::OzoneConfig, |
| 19 | constant, |
| 20 | }, |
| 21 | comm::response::Wait, |
| 22 | test::{ |
| 23 | hooks, |
| 24 | setup::{ |
| 25 | self, |
| 26 | Uid, |
| 27 | }, |
| 28 | }, |
| 29 | }; |
| 30 | |
| 31 | use std::{ |
| 32 | collections::BTreeMap, |
| 33 | fs, |
| 34 | path::{ |
| 35 | Path, |
| 36 | PathBuf, |
| 37 | }, |
| 38 | sync::{ |
| 39 | Mutex, |
| 40 | MutexGuard, |
| 41 | }, |
| 42 | thread, |
| 43 | time::{ |
| 44 | Duration, |
| 45 | Instant, |
| 46 | }, |
| 47 | }; |
| 48 | |
| 49 | static HOOKS: Mutex<()> = Mutex::new(()); |
| 50 | |
| 51 | const PERIOD_MS: u64 = 200; |
| 52 | const CHUNK: u64 = 400; |
| 53 | const GIANT: usize = 600_000; // 1,500 chunks |
| 54 | const LAG: Duration = Duration::from_millis(2); // per chunk, so about 3 s of syncer time |
| 55 | const CUT_AFTER: Duration = Duration::from_millis(800); |
| 56 | |
| 57 | fn lock() -> MutexGuard<'static, ()> { |
| 58 | let g = match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() }; |
| 59 | hooks::set_barrier_delay(Duration::ZERO); |
| 60 | hooks::set_head_hold(false); |
| 61 | hooks::set_syncer_chunk_delay(Duration::ZERO); |
| 62 | g |
| 63 | } |
| 64 | |
| 65 | fn cfg() -> Outcome<OzoneConfig> { |
| 66 | let mut c = res!(setup::default_cfg()); |
| 67 | c.num_zones = 1; |
| 68 | c.num_cbots_per_zone = 2; |
| 69 | c.num_fbots_per_zone = 1; |
| 70 | c.num_wbots_per_zone = 1; |
| 71 | c.zone_overrides = BTreeMap::new(); |
| 72 | c.rest_chunk_threshold = 300; |
| 73 | c.rest_chunk_bytes = CHUNK; |
| 74 | c.data_file_max_bytes = 16 * 1_048_576; |
| 75 | c.sync_on_write = false; |
| 76 | c.sync_every_n_writes = 0; |
| 77 | c.sync_interval_ms = PERIOD_MS; |
| 78 | Ok(c) |
| 79 | } |
| 80 | |
| 81 | fn fresh(dir: &str) -> Outcome<PathBuf> { |
| 82 | let _ = fs::remove_dir_all(dir); |
| 83 | res!(fs::create_dir_all(dir)); |
| 84 | Ok(res!(Path::new(dir).canonicalize())) |
| 85 | } |
| 86 | |
| 87 | fn open(root: &Path, c: &OzoneConfig, wipe: bool) -> Outcome<TestDb> { |
| 88 | log_set_level!("error"); |
| 89 | setup::start_db(root.to_path_buf(), Some(c.clone()), schemes(), None, false, wipe) |
| 90 | } |
| 91 | |
| 92 | fn val(len: usize, seed: u8) -> Dat { |
| 93 | Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect()) |
| 94 | } |
| 95 | |
| 96 | fn wait() -> Wait { |
| 97 | Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL } |
| 98 | } |
| 99 | |
| 100 | // Q lands with the period due and barriers; P follows at once, finds it not due, and is left to |
| 101 | // the timer. A long put then keeps the syncer fed with chunks, slowed by the lag hook, well past |
| 102 | // P's period. A cut taken while the put is still in its chunk phase must find P. |
| 103 | #[test] |
| 104 | fn a_plain_records_period_outlasts_a_long_puts_chunks() -> Outcome<()> { |
| 105 | let _lock = lock(); |
| 106 | let c = res!(cfg()); |
| 107 | let root = res!(fresh("./test_db_chunk_period")); |
| 108 | let db = res!(open(&root, &c, true)); |
| 109 | res!(db.insert(dat!("warm"), dat!("up"), Uid::default(), None)); |
| 110 | res!(db.api().sync_now(wait())); |
| 111 | hooks::set_syncer_chunk_delay(LAG); |
| 112 | thread::sleep(Duration::from_millis(PERIOD_MS + 100)); |
| 113 | res!(db.insert(dat!("plain q"), val(50, 5), Uid::default(), None)); |
| 114 | let k = dat!("plain p"); |
| 115 | let v = val(50, 3); |
| 116 | res!(db.insert(k.clone(), v.clone(), Uid::default(), None)); |
| 117 | let acked = Instant::now(); |
| 118 | let giant = { |
| 119 | let db = db.clone(); |
| 120 | thread::spawn(move || db.insert(dat!("giant"), val(GIANT, 9), Uid::default(), None).map(|_| ())) |
| 121 | }; |
| 122 | thread::sleep(CUT_AFTER); |
| 123 | let running = !giant.is_finished(); |
| 124 | let cut = res!(fresh("./test_db_chunk_period_cut")); |
| 125 | let cut_result = instant_cut(&db, &root, &c, &cut); |
| 126 | let at = acked.elapsed(); |
| 127 | hooks::set_syncer_chunk_delay(Duration::ZERO); |
| 128 | let giant = match giant.join() { Ok(r) => r, Err(_) => Err(err!("The long put panicked."; Test)) }; |
| 129 | res!(cut_result); |
| 130 | let got = { |
| 131 | let db2 = res!(open(&cut, &c, false)); |
| 132 | let got = db2.get(&k, None); |
| 133 | res!(db2.close()); |
| 134 | got |
| 135 | }; |
| 136 | let _ = fs::remove_dir_all(&cut); |
| 137 | res!(giant); |
| 138 | res!(db.close()); |
| 139 | let _ = fs::remove_dir_all(&root); |
| 140 | if !running { |
| 141 | return Err(err!( |
| 142 | "The long put had finished before the cut {} ms after P's ack, so its own SyncNow \ |
| 143 | could have covered P and the test shows nothing.", at.as_millis(); Test)); |
| 144 | } |
| 145 | match res!(got) { |
| 146 | Some((d, _)) if d == v => Ok(()), |
| 147 | Some((d, _)) => Err(err!("P read back as another value at a cut {} ms after its ack: {:?}", |
| 148 | at.as_millis(), d; Test)), |
| 149 | None => Err(err!( |
| 150 | "P, acknowledged under Interval({}), was lost at a cut {} ms after its ack, while a \ |
| 151 | long put's chunks held the syncer.", PERIOD_MS, at.as_millis(); Test)), |
| 152 | } |
| 153 | } |