oxedyne/fe2o3/fe2o3_o3db_sync/tests/chunk_barriers.rs
9.5 KiB, 1 run
created by r1870400018:74443, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! Barriers on a chunked put's path (Ozone chunk barriers, 2026-10-11). A store's chunks are |
| 2 | //! never answered before a barrier covers them, and the store asks each writer it used for one |
| 3 | //! behind them, so a chunk must not trip the timed or every-N barrier as well: that barrier would |
| 4 | //! cover only the chunks taken so far, and the rest would wait on a second one, in series (M1). |
| 5 | //! Nor does one put spread its chunks over every writer of a zone (M3). Barriers are counted |
| 6 | //! with `hooks::barriers_by_writer`, not timed. The chunk phase is read at the head hold, once |
| 7 | //! every chunk is answered and before the head is sent. The hooks are process-wide, so the |
| 8 | //! tests run one at a time. |
| 9 | |
| 10 | mod gc_pair; |
| 11 | |
| 12 | use gc_pair::*; |
| 13 | |
| 14 | use oxedyne_fe2o3_core::prelude::*; |
| 15 | use oxedyne_fe2o3_iop_db::api::Database; |
| 16 | use oxedyne_fe2o3_jdat::prelude::*; |
| 17 | use oxedyne_fe2o3_o3db_sync::{ |
| 18 | base::{ |
| 19 | cfg::OzoneConfig, |
| 20 | constant, |
| 21 | }, |
| 22 | comm::response::Wait, |
| 23 | test::{ |
| 24 | hooks, |
| 25 | setup::{ |
| 26 | self, |
| 27 | Uid, |
| 28 | }, |
| 29 | }, |
| 30 | }; |
| 31 | |
| 32 | use std::{ |
| 33 | collections::{ |
| 34 | BTreeMap, |
| 35 | BTreeSet, |
| 36 | }, |
| 37 | path::Path, |
| 38 | sync::{ |
| 39 | Mutex, |
| 40 | MutexGuard, |
| 41 | }, |
| 42 | thread, |
| 43 | time::{ |
| 44 | Duration, |
| 45 | Instant, |
| 46 | }, |
| 47 | }; |
| 48 | |
| 49 | static HOOKS: Mutex<()> = Mutex::new(()); |
| 50 | |
| 51 | const CHUNK: u64 = 400; |
| 52 | const LENS: [usize; 4] = [350, 1_100, 3_000, 6_400]; // 1, 3, 8 and 16 chunks |
| 53 | |
| 54 | #[derive(Clone, Copy, Debug)] |
| 55 | enum Policy { EveryN(u32), EveryWrite, Interval(u64), Never } |
| 56 | |
| 57 | fn lock() -> MutexGuard<'static, ()> { |
| 58 | let g = match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() }; |
| 59 | hooks::set_barrier_delay(Duration::ZERO); |
| 60 | hooks::set_head_hold(false); |
| 61 | g |
| 62 | } |
| 63 | |
| 64 | fn cfg(p: Policy, zones: u16, wbots: u16, file_max: u64) -> Outcome<OzoneConfig> { |
| 65 | let mut c = res!(setup::default_cfg()); |
| 66 | c.num_zones = zones; |
| 67 | c.num_cbots_per_zone = 2; |
| 68 | c.num_fbots_per_zone = 1; |
| 69 | c.num_wbots_per_zone = wbots; |
| 70 | c.zone_overrides = BTreeMap::new(); |
| 71 | c.rest_chunk_threshold = 300; |
| 72 | c.rest_chunk_bytes = CHUNK; |
| 73 | c.data_file_max_bytes = file_max; |
| 74 | c.sync_on_write = false; |
| 75 | c.sync_every_n_writes = 0; |
| 76 | c.sync_interval_ms = 0; |
| 77 | match p { |
| 78 | Policy::EveryN(n) => c.sync_every_n_writes = n, |
| 79 | Policy::EveryWrite => c.sync_on_write = true, |
| 80 | Policy::Interval(t) => c.sync_interval_ms = t, |
| 81 | Policy::Never => (), |
| 82 | } |
| 83 | Ok(c) |
| 84 | } |
| 85 | |
| 86 | fn start(dir: &str, c: &OzoneConfig) -> Outcome<TestDb> { |
| 87 | log_set_level!("error"); |
| 88 | let _ = std::fs::remove_dir_all(dir); |
| 89 | res!(std::fs::create_dir_all(dir)); |
| 90 | let root = res!(Path::new(dir).canonicalize()); |
| 91 | setup::start_db(root, Some(c.clone()), schemes(), None, false, true) |
| 92 | } |
| 93 | |
| 94 | fn val(len: usize, seed: u8) -> Dat { |
| 95 | Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect()) |
| 96 | } |
| 97 | |
| 98 | fn wait() -> Wait { |
| 99 | Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL } |
| 100 | } |
| 101 | |
| 102 | fn wait_until(limit: Duration, mut cond: impl FnMut() -> bool) -> bool { |
| 103 | let deadline = Instant::now() + limit; |
| 104 | loop { |
| 105 | if cond() { |
| 106 | return true; |
| 107 | } |
| 108 | if Instant::now() >= deadline { |
| 109 | return false; |
| 110 | } |
| 111 | thread::sleep(Duration::from_millis(5)); |
| 112 | } |
| 113 | } |
| 114 | |
| 115 | // The barriers each writer made between two readings of the gauge. |
| 116 | fn since(before: &BTreeMap<(usize, usize), u64>) -> BTreeMap<(usize, usize), u64> { |
| 117 | hooks::barriers_by_writer().into_iter() |
| 118 | .map(|(w, n)| (w, n - before.get(&w).copied().unwrap_or(0))) |
| 119 | .filter(|(_, n)| *n > 0) |
| 120 | .collect() |
| 121 | } |
| 122 | |
| 123 | // Puts a fresh key and returns the barriers made, by writer, before its head was sent. |
| 124 | fn chunk_phase(db: &TestDb, k: Dat, v: Dat) -> Outcome<BTreeMap<(usize, usize), u64>> { |
| 125 | let held = hooks::heads_held(); |
| 126 | let before = hooks::barriers_by_writer(); |
| 127 | hooks::set_head_hold(true); |
| 128 | let put = { let db = db.clone(); thread::spawn(move || db.insert(k, v, Uid::default(), None).map(|_| ())) }; |
| 129 | let reached = wait_until(Duration::from_secs(30), || hooks::heads_held() > held); |
| 130 | let phase = since(&before); |
| 131 | hooks::set_head_hold(false); |
| 132 | let put = match put.join() { Ok(r) => r, Err(_) => Err(err!("The put panicked."; Test)) }; |
| 133 | res!(put); |
| 134 | if !reached { |
| 135 | return Err(err!("The put never reached the head hold."; Test)); |
| 136 | } |
| 137 | Ok(phase) |
| 138 | } |
| 139 | |
| 140 | // Each layout puts values of 1 to 16 chunks to fresh keys, each after the writers have been |
| 141 | // idle past the period, so that a chunk would find the timed barrier due. |
| 142 | fn run_layouts(p: Policy, settle: Duration, tag: &str) -> Outcome<Vec<String>> { |
| 143 | let mut bad = Vec::new(); |
| 144 | for (z, w) in [(1u16, 1u16), (1, 4), (2, 2)] { |
| 145 | let c = res!(cfg(p, z, w, 16 * 1_048_576)); |
| 146 | let dir = fmt!("./test_db_chunk_barriers_{}_{}x{}", tag, z, w); |
| 147 | let db = res!(start(&dir, &c)); |
| 148 | res!(db.insert(dat!("warm"), dat!("up"), Uid::default(), None)); |
| 149 | let mut counts = Vec::new(); |
| 150 | for (i, len) in LENS.iter().enumerate() { |
| 151 | res!(db.api().sync_now(wait())); |
| 152 | thread::sleep(settle); |
| 153 | let phase = res!(chunk_phase(&db, dat!(fmt!("chunked {}", i)), val(*len, i as u8))); |
| 154 | let total: u64 = phase.values().sum(); |
| 155 | let worst = phase.values().copied().max().unwrap_or(0); |
| 156 | counts.push(fmt!("{}B:{}/{}w", len, total, phase.len())); |
| 157 | // R1: the chunks are durable before the head goes out. |
| 158 | if total == 0 { |
| 159 | bad.push(fmt!("{:?} {}x{} {} bytes: no barrier before the head", p, z, w, len)); |
| 160 | } |
| 161 | if let Policy::EveryWrite = p { |
| 162 | continue; |
| 163 | } |
| 164 | // M1: one barrier per writer touched. |
| 165 | if worst > 1 { |
| 166 | bad.push(fmt!("{:?} {}x{} {} bytes: a writer made {} barriers in the chunk phase {:?}", |
| 167 | p, z, w, len, worst, phase)); |
| 168 | } |
| 169 | // M3: one writer per zone per put, each writer touched in a zone of its own. |
| 170 | let zones: BTreeSet<usize> = phase.keys().map(|(zone, _)| *zone).collect(); |
| 171 | if zones.len() != phase.len() || phase.len() > z as usize { |
| 172 | bad.push(fmt!("{:?} {}x{} {} bytes: {} writers barriered in {} distinct zones of {} {:?}", |
| 173 | p, z, w, len, phase.len(), zones.len(), z, phase)); |
| 174 | } |
| 175 | } |
| 176 | println!("BARRIERS {:?} {}x{}: {}", p, z, w, counts.join(" ")); |
| 177 | res!(db.close()); |
| 178 | let _ = std::fs::remove_dir_all(&dir); |
| 179 | } |
| 180 | Ok(bad) |
| 181 | } |
| 182 | |
| 183 | #[test] |
| 184 | fn chunk_phase_interval() -> Outcome<()> { |
| 185 | let _lock = lock(); |
| 186 | let bad = res!(run_layouts(Policy::Interval(200), Duration::from_millis(400), "int")); |
| 187 | if !bad.is_empty() { |
| 188 | return Err(err!("{:?}", bad; Test)); |
| 189 | } |
| 190 | Ok(()) |
| 191 | } |
| 192 | |
| 193 | // The chunks do not count towards every n writes either, so a put of more chunks than n still |
| 194 | // takes one barrier per writer touched. |
| 195 | #[test] |
| 196 | fn chunk_phase_every_n() -> Outcome<()> { |
| 197 | let _lock = lock(); |
| 198 | let bad = res!(run_layouts(Policy::EveryN(3), Duration::ZERO, "everyn")); |
| 199 | if !bad.is_empty() { |
| 200 | return Err(err!("{:?}", bad; Test)); |
| 201 | } |
| 202 | Ok(()) |
| 203 | } |
| 204 | |
| 205 | #[test] |
| 206 | fn chunk_phase_never() -> Outcome<()> { |
| 207 | let _lock = lock(); |
| 208 | let bad = res!(run_layouts(Policy::Never, Duration::ZERO, "never")); |
| 209 | if !bad.is_empty() { |
| 210 | return Err(err!("{:?}", bad; Test)); |
| 211 | } |
| 212 | Ok(()) |
| 213 | } |
| 214 | |
| 215 | // Every write waits on a barrier of its own, as before: counted, and held to R1 only. |
| 216 | #[test] |
| 217 | fn chunk_phase_every_write() -> Outcome<()> { |
| 218 | let _lock = lock(); |
| 219 | let bad = res!(run_layouts(Policy::EveryWrite, Duration::ZERO, "every")); |
| 220 | if !bad.is_empty() { |
| 221 | return Err(err!("{:?}", bad; Test)); |
| 222 | } |
| 223 | Ok(()) |
| 224 | } |
| 225 | |
| 226 | // The interval still bounds the loss of acknowledged writes: a stream of small inserts over a |
| 227 | // second, one every 50 ms, barriers about once per 200 ms period. |
| 228 | #[test] |
| 229 | fn interval_floor_holds_for_plain_writes() -> Outcome<()> { |
| 230 | let _lock = lock(); |
| 231 | let c = res!(cfg(Policy::Interval(200), 1, 1, 16 * 1_048_576)); |
| 232 | let dir = "./test_db_chunk_barriers_floor"; |
| 233 | let db = res!(start(dir, &c)); |
| 234 | res!(db.api().sync_now(wait())); |
| 235 | let before = hooks::barriers_by_writer(); |
| 236 | for i in 0..20u32 { |
| 237 | res!(db.insert(dat!(fmt!("small {}", i)), dat!(i), Uid::default(), None)); |
| 238 | thread::sleep(Duration::from_millis(50)); |
| 239 | } |
| 240 | thread::sleep(Duration::from_millis(300)); |
| 241 | let made: u64 = since(&before).values().sum(); |
| 242 | println!("FLOOR Interval(200) 20 inserts over 1 s: {} barriers", made); |
| 243 | res!(db.close()); |
| 244 | let _ = std::fs::remove_dir_all(dir); |
| 245 | if !(3..=10).contains(&made) { |
| 246 | return Err(err!("{} barriers for a second of writes under a 200 ms interval.", made; Test)); |
| 247 | } |
| 248 | Ok(()) |
| 249 | } |
| 250 | |
| 251 | // A chunk that does not fit rolls the live file, and the dirty old file is sealed with a barrier |
| 252 | // of its own (M2). Documented, not bounded: the count is one plus the rollovers. |
| 253 | #[test] |
| 254 | fn rollover_seals_counted() -> Outcome<()> { |
| 255 | let _lock = lock(); |
| 256 | let c = res!(cfg(Policy::Never, 1, 1, 2_500)); |
| 257 | let dir = "./test_db_chunk_barriers_roll"; |
| 258 | let db = res!(start(dir, &c)); |
| 259 | let mut counts = Vec::new(); |
| 260 | for i in 0..6u8 { |
| 261 | let phase = res!(chunk_phase(&db, dat!(fmt!("roll {}", i)), val(3_000, i))); |
| 262 | counts.push(phase.values().sum::<u64>()); |
| 263 | } |
| 264 | println!("ROLLOVER Never 1x1 2500-byte files, 8-chunk puts: {:?}", counts); |
| 265 | res!(db.close()); |
| 266 | let _ = std::fs::remove_dir_all(dir); |
| 267 | if counts.iter().any(|n| *n == 0) { |
| 268 | return Err(err!("A put's head went out with no barrier over its chunks: {:?}", counts; Test)); |
| 269 | } |
| 270 | Ok(()) |
| 271 | } |