oxedyne/fe2o3/fe2o3_o3db_sync/tests/start_barrier.rs
12.7 KiB, 1 run
created by r1870400018:70621, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! The start barrier (2026-10-09). `O3db::start` returns once every record on disk is applied |
| 2 | //! to its cache, so a get, a put or a ping made the moment it returns waits on no replay. Each |
| 3 | //! cache insert is held 1 ms, so the replay takes about two seconds to apply, and each file's |
| 4 | //! replay is held too, so it is still being queued for a while after the zones begin. The stamp |
| 5 | //! clock is process-wide, so every store is written by a child process of this binary with its |
| 6 | //! clock a minute ahead (`sb_child`, env SB_MODE / SB_DIR / SB_T), and the put after a reopen is |
| 7 | //! made by another child. |
| 8 | |
| 9 | mod gc_pair; |
| 10 | |
| 11 | use gc_pair::*; |
| 12 | |
| 13 | use oxedyne_fe2o3_core::prelude::*; |
| 14 | use oxedyne_fe2o3_iop_db::api::Database; |
| 15 | use oxedyne_fe2o3_jdat::prelude::*; |
| 16 | use oxedyne_fe2o3_o3db_sync::{ |
| 17 | base::{ |
| 18 | cfg::OzoneConfig, |
| 19 | constant, |
| 20 | }, |
| 21 | db::O3db, |
| 22 | test::{ |
| 23 | hooks, |
| 24 | setup::{ |
| 25 | self, |
| 26 | Uid, |
| 27 | }, |
| 28 | }, |
| 29 | }; |
| 30 | |
| 31 | use std::{ |
| 32 | path::Path, |
| 33 | process::Command, |
| 34 | sync::{ |
| 35 | Arc, |
| 36 | Mutex, |
| 37 | MutexGuard, |
| 38 | atomic::{ |
| 39 | AtomicUsize, |
| 40 | Ordering, |
| 41 | }, |
| 42 | }, |
| 43 | thread, |
| 44 | time::{ |
| 45 | Duration, |
| 46 | Instant, |
| 47 | SystemTime, |
| 48 | UNIX_EPOCH, |
| 49 | }, |
| 50 | }; |
| 51 | |
| 52 | const NSMALL: usize = 3_000; // small keys, each written twice |
| 53 | const NCHUNKED: usize = 10; // chunked keys, each written twice |
| 54 | const NPROBE: usize = 100; // the last small keys written, put again after a reopen |
| 55 | const ZONES: u16 = 2; |
| 56 | const CBOTS: u16 = 2; // per zone |
| 57 | const INSERT_MS: u64 = 1; // each cache insert is held this long |
| 58 | const FILE_MS: u64 = 25; // each file's replay is held this long |
| 59 | |
| 60 | static HOOKS: Mutex<()> = Mutex::new(()); |
| 61 | |
| 62 | fn lock() -> MutexGuard<'static, ()> { |
| 63 | match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() } |
| 64 | } |
| 65 | |
| 66 | fn cfg() -> Outcome<OzoneConfig> { |
| 67 | let mut c = res!(setup::default_cfg()); |
| 68 | c.num_zones = ZONES; |
| 69 | c.num_cbots_per_zone = CBOTS; |
| 70 | c.num_igbots_per_zone = 2; |
| 71 | c.num_wbots_per_zone = 1; |
| 72 | c.data_file_max_bytes = 8_192; |
| 73 | c.rest_chunk_threshold = 1_000; |
| 74 | c.rest_chunk_bytes = 400; |
| 75 | c.sync_on_write = false; |
| 76 | c.zone_overrides = mapdat!{ |
| 77 | 1u16 => mapdat!{ "dir" => "", "max_size" => 100_000_000u64 }, |
| 78 | 2u16 => mapdat!{ "dir" => "", "max_size" => 100_000_000u64 }, |
| 79 | }.get_map().unwrap_or_default(); |
| 80 | Ok(c) |
| 81 | } |
| 82 | |
| 83 | fn small_key(i: usize) -> Dat { dat!(fmt!("sb {:04}", i)) } |
| 84 | fn small_val(i: usize, ver: u8) -> Dat { dat!(fmt!("v{} of {:04}", ver, i)) } |
| 85 | fn chunked_key(i: usize) -> Dat { dat!(fmt!("sb chunked {}", i)) } |
| 86 | |
| 87 | fn chunked_val(i: usize, ver: u8) -> Dat { |
| 88 | let s = (i as u8).wrapping_mul(16) ^ ver; |
| 89 | Dat::BU32((0..2_500usize).map(|j| s ^ (j as u8)).collect()) |
| 90 | } |
| 91 | |
| 92 | fn hooks_on() { |
| 93 | hooks::set_insert_delay(Duration::from_millis(INSERT_MS)); |
| 94 | hooks::set_replay_file_delay(Duration::from_millis(FILE_MS)); |
| 95 | } |
| 96 | |
| 97 | fn hooks_off() { |
| 98 | hooks::set_insert_delay(Duration::ZERO); |
| 99 | hooks::set_replay_file_delay(Duration::ZERO); |
| 100 | } |
| 101 | |
| 102 | /// Opens the store as the gateway does: `start`, then garbage collection switched at once. The |
| 103 | /// write phase leaves it off, so every replaced record is still on disk to be replayed. |
| 104 | fn open(dir: &str, wipe: bool, gc: bool) -> Outcome<(TestDb, Duration)> { |
| 105 | log_set_level!("error"); |
| 106 | if wipe { |
| 107 | let _ = std::fs::remove_dir_all(dir); |
| 108 | } |
| 109 | res!(std::fs::create_dir_all(dir)); |
| 110 | let root = res!(Path::new(dir).canonicalize()); |
| 111 | if wipe { |
| 112 | let _ = std::fs::remove_file(OzoneConfig::config_path(&root)); |
| 113 | } |
| 114 | let mut db = res!(O3db::new(root, Some(res!(cfg())), schemes(), Uid::default())); |
| 115 | let t = Instant::now(); |
| 116 | res!(db.start("start_barrier")); |
| 117 | let took = t.elapsed(); |
| 118 | res!(ok!(db.updated_api()).activate_gc(gc)); |
| 119 | Ok((db, took)) |
| 120 | } |
| 121 | |
| 122 | // The replayed records a cache bot holds on average: every write, with each chunked value's |
| 123 | // bunch key and chunks. |
| 124 | fn inserts_per_cbot() -> u64 { |
| 125 | let per_chunked = 1 + 2_500 / 400 + 1; |
| 126 | (2 * (NSMALL + NCHUNKED * per_chunked) / (ZONES * CBOTS) as usize) as u64 |
| 127 | } |
| 128 | |
| 129 | // --------------------------------------------------------------------------------------------- |
| 130 | // Child-process phases. |
| 131 | |
| 132 | fn child(mode: &str, dir: &str) -> Outcome<String> { |
| 133 | let exe = res!(std::env::current_exe()); |
| 134 | let ahead = res!(SystemTime::now().duration_since(UNIX_EPOCH)) + Duration::from_secs(60); |
| 135 | let out = res!(Command::new(exe) |
| 136 | .args(["sb_child", "--exact", "--ignored", "--nocapture", "--test-threads=1"]) |
| 137 | .env("SB_MODE", mode) |
| 138 | .env("SB_DIR", dir) |
| 139 | .env("SB_T", fmt!("{}", ahead.as_nanos())) |
| 140 | .output()); |
| 141 | let so = String::from_utf8_lossy(&out.stdout).to_string(); |
| 142 | // The harness prints `test sb_child ... ` with no newline, so a child's first line follows it. |
| 143 | let lines: Vec<&str> = so.lines().filter_map(|l| l.find("SB ").map(|i| &l[i..])).collect(); |
| 144 | Ok(fmt!("status={:?}\n{}", out.status.code(), lines.join("\n"))) |
| 145 | } |
| 146 | |
| 147 | // Each probe key read back, "ok" when it holds the value put after the reopen. |
| 148 | fn probes(db: &TestDb) -> (usize, Vec<String>) { |
| 149 | let mut ok = 0; |
| 150 | let mut bad = Vec::new(); |
| 151 | for i in NSMALL - NPROBE..NSMALL { |
| 152 | match db.get(&small_key(i), None) { |
| 153 | Ok(Some((v, _))) if v == small_val(i, 3) => ok += 1, |
| 154 | other => bad.push(fmt!("{}:{:?}", i, other.map(|o| o.map(|(v, _)| v)))), |
| 155 | } |
| 156 | } |
| 157 | match db.get(&chunked_key(0), None) { |
| 158 | Ok(Some((v, _))) if v == chunked_val(0, 3) => ok += 1, |
| 159 | _ => bad.push(fmt!("chunked 0")), |
| 160 | } |
| 161 | bad.truncate(5); |
| 162 | (ok, bad) |
| 163 | } |
| 164 | |
| 165 | #[test] |
| 166 | #[ignore] |
| 167 | fn sb_child() -> Outcome<()> { |
| 168 | let mode = match std::env::var("SB_MODE") { Ok(m) => m, Err(_) => return Ok(()) }; |
| 169 | let dir = res!(std::env::var("SB_DIR")); |
| 170 | match mode.as_str() { |
| 171 | // The store, written while the clock read a minute ahead. The chunked keys go first, so |
| 172 | // the last records written, the highest stamps, are the probe keys' second versions. |
| 173 | "write" => { |
| 174 | let t: u128 = res!(res!(std::env::var("SB_T")).parse::<u128>()); |
| 175 | let (db, _) = res!(open(&dir, true, false)); |
| 176 | hooks::set_fixed_stamp(Some(Duration::new( |
| 177 | (t / 1_000_000_000) as u64, (t % 1_000_000_000) as u32))); |
| 178 | for ver in 1..3u8 { |
| 179 | for i in 0..NCHUNKED { |
| 180 | res!(db.insert(chunked_key(i), chunked_val(i, ver), Uid::default(), None)); |
| 181 | } |
| 182 | } |
| 183 | for ver in 1..3u8 { |
| 184 | let mut ts = Vec::new(); |
| 185 | for w in 0..4usize { |
| 186 | let db = db.clone(); |
| 187 | ts.push(thread::spawn(move || -> Outcome<()> { |
| 188 | for i in (w..NSMALL - NPROBE).step_by(4) { |
| 189 | res!(db.insert(small_key(i), small_val(i, ver), Uid::default(), None)); |
| 190 | } |
| 191 | Ok(()) |
| 192 | })); |
| 193 | } |
| 194 | for t in ts { |
| 195 | match t.join() { |
| 196 | Ok(r) => res!(r), |
| 197 | Err(_) => return Err(err!("A writer thread panicked."; Bug)), |
| 198 | } |
| 199 | } |
| 200 | } |
| 201 | for ver in 1..3u8 { |
| 202 | for i in NSMALL - NPROBE..NSMALL { |
| 203 | res!(db.insert(small_key(i), small_val(i, ver), Uid::default(), None)); |
| 204 | } |
| 205 | } |
| 206 | hooks::set_fixed_stamp(None); |
| 207 | res!(db.close()); |
| 208 | println!("SB written"); |
| 209 | }, |
| 210 | // A put of every probe key the moment `start` returns, with the replay still to apply |
| 211 | // were `start` not to wait for it, then the same reads after a restart. |
| 212 | "put" => { |
| 213 | hooks_on(); |
| 214 | let (db, took) = res!(open(&dir, false, true)); |
| 215 | for i in NSMALL - NPROBE..NSMALL { |
| 216 | res!(db.insert(small_key(i), small_val(i, 3), Uid::default(), None)); |
| 217 | } |
| 218 | res!(db.insert(chunked_key(0), chunked_val(0, 3), Uid::default(), None)); |
| 219 | let (ok, bad) = probes(&db); |
| 220 | println!("SB put start_ms={} ok={} bad={:?}", took.as_millis(), ok, bad); |
| 221 | res!(db.close()); |
| 222 | hooks_off(); |
| 223 | let (db, _) = res!(open(&dir, false, true)); |
| 224 | let (ok, bad) = probes(&db); |
| 225 | println!("SB restart ok={} bad={:?}", ok, bad); |
| 226 | res!(db.close()); |
| 227 | }, |
| 228 | m => return Err(err!("Unknown child mode {}.", m; Invalid, Input)), |
| 229 | } |
| 230 | Ok(()) |
| 231 | } |
| 232 | |
| 233 | // --------------------------------------------------------------------------------------------- |
| 234 | |
| 235 | /// Every key, read by 8 threads the moment `start` returns, holds its second version, no read |
| 236 | /// errs, and a ping then is answered at once. `start` itself must have taken at least half the |
| 237 | /// replay's apply time, or the delay that makes the window did not act. |
| 238 | #[test] |
| 239 | fn a_get_at_once_after_start_finds_every_key() -> Outcome<()> { |
| 240 | let _g = lock(); |
| 241 | let dir = "./test_db_start_barrier_get"; |
| 242 | let out = res!(child("write", dir)); |
| 243 | assert!(out.contains("SB written"), "The write phase failed: {}", out); |
| 244 | |
| 245 | hooks_on(); |
| 246 | let (db, took) = res!(open(dir, false, true)); |
| 247 | let t = Instant::now(); |
| 248 | let ping = db.api().ping_bots(constant::USER_REQUEST_WAIT); |
| 249 | let ping_ms = t.elapsed().as_millis(); |
| 250 | |
| 251 | let absent = Arc::new(AtomicUsize::new(0)); |
| 252 | let stale = Arc::new(AtomicUsize::new(0)); |
| 253 | let errs = Arc::new(AtomicUsize::new(0)); |
| 254 | let first = Arc::new(Mutex::new(Vec::<String>::new())); |
| 255 | let mut ts = Vec::new(); |
| 256 | for w in 0..8usize { |
| 257 | let (db, absent, stale, errs, first) = |
| 258 | (db.clone(), absent.clone(), stale.clone(), errs.clone(), first.clone()); |
| 259 | ts.push(thread::spawn(move || { |
| 260 | let note = |what: String| { |
| 261 | if let Ok(mut f) = first.lock() { if f.len() < 5 { f.push(what); } } |
| 262 | }; |
| 263 | // Newest first: those keys are in the files replayed last. |
| 264 | for i in (w..NSMALL + NCHUNKED).step_by(8).rev() { |
| 265 | let (k, want) = match i < NSMALL { |
| 266 | true => (small_key(i), small_val(i, 2)), |
| 267 | false => (chunked_key(i - NSMALL), chunked_val(i - NSMALL, 2)), |
| 268 | }; |
| 269 | match db.get(&k, None) { |
| 270 | Ok(Some((v, _))) if v == want => (), |
| 271 | Ok(Some(_)) => { stale.fetch_add(1, Ordering::SeqCst); note(fmt!("stale {}", i)); }, |
| 272 | Ok(None) => { absent.fetch_add(1, Ordering::SeqCst); note(fmt!("absent {}", i)); }, |
| 273 | Err(e) => { errs.fetch_add(1, Ordering::SeqCst); note(fmt!("error {}: {}", i, e)); }, |
| 274 | } |
| 275 | } |
| 276 | })); |
| 277 | } |
| 278 | for t in ts { |
| 279 | assert!(t.join().is_ok(), "A reader thread panicked."); |
| 280 | } |
| 281 | let gets_ms = t.elapsed().as_millis(); |
| 282 | hooks_off(); |
| 283 | res!(db.close()); |
| 284 | |
| 285 | let apply_ms = inserts_per_cbot() * INSERT_MS; |
| 286 | println!("start_barrier: start {} ms (replay apply about {} ms per cache bot), ping {} ms, \ |
| 287 | every get done {} ms after start", took.as_millis(), apply_ms, ping_ms, gets_ms); |
| 288 | let first = match first.lock() { Ok(f) => f.clone(), Err(p) => p.into_inner().clone() }; |
| 289 | assert_eq!((absent.load(Ordering::SeqCst), stale.load(Ordering::SeqCst), errs.load(Ordering::SeqCst)), |
| 290 | (0, 0, 0), "Reads the moment start returned (absent, stale, errors): {:?}", first); |
| 291 | assert!(ping.is_ok(), "The ping right after start failed: {:?}", ping.err()); |
| 292 | assert!(ping_ms < 250, |
| 293 | "The ping right after start took {} ms: start returned with replayed inserts still queued \ |
| 294 | at the cache bots.", ping_ms); |
| 295 | assert!(took.as_millis() as u64 >= apply_ms / 2, |
| 296 | "start took {} ms, under half the {} ms the held inserts take to apply, so the hook did \ |
| 297 | not hold them.", took.as_millis(), apply_ms); |
| 298 | let _ = std::fs::remove_dir_all(dir); |
| 299 | Ok(()) |
| 300 | } |
| 301 | |
| 302 | /// A put the moment `start` returns, of keys whose last records on disk carry the highest stamps |
| 303 | /// and were written a minute ahead of this clock, outranks them, live and after a restart. |
| 304 | #[test] |
| 305 | fn a_put_at_once_after_start_outranks_its_replayed_record() -> Outcome<()> { |
| 306 | let _g = lock(); |
| 307 | let dir = "./test_db_start_barrier_put"; |
| 308 | let out = res!(child("write", dir)); |
| 309 | assert!(out.contains("SB written"), "The write phase failed: {}", out); |
| 310 | let out = res!(child("put", dir)); |
| 311 | println!("start_barrier put: {}", out.replace('\n', " | ")); |
| 312 | let want = fmt!("ok={} bad=[]", NPROBE + 1); |
| 313 | assert!(out.lines().any(|l| l.starts_with("SB put ") && l.ends_with(&want)), |
| 314 | "A put the moment start returned lost to its replayed record: {}", out); |
| 315 | assert!(out.lines().any(|l| l.starts_with("SB restart ") && l.ends_with(&want)), |
| 316 | "A put the moment start returned lost to its replayed record after a restart: {}", out); |
| 317 | let _ = std::fs::remove_dir_all(dir); |
| 318 | Ok(()) |
| 319 | } |