Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/start_barrier.rs

12.7 KiB, 1 run

created by r1870400018:70621, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! The start barrier (2026-10-09). `O3db::start` returns once every record on disk is applied
2//! to its cache, so a get, a put or a ping made the moment it returns waits on no replay. Each
3//! cache insert is held 1 ms, so the replay takes about two seconds to apply, and each file's
4//! replay is held too, so it is still being queued for a while after the zones begin. The stamp
5//! clock is process-wide, so every store is written by a child process of this binary with its
6//! clock a minute ahead (`sb_child`, env SB_MODE / SB_DIR / SB_T), and the put after a reopen is
7//! made by another child.
8
9mod gc_pair;
10
11use gc_pair::*;
12
13use oxedyne_fe2o3_core::prelude::*;
14use oxedyne_fe2o3_iop_db::api::Database;
15use oxedyne_fe2o3_jdat::prelude::*;
16use oxedyne_fe2o3_o3db_sync::{
17 base::{
18 cfg::OzoneConfig,
19 constant,
20 },
21 db::O3db,
22 test::{
23 hooks,
24 setup::{
25 self,
26 Uid,
27 },
28 },
29};
30
31use std::{
32 path::Path,
33 process::Command,
34 sync::{
35 Arc,
36 Mutex,
37 MutexGuard,
38 atomic::{
39 AtomicUsize,
40 Ordering,
41 },
42 },
43 thread,
44 time::{
45 Duration,
46 Instant,
47 SystemTime,
48 UNIX_EPOCH,
49 },
50};
51
52const NSMALL: usize = 3_000; // small keys, each written twice
53const NCHUNKED: usize = 10; // chunked keys, each written twice
54const NPROBE: usize = 100; // the last small keys written, put again after a reopen
55const ZONES: u16 = 2;
56const CBOTS: u16 = 2; // per zone
57const INSERT_MS: u64 = 1; // each cache insert is held this long
58const FILE_MS: u64 = 25; // each file's replay is held this long
59
60static HOOKS: Mutex<()> = Mutex::new(());
61
62fn lock() -> MutexGuard<'static, ()> {
63 match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() }
64}
65
66fn cfg() -> Outcome<OzoneConfig> {
67 let mut c = res!(setup::default_cfg());
68 c.num_zones = ZONES;
69 c.num_cbots_per_zone = CBOTS;
70 c.num_igbots_per_zone = 2;
71 c.num_wbots_per_zone = 1;
72 c.data_file_max_bytes = 8_192;
73 c.rest_chunk_threshold = 1_000;
74 c.rest_chunk_bytes = 400;
75 c.sync_on_write = false;
76 c.zone_overrides = mapdat!{
77 1u16 => mapdat!{ "dir" => "", "max_size" => 100_000_000u64 },
78 2u16 => mapdat!{ "dir" => "", "max_size" => 100_000_000u64 },
79 }.get_map().unwrap_or_default();
80 Ok(c)
81}
82
83fn small_key(i: usize) -> Dat { dat!(fmt!("sb {:04}", i)) }
84fn small_val(i: usize, ver: u8) -> Dat { dat!(fmt!("v{} of {:04}", ver, i)) }
85fn chunked_key(i: usize) -> Dat { dat!(fmt!("sb chunked {}", i)) }
86
87fn chunked_val(i: usize, ver: u8) -> Dat {
88 let s = (i as u8).wrapping_mul(16) ^ ver;
89 Dat::BU32((0..2_500usize).map(|j| s ^ (j as u8)).collect())
90}
91
92fn hooks_on() {
93 hooks::set_insert_delay(Duration::from_millis(INSERT_MS));
94 hooks::set_replay_file_delay(Duration::from_millis(FILE_MS));
95}
96
97fn hooks_off() {
98 hooks::set_insert_delay(Duration::ZERO);
99 hooks::set_replay_file_delay(Duration::ZERO);
100}
101
102/// Opens the store as the gateway does: `start`, then garbage collection switched at once. The
103/// write phase leaves it off, so every replaced record is still on disk to be replayed.
104fn open(dir: &str, wipe: bool, gc: bool) -> Outcome<(TestDb, Duration)> {
105 log_set_level!("error");
106 if wipe {
107 let _ = std::fs::remove_dir_all(dir);
108 }
109 res!(std::fs::create_dir_all(dir));
110 let root = res!(Path::new(dir).canonicalize());
111 if wipe {
112 let _ = std::fs::remove_file(OzoneConfig::config_path(&root));
113 }
114 let mut db = res!(O3db::new(root, Some(res!(cfg())), schemes(), Uid::default()));
115 let t = Instant::now();
116 res!(db.start("start_barrier"));
117 let took = t.elapsed();
118 res!(ok!(db.updated_api()).activate_gc(gc));
119 Ok((db, took))
120}
121
122// The replayed records a cache bot holds on average: every write, with each chunked value's
123// bunch key and chunks.
124fn inserts_per_cbot() -> u64 {
125 let per_chunked = 1 + 2_500 / 400 + 1;
126 (2 * (NSMALL + NCHUNKED * per_chunked) / (ZONES * CBOTS) as usize) as u64
127}
128
129// ---------------------------------------------------------------------------------------------
130// Child-process phases.
131
132fn child(mode: &str, dir: &str) -> Outcome<String> {
133 let exe = res!(std::env::current_exe());
134 let ahead = res!(SystemTime::now().duration_since(UNIX_EPOCH)) + Duration::from_secs(60);
135 let out = res!(Command::new(exe)
136 .args(["sb_child", "--exact", "--ignored", "--nocapture", "--test-threads=1"])
137 .env("SB_MODE", mode)
138 .env("SB_DIR", dir)
139 .env("SB_T", fmt!("{}", ahead.as_nanos()))
140 .output());
141 let so = String::from_utf8_lossy(&out.stdout).to_string();
142 // The harness prints `test sb_child ... ` with no newline, so a child's first line follows it.
143 let lines: Vec<&str> = so.lines().filter_map(|l| l.find("SB ").map(|i| &l[i..])).collect();
144 Ok(fmt!("status={:?}\n{}", out.status.code(), lines.join("\n")))
145}
146
147// Each probe key read back, "ok" when it holds the value put after the reopen.
148fn probes(db: &TestDb) -> (usize, Vec<String>) {
149 let mut ok = 0;
150 let mut bad = Vec::new();
151 for i in NSMALL - NPROBE..NSMALL {
152 match db.get(&small_key(i), None) {
153 Ok(Some((v, _))) if v == small_val(i, 3) => ok += 1,
154 other => bad.push(fmt!("{}:{:?}", i, other.map(|o| o.map(|(v, _)| v)))),
155 }
156 }
157 match db.get(&chunked_key(0), None) {
158 Ok(Some((v, _))) if v == chunked_val(0, 3) => ok += 1,
159 _ => bad.push(fmt!("chunked 0")),
160 }
161 bad.truncate(5);
162 (ok, bad)
163}
164
165#[test]
166#[ignore]
167fn sb_child() -> Outcome<()> {
168 let mode = match std::env::var("SB_MODE") { Ok(m) => m, Err(_) => return Ok(()) };
169 let dir = res!(std::env::var("SB_DIR"));
170 match mode.as_str() {
171 // The store, written while the clock read a minute ahead. The chunked keys go first, so
172 // the last records written, the highest stamps, are the probe keys' second versions.
173 "write" => {
174 let t: u128 = res!(res!(std::env::var("SB_T")).parse::<u128>());
175 let (db, _) = res!(open(&dir, true, false));
176 hooks::set_fixed_stamp(Some(Duration::new(
177 (t / 1_000_000_000) as u64, (t % 1_000_000_000) as u32)));
178 for ver in 1..3u8 {
179 for i in 0..NCHUNKED {
180 res!(db.insert(chunked_key(i), chunked_val(i, ver), Uid::default(), None));
181 }
182 }
183 for ver in 1..3u8 {
184 let mut ts = Vec::new();
185 for w in 0..4usize {
186 let db = db.clone();
187 ts.push(thread::spawn(move || -> Outcome<()> {
188 for i in (w..NSMALL - NPROBE).step_by(4) {
189 res!(db.insert(small_key(i), small_val(i, ver), Uid::default(), None));
190 }
191 Ok(())
192 }));
193 }
194 for t in ts {
195 match t.join() {
196 Ok(r) => res!(r),
197 Err(_) => return Err(err!("A writer thread panicked."; Bug)),
198 }
199 }
200 }
201 for ver in 1..3u8 {
202 for i in NSMALL - NPROBE..NSMALL {
203 res!(db.insert(small_key(i), small_val(i, ver), Uid::default(), None));
204 }
205 }
206 hooks::set_fixed_stamp(None);
207 res!(db.close());
208 println!("SB written");
209 },
210 // A put of every probe key the moment `start` returns, with the replay still to apply
211 // were `start` not to wait for it, then the same reads after a restart.
212 "put" => {
213 hooks_on();
214 let (db, took) = res!(open(&dir, false, true));
215 for i in NSMALL - NPROBE..NSMALL {
216 res!(db.insert(small_key(i), small_val(i, 3), Uid::default(), None));
217 }
218 res!(db.insert(chunked_key(0), chunked_val(0, 3), Uid::default(), None));
219 let (ok, bad) = probes(&db);
220 println!("SB put start_ms={} ok={} bad={:?}", took.as_millis(), ok, bad);
221 res!(db.close());
222 hooks_off();
223 let (db, _) = res!(open(&dir, false, true));
224 let (ok, bad) = probes(&db);
225 println!("SB restart ok={} bad={:?}", ok, bad);
226 res!(db.close());
227 },
228 m => return Err(err!("Unknown child mode {}.", m; Invalid, Input)),
229 }
230 Ok(())
231}
232
233// ---------------------------------------------------------------------------------------------
234
235/// Every key, read by 8 threads the moment `start` returns, holds its second version, no read
236/// errs, and a ping then is answered at once. `start` itself must have taken at least half the
237/// replay's apply time, or the delay that makes the window did not act.
238#[test]
239fn a_get_at_once_after_start_finds_every_key() -> Outcome<()> {
240 let _g = lock();
241 let dir = "./test_db_start_barrier_get";
242 let out = res!(child("write", dir));
243 assert!(out.contains("SB written"), "The write phase failed: {}", out);
244
245 hooks_on();
246 let (db, took) = res!(open(dir, false, true));
247 let t = Instant::now();
248 let ping = db.api().ping_bots(constant::USER_REQUEST_WAIT);
249 let ping_ms = t.elapsed().as_millis();
250
251 let absent = Arc::new(AtomicUsize::new(0));
252 let stale = Arc::new(AtomicUsize::new(0));
253 let errs = Arc::new(AtomicUsize::new(0));
254 let first = Arc::new(Mutex::new(Vec::<String>::new()));
255 let mut ts = Vec::new();
256 for w in 0..8usize {
257 let (db, absent, stale, errs, first) =
258 (db.clone(), absent.clone(), stale.clone(), errs.clone(), first.clone());
259 ts.push(thread::spawn(move || {
260 let note = |what: String| {
261 if let Ok(mut f) = first.lock() { if f.len() < 5 { f.push(what); } }
262 };
263 // Newest first: those keys are in the files replayed last.
264 for i in (w..NSMALL + NCHUNKED).step_by(8).rev() {
265 let (k, want) = match i < NSMALL {
266 true => (small_key(i), small_val(i, 2)),
267 false => (chunked_key(i - NSMALL), chunked_val(i - NSMALL, 2)),
268 };
269 match db.get(&k, None) {
270 Ok(Some((v, _))) if v == want => (),
271 Ok(Some(_)) => { stale.fetch_add(1, Ordering::SeqCst); note(fmt!("stale {}", i)); },
272 Ok(None) => { absent.fetch_add(1, Ordering::SeqCst); note(fmt!("absent {}", i)); },
273 Err(e) => { errs.fetch_add(1, Ordering::SeqCst); note(fmt!("error {}: {}", i, e)); },
274 }
275 }
276 }));
277 }
278 for t in ts {
279 assert!(t.join().is_ok(), "A reader thread panicked.");
280 }
281 let gets_ms = t.elapsed().as_millis();
282 hooks_off();
283 res!(db.close());
284
285 let apply_ms = inserts_per_cbot() * INSERT_MS;
286 println!("start_barrier: start {} ms (replay apply about {} ms per cache bot), ping {} ms, \
287 every get done {} ms after start", took.as_millis(), apply_ms, ping_ms, gets_ms);
288 let first = match first.lock() { Ok(f) => f.clone(), Err(p) => p.into_inner().clone() };
289 assert_eq!((absent.load(Ordering::SeqCst), stale.load(Ordering::SeqCst), errs.load(Ordering::SeqCst)),
290 (0, 0, 0), "Reads the moment start returned (absent, stale, errors): {:?}", first);
291 assert!(ping.is_ok(), "The ping right after start failed: {:?}", ping.err());
292 assert!(ping_ms < 250,
293 "The ping right after start took {} ms: start returned with replayed inserts still queued \
294 at the cache bots.", ping_ms);
295 assert!(took.as_millis() as u64 >= apply_ms / 2,
296 "start took {} ms, under half the {} ms the held inserts take to apply, so the hook did \
297 not hold them.", took.as_millis(), apply_ms);
298 let _ = std::fs::remove_dir_all(dir);
299 Ok(())
300}
301
302/// A put the moment `start` returns, of keys whose last records on disk carry the highest stamps
303/// and were written a minute ahead of this clock, outranks them, live and after a restart.
304#[test]
305fn a_put_at_once_after_start_outranks_its_replayed_record() -> Outcome<()> {
306 let _g = lock();
307 let dir = "./test_db_start_barrier_put";
308 let out = res!(child("write", dir));
309 assert!(out.contains("SB written"), "The write phase failed: {}", out);
310 let out = res!(child("put", dir));
311 println!("start_barrier put: {}", out.replace('\n', " | "));
312 let want = fmt!("ok={} bad=[]", NPROBE + 1);
313 assert!(out.lines().any(|l| l.starts_with("SB put ") && l.ends_with(&want)),
314 "A put the moment start returned lost to its replayed record: {}", out);
315 assert!(out.lines().any(|l| l.starts_with("SB restart ") && l.ends_with(&want)),
316 "A put the moment start returned lost to its replayed record after a restart: {}", out);
317 let _ = std::fs::remove_dir_all(dir);
318 Ok(())
319}