Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/durable_mark_races.rs

15.6 KiB, 8 runs

created by r1870400018:70850, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! Races of the durable mark (2026-10-10, from QA agent 2's probes P1, P5 and P6). A chunked
2//! put's answer waits for the barrier over its chunks, a power cut taken at random through a run
3//! with automatic collection on never reads a torn, lost or stale value, and a close racing a
4//! writer leaves every live file covered by its mark. The hooks are process-wide, so the tests
5//! run one at a time.
6
7mod gc_pair;
8
9use gc_pair::*;
10
11use oxedyne_fe2o3_core::prelude::*;
12use oxedyne_fe2o3_iop_db::api::{
13 Database,
14 Meta,
15};
16use oxedyne_fe2o3_jdat::prelude::*;
17use oxedyne_fe2o3_o3db_sync::{
18 base::{
19 cfg::OzoneConfig,
20 constant,
21 },
22 comm::response::Wait,
23 file::{
24 core::FileType,
25 zdir::ZoneDir,
26 },
27 sweep,
28 test::{
29 hooks,
30 setup::{
31 self,
32 Uid,
33 UID_LEN,
34 },
35 },
36 verify,
37};
38
39use std::{
40 collections::BTreeMap,
41 fs,
42 path::{
43 Path,
44 PathBuf,
45 },
46 sync::{
47 Arc,
48 Mutex,
49 atomic::{
50 AtomicBool,
51 Ordering,
52 },
53 },
54 time::{
55 Duration,
56 Instant,
57 },
58};
59
60static SERIAL: Mutex<()> = Mutex::new(());
61
62#[derive(Clone, Copy, Debug)]
63enum Policy {
64 EveryWrite,
65 EveryN(u32),
66 Interval(u64),
67 Never,
68}
69
70const POLICIES: [Policy; 4] = [
71 Policy::EveryWrite,
72 Policy::EveryN(100_000),
73 Policy::Interval(3_600_000),
74 Policy::Never,
75];
76
77fn cfg(p: Policy, zones: u16, wbots: u16) -> Outcome<OzoneConfig> {
78 let mut c = res!(setup::default_cfg());
79 c.num_zones = zones;
80 c.num_cbots_per_zone = 4;
81 c.num_fbots_per_zone = 1;
82 c.num_wbots_per_zone = wbots;
83 c.zone_overrides = BTreeMap::new();
84 c.rest_chunk_threshold = 1_000;
85 c.rest_chunk_bytes = 400;
86 c.sync_on_write = false;
87 c.sync_every_n_writes = 0;
88 c.sync_interval_ms = 0;
89 match p {
90 Policy::EveryWrite => c.sync_on_write = true,
91 Policy::EveryN(n) => c.sync_every_n_writes = n,
92 Policy::Interval(t) => c.sync_interval_ms = t,
93 Policy::Never => (),
94 }
95 Ok(c)
96}
97
98fn lock() -> std::sync::MutexGuard<'static, ()> {
99 let g = match SERIAL.lock() { Ok(g) => g, Err(p) => p.into_inner() };
100 hooks::set_barrier_failure(false);
101 hooks::set_barrier_failure_for(None);
102 hooks::set_fault_record_fails(false);
103 hooks::set_owed_drop(false);
104 hooks::set_barrier_delay(Duration::ZERO);
105 g
106}
107
108fn val(len: usize, seed: u8) -> Dat {
109 Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect())
110}
111
112fn fresh(name: &str) -> Outcome<PathBuf> {
113 let dir = fmt!("./test_db_races_{}", name);
114 let _ = fs::remove_dir_all(&dir);
115 res!(fs::create_dir_all(&dir));
116 Ok(res!(Path::new(&dir).canonicalize()))
117}
118
119fn open(root: &Path, c: &OzoneConfig, gc: bool, wipe: bool) -> Outcome<TestDb> {
120 log_set_level!("error");
121 setup::start_db(root.to_path_buf(), Some(c.clone()), schemes(), None, gc, wipe)
122}
123
124fn wait() -> Wait {
125 Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL }
126}
127
128fn put(db: &TestDb, k: &Dat, v: Dat) -> Outcome<()> {
129 db.insert(k.clone(), v, Uid::default(), None).map(|_| ())
130}
131
132fn describe(r: &Outcome<Option<(Dat, Meta<{ UID_LEN }, Uid>)>>) -> String {
133 match r {
134 Ok(None) => fmt!("none"),
135 Ok(Some((Dat::BU32(b), _))) | Ok(Some((Dat::BU8(b), _))) | Ok(Some((Dat::BU16(b), _))) |
136 Ok(Some((Dat::BU64(b), _))) if !b.is_empty()
137 && b.iter().enumerate().all(|(j, x)| *x == b[0] ^ (j as u8)) => fmt!("{}/{}", b.len(), b[0]),
138 Ok(Some((d, _))) => fmt!("TORN:{:?}", d).chars().take(100).collect(),
139 Err(e) => fmt!("ERR:{}", fmt!("{}", e).chars().take(200).collect::<String>()),
140 }
141}
142
143fn read(db: &TestDb, k: &Dat) -> String {
144 describe(&db.get(k, None))
145}
146
147fn zdir(root: &Path, c: &OzoneConfig, z: usize) -> PathBuf {
148 c.zone_root(root).join(fmt!("zone_{:03}", z + 1))
149}
150
151/// The power cut of `durable_mark.rs`, at one instant (`gc_pair::instant_cut`).
152fn power_cut(db: &TestDb, root: &Path, c: &OzoneConfig, to: &str) -> Outcome<PathBuf> {
153 let dst = res!(fresh(to));
154 res!(instant_cut(db, root, c, &dst));
155 Ok(dst)
156}
157
158fn read_after_cut(db: &TestDb, root: &Path, c: &OzoneConfig, to: &str, keys: &[Dat]) -> Outcome<Vec<String>> {
159 let cut = res!(power_cut(db, root, c, to));
160 let db2 = res!(open(&cut, c, false, false));
161 let got = keys.iter().map(|k| read(&db2, k)).collect();
162 res!(db2.close());
163 let _ = fs::remove_dir_all(&cut);
164 Ok(got)
165}
166
167// ---------------------------------------------------------------------------------------------
168// R1 (P1, M8). A chunked put's caller waits on the barrier over its chunks (M8 closer), and a power
169// cut while that barrier is in progress gives the old value, never the new head without chunks.
170#[test]
171fn a_chunk_answer_waits_for_its_barrier() -> Outcome<()> {
172 let _g = lock();
173 for p in [Policy::Interval(3_600_000), Policy::Interval(200), Policy::Never, Policy::EveryN(100_000)] {
174 for (z, w) in [(1u16, 1u16), (2, 2)] {
175 let label = fmt!("{:?} {}x{}", p, z, w);
176 let c = res!(cfg(p, z, w));
177 let root = res!(fresh("p1"));
178 let db = Arc::new(res!(open(&root, &c, false, true)));
179 let k = dat!("races p1 chunked");
180 res!(put(&db, &k, val(3_000, 1)));
181 res!(db.api().sync_now(wait()));
182 let delay = Duration::from_millis(900);
183 hooks::set_barrier_delay(delay);
184 let db2 = db.clone();
185 let k2 = k.clone();
186 let t0 = Instant::now();
187 let h = std::thread::spawn(move || {
188 let r = put(&db2, &k2, val(3_000, 2));
189 (r, Instant::now())
190 });
191 std::thread::sleep(Duration::from_millis(300));
192 let mid = res!(read_after_cut(&db, &root, &c, "p1cut", &[k.clone()]));
193 let (r, t1) = match h.join() {
194 Ok(x) => x,
195 Err(_) => return Err(err!("put thread panicked"; Test)),
196 };
197 hooks::set_barrier_delay(Duration::ZERO);
198 res!(r);
199 let took = t1.duration_since(t0);
200 println!("{}: chunked put took {:?}; cut mid-barrier read {}", label, took, mid[0]);
201 assert!(took >= delay - Duration::from_millis(50),
202 "{}: the chunked put returned in {:?}, before the barrier over its chunks ({:?})", label, took, delay);
203 assert_eq!(mid[0], "3000/1", "{}: a cut during the chunk barrier did not give the old value", label);
204 let after = res!(read_after_cut(&db, &root, &c, "p1cut", &[k.clone()]));
205 // The head is past its mark under these policies unless an owed barrier covered it.
206 assert!(after[0] == "3000/1" || after[0] == "3000/2", "{}: after: {}", label, after[0]);
207 assert_eq!(read(&db, &k), "3000/2", "{}", label);
208 match Arc::try_unwrap(db) {
209 Ok(db) => res!(db.close()),
210 Err(_) => return Err(err!("db still shared"; Test)),
211 }
212 let _ = fs::remove_dir_all(&root);
213 }
214 }
215 Ok(())
216}
217
218// M8. A writer finishes its barrier before it takes the next record, so a head sharing a writer
219// with a chunk waits on the chunk's barrier whether or not the chunk's answer was held. With one
220// writer in each of sixteen zones a head usually lands in a zone that took none of its three
221// chunks; only the held chunk answers then keep the put from returning, and the head from being
222// written, before its chunks are durable.
223#[test]
224fn a_head_with_its_own_writer_waits_for_its_chunks() -> Outcome<()> {
225 let _g = lock();
226 let delay = Duration::from_millis(600);
227 for p in [Policy::Never, Policy::Interval(3_600_000)] {
228 let c = res!(cfg(p, 16, 1));
229 let root = res!(fresh("m8"));
230 let db = res!(open(&root, &c, false, true));
231 for i in 0..8 {
232 let label = fmt!("{:?} key {}", p, i);
233 let k = dat!(fmt!("races m8 chunked {}", i));
234 res!(put(&db, &k, val(1_200, 1)));
235 res!(db.api().sync_now(wait()));
236 hooks::set_barrier_delay(delay);
237 let t0 = Instant::now();
238 let r = put(&db, &k, val(1_200, 2));
239 let took = t0.elapsed();
240 hooks::set_barrier_delay(Duration::ZERO);
241 res!(r);
242 assert!(took >= delay - Duration::from_millis(50),
243 "{}: the chunked put returned in {:?}, before the barrier over its chunks ({:?})",
244 label, took, delay);
245 assert_eq!(read(&db, &k), "1200/2", "{}", label);
246 }
247 res!(db.close());
248 let _ = fs::remove_dir_all(&root);
249 }
250 Ok(())
251}
252
253// P5: the power-cut oracle with automatic collection on, small files and more ops, every policy.
254fn model(seed: u64, p: Policy, zones: u16, wbots: u16) -> Outcome<()> {
255 let label = fmt!("seed {} {:?} {}x{}", seed, p, zones, wbots);
256 let mut c = res!(cfg(p, zones, wbots));
257 c.data_file_max_bytes = 5_000;
258 let root = res!(fresh("p5"));
259 let db = res!(open(&root, &c, true, true));
260 let keys: Vec<Dat> = (0..8).map(|i| dat!(fmt!("races model {}", i))).collect();
261 let mut hist: Vec<Vec<String>> = vec![vec![fmt!("none")]; keys.len()];
262 let mut covered: Vec<usize> = vec![1; keys.len()];
263 let mut rng = seed | 1;
264 let mut next = |n: u64| { rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17; rng % n };
265 let mut wseed = 0u8;
266 let mut cut_n = 0;
267 for step in 0..120 {
268 let i = next(keys.len() as u64) as usize;
269 match next(24) {
270 0..=6 => {
271 wseed = wseed.wrapping_add(1).max(1);
272 let len = 1_100 + next(500) as usize;
273 res!(put(&db, &keys[i], val(len, wseed)));
274 hist[i].push(fmt!("{}/{}", len, wseed));
275 },
276 7..=15 => {
277 wseed = wseed.wrapping_add(1).max(1);
278 let len = 50 + next(250) as usize;
279 res!(put(&db, &keys[i], val(len, wseed)));
280 hist[i].push(fmt!("{}/{}", len, wseed));
281 },
282 16..=17 => {
283 res!(db.delete(&keys[i], Uid::default(), None));
284 hist[i].push(fmt!("none"));
285 },
286 18 => {
287 res!(db.api().sync_now(wait()));
288 for (h, n) in hist.iter().zip(covered.iter_mut()) { *n = h.len(); }
289 },
290 19 => { res!(db.api().new_live_files()); },
291 20 => { res!(sweep::sweep_orphans(db.api(), Uid::default(), None, wait())); },
292 21 => {
293 // Let owed barriers and collections run, then cut mid-run.
294 std::thread::sleep(Duration::from_millis(next(400)));
295 let got = res!(read_after_cut(&db, &root, &c, "p5cut", &keys));
296 for (j, g) in got.iter().enumerate() {
297 let ok = &hist[j][covered[j] - 1..];
298 assert!(ok.contains(g), "{} step {}: key {} reads {} after a mid-run cut; allowed {:?}", label, step, j, g, ok);
299 }
300 cut_n += 1;
301 },
302 _ => {
303 // A cut right now, with no pause.
304 let got = res!(read_after_cut(&db, &root, &c, "p5cut", &keys));
305 for (j, g) in got.iter().enumerate() {
306 let ok = &hist[j][covered[j] - 1..];
307 assert!(ok.contains(g), "{} step {}: key {} reads {} after an immediate cut; allowed {:?}", label, step, j, g, ok);
308 }
309 cut_n += 1;
310 },
311 }
312 if let Policy::EveryWrite = p {
313 for (h, n) in hist.iter().zip(covered.iter_mut()) { *n = h.len(); }
314 }
315 }
316 res!(db.api().settle_for_test(Duration::from_secs(20)));
317 let got = res!(read_after_cut(&db, &root, &c, "p5cut", &keys));
318 for (i, g) in got.iter().enumerate() {
319 let ok = &hist[i][covered[i] - 1..];
320 assert!(ok.contains(g), "{}: key {} reads {} after the final cut; allowed {:?}", label, i, g, ok);
321 }
322 for (i, k) in keys.iter().enumerate() {
323 let want = match hist[i].last() { Some(w) => w.clone(), None => fmt!("none") };
324 assert_eq!(read(&db, k), want, "{}: live read of key {}", label, i);
325 }
326 let v = res!(verify::verify_live_set(db.api(), None, wait()));
327 assert!(v.clean(), "{}: {}", label, v.summary(4));
328 res!(db.close());
329 let db = res!(open(&root, &c, true, false));
330 for (i, k) in keys.iter().enumerate() {
331 let want = match hist[i].last() { Some(w) => w.clone(), None => fmt!("none") };
332 assert_eq!(read(&db, k), want, "{}: read of key {} after a clean restart", label, i);
333 }
334 res!(db.close());
335 let _ = fs::remove_dir_all(&root);
336 println!("{}: {} mid-run cuts passed", label, cut_n);
337 Ok(())
338}
339
340#[test]
341fn the_power_cut_oracle_with_collection_on() -> Outcome<()> {
342 let _g = lock();
343 let base: u64 = std::env::var("RACES_SEED").ok().and_then(|s| s.parse().ok()).unwrap_or(0xa2a2_0001);
344 let reps: u64 = std::env::var("RACES_REPS").ok().and_then(|s| s.parse().ok()).unwrap_or(1);
345 for n in 0..reps {
346 for p in POLICIES {
347 for (z, w) in [(1u16, 1u16), (2, 2)] {
348 res!(model(base + n * 7_919, p, z, w));
349 }
350 }
351 }
352 Ok(())
353}
354
355// ---------------------------------------------------------------------------------------------
356// M5 (P6): close with writes racing it under Never (M5). After close, every live file's mark must
357// cover the file's length on disk: the announced close barriered what was written after the
358// close's own sync_now.
359#[test]
360fn a_close_covers_racing_writes() -> Outcome<()> {
361 let _g = lock();
362 for (z, w) in [(1u16, 1u16), (2, 2)] {
363 for round in 0..5 {
364 let label = fmt!("Never {}x{} round {}", z, w, round);
365 let mut c = res!(cfg(Policy::Never, z, w));
366 c.data_file_max_bytes = 10_000_000;
367 let root = res!(fresh("p6"));
368 let db = Arc::new(res!(open(&root, &c, false, true)));
369 let stop = Arc::new(AtomicBool::new(false));
370 let (db2, stop2) = (db.clone(), stop.clone());
371 let h = std::thread::spawn(move || {
372 let mut i = 0usize;
373 let mut ok = 0usize;
374 while !stop2.load(Ordering::SeqCst) && i < 200_000 {
375 if put(&db2, &dat!(fmt!("races p6 {}", i % 50)), val(60, (i % 200) as u8 + 1)).is_ok() {
376 ok += 1;
377 } else {
378 break;
379 }
380 i += 1;
381 }
382 ok
383 });
384 std::thread::sleep(Duration::from_millis(300));
385 let marks = db.api().chans().marks().clone();
386 let r = db.close();
387 stop.store(true, Ordering::SeqCst);
388 let ok = h.join().unwrap_or(0);
389 let mut bad = Vec::new();
390 for zi in 0..z as usize {
391 for (f, (d, i)) in res!(marks.files(zi)) {
392 let dl = fs::metadata(zdir(&root, &c, zi).join(ZoneDir::relative_file_path(&FileType::Data, f))).map(|m| m.len()).unwrap_or(0);
393 let il = fs::metadata(zdir(&root, &c, zi).join(ZoneDir::relative_file_path(&FileType::Index, f))).map(|m| m.len()).unwrap_or(0);
394 if d < dl || i < il {
395 bad.push(fmt!("zone {} file {}: mark ({}, {}) disk ({}, {})", zi, f, d, i, dl, il));
396 }
397 }
398 }
399 println!("{}: close {:?}, {} writes ok, uncovered: {:?}", label, r.is_ok(), ok, bad);
400 assert!(bad.is_empty(), "{}: close left written bytes past the mark: {:?}", label, bad);
401 drop(db);
402 let _ = fs::remove_dir_all(&root);
403 }
404 }
405 Ok(())
406}