oxedyne/fe2o3/fe2o3_o3db_sync/tests/qa_a3r2a_chunks.rs
11.7 KiB, 1 run
created by r1870400018:69931, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! Opus QA agent 2 of A3 round 2a (2026-10-09): chunked values. Each test is a finding's proof |
| 2 | //! and fails on bebb931d. The hooks are process-wide, so every test takes one lock. |
| 3 | |
| 4 | mod gc_pair; |
| 5 | |
| 6 | use gc_pair::*; |
| 7 | |
| 8 | use oxedyne_fe2o3_core::prelude::*; |
| 9 | use oxedyne_fe2o3_iop_db::api::Database; |
| 10 | use oxedyne_fe2o3_jdat::prelude::*; |
| 11 | use oxedyne_fe2o3_o3db_sync::{ |
| 12 | base::{ |
| 13 | cfg::OzoneConfig, |
| 14 | constant, |
| 15 | }, |
| 16 | test::{ |
| 17 | hooks, |
| 18 | setup::{ |
| 19 | self, |
| 20 | Uid, |
| 21 | }, |
| 22 | }, |
| 23 | }; |
| 24 | |
| 25 | use std::{ |
| 26 | collections::BTreeMap, |
| 27 | path::Path, |
| 28 | sync::{ |
| 29 | Arc, |
| 30 | Mutex, |
| 31 | MutexGuard, |
| 32 | atomic::{ |
| 33 | AtomicBool, |
| 34 | Ordering, |
| 35 | }, |
| 36 | }, |
| 37 | thread, |
| 38 | time::Duration, |
| 39 | }; |
| 40 | |
| 41 | static HOOKS: Mutex<()> = Mutex::new(()); |
| 42 | |
| 43 | fn lock() -> MutexGuard<'static, ()> { |
| 44 | match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() } |
| 45 | } |
| 46 | |
| 47 | fn cfg(cbots: u16) -> Outcome<OzoneConfig> { |
| 48 | let mut c = res!(setup::default_cfg()); |
| 49 | c.num_zones = 1; |
| 50 | c.num_cbots_per_zone = cbots; |
| 51 | c.num_fbots_per_zone = 1; |
| 52 | c.num_wbots_per_zone = 2; |
| 53 | c.zone_overrides = BTreeMap::new(); |
| 54 | c.rest_chunk_threshold = 1_000; |
| 55 | c.rest_chunk_bytes = 400; |
| 56 | c.sync_on_write = true; |
| 57 | Ok(c) |
| 58 | } |
| 59 | |
| 60 | // A value of `len` bytes whose every byte is `seed ^ j`, so a value assembled from the chunks of |
| 61 | // two values is told by `whole`. |
| 62 | fn val(len: usize, seed: u8) -> Dat { |
| 63 | Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect()) |
| 64 | } |
| 65 | |
| 66 | // Is this one value, whole, of one of the lengths stored? |
| 67 | fn whole(v: &Dat, lens: &[usize]) -> bool { |
| 68 | match v { |
| 69 | Dat::BU32(b) | Dat::BU64(b) | Dat::BU8(b) | Dat::BU16(b) => { |
| 70 | if !lens.contains(&b.len()) || b.is_empty() { return false; } |
| 71 | let seed = b[0]; |
| 72 | b.iter().enumerate().all(|(j, x)| *x == seed ^ (j as u8)) |
| 73 | }, |
| 74 | _ => false, |
| 75 | } |
| 76 | } |
| 77 | |
| 78 | fn start(dir: &str, c: OzoneConfig, wipe: bool) -> Outcome<TestDb> { |
| 79 | log_set_level!("error"); |
| 80 | if wipe { |
| 81 | let _ = std::fs::remove_dir_all(dir); |
| 82 | } |
| 83 | res!(std::fs::create_dir_all(dir)); |
| 84 | let root = res!(Path::new(dir).canonicalize()); |
| 85 | setup::start_db(root, Some(c), schemes(), None, false, wipe) |
| 86 | } |
| 87 | |
| 88 | struct Seen { none: usize, ok: usize, torn: usize, err: usize, sample: Option<String> } |
| 89 | |
| 90 | fn reader(db: TestDb, k: Dat, lens: Vec<usize>, done: Arc<AtomicBool>) -> thread::JoinHandle<Seen> { |
| 91 | thread::spawn(move || { |
| 92 | let mut s = Seen { none: 0, ok: 0, torn: 0, err: 0, sample: None }; |
| 93 | while !done.load(Ordering::Relaxed) { |
| 94 | match db.get(&k, None) { |
| 95 | Ok(None) => s.none += 1, |
| 96 | Ok(Some((v, _))) if whole(&v, &lens) => s.ok += 1, |
| 97 | Ok(Some((v, _))) => { |
| 98 | s.torn += 1; |
| 99 | if s.sample.is_none() { |
| 100 | s.sample = Some(fmt!("{:?}", v).chars().take(160).collect()); |
| 101 | } |
| 102 | }, |
| 103 | Err(e) => { |
| 104 | s.err += 1; |
| 105 | if s.sample.is_none() { |
| 106 | s.sample = Some(fmt!("{}", e).chars().take(300).collect()); |
| 107 | } |
| 108 | }, |
| 109 | } |
| 110 | thread::sleep(Duration::from_millis(1)); |
| 111 | } |
| 112 | s |
| 113 | }) |
| 114 | } |
| 115 | |
| 116 | fn join(h: thread::JoinHandle<Seen>) -> Outcome<Seen> { |
| 117 | match h.join() { |
| 118 | Ok(s) => Ok(s), |
| 119 | Err(_) => Err(err!("A reader thread panicked."; Test, Invalid)), |
| 120 | } |
| 121 | } |
| 122 | |
| 123 | // F1. A value overwritten by one of the same length is written over its own chunk records in |
| 124 | // place: the chunk set identifier comes from the key, and a chunk key carries only the set, the |
| 125 | // index, the length, the count and the size. The new bunch key is the old one byte for byte, so |
| 126 | // publishing it last protects nothing, and a reader finds the old bunch key over a mixture of old |
| 127 | // and new chunks for as long as the chunks take to land. |
| 128 | #[test] |
| 129 | fn f1_same_length_overwrite_is_never_read_torn() -> Outcome<()> { |
| 130 | let _lock = lock(); |
| 131 | let dir = "./test_db_qa2_f1"; |
| 132 | // No hook: the chunks go to two writers at once, and each writer's barrier releases its own. |
| 133 | let db = res!(start(dir, res!(cfg(8)), true)); |
| 134 | let k = dat!("same length overwrite"); |
| 135 | let len = 40_000usize; // 100 chunks |
| 136 | let lens = vec![len]; |
| 137 | res!(db.insert(k.clone(), val(len, 0), Uid::default(), None)); |
| 138 | |
| 139 | let done = Arc::new(AtomicBool::new(false)); |
| 140 | let rs: Vec<_> = (0..3).map(|_| reader(db.clone(), k.clone(), lens.clone(), done.clone())).collect(); |
| 141 | let mut put = Ok(()); |
| 142 | for seed in 1..60u8 { |
| 143 | if let Err(e) = db.insert(k.clone(), val(len, seed), Uid::default(), None) { |
| 144 | put = Err(e); |
| 145 | } |
| 146 | } |
| 147 | done.store(true, Ordering::Relaxed); |
| 148 | let (mut torn, mut errs, mut oks, mut sample) = (0, 0, 0, None); |
| 149 | for r in rs { |
| 150 | let s = res!(join(r)); |
| 151 | torn += s.torn; errs += s.err; oks += s.ok; |
| 152 | if sample.is_none() { sample = s.sample; } |
| 153 | } |
| 154 | res!(put); |
| 155 | msg!("F1: whole {} torn {} err {}; first bad: {:?}", oks, torn, errs, sample); |
| 156 | res!(db.close()); |
| 157 | let _ = std::fs::remove_dir_all(dir); |
| 158 | assert_eq!(torn + errs, 0, "a reader found a value assembled from two values' chunks: {:?}", sample); |
| 159 | Ok(()) |
| 160 | } |
| 161 | |
| 162 | // F2. The same in-place overwrite interrupted after one chunk has landed -- the process stopped, |
| 163 | // which the new store makes more likely by sending every chunk at once and the bunch key only |
| 164 | // later -- leaves the old bunch key over one new chunk and the rest old, for good. The control |
| 165 | // does the same with a value of another length, whose chunks have keys of their own: the old value |
| 166 | // survives whole. |
| 167 | #[test] |
| 168 | fn f2_overwrite_interrupted_after_one_chunk_keeps_a_whole_value() -> Outcome<()> { |
| 169 | let _lock = lock(); |
| 170 | let mut results = Vec::new(); |
| 171 | for (label, newlen) in [("control, other length", 1_500usize), ("same length", 1_100usize)] { |
| 172 | let dir = "./test_db_qa2_f2"; |
| 173 | let k = dat!("interrupted overwrite"); |
| 174 | { |
| 175 | let db = res!(start(dir, res!(cfg(4)), true)); |
| 176 | res!(db.insert(k.clone(), val(1_100, 1), Uid::default(), None)); |
| 177 | // The new value, prepared as a store prepares it; only its first chunk reaches a |
| 178 | // writer before the "crash". |
| 179 | let resp = db.api().responder(); |
| 180 | let mut msgs = res!(db.api().prepare_write_dat( |
| 181 | k.clone(), val(newlen, 2), Uid::default(), None, resp.clone(), None)); |
| 182 | assert!(msgs.len() >= 3, "the value was not chunked"); |
| 183 | let first = msgs.remove(1); |
| 184 | res!(db.api().store_bytes(vec![first])); |
| 185 | res!(resp.recv_write_acks(1, constant::USER_REQUEST_TIMEOUT, constant::DURABILITY_TIMEOUT)); |
| 186 | let live = res!(db.get(&k, None)); |
| 187 | let live_whole = match &live { Some((v, _)) => whole(v, &[1_100, newlen]), None => false }; |
| 188 | msg!("F2 {}: before the restart a reader gets a whole value: {}", label, live_whole); |
| 189 | res!(db.close()); |
| 190 | } |
| 191 | let db = res!(start(dir, res!(cfg(4)), false)); |
| 192 | let got = db.get(&k, None); |
| 193 | let ok = match &got { |
| 194 | Ok(Some((v, _))) => *v == val(1_100, 1), |
| 195 | _ => false, |
| 196 | }; |
| 197 | msg!("F2 {}: after the restart the old value reads back whole: {}", label, ok); |
| 198 | results.push((label, ok)); |
| 199 | res!(db.close()); |
| 200 | let _ = std::fs::remove_dir_all(dir); |
| 201 | } |
| 202 | assert!(results[0].1, "the control failed: an interrupted overwrite of another length lost the old value"); |
| 203 | assert!(results[1].1, "an overwrite interrupted after one chunk left a value of mixed chunks, read as if whole"); |
| 204 | Ok(()) |
| 205 | } |
| 206 | |
| 207 | // F4. Many puts and deletes at once on one key, of three lengths (one not chunked). Afterwards |
| 208 | // the key holds nothing or one value whole, and every read during the race gets the same. |
| 209 | #[test] |
| 210 | fn f4_concurrent_puts_and_deletes_on_one_key_settle_whole() -> Outcome<()> { |
| 211 | let _lock = lock(); |
| 212 | let dir = "./test_db_qa2_f4"; |
| 213 | let db = res!(start(dir, res!(cfg(8)), true)); |
| 214 | let k = dat!("contended"); |
| 215 | let lens = vec![300usize, 1_100, 1_500]; |
| 216 | let mut bad_final = Vec::new(); |
| 217 | let mut failed = Vec::new(); |
| 218 | let (mut torn, mut errs, mut sample) = (0, 0, None); |
| 219 | hooks::set_chunk_insert_delay(Duration::from_millis(20)); |
| 220 | for round in 0..6u8 { |
| 221 | let done = Arc::new(AtomicBool::new(false)); |
| 222 | let r = reader(db.clone(), k.clone(), lens.clone(), done.clone()); |
| 223 | let ws: Vec<_> = (0..16usize).map(|i| { |
| 224 | let (db, k, lens) = (db.clone(), k.clone(), lens.clone()); |
| 225 | thread::spawn(move || -> Result<(), String> { |
| 226 | if i % 5 == 4 { |
| 227 | db.delete(&k, Uid::default(), None).map(|_| ()).map_err(|e| fmt!("{}", e)) |
| 228 | } else { |
| 229 | let seed = round.wrapping_mul(31).wrapping_add(i as u8); |
| 230 | db.insert(k, val(lens[i % 3], seed), Uid::default(), None) |
| 231 | .map(|_| ()).map_err(|e| fmt!("{}", e)) |
| 232 | } |
| 233 | }) |
| 234 | }).collect(); |
| 235 | for w in ws { |
| 236 | match w.join() { |
| 237 | Ok(Ok(())) => (), |
| 238 | Ok(Err(e)) => failed.push(e.chars().take(200).collect::<String>()), |
| 239 | Err(_) => failed.push(fmt!("panicked")), |
| 240 | } |
| 241 | } |
| 242 | done.store(true, Ordering::Relaxed); |
| 243 | let s = res!(join(r)); |
| 244 | torn += s.torn; errs += s.err; |
| 245 | if sample.is_none() { sample = s.sample; } |
| 246 | match db.get(&k, None) { |
| 247 | Ok(None) => (), |
| 248 | Ok(Some((v, _))) if whole(&v, &lens) => (), |
| 249 | other => bad_final.push(fmt!("{:?}", other).chars().take(200).collect::<String>()), |
| 250 | } |
| 251 | } |
| 252 | hooks::set_chunk_insert_delay(Duration::ZERO); |
| 253 | msg!("F4: torn {} err {} during; finals bad {:?}; failed calls {:?}; first bad read {:?}", |
| 254 | torn, errs, bad_final, failed, sample); |
| 255 | res!(db.close()); |
| 256 | let _ = std::fs::remove_dir_all(dir); |
| 257 | assert!(bad_final.is_empty(), "the key settled on a value that is not whole: {:?}", bad_final); |
| 258 | assert!(failed.is_empty(), "calls failed: {:?}", failed); |
| 259 | assert_eq!(torn + errs, 0, "reads during the race were torn or failed: {:?}", sample); |
| 260 | Ok(()) |
| 261 | } |
| 262 | |
| 263 | // F3b. F3 with reads that go to the files and take their time: the cache holds no values, and |
| 264 | // each file read is held 15 ms, as a loaded disk would hold it. |
| 265 | #[test] |
| 266 | fn f3b_slow_reader_never_errs_while_a_key_is_overwritten_with_a_new_length() -> Outcome<()> { |
| 267 | let _lock = lock(); |
| 268 | let dir = "./test_db_qa2_f3b"; |
| 269 | let mut c = res!(cfg(8)); |
| 270 | c.cache_size_limit_bytes = 1; |
| 271 | let db = res!(start(dir, c, true)); |
| 272 | let k = dat!("changing length, slow reader"); |
| 273 | let lens = vec![1_100usize, 1_500]; |
| 274 | res!(db.insert(k.clone(), val(lens[0], 0), Uid::default(), None)); |
| 275 | |
| 276 | let done = Arc::new(AtomicBool::new(false)); |
| 277 | hooks::set_read_delay(Duration::from_millis(15)); |
| 278 | let rs: Vec<_> = (0..3).map(|_| reader(db.clone(), k.clone(), lens.clone(), done.clone())).collect(); |
| 279 | let mut put = Ok(()); |
| 280 | for i in 1..40usize { |
| 281 | if let Err(e) = db.insert(k.clone(), val(lens[i % 2], i as u8), Uid::default(), None) { |
| 282 | put = Err(e); |
| 283 | } |
| 284 | thread::sleep(Duration::from_millis(20)); |
| 285 | } |
| 286 | done.store(true, Ordering::Relaxed); |
| 287 | let (mut torn, mut errs, mut oks, mut nones, mut sample) = (0, 0, 0, 0, None); |
| 288 | for r in rs { |
| 289 | let s = res!(join(r)); |
| 290 | torn += s.torn; errs += s.err; oks += s.ok; nones += s.none; |
| 291 | if sample.is_none() { sample = s.sample; } |
| 292 | } |
| 293 | hooks::set_read_delay(Duration::ZERO); |
| 294 | res!(put); |
| 295 | msg!("F3b: whole {} none {} torn {} err {}; first bad: {:?}", oks, nones, torn, errs, sample); |
| 296 | res!(db.close()); |
| 297 | let _ = std::fs::remove_dir_all(dir); |
| 298 | assert_eq!(torn + errs + nones, 0, "a slow reader of a live key failed while it was overwritten: {:?}", sample); |
| 299 | Ok(()) |
| 300 | } |