oxedyne/fe2o3/fe2o3_o3db_sync/tests/lone_index_campaign.rs
13.9 KiB, 1 run
created by r1870400018:72470, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! The review's crash campaign as a regression test (review of 9f64d2fc, 2026-10-11). A child, |
| 2 | //! this test binary run again, writes a store with collection on, deleting whole files in the |
| 3 | //! order of builds before 2026-10-11 (data file first), and aborts after the first unlink of its |
| 4 | //! nth whole-file delete (`test::hooks::set_delete_cut`). The parent opens the image three times |
| 5 | //! with collection off: each open must complete, every key must read as the child's log of |
| 6 | //! operations says (the one in flight may land either way), and no live file may take the number |
| 7 | //! of an index that stood alone. The default runs a few dozen images; O3DB_LONE_CAMPAIGN_CUTS |
| 8 | //! raises the highest cut tried and O3DB_LONE_CAMPAIGN_SEEDS names the random scenarios' seeds. |
| 9 | |
| 10 | mod gc_pair; |
| 11 | |
| 12 | use gc_pair::*; |
| 13 | |
| 14 | use oxedyne_fe2o3_core::prelude::*; |
| 15 | use oxedyne_fe2o3_iop_db::api::Database; |
| 16 | use oxedyne_fe2o3_jdat::prelude::*; |
| 17 | use oxedyne_fe2o3_o3db_sync::{ |
| 18 | base::cfg::OzoneConfig, |
| 19 | test::{ |
| 20 | hooks, |
| 21 | setup::{ |
| 22 | self, |
| 23 | Uid, |
| 24 | }, |
| 25 | }, |
| 26 | }; |
| 27 | |
| 28 | use std::{ |
| 29 | collections::BTreeMap, |
| 30 | fs, |
| 31 | io::Write, |
| 32 | path::{ |
| 33 | Path, |
| 34 | PathBuf, |
| 35 | }, |
| 36 | process::{ |
| 37 | Command, |
| 38 | Stdio, |
| 39 | }, |
| 40 | time::{ |
| 41 | Duration, |
| 42 | Instant, |
| 43 | }, |
| 44 | }; |
| 45 | |
| 46 | const DIR: &str = "O3DB_LONE_CAMPAIGN_DIR"; |
| 47 | const SCEN: &str = "O3DB_LONE_CAMPAIGN_SCEN"; |
| 48 | const CUT: &str = "O3DB_LONE_CAMPAIGN_CUT"; |
| 49 | const CUTS: &str = "O3DB_LONE_CAMPAIGN_CUTS"; |
| 50 | const SEEDS: &str = "O3DB_LONE_CAMPAIGN_SEEDS"; |
| 51 | const NK: u64 = 12; |
| 52 | |
| 53 | fn cfg() -> Outcome<OzoneConfig> { |
| 54 | let mut c = res!(config(1_500)); |
| 55 | c.num_zones = 2; |
| 56 | c.num_cbots_per_zone = 2; |
| 57 | c.rest_chunk_threshold = 1_000; |
| 58 | Ok(c) |
| 59 | } |
| 60 | |
| 61 | fn key_of(i: u64) -> Dat { dat!(fmt!("campaign key {:03}", i)) } |
| 62 | |
| 63 | // The value carries its key, version and size, so a value read back names its write. |
| 64 | fn val(i: u64, ver: u64, big: bool) -> Dat { |
| 65 | let n = if big { 2_600 } else { 90 }; |
| 66 | let mut v = vec![0u8; n]; |
| 67 | v[0] = i as u8; |
| 68 | v[1] = ver as u8; |
| 69 | v[2] = big as u8; |
| 70 | for j in 3..n { |
| 71 | v[j] = (i as u8) ^ (ver as u8).wrapping_mul(7) ^ (j as u8); |
| 72 | } |
| 73 | Dat::BU32(v) |
| 74 | } |
| 75 | |
| 76 | struct Rng(u64); |
| 77 | |
| 78 | impl Rng { |
| 79 | fn next(&mut self) -> u64 { |
| 80 | self.0 ^= self.0 << 13; |
| 81 | self.0 ^= self.0 >> 7; |
| 82 | self.0 ^= self.0 << 17; |
| 83 | self.0 |
| 84 | } |
| 85 | } |
| 86 | |
| 87 | fn ops_path(root: &Path) -> PathBuf { PathBuf::from(fmt!("{}.ops", root.display())) } |
| 88 | |
| 89 | // Each line is synced before the operation it names goes on, so the log outlives the abort. |
| 90 | fn log_op(f: &mut fs::File, line: &str) -> Outcome<()> { |
| 91 | res!(writeln!(f, "{}", line)); |
| 92 | res!(f.sync_data()); |
| 93 | Ok(()) |
| 94 | } |
| 95 | |
| 96 | fn read_value(db: &TestDb, k: &Dat) -> Outcome<Option<Dat>> { |
| 97 | match res!(db.get(k, None)) { |
| 98 | None => Ok(None), |
| 99 | Some((Dat::Tup5u64(t), _)) => Ok(Some(res!(db.api().fetch_chunks(&Dat::Tup5u64(t), None)))), |
| 100 | Some((v, _)) => Ok(Some(v)), |
| 101 | } |
| 102 | } |
| 103 | |
| 104 | struct Ops<'a> { |
| 105 | db: &'a TestDb, |
| 106 | log: fs::File, |
| 107 | } |
| 108 | |
| 109 | impl<'a> Ops<'a> { |
| 110 | fn put(&mut self, i: u64, ver: u64, big: bool) -> Outcome<()> { |
| 111 | res!(log_op(&mut self.log, &fmt!("try put {} {} {}", i, ver, big as u8))); |
| 112 | res!(self.db.insert(key_of(i), val(i, ver, big), Uid::default(), None)); |
| 113 | log_op(&mut self.log, &fmt!("put {} {} {}", i, ver, big as u8)) |
| 114 | } |
| 115 | fn del(&mut self, i: u64) -> Outcome<()> { |
| 116 | res!(log_op(&mut self.log, &fmt!("try del {}", i))); |
| 117 | let _ = res!(self.db.delete(&key_of(i), Uid::default(), None)); |
| 118 | log_op(&mut self.log, &fmt!("del {}", i)) |
| 119 | } |
| 120 | } |
| 121 | |
| 122 | /// The child: writes one scenario with collection on and the old delete order, aborting at the |
| 123 | /// cut. A child that reaches the end without it closes the store and says so in the log. |
| 124 | fn child(root: PathBuf, scen: String, cut: u64) -> Outcome<()> { |
| 125 | log_set_level!("error"); |
| 126 | let _ = fs::remove_dir_all(&root); |
| 127 | res!(fs::create_dir_all(&root)); |
| 128 | hooks::set_delete_cut(true, cut); |
| 129 | let db: TestDb = res!(setup::start_db(root.clone(), Some(res!(cfg())), schemes(), None, true, true)); |
| 130 | let log = res!(fs::OpenOptions::new().create(true).truncate(true).write(true).open(ops_path(&root))); |
| 131 | let mut ops = Ops { db: &db, log }; |
| 132 | match scen.as_str() { |
| 133 | "over" => { |
| 134 | for i in 0..NK { res!(ops.put(i, 1, false)); } |
| 135 | for i in 0..NK { res!(ops.put(i, 2, false)); } |
| 136 | }, |
| 137 | "del" => { |
| 138 | for i in 0..NK { res!(ops.put(i, 1, false)); } |
| 139 | for i in 0..NK / 2 { res!(ops.del(i)); } |
| 140 | for i in NK / 2..NK { res!(ops.put(i, 2, false)); } |
| 141 | // Pads, so that the files of tombstones seal. |
| 142 | for i in NK..NK + 12 { res!(ops.put(i, 1, false)); } |
| 143 | }, |
| 144 | "chunk" => { |
| 145 | for i in 0..4 { res!(ops.put(i, 1, true)); } |
| 146 | res!(ops.put(0, 2, true)); |
| 147 | res!(ops.del(1)); |
| 148 | res!(ops.put(2, 3, false)); |
| 149 | res!(ops.del(3)); |
| 150 | res!(ops.put(3, 4, true)); |
| 151 | for i in 4..NK + 4 { res!(ops.put(i, 1, false)); } |
| 152 | for i in 4..NK + 4 { res!(ops.put(i, 2, false)); } |
| 153 | }, |
| 154 | seed => { |
| 155 | let mut r = Rng(res!(seed.parse::<u64>()) | 1); |
| 156 | for s in 0..150 { |
| 157 | let i = r.next() % NK; |
| 158 | match r.next() % 10 { |
| 159 | 0 | 1 => res!(ops.del(i)), |
| 160 | 2 | 3 => res!(ops.put(i, 10 + s, true)), |
| 161 | _ => res!(ops.put(i, 10 + s, false)), |
| 162 | } |
| 163 | } |
| 164 | }, |
| 165 | } |
| 166 | let mut log = ops.log; |
| 167 | res!(log_op(&mut log, "writes done")); |
| 168 | let _ = db.api().settle_for_test(Duration::from_secs(20)); |
| 169 | std::thread::sleep(Duration::from_secs(3)); |
| 170 | let _ = db.api().settle_for_test(Duration::from_secs(20)); |
| 171 | res!(log_op(&mut log, "closing")); |
| 172 | res!(db.close()); |
| 173 | log_op(&mut log, "closed") |
| 174 | } |
| 175 | |
| 176 | type Want = BTreeMap<u64, Option<(u64, bool)>>; |
| 177 | |
| 178 | // What each key must read, and the operation in flight at the cut, if any. |
| 179 | fn expected(root: &Path) -> Outcome<(Want, Option<(u64, Option<(u64, bool)>)>)> { |
| 180 | let text = res!(fs::read_to_string(ops_path(root))); |
| 181 | let mut want = BTreeMap::new(); |
| 182 | let mut pend = None; |
| 183 | let num = |w: Option<&&str>| -> Outcome<u64> { |
| 184 | let w = res!(w.ok_or_else(|| err!("A short line in the log of operations."; Test))); |
| 185 | Ok(res!(w.parse::<u64>())) |
| 186 | }; |
| 187 | for line in text.lines() { |
| 188 | let w: Vec<&str> = line.split(' ').collect(); |
| 189 | match w.as_slice() { |
| 190 | ["try", "put", ..] => pend = Some((res!(num(w.get(2))), Some((res!(num(w.get(3))), w.get(4) == Some(&"1"))))), |
| 191 | ["try", "del", ..] => pend = Some((res!(num(w.get(2))), None)), |
| 192 | ["put", ..] => { |
| 193 | pend = None; |
| 194 | want.insert(res!(num(w.get(1))), Some((res!(num(w.get(2))), w.get(3) == Some(&"1")))); |
| 195 | }, |
| 196 | ["del", ..] => { |
| 197 | pend = None; |
| 198 | want.insert(res!(num(w.get(1))), None); |
| 199 | }, |
| 200 | _ => (), |
| 201 | } |
| 202 | } |
| 203 | Ok((want, pend)) |
| 204 | } |
| 205 | |
| 206 | fn walk(dir: &Path, out: &mut Vec<PathBuf>) { |
| 207 | if let Ok(rd) = fs::read_dir(dir) { |
| 208 | for e in rd.flatten() { |
| 209 | let p = e.path(); |
| 210 | match p.is_dir() { |
| 211 | true => walk(&p, out), |
| 212 | false => out.push(p), |
| 213 | } |
| 214 | } |
| 215 | } |
| 216 | } |
| 217 | |
| 218 | // The indexes standing without a data file, and every index, by path. |
| 219 | fn indexes(root: &Path) -> (Vec<PathBuf>, Vec<PathBuf>) { |
| 220 | let mut all = Vec::new(); |
| 221 | walk(root, &mut all); |
| 222 | let inds: Vec<PathBuf> = all.into_iter() |
| 223 | .filter(|p| p.extension().map_or(false, |x| x == "ind")) |
| 224 | .collect(); |
| 225 | let lone = inds.iter().filter(|p| !p.with_extension("dat").exists()).cloned().collect(); |
| 226 | (lone, inds) |
| 227 | } |
| 228 | |
| 229 | fn check(db: &TestDb, want: &Want, pend: &Option<(u64, Option<(u64, bool)>)>) -> Vec<String> { |
| 230 | let mut bad = Vec::new(); |
| 231 | let mut all = want.clone(); |
| 232 | if let Some((k, _)) = pend { |
| 233 | all.entry(*k).or_insert(None); |
| 234 | } |
| 235 | for (i, w) in &all { |
| 236 | let got = read_value(db, &key_of(*i)); |
| 237 | let fits = |w: &Option<(u64, bool)>| match (&got, w) { |
| 238 | (Ok(Some(v)), Some((ver, big))) => *v == val(*i, *ver, *big), |
| 239 | (Ok(None), None) => true, |
| 240 | _ => false, |
| 241 | }; |
| 242 | let alt = match pend { |
| 243 | Some((k, a)) if k == i => fits(a), |
| 244 | _ => false, |
| 245 | }; |
| 246 | if !fits(w) && !alt { |
| 247 | bad.push(fmt!("key {} wants {:?} and reads {}", i, w, match &got { |
| 248 | Ok(Some(v)) => fmt!("version {} big {}", |
| 249 | v.bytes_ref().and_then(|b| b.get(1).copied()).unwrap_or(0), |
| 250 | v.bytes_ref().and_then(|b| b.get(2).copied()).unwrap_or(0)), |
| 251 | Ok(None) => fmt!("nothing"), |
| 252 | Err(e) => fmt!("an error: {}", e), |
| 253 | })); |
| 254 | } |
| 255 | } |
| 256 | bad |
| 257 | } |
| 258 | |
| 259 | // Runs one image: the child to its cut, then three opens. Whether the child was cut. |
| 260 | fn image(base: &Path, scen: &str, cut: u64, fails: &mut Vec<String>) -> Outcome<bool> { |
| 261 | let root = base.join(fmt!("{}_{}", scen, cut)); |
| 262 | let tag = fmt!("{} cut {}", scen, cut); |
| 263 | let log = base.join(fmt!("{}_{}.log", scen, cut)); |
| 264 | let out = res!(fs::File::create(&log)); |
| 265 | let err = res!(out.try_clone()); |
| 266 | // No core file: the abort is the point. |
| 267 | let mut kid = res!(Command::new("bash") |
| 268 | .args(["-c", "ulimit -c 0; exec \"$0\" \"$@\""]) |
| 269 | .arg(res!(std::env::current_exe())) |
| 270 | .args(["main", "--exact", "--nocapture", "--test-threads=1"]) |
| 271 | .env(DIR, &root) |
| 272 | .env(SCEN, scen) |
| 273 | .env(CUT, fmt!("{}", cut)) |
| 274 | .stdout(Stdio::from(out)) |
| 275 | .stderr(Stdio::from(err)) |
| 276 | .spawn()); |
| 277 | let start = Instant::now(); |
| 278 | let status = loop { |
| 279 | if let Some(s) = res!(kid.try_wait()) { |
| 280 | break s; |
| 281 | } |
| 282 | if start.elapsed() > Duration::from_secs(120) { |
| 283 | let _ = kid.kill(); |
| 284 | let _ = kid.wait(); |
| 285 | fails.push(fmt!("{}: the child ran past two minutes; see {:?}", tag, log)); |
| 286 | return Ok(false); |
| 287 | } |
| 288 | std::thread::sleep(Duration::from_millis(50)); |
| 289 | }; |
| 290 | let ops = res!(fs::read_to_string(ops_path(&root)).or_else(|_| Ok::<_, std::io::Error>(String::new()))); |
| 291 | let cut_hit = !ops.contains("closed"); |
| 292 | if !cut_hit { |
| 293 | if !status.success() { |
| 294 | fails.push(fmt!("{}: the child failed without a cut; see {:?}", tag, log)); |
| 295 | } |
| 296 | let _ = fs::remove_dir_all(&root); |
| 297 | let _ = fs::remove_file(ops_path(&root)); |
| 298 | let _ = fs::remove_file(&log); |
| 299 | return Ok(false); |
| 300 | } |
| 301 | let (want, pend) = res!(expected(&root)); |
| 302 | let (lone, _) = indexes(&root); |
| 303 | let mut fine = true; |
| 304 | if lone.is_empty() { |
| 305 | fails.push(fmt!("{}: positive control: the cut left no index alone", tag)); |
| 306 | fine = false; |
| 307 | } |
| 308 | let mut want = want; |
| 309 | for round in 0..3 { |
| 310 | let db: TestDb = match setup::start_db(root.clone(), Some(res!(cfg())), schemes(), None, false, false) { |
| 311 | Ok(db) => db, |
| 312 | Err(e) => { |
| 313 | fails.push(fmt!("{}: open {} refused: {}", tag, round, e)); |
| 314 | fine = false; |
| 315 | break; |
| 316 | }, |
| 317 | }; |
| 318 | let bad = check(&db, &want, &pend); |
| 319 | if !bad.is_empty() { |
| 320 | fails.push(fmt!("{}: open {} reads {} keys wrong: {:?}", tag, round, bad.len(), bad)); |
| 321 | fine = false; |
| 322 | } |
| 323 | let (still, before) = indexes(&root); |
| 324 | if !still.is_empty() { |
| 325 | fails.push(fmt!("{}: open {} left indexes alone: {:?}", tag, round, still)); |
| 326 | fine = false; |
| 327 | } |
| 328 | if round == 0 { |
| 329 | // Enough writes to roll over to new files, none of which may take a lone number. |
| 330 | for j in 900..912 { |
| 331 | res!(db.insert(key_of(j), val(j, 1, false), Uid::default(), None)); |
| 332 | want.insert(j, Some((1, false))); |
| 333 | } |
| 334 | let (_, after) = indexes(&root); |
| 335 | let reused: Vec<&PathBuf> = after.iter() |
| 336 | .filter(|p| !before.contains(p) && lone.contains(p)) |
| 337 | .collect(); |
| 338 | if !reused.is_empty() { |
| 339 | fails.push(fmt!("{}: new files took the numbers of lone indexes: {:?}", tag, reused)); |
| 340 | fine = false; |
| 341 | } |
| 342 | } |
| 343 | res!(db.close()); |
| 344 | } |
| 345 | if fine { |
| 346 | let _ = fs::remove_dir_all(&root); |
| 347 | let _ = fs::remove_file(ops_path(&root)); |
| 348 | let _ = fs::remove_file(&log); |
| 349 | } |
| 350 | Ok(true) |
| 351 | } |
| 352 | |
| 353 | fn parent() -> Outcome<()> { |
| 354 | log_set_level!("error"); |
| 355 | let dir = "./test_db_lone_index_campaign"; |
| 356 | let _ = fs::remove_dir_all(dir); |
| 357 | res!(fs::create_dir_all(dir)); |
| 358 | let base = res!(Path::new(dir).canonicalize()); |
| 359 | let cuts = std::env::var(CUTS).ok().and_then(|v| v.parse::<u64>().ok()).unwrap_or(5); |
| 360 | let seeds = std::env::var(SEEDS).unwrap_or(fmt!("7,11")); |
| 361 | let mut scens: Vec<String> = vec![fmt!("chunk"), fmt!("over"), fmt!("del")]; |
| 362 | scens.extend(seeds.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty())); |
| 363 | let mut fails = Vec::new(); |
| 364 | let mut images = 0; |
| 365 | for scen in &scens { |
| 366 | for cut in 1..(cuts + 1) { |
| 367 | match res!(image(&base, scen, cut, &mut fails)) { |
| 368 | true => images += 1, |
| 369 | false => break, // fewer deletes than the cut: higher cuts are the same run |
| 370 | } |
| 371 | } |
| 372 | } |
| 373 | log_finish_wait!(); |
| 374 | println!("{} cut images, {} failures.", images, fails.len()); |
| 375 | for f in &fails { |
| 376 | println!("FAIL {}", f); |
| 377 | } |
| 378 | assert!(images >= scens.len(), "positive control: only {} images were cut", images); |
| 379 | assert!(fails.is_empty(), "{} failures over {} images: {:?}", fails.len(), images, fails); |
| 380 | let _ = fs::remove_dir_all(dir); |
| 381 | Ok(()) |
| 382 | } |
| 383 | |
| 384 | #[test] |
| 385 | fn main() -> Outcome<()> { |
| 386 | match (std::env::var(DIR), std::env::var(SCEN), std::env::var(CUT)) { |
| 387 | (Ok(dir), Ok(scen), Ok(cut)) => child(PathBuf::from(dir), scen, res!(cut.parse::<u64>())), |
| 388 | _ => parent(), |
| 389 | } |
| 390 | } |