oxedyne/fe2o3/fe2o3_o3db_sync/tests/qa_appendseek.rs
27.9 KiB, 1 run
created by r1870400018:71730, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! The APPENDSEEK QA probes (rc-b, 2026-10-11, written against ac09ec89), adopted with the |
| 2 | //! fix-forward of findings F1 and F3: a stop after a failed cut back is named as such, never as a |
| 3 | //! failed sync, and a tear hook that an append outran is not counted as a tear. Each test gathers |
| 4 | //! every deviation into `bad` and prints its observations, so one run shows the whole picture. |
| 5 | //! The hooks are process-wide, so the tests run one at a time. |
| 6 | |
| 7 | mod gc_pair; |
| 8 | |
| 9 | use gc_pair::*; |
| 10 | |
| 11 | use oxedyne_fe2o3_core::prelude::*; |
| 12 | use oxedyne_fe2o3_iop_db::api::Database; |
| 13 | use oxedyne_fe2o3_jdat::prelude::*; |
| 14 | use oxedyne_fe2o3_o3db_sync::{ |
| 15 | base::{ |
| 16 | cfg::OzoneConfig, |
| 17 | constant, |
| 18 | }, |
| 19 | comm::response::Wait, |
| 20 | file::{ |
| 21 | core::FileType, |
| 22 | refusal::Refusal, |
| 23 | }, |
| 24 | sweep, |
| 25 | test::{ |
| 26 | hooks, |
| 27 | setup::{ |
| 28 | self, |
| 29 | Uid, |
| 30 | }, |
| 31 | }, |
| 32 | }; |
| 33 | |
| 34 | use std::{ |
| 35 | collections::BTreeMap, |
| 36 | fs, |
| 37 | path::{ |
| 38 | Path, |
| 39 | PathBuf, |
| 40 | }, |
| 41 | sync::{ |
| 42 | Arc, |
| 43 | Mutex, |
| 44 | atomic::{ |
| 45 | AtomicBool, |
| 46 | Ordering, |
| 47 | }, |
| 48 | }, |
| 49 | time::Duration, |
| 50 | }; |
| 51 | |
| 52 | static SERIAL: Mutex<()> = Mutex::new(()); |
| 53 | |
| 54 | fn lock() -> std::sync::MutexGuard<'static, ()> { |
| 55 | let g = match SERIAL.lock() { Ok(g) => g, Err(p) => p.into_inner() }; |
| 56 | disarm(); |
| 57 | g |
| 58 | } |
| 59 | |
| 60 | fn disarm() { |
| 61 | hooks::set_write_tear(FileType::Data, None); |
| 62 | hooks::set_write_tear(FileType::Index, None); |
| 63 | hooks::set_cut_back_failure(false); |
| 64 | hooks::set_barrier_failure(false); |
| 65 | hooks::set_fault_record_fails_at(0); |
| 66 | hooks::set_owed_drop(false); |
| 67 | } |
| 68 | |
| 69 | // One zone, one writer. `held` holds the sync back for an hour, else every write syncs. |
| 70 | fn cfg(held: bool, chunked: bool, max: u64) -> Outcome<OzoneConfig> { |
| 71 | let mut c = res!(setup::default_cfg()); |
| 72 | c.num_zones = 1; |
| 73 | c.num_wbots_per_zone = 1; |
| 74 | c.num_cbots_per_zone = 4; |
| 75 | c.num_fbots_per_zone = 1; |
| 76 | c.zone_overrides = BTreeMap::new(); |
| 77 | c.sync_on_write = !held; |
| 78 | c.sync_every_n_writes = 0; |
| 79 | c.sync_interval_ms = if held { 3_600_000 } else { 0 }; |
| 80 | c.data_file_max_bytes = max; |
| 81 | if chunked { |
| 82 | c.rest_chunk_threshold = 1_000; |
| 83 | c.rest_chunk_bytes = 400; |
| 84 | } |
| 85 | Ok(c) |
| 86 | } |
| 87 | |
| 88 | fn fresh(dir: &str) -> Outcome<PathBuf> { |
| 89 | let _ = fs::remove_dir_all(dir); |
| 90 | res!(fs::create_dir_all(dir)); |
| 91 | Ok(res!(Path::new(dir).canonicalize())) |
| 92 | } |
| 93 | |
| 94 | fn open(root: &Path, c: &OzoneConfig, wipe: bool) -> Outcome<TestDb> { |
| 95 | log_set_level!("error"); |
| 96 | setup::start_db(root.to_path_buf(), Some(c.clone()), schemes(), None, false, wipe) |
| 97 | } |
| 98 | |
| 99 | fn wait() -> Wait { |
| 100 | Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL } |
| 101 | } |
| 102 | |
| 103 | fn val(len: usize, seed: u8) -> Dat { |
| 104 | Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect()) |
| 105 | } |
| 106 | |
| 107 | fn put(db: &TestDb, k: &str, len: usize, seed: u8) -> Outcome<()> { |
| 108 | db.insert(dat!(k), val(len, seed), Uid::default(), None).map(|_| ()) |
| 109 | } |
| 110 | |
| 111 | // "L/S" for a whole value, "none", "TORN" or "ERR:...". |
| 112 | fn read(db: &TestDb, k: &str) -> String { |
| 113 | match db.get(&dat!(k), None) { |
| 114 | Ok(None) => fmt!("none"), |
| 115 | Ok(Some((Dat::BU32(b), _))) if !b.is_empty() |
| 116 | && b.iter().enumerate().all(|(j, x)| *x == b[0] ^ (j as u8)) => fmt!("{}/{}", b.len(), b[0]), |
| 117 | Ok(Some(_)) => fmt!("TORN"), |
| 118 | Err(e) => fmt!("ERR:{}", plain(&fmt!("{}", e))), |
| 119 | } |
| 120 | } |
| 121 | |
| 122 | // Text without colour codes. |
| 123 | fn bare(s: &str) -> String { |
| 124 | let mut out = String::new(); |
| 125 | let mut esc = false; |
| 126 | for ch in s.chars() { |
| 127 | if ch == '\u{1b}' { esc = true; continue; } |
| 128 | if esc { if ch == 'm' { esc = false; } continue; } |
| 129 | out.push(ch); |
| 130 | } |
| 131 | out |
| 132 | } |
| 133 | |
| 134 | // The innermost message of an error, without colour codes. |
| 135 | fn plain(s: &str) -> String { |
| 136 | let out = bare(s); |
| 137 | match out.rfind("LocalErr") { |
| 138 | Some(i) => out[i..].chars().take(400).collect(), |
| 139 | None => out.chars().take(400).collect(), |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | fn live(root: &Path, c: &OzoneConfig) -> Outcome<(u32, PathBuf, PathBuf)> { |
| 144 | let n = res!((1..10_000u32).rev().find(|n| file(root, c, FileType::Data, *n).exists()) |
| 145 | .ok_or_else(|| err!("No data file is live."; Test))); |
| 146 | Ok((n, file(root, c, FileType::Data, n), file(root, c, FileType::Index, n))) |
| 147 | } |
| 148 | |
| 149 | fn marker(root: &Path) -> String { |
| 150 | fs::read_to_string(Refusal::path(root)).unwrap_or_else(|_| fmt!("<no marker>")) |
| 151 | } |
| 152 | |
| 153 | fn record(root: &Path) -> String { |
| 154 | fs::read_to_string(root.join(constant::FAULT_FILENAME)).unwrap_or_else(|_| fmt!("<no record>")) |
| 155 | } |
| 156 | |
| 157 | fn check(bad: &mut Vec<String>, db: &TestDb, want: &[(&str, &str)], when: &str) { |
| 158 | for (k, v) in want { |
| 159 | let got = read(db, k); |
| 160 | if got != *v { |
| 161 | bad.push(fmt!("{}: {} reads {} not {}", when, k, got, v)); |
| 162 | } |
| 163 | } |
| 164 | } |
| 165 | |
| 166 | fn finish(name: &str, bad: Vec<String>) -> Outcome<()> { |
| 167 | eprintln!("QA[{}] {} deviation(s)", name, bad.len()); |
| 168 | for b in &bad { |
| 169 | eprintln!("QA[{}] BAD {}", name, b); |
| 170 | } |
| 171 | if !bad.is_empty() { |
| 172 | return Err(err!("{}: {:?}", name, bad; Test)); |
| 173 | } |
| 174 | Ok(()) |
| 175 | } |
| 176 | |
| 177 | // F1: a refusal on a store stopped by a failed cut back names the cut back, not a sync. |
| 178 | fn names_cut_back(bad: &mut Vec<String>, what: &str, r: &Outcome<()>) { |
| 179 | match r { |
| 180 | Ok(()) => bad.push(fmt!("{} succeeded on a stopped store", what)), |
| 181 | Err(e) => { |
| 182 | let t = bare(&fmt!("{}", e)); |
| 183 | if !t.contains("stopped writing after a failed cut back of a torn append") { |
| 184 | bad.push(fmt!("{} does not name the failed cut back: {}", what, plain(&t))); |
| 185 | } |
| 186 | if t.contains("failed sync") { |
| 187 | bad.push(fmt!("{} names a failed sync: {}", what, plain(&t))); |
| 188 | } |
| 189 | }, |
| 190 | } |
| 191 | } |
| 192 | |
| 193 | // F1: the marker and the record left by a failed cut back carry its cause. |
| 194 | fn marks_cut_back(bad: &mut Vec<String>, root: &Path) { |
| 195 | let mk = marker(root); |
| 196 | if !mk.lines().any(|l| l.starts_with("why a torn append could not be cut back at run time: ")) { |
| 197 | bad.push(fmt!("the marker's why line does not name the cut back:\n{}", mk)); |
| 198 | } |
| 199 | if !mk.lines().any(|l| l == "cause cut-back") { |
| 200 | bad.push(fmt!("the marker names no cut-back cause:\n{}", mk)); |
| 201 | } |
| 202 | let rc = record(root); |
| 203 | if !rc.lines().any(|l| l == "cause cut-back") { |
| 204 | bad.push(fmt!("the record names no cut-back cause:\n{}", rc)); |
| 205 | } |
| 206 | } |
| 207 | |
| 208 | // Surface 1: sync held back, k1 k2 acknowledged past the mark, k3 torn on `typ`, cut fails. |
| 209 | fn torn_under_held_sync(name: &str, typ: FileType, crash_to: Option<i64>) -> Outcome<()> { |
| 210 | let mut bad = Vec::new(); |
| 211 | let root = res!(fresh(&fmt!("./test_db_qa_seek_{}", name))); |
| 212 | let c = res!(cfg(true, false, 1_000_000)); |
| 213 | let db = res!(open(&root, &c, true)); |
| 214 | res!(put(&db, "k0", 100, 1)); |
| 215 | res!(db.api().sync_now(wait())); |
| 216 | let (_, dat, ind) = res!(live(&root, &c)); |
| 217 | let mark = res!(db.api().chans().marks().files(0)); |
| 218 | let m0 = size(&dat); |
| 219 | for (k, s) in [("k1", 2u8), ("k2", 3)] { |
| 220 | if let Err(e) = put(&db, k, 100, s) { |
| 221 | bad.push(fmt!("{} under a held sync was refused: {}", k, e)); |
| 222 | } |
| 223 | } |
| 224 | let (dlen, ilen) = (size(&dat), size(&ind)); |
| 225 | let rec = dlen - m0; |
| 226 | hooks::set_cut_back_failure(true); |
| 227 | match typ { |
| 228 | FileType::Data => hooks::set_write_tear(FileType::Data, Some(rec / 4)), |
| 229 | FileType::Index => hooks::set_write_tear(FileType::Index, Some(3)), |
| 230 | } |
| 231 | let failed = put(&db, "k3", 100, 4); |
| 232 | disarm(); |
| 233 | if failed.is_ok() { bad.push(fmt!("the torn k3 was confirmed")); } |
| 234 | eprintln!("QA[{}] marks {:?} m0 {} dlen {} ilen {} now ({}, {})", |
| 235 | name, mark, m0, dlen, ilen, size(&dat), size(&ind)); |
| 236 | let report = db.api().fault(); |
| 237 | match &report { |
| 238 | None => bad.push(fmt!("no fault after a failed cut back")), |
| 239 | Some(r) => { |
| 240 | eprintln!("QA[{}] cuts {:?}", name, r.cuts); |
| 241 | if !r.cuts.iter().any(|x| x.data_upto == dlen && x.index_upto == ilen) { |
| 242 | bad.push(fmt!("fault cuts {:?} do not name ({}, {})", r.cuts, dlen, ilen)); |
| 243 | } |
| 244 | }, |
| 245 | } |
| 246 | // Observation only: the durable-mark design refuses, in process, reads past the mark once |
| 247 | // the store has stopped. |
| 248 | for k in ["k0", "k1", "k2", "k3"] { |
| 249 | eprintln!("QA[{}] OBS in process after the fault {} = {}", name, k, read(&db, k)); |
| 250 | } |
| 251 | names_cut_back(&mut bad, "k4's put", &put(&db, "k4", 100, 5)); |
| 252 | names_cut_back(&mut bad, "sync_now", &db.api().sync_now(wait())); |
| 253 | let _ = db.close(); |
| 254 | eprintln!("QA[{}] after close ({}, {}) marker:\n{}record:\n{}", name, size(&dat), size(&ind), |
| 255 | marker(&root), record(&root)); |
| 256 | marks_cut_back(&mut bad, &root); |
| 257 | // A power cut leaves the files somewhere between the mark and their lengths. |
| 258 | if let Some(to) = crash_to { |
| 259 | let upto = (m0 as i64 + to) as u64; |
| 260 | let f = res!(fs::OpenOptions::new().write(true).open(&dat)); |
| 261 | res!(f.set_len(upto)); |
| 262 | let ilen0 = mark.iter().map(|(_, m)| m.1).max().unwrap_or(0); |
| 263 | let f = res!(fs::OpenOptions::new().write(true).open(&ind)); |
| 264 | res!(f.set_len(ilen0)); |
| 265 | eprintln!("QA[{}] crash: data to {}, index to {}", name, upto, ilen0); |
| 266 | } |
| 267 | // The marker refuses the open until cleared. |
| 268 | match open(&root, &c, false) { |
| 269 | Ok(db) => { bad.push(fmt!("the store opened over its refusal marker")); let _ = db.close(); }, |
| 270 | Err(e) => eprintln!("QA[{}] refused as wanted: {}", name, fmt!("{}", e).chars().take(120).collect::<String>()), |
| 271 | } |
| 272 | match Refusal::clear(&root) { |
| 273 | Ok(true) => (), |
| 274 | other => bad.push(fmt!("clear returned {:?}", other)), |
| 275 | } |
| 276 | let db = res!(open(&root, &c, false)); |
| 277 | eprintln!("QA[{}] after open ({}, {})", name, size(&dat), size(&ind)); |
| 278 | let want: Vec<(&str, &str)> = match crash_to { |
| 279 | None => vec![("k0", "100/1"), ("k1", "100/2"), ("k2", "100/3"), ("k3", "none")], |
| 280 | // k1, k2 were never synced: lost or whole, never torn; k0 was synced. |
| 281 | Some(_) => vec![("k0", "100/1"), ("k3", "none")], |
| 282 | }; |
| 283 | check(&mut bad, &db, &want, "after the cleared open"); |
| 284 | if crash_to.is_none() && size(&dat) != dlen { |
| 285 | bad.push(fmt!("data file {} after open, not the cut {}", size(&dat), dlen)); |
| 286 | } |
| 287 | if crash_to.is_some() { |
| 288 | for k in ["k1", "k2"] { |
| 289 | let r = read(&db, k); |
| 290 | if r != "none" && !r.starts_with("100/") { bad.push(fmt!("after crash {} reads {}", k, r)); } |
| 291 | eprintln!("QA[{}] after crash {} = {}", name, k, r); |
| 292 | } |
| 293 | } |
| 294 | if let Err(e) = put(&db, "k5", 100, 6) { bad.push(fmt!("k5 after the open failed: {}", e)); } |
| 295 | if Refusal::path(&root).exists() { bad.push(fmt!("a marker stands after a clean open")); } |
| 296 | res!(db.close()); |
| 297 | let db = res!(open(&root, &c, false)); |
| 298 | check(&mut bad, &db, &[("k0", "100/1"), ("k3", "none"), ("k5", "100/6")], "after a second open"); |
| 299 | res!(db.close()); |
| 300 | let _ = fs::remove_dir_all(&root); |
| 301 | finish(name, bad) |
| 302 | } |
| 303 | |
| 304 | #[test] |
| 305 | fn s1_torn_data_failed_cut_under_held_sync() -> Outcome<()> { |
| 306 | let _g = lock(); |
| 307 | torn_under_held_sync("s1d", FileType::Data, None) |
| 308 | } |
| 309 | |
| 310 | #[test] |
| 311 | fn s1_torn_index_failed_cut_under_held_sync() -> Outcome<()> { |
| 312 | let _g = lock(); |
| 313 | torn_under_held_sync("s1i", FileType::Index, None) |
| 314 | } |
| 315 | |
| 316 | // The power is cut part-way through k1 before the restart. |
| 317 | #[test] |
| 318 | fn s1_torn_data_failed_cut_then_power_cut_mid_record() -> Outcome<()> { |
| 319 | let _g = lock(); |
| 320 | torn_under_held_sync("s1c", FileType::Data, Some(37)) |
| 321 | } |
| 322 | |
| 323 | // Surface 1 at rollover: the torn append starts a new live pair, and its cut fails. |
| 324 | #[test] |
| 325 | fn s1_torn_first_append_of_a_new_pair_failed_cut() -> Outcome<()> { |
| 326 | let _g = lock(); |
| 327 | let mut bad = Vec::new(); |
| 328 | let root = res!(fresh("./test_db_qa_seek_s1r")); |
| 329 | let c = res!(cfg(false, false, 2_000)); |
| 330 | let db = res!(open(&root, &c, true)); |
| 331 | res!(put(&db, "k0", 100, 1)); |
| 332 | res!(put(&db, "k1", 100, 2)); |
| 333 | let (n0, d0, _) = res!(live(&root, &c)); |
| 334 | let rec = size(&d0) / 2 + 1; |
| 335 | let mut k = 2u8; |
| 336 | // Fill until the next put cannot fit, so that it rolls over. |
| 337 | loop { |
| 338 | let before = size(&d0); |
| 339 | if before + rec > 2_000 { break; } |
| 340 | res!(put(&db, &fmt!("k{}", k), 100, k + 1)); |
| 341 | k += 1; |
| 342 | } |
| 343 | hooks::set_cut_back_failure(true); |
| 344 | hooks::set_write_tear(FileType::Data, Some(20)); |
| 345 | let failed = put(&db, "kr", 100, 99); |
| 346 | disarm(); |
| 347 | if failed.is_ok() { bad.push(fmt!("the torn kr was confirmed")); } |
| 348 | let (n1, d1, i1) = res!(live(&root, &c)); |
| 349 | eprintln!("QA[s1r] pair {} -> {}; new data {} index {}; cuts {:?}", n0, n1, |
| 350 | size(&d1), size(&i1), db.api().fault().map(|r| r.cuts)); |
| 351 | if n1 == n0 { bad.push(fmt!("positive control: no rollover happened")); } |
| 352 | let _ = db.close(); |
| 353 | match Refusal::clear(&root) { |
| 354 | Ok(true) => (), |
| 355 | other => bad.push(fmt!("clear returned {:?}", other)), |
| 356 | } |
| 357 | let db = res!(open(&root, &c, false)); |
| 358 | eprintln!("QA[s1r] after open new data {} index {}", size(&d1), size(&i1)); |
| 359 | let mut want = vec![("kr".to_string(), "none".to_string())]; |
| 360 | for j in 0..k { want.push((fmt!("k{}", j), fmt!("100/{}", j + 1))); } |
| 361 | let want: Vec<(&str, &str)> = want.iter().map(|(a, b)| (a.as_str(), b.as_str())).collect(); |
| 362 | check(&mut bad, &db, &want, "after the cleared open"); |
| 363 | if let Err(e) = put(&db, "kz", 100, 50) { bad.push(fmt!("kz after the open failed: {}", e)); } |
| 364 | res!(db.close()); |
| 365 | let db = res!(open(&root, &c, false)); |
| 366 | check(&mut bad, &db, &[("kz", "100/50"), ("kr", "none")], "after a second open"); |
| 367 | res!(db.close()); |
| 368 | let _ = fs::remove_dir_all(&root); |
| 369 | finish("s1r", bad) |
| 370 | } |
| 371 | |
| 372 | // Surface 2: the writer's cut above the mark, then a barrier fails, so the syncer cuts to the |
| 373 | // mark as well. The two must compose to the lower; the marker must carry both. |
| 374 | #[test] |
| 375 | fn s2_writer_cut_above_mark_with_syncer_cut_to_mark() -> Outcome<()> { |
| 376 | let _g = lock(); |
| 377 | let mut bad = Vec::new(); |
| 378 | let root = res!(fresh("./test_db_qa_seek_s2")); |
| 379 | let c = res!(cfg(true, false, 1_000_000)); |
| 380 | let db = res!(open(&root, &c, true)); |
| 381 | res!(put(&db, "k0", 100, 1)); |
| 382 | res!(db.api().sync_now(wait())); |
| 383 | let (_, dat, ind) = res!(live(&root, &c)); |
| 384 | let mark = res!(db.api().chans().marks().files(0)); |
| 385 | res!(put(&db, "k1", 100, 2)); |
| 386 | let (dlen, ilen) = (size(&dat), size(&ind)); |
| 387 | hooks::set_cut_back_failure(true); |
| 388 | hooks::set_write_tear(FileType::Data, Some(30)); |
| 389 | let _ = put(&db, "k2", 100, 3); |
| 390 | disarm(); |
| 391 | // The syncer's barrier now fails. sync_now refuses on a stopped store, so the barrier comes |
| 392 | // from the close; report what each does. |
| 393 | hooks::set_barrier_failure(true); |
| 394 | let sn = db.api().sync_now(wait()); |
| 395 | let cl = db.close(); |
| 396 | hooks::set_barrier_failure(false); |
| 397 | eprintln!("QA[s2] mark {:?} writer cut ({}, {}); sync_now {:?}; close ok {}", |
| 398 | mark, dlen, ilen, sn.as_ref().map_err(|e| fmt!("{}", e).chars().take(80).collect::<String>()), cl.is_ok()); |
| 399 | let mk = marker(&root); |
| 400 | let rc = record(&root); |
| 401 | eprintln!("QA[s2] marker:\n{}record:\n{}", mk, rc); |
| 402 | let cuts = |t: &str| t.lines().filter(|l| l.starts_with("cut ")).count(); |
| 403 | if cuts(&mk) != cuts(&rc) { bad.push(fmt!("marker and record disagree on cuts")); } |
| 404 | if open(&root, &c, false).map(|db| { let _ = db.close(); }).is_ok() { |
| 405 | bad.push(fmt!("the store opened over its refusal marker")); |
| 406 | } |
| 407 | let _ = Refusal::clear(&root); |
| 408 | let db = res!(open(&root, &c, false)); |
| 409 | let low = mark.iter().map(|(_, m)| m.0).max().unwrap_or(0); |
| 410 | eprintln!("QA[s2] after open data {} (mark {}, writer cut {})", size(&dat), low, dlen); |
| 411 | if size(&dat) > dlen { bad.push(fmt!("data {} above the writer's cut {}", size(&dat), dlen)); } |
| 412 | let k1 = read(&db, "k1"); |
| 413 | eprintln!("QA[s2] k1 = {}", k1); |
| 414 | if k1 != "none" && k1 != "100/2" { bad.push(fmt!("k1 reads {}", k1)); } |
| 415 | check(&mut bad, &db, &[("k0", "100/1"), ("k2", "none")], "after the cleared open"); |
| 416 | res!(db.close()); |
| 417 | let _ = fs::remove_dir_all(&root); |
| 418 | finish("s2", bad) |
| 419 | } |
| 420 | |
| 421 | // Surface 2, "nothing more": a torn append cut back cleanly leaves no marker, no record, and the |
| 422 | // store opens without a clear. |
| 423 | #[test] |
| 424 | fn s2_clean_cut_back_leaves_no_refusal() -> Outcome<()> { |
| 425 | let _g = lock(); |
| 426 | let mut bad = Vec::new(); |
| 427 | let root = res!(fresh("./test_db_qa_seek_s2n")); |
| 428 | let c = res!(cfg(true, false, 1_000_000)); |
| 429 | let db = res!(open(&root, &c, true)); |
| 430 | res!(put(&db, "k0", 100, 1)); |
| 431 | hooks::set_write_tear(FileType::Data, Some(30)); |
| 432 | let _ = put(&db, "k1", 100, 2); |
| 433 | hooks::set_write_tear(FileType::Index, Some(3)); |
| 434 | let _ = put(&db, "k2", 100, 3); |
| 435 | disarm(); |
| 436 | res!(put(&db, "k3", 100, 4)); |
| 437 | if db.api().fault().is_some() { bad.push(fmt!("a clean cut back stopped the store")); } |
| 438 | if let Err(e) = db.api().sync_now(wait()) { bad.push(fmt!("sync_now after clean cuts: {}", e)); } |
| 439 | res!(db.close()); |
| 440 | if Refusal::path(&root).exists() { bad.push(fmt!("a clean cut back wrote a marker")); } |
| 441 | if root.join(constant::FAULT_FILENAME).exists() { bad.push(fmt!("a clean cut back wrote a record")); } |
| 442 | let db = res!(open(&root, &c, false)); |
| 443 | check(&mut bad, &db, &[("k0", "100/1"), ("k1", "none"), ("k2", "none"), ("k3", "100/4")], "after reopen"); |
| 444 | res!(db.close()); |
| 445 | let _ = fs::remove_dir_all(&root); |
| 446 | finish("s2n", bad) |
| 447 | } |
| 448 | |
| 449 | // Surface 3: a chunked put whose middle chunk is torn and whose cut fails. The set stays held in |
| 450 | // the stopped process; after the clear and the open, the old value reads whole and the sweep |
| 451 | // retires what the failed put left. The tear is armed by a thread watching the file grow, which |
| 452 | // under load can arm it only after the put has finished; such a try proves nothing and is run |
| 453 | // again on a fresh store. |
| 454 | fn chunk_case(name: &str, fail_cut: bool, arm_after: u64, tear: u64) -> Outcome<()> { |
| 455 | for attempt in 1..=5 { |
| 456 | match res!(chunk_try(name, fail_cut, arm_after, tear)) { |
| 457 | Some(bad) => return finish(name, bad), |
| 458 | None => eprintln!("QA[{}] try {}: the tear missed the put, trying again", |
| 459 | name, attempt), |
| 460 | } |
| 461 | } |
| 462 | Err(err!("{}: the tear missed the put in every try.", name; Test)) |
| 463 | } |
| 464 | |
| 465 | // One try of `chunk_case`: `None` when the tear did not fire within the put. |
| 466 | fn chunk_try(name: &str, fail_cut: bool, arm_after: u64, tear: u64) |
| 467 | -> Outcome<Option<Vec<String>>> |
| 468 | { |
| 469 | let mut bad = Vec::new(); |
| 470 | let root = res!(fresh(&fmt!("./test_db_qa_seek_{}", name))); |
| 471 | let c = res!(cfg(false, true, 1_000_000)); |
| 472 | let db = res!(open(&root, &c, true)); |
| 473 | res!(put(&db, "K", 3_000, 1)); |
| 474 | res!(put(&db, "S", 50, 9)); |
| 475 | let (_, dat, _) = res!(live(&root, &c)); |
| 476 | let base = size(&dat); |
| 477 | // Arm the tear once the put's first chunk is in, so a middle chunk tears. |
| 478 | let stop = Arc::new(AtomicBool::new(false)); |
| 479 | let armer = { |
| 480 | let stop = stop.clone(); |
| 481 | let dat = dat.clone(); |
| 482 | std::thread::spawn(move || { |
| 483 | while !stop.load(Ordering::SeqCst) { |
| 484 | if size(&dat) > base + arm_after { |
| 485 | hooks::set_write_tear(FileType::Data, Some(tear)); |
| 486 | return true; |
| 487 | } |
| 488 | std::hint::spin_loop(); |
| 489 | } |
| 490 | false |
| 491 | }) |
| 492 | }; |
| 493 | if fail_cut { hooks::set_cut_back_failure(true); } |
| 494 | let torn0 = hooks::writes_torn(); |
| 495 | let failed = put(&db, "K", 3_000, 2); |
| 496 | eprintln!("QA[{}] data grew {} bytes", name, size(&dat) - base); |
| 497 | stop.store(true, Ordering::SeqCst); |
| 498 | let armed = armer.join().unwrap_or(false); |
| 499 | let torn = hooks::writes_torn() - torn0; |
| 500 | disarm(); |
| 501 | eprintln!("QA[{}] armed {} torn {} put {:?}", name, armed, torn, |
| 502 | failed.as_ref().map_err(|e| fmt!("{}", e).chars().take(100).collect::<String>())); |
| 503 | if torn == 0 { |
| 504 | let _ = db.close(); |
| 505 | let _ = fs::remove_dir_all(&root); |
| 506 | return Ok(None); |
| 507 | } |
| 508 | if failed.is_ok() { bad.push(fmt!("a chunked put with a torn chunk was confirmed")); } |
| 509 | let held = |db: &TestDb| -> bool { |
| 510 | match db.api().chans().pending_sets().lock() { Ok(g) => !g.is_empty(), Err(p) => !p.into_inner().is_empty() } |
| 511 | }; |
| 512 | std::thread::sleep(Duration::from_millis(500)); |
| 513 | let h1 = held(&db); |
| 514 | eprintln!("QA[{}] fault {} held {}", name, db.api().fault().is_some(), h1); |
| 515 | let head = arm_after > 3_000; |
| 516 | if fail_cut && head && !h1 { bad.push(fmt!("the head's set was released after a failed cut")); } |
| 517 | if !fail_cut && h1 { bad.push(fmt!("the set stays held after a clean cut")); } |
| 518 | check(&mut bad, &db, &[("K", "3000/1"), ("S", "50/9")], "in process after the failed put"); |
| 519 | if fail_cut { |
| 520 | match sweep::sweep_orphans(db.api(), Uid::default(), None, wait()) { |
| 521 | Ok(r) => bad.push(fmt!("the sweep ran on a stopped store: {:?}", r.orphans_found)), |
| 522 | Err(_) => (), |
| 523 | } |
| 524 | if head && !held(&db) { bad.push(fmt!("the hold went before the clear")); } |
| 525 | let _ = db.close(); |
| 526 | if open(&root, &c, false).map(|db| { let _ = db.close(); }).is_ok() { |
| 527 | bad.push(fmt!("the store opened over its refusal marker")); |
| 528 | } |
| 529 | match Refusal::clear(&root) { |
| 530 | Ok(true) => (), |
| 531 | other => bad.push(fmt!("clear returned {:?}", other)), |
| 532 | } |
| 533 | } else { |
| 534 | res!(db.close()); |
| 535 | } |
| 536 | let db = res!(open(&root, &c, false)); |
| 537 | check(&mut bad, &db, &[("K", "3000/1"), ("S", "50/9")], "after the open"); |
| 538 | match sweep::sweep_orphans(db.api(), Uid::default(), None, wait()) { |
| 539 | Ok(r) => eprintln!("QA[{}] sweep after open: orphans {} skipped_pending {}", name, |
| 540 | r.orphans_found, r.skipped_pending), |
| 541 | Err(e) => bad.push(fmt!("sweep after open failed: {}", e)), |
| 542 | } |
| 543 | if held(&db) { bad.push(fmt!("a set is held in the new process")); } |
| 544 | check(&mut bad, &db, &[("K", "3000/1"), ("S", "50/9")], "after the sweep"); |
| 545 | if let Err(e) = put(&db, "K", 3_000, 3) { bad.push(fmt!("a new chunked put failed: {}", e)); } |
| 546 | res!(db.close()); |
| 547 | let db = res!(open(&root, &c, false)); |
| 548 | check(&mut bad, &db, &[("K", "3000/3"), ("S", "50/9")], "after the second open"); |
| 549 | match sweep::sweep_orphans(db.api(), Uid::default(), None, wait()) { |
| 550 | Ok(r) => if r.orphans_found > 0 { bad.push(fmt!("{} orphans after the second open", r.orphans_found)); }, |
| 551 | Err(e) => bad.push(fmt!("second sweep failed: {}", e)), |
| 552 | } |
| 553 | check(&mut bad, &db, &[("K", "3000/3"), ("S", "50/9")], "after the second sweep"); |
| 554 | res!(db.close()); |
| 555 | let _ = fs::remove_dir_all(&root); |
| 556 | Ok(Some(bad)) |
| 557 | } |
| 558 | |
| 559 | #[test] |
| 560 | fn s3_chunk_torn_failed_cut_holds_then_clears() -> Outcome<()> { |
| 561 | let _g = lock(); |
| 562 | chunk_case("s3f", true, 400, 100) |
| 563 | } |
| 564 | |
| 565 | #[test] |
| 566 | fn s3_chunk_torn_clean_cut_releases() -> Outcome<()> { |
| 567 | let _g = lock(); |
| 568 | chunk_case("s3c", false, 400, 100) |
| 569 | } |
| 570 | |
| 571 | // The head, the last append of the put, torn: the writer keeps its set's hold when its cut fails. |
| 572 | #[test] |
| 573 | fn s3_head_torn_failed_cut_holds_then_clears() -> Outcome<()> { |
| 574 | let _g = lock(); |
| 575 | chunk_case("s3hf", true, 3_951, 10) |
| 576 | } |
| 577 | |
| 578 | #[test] |
| 579 | fn s3_head_torn_clean_cut_releases() -> Outcome<()> { |
| 580 | let _g = lock(); |
| 581 | chunk_case("s3hc", false, 3_951, 10) |
| 582 | } |
| 583 | |
| 584 | // Surface 1, pipelined: under sync_on_write, k1's barrier is held back (delayed) when k2 tears and |
| 585 | // its cut fails. k1's caller's answer must agree with what the restart serves: confirmed and |
| 586 | // present, or failed and absent. |
| 587 | #[test] |
| 588 | fn s1_pipelined_barrier_held_when_cut_fails() -> Outcome<()> { |
| 589 | let _g = lock(); |
| 590 | let mut bad = Vec::new(); |
| 591 | let root = res!(fresh("./test_db_qa_seek_s1p")); |
| 592 | let c = res!(cfg(false, false, 1_000_000)); |
| 593 | let db = Arc::new(res!(open(&root, &c, true))); |
| 594 | res!(put(&db, "k0", 100, 1)); |
| 595 | let (_, dat, ind) = res!(live(&root, &c)); |
| 596 | let d0 = size(&dat); |
| 597 | hooks::set_barrier_delay(Duration::from_millis(1_500)); |
| 598 | let t = { |
| 599 | let db = db.clone(); |
| 600 | std::thread::spawn(move || put(&db, "k1", 100, 2).map_err(|e| plain(&fmt!("{}", e)))) |
| 601 | }; |
| 602 | // Wait for k1 to reach the files, its barrier still to come. |
| 603 | let landed = wait_until(Duration::from_secs(5), || size(&dat) > d0); |
| 604 | let (dlen, ilen) = (size(&dat), size(&ind)); |
| 605 | hooks::set_cut_back_failure(true); |
| 606 | hooks::set_write_tear(FileType::Data, Some(30)); |
| 607 | let t0 = std::time::Instant::now(); |
| 608 | let failed = put(&db, "k2", 100, 3); |
| 609 | let k2_ms = t0.elapsed().as_millis(); |
| 610 | let k1_done = t.is_finished(); |
| 611 | hooks::set_cut_back_failure(false); |
| 612 | hooks::set_write_tear(FileType::Data, None); |
| 613 | eprintln!("QA[s1p] k2 answered in {} ms; k1 answered before k2: {}", k2_ms, k1_done); |
| 614 | if k1_done { bad.push(fmt!("positive control: k1's barrier was not pending when k2 tore")); } |
| 615 | let k1 = t.join().unwrap_or_else(|_| Err(fmt!("panicked"))); |
| 616 | hooks::set_barrier_delay(Duration::from_millis(0)); |
| 617 | eprintln!("QA[s1p] landed {} k1 answer {:?} k2 failed {} cuts {:?}", landed, k1, failed.is_err(), |
| 618 | db.api().fault().map(|r| r.cuts.iter().map(|x| x.line()).collect::<Vec<_>>())); |
| 619 | if !landed { bad.push(fmt!("positive control: k1 never reached the file")); } |
| 620 | if failed.is_ok() { bad.push(fmt!("the torn k2 was confirmed")); } |
| 621 | eprintln!("QA[s1p] OBS in process k1 = {}", read(&db, "k1")); |
| 622 | let db = match Arc::try_unwrap(db) { Ok(db) => db, Err(_) => return Err(err!("db still shared"; Test)) }; |
| 623 | let _ = db.close(); |
| 624 | let _ = Refusal::clear(&root); |
| 625 | let db = res!(open(&root, &c, false)); |
| 626 | let k1r = read(&db, "k1"); |
| 627 | eprintln!("QA[s1p] after open k1 = {} data {} (cut {}) index {} (cut {})", k1r, size(&dat), dlen, size(&ind), ilen); |
| 628 | match (&k1, k1r.as_str()) { |
| 629 | (Ok(()), "100/2") | (Err(_), "none") => (), |
| 630 | (Ok(()), r) => bad.push(fmt!("k1 confirmed but reads {} after the restart", r)), |
| 631 | (Err(e), r) => bad.push(fmt!("k1 reported failed ({}) but reads {} after the restart", e, r)), |
| 632 | } |
| 633 | check(&mut bad, &db, &[("k0", "100/1"), ("k2", "none")], "after the cleared open"); |
| 634 | res!(db.close()); |
| 635 | let _ = fs::remove_dir_all(&root); |
| 636 | finish("s1p", bad) |
| 637 | } |
| 638 | |
| 639 | // F1, on its own: the stop after a failed cut back is reported as one by every refusal, by the |
| 640 | // marker and by the record, under a sync on every write. |
| 641 | #[test] |
| 642 | fn f1_failed_cut_back_is_named_as_a_cut_back() -> Outcome<()> { |
| 643 | let _g = lock(); |
| 644 | let mut bad = Vec::new(); |
| 645 | let root = res!(fresh("./test_db_qa_seek_f1")); |
| 646 | let c = res!(cfg(false, false, 1_000_000)); |
| 647 | let db = res!(open(&root, &c, true)); |
| 648 | res!(put(&db, "k0", 100, 1)); |
| 649 | hooks::set_cut_back_failure(true); |
| 650 | hooks::set_write_tear(FileType::Data, Some(30)); |
| 651 | let failed = put(&db, "k1", 100, 2); |
| 652 | disarm(); |
| 653 | if failed.is_ok() { bad.push(fmt!("the torn k1 was confirmed")); } |
| 654 | names_cut_back(&mut bad, "k2's put", &put(&db, "k2", 100, 3)); |
| 655 | names_cut_back(&mut bad, "sync_now", &db.api().sync_now(wait())); |
| 656 | names_cut_back(&mut bad, "the sweep", |
| 657 | &sweep::sweep_orphans(db.api(), Uid::default(), None, wait()).map(|_| ())); |
| 658 | let _ = db.close(); |
| 659 | marks_cut_back(&mut bad, &root); |
| 660 | let _ = Refusal::clear(&root); |
| 661 | let db = res!(open(&root, &c, false)); |
| 662 | check(&mut bad, &db, &[("k0", "100/1"), ("k1", "none"), ("k2", "none")], "after the cleared open"); |
| 663 | res!(db.close()); |
| 664 | let _ = fs::remove_dir_all(&root); |
| 665 | finish("f1", bad) |
| 666 | } |
| 667 | |
| 668 | // F3: a tear armed past the end of the append writes it whole, so it is not counted as torn. |
| 669 | #[test] |
| 670 | fn f3_an_append_shorter_than_the_tear_is_not_counted_torn() -> Outcome<()> { |
| 671 | let _g = lock(); |
| 672 | let mut bad = Vec::new(); |
| 673 | let root = res!(fresh("./test_db_qa_seek_f3")); |
| 674 | let c = res!(cfg(false, false, 1_000_000)); |
| 675 | let db = res!(open(&root, &c, true)); |
| 676 | let before = hooks::writes_torn(); |
| 677 | hooks::set_write_tear(FileType::Data, Some(100_000)); |
| 678 | if let Err(e) = put(&db, "k0", 100, 1) { bad.push(fmt!("k0 under an outrun tear failed: {}", e)); } |
| 679 | disarm(); |
| 680 | if hooks::writes_torn() != before { |
| 681 | bad.push(fmt!("an append the tear never reached was counted torn ({} -> {})", |
| 682 | before, hooks::writes_torn())); |
| 683 | } |
| 684 | // Positive control: a tear the append reaches is counted. |
| 685 | hooks::set_write_tear(FileType::Data, Some(10)); |
| 686 | if put(&db, "k1", 100, 2).is_ok() { bad.push(fmt!("the torn k1 was confirmed")); } |
| 687 | disarm(); |
| 688 | if hooks::writes_torn() != before + 1 { |
| 689 | bad.push(fmt!("a real tear was not counted once ({} -> {})", before, hooks::writes_torn())); |
| 690 | } |
| 691 | check(&mut bad, &db, &[("k0", "100/1"), ("k1", "none")], "in process"); |
| 692 | res!(db.close()); |
| 693 | let _ = fs::remove_dir_all(&root); |
| 694 | finish("f3", bad) |
| 695 | } |