oxedyne/fe2o3/fe2o3_o3db_sync/tests/fault_stop.rs
11.7 KiB, 1 run
created by r1870400018:70854, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! The stop after a failed barrier (2026-10-10, QA F1, F2 and F3). The fault record names each |
| 2 | //! file in doubt by zone and number, so a store moved or restored elsewhere is cut where it now |
| 3 | //! stands and a copy left at the old path is not touched; a file in doubt that is missing refuses |
| 4 | //! the open; the file a writer rolled over to while the failing barrier was held is cut too; and |
| 5 | //! the stop is written down, as the record and the refusal marker, before a process may exit on |
| 6 | //! it. The hooks are process-wide, so the tests run one at a time. |
| 7 | |
| 8 | mod gc_pair; |
| 9 | |
| 10 | use gc_pair::*; |
| 11 | |
| 12 | use oxedyne_fe2o3_core::prelude::*; |
| 13 | use oxedyne_fe2o3_iop_db::api::Database; |
| 14 | use oxedyne_fe2o3_jdat::prelude::*; |
| 15 | use oxedyne_fe2o3_o3db_sync::{ |
| 16 | base::{ |
| 17 | cfg::OzoneConfig, |
| 18 | constant, |
| 19 | }, |
| 20 | comm::{ |
| 21 | channels::FaultReport, |
| 22 | response::Wait, |
| 23 | }, |
| 24 | file::{ |
| 25 | core::FileType, |
| 26 | refusal::Refusal, |
| 27 | zdir::ZoneDir, |
| 28 | }, |
| 29 | test::{ |
| 30 | hooks, |
| 31 | setup::{ |
| 32 | self, |
| 33 | Uid, |
| 34 | }, |
| 35 | }, |
| 36 | }; |
| 37 | |
| 38 | use std::{ |
| 39 | collections::BTreeMap, |
| 40 | fs, |
| 41 | io::Write, |
| 42 | path::{ |
| 43 | Path, |
| 44 | PathBuf, |
| 45 | }, |
| 46 | sync::{ |
| 47 | Arc, |
| 48 | Mutex, |
| 49 | }, |
| 50 | time::Duration, |
| 51 | }; |
| 52 | |
| 53 | static SERIAL: Mutex<()> = Mutex::new(()); |
| 54 | |
| 55 | fn lock() -> std::sync::MutexGuard<'static, ()> { |
| 56 | let g = match SERIAL.lock() { Ok(g) => g, Err(p) => p.into_inner() }; |
| 57 | hooks::set_barrier_failure(false); |
| 58 | hooks::set_barrier_failure_for(None); |
| 59 | hooks::set_fault_record_fails(false); |
| 60 | hooks::set_barrier_delay(Duration::ZERO); |
| 61 | g |
| 62 | } |
| 63 | |
| 64 | // One zone of one writer, syncing on every write unless `never`. |
| 65 | fn cfg(never: bool, max: u64) -> Outcome<OzoneConfig> { |
| 66 | let mut c = res!(setup::default_cfg()); |
| 67 | c.num_zones = 1; |
| 68 | c.num_wbots_per_zone = 1; |
| 69 | c.zone_overrides = BTreeMap::new(); |
| 70 | c.sync_on_write = !never; |
| 71 | c.sync_every_n_writes = 0; |
| 72 | c.sync_interval_ms = 0; |
| 73 | c.data_file_max_bytes = max; |
| 74 | Ok(c) |
| 75 | } |
| 76 | |
| 77 | fn fresh(dir: &str) -> Outcome<PathBuf> { |
| 78 | let _ = fs::remove_dir_all(dir); |
| 79 | res!(fs::create_dir_all(dir)); |
| 80 | Ok(res!(Path::new(dir).canonicalize())) |
| 81 | } |
| 82 | |
| 83 | fn open(root: &Path, c: &OzoneConfig, wipe: bool) -> Outcome<TestDb> { |
| 84 | log_set_level!("error"); |
| 85 | setup::start_db(root.to_path_buf(), Some(c.clone()), schemes(), None, false, wipe) |
| 86 | } |
| 87 | |
| 88 | fn wait() -> Wait { |
| 89 | Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL } |
| 90 | } |
| 91 | |
| 92 | fn val(seed: u8) -> Dat { |
| 93 | Dat::BU32((0..100).map(|j| seed ^ (j as u8)).collect()) |
| 94 | } |
| 95 | |
| 96 | fn put(db: &TestDb, k: &str, seed: u8) -> Outcome<()> { |
| 97 | db.insert(dat!(k), val(seed), Uid::default(), None).map(|_| ()) |
| 98 | } |
| 99 | |
| 100 | fn reads(db: &TestDb, k: &str) -> Outcome<Option<Dat>> { |
| 101 | Ok(res!(db.get(&dat!(k), None)).map(|(v, _)| v)) |
| 102 | } |
| 103 | |
| 104 | fn len(p: &Path) -> u64 { |
| 105 | fs::metadata(p).map(|m| m.len()).unwrap_or(0) |
| 106 | } |
| 107 | |
| 108 | fn copy_tree(from: &Path, to: &Path) -> Outcome<()> { |
| 109 | res!(fs::create_dir_all(to)); |
| 110 | for e in res!(fs::read_dir(from)) { |
| 111 | let p = res!(e).path(); |
| 112 | let name = res!(p.file_name().ok_or_else(|| err!("{:?} has no name.", p; Test))); |
| 113 | match p.is_dir() { |
| 114 | true => res!(copy_tree(&p, &to.join(name))), |
| 115 | false => { res!(fs::copy(&p, to.join(name))); }, |
| 116 | } |
| 117 | } |
| 118 | Ok(()) |
| 119 | } |
| 120 | |
| 121 | // A store at `dir` holding a durable "kept", stopped by a failed barrier over "lost". |
| 122 | fn stopped(dir: &str) -> Outcome<(PathBuf, OzoneConfig, FaultReport)> { |
| 123 | let root = res!(fresh(dir)); |
| 124 | let c = res!(cfg(false, 1_000_000)); |
| 125 | let db = res!(open(&root, &c, true)); |
| 126 | res!(put(&db, "kept", 1)); |
| 127 | hooks::set_barrier_failure(true); |
| 128 | let lost = put(&db, "lost", 2); |
| 129 | hooks::set_barrier_failure(false); |
| 130 | assert!(lost.is_err(), "a write whose barrier failed was confirmed"); |
| 131 | let report = res!(db.api().fault().ok_or_else(|| err!("The failed barrier raised no fault."; Test))); |
| 132 | let _ = db.close(); |
| 133 | assert!(!report.cuts.is_empty(), "the fault names no file in doubt"); |
| 134 | Ok((root, c, report)) |
| 135 | } |
| 136 | |
| 137 | /// The stop is written down as the record and the marker, so `recorded()` lets a gateway exit, |
| 138 | /// and the store refuses to open until the marker is cleared. |
| 139 | #[test] |
| 140 | fn a_stop_is_recorded_and_refuses_the_open() -> Outcome<()> { |
| 141 | let _g = lock(); |
| 142 | let (root, c, report) = res!(stopped("./test_db_fault_stop_recorded")); |
| 143 | assert!(report.recorded(), "a stop written down says it is not"); |
| 144 | assert_eq!(report.record, Some(root.join(constant::FAULT_FILENAME))); |
| 145 | assert_eq!(report.refusal, Some(Refusal::path(&root))); |
| 146 | assert!(Refusal::path(&root).exists(), "the stop left no refusal marker"); |
| 147 | match open(&root, &c, false) { |
| 148 | Ok(db) => { |
| 149 | let _ = db.close(); |
| 150 | return Err(err!("The store opened over its refusal marker."; Test)); |
| 151 | }, |
| 152 | Err(e) => assert!(fmt!("{}", e).contains(constant::REFUSAL_FILENAME), |
| 153 | "the refusal does not name the marker: {}", e), |
| 154 | } |
| 155 | assert!(res!(Refusal::clear(&root))); |
| 156 | let db = res!(open(&root, &c, false)); |
| 157 | assert_eq!(res!(reads(&db, "kept")), Some(val(1)), "the durable value was lost"); |
| 158 | assert_eq!(res!(reads(&db, "lost")), None, "the unconfirmed value was replayed"); |
| 159 | res!(db.close()); |
| 160 | let _ = fs::remove_dir_all(&root); |
| 161 | Ok(()) |
| 162 | } |
| 163 | |
| 164 | /// A stop that could not be written down says so, and a gateway must then stay up. |
| 165 | #[test] |
| 166 | fn a_stop_not_written_down_is_not_recorded() -> Outcome<()> { |
| 167 | let _g = lock(); |
| 168 | hooks::set_fault_record_fails(true); |
| 169 | let stopped = stopped("./test_db_fault_stop_unrecorded"); |
| 170 | hooks::set_fault_record_fails(false); |
| 171 | let (root, _, report) = res!(stopped); |
| 172 | assert!(!report.recorded(), "a stop written nowhere says it is recorded"); |
| 173 | assert!(report.record.is_none() && report.refusal.is_none()); |
| 174 | assert!(!root.join(constant::FAULT_FILENAME).exists() && !Refusal::path(&root).exists()); |
| 175 | let _ = fs::remove_dir_all(&root); |
| 176 | Ok(()) |
| 177 | } |
| 178 | |
| 179 | /// A stopped store moved elsewhere is cut where it now stands, and the copy left at the old path |
| 180 | /// is not touched (F2). Each file in doubt is given a tail past its mark, as a crash before the |
| 181 | /// in-process cut would leave it. |
| 182 | #[test] |
| 183 | fn a_moved_store_is_cut_where_it_stands() -> Outcome<()> { |
| 184 | let _g = lock(); |
| 185 | let (old, c, report) = res!(stopped("./test_db_fault_stop_old")); |
| 186 | let new = res!(fresh("./test_db_fault_stop_new")); |
| 187 | res!(copy_tree(&old, &new)); |
| 188 | let mut tails = Vec::new(); |
| 189 | for cut in &report.cuts { |
| 190 | for (p, upto) in [(&cut.data, cut.data_upto), (&cut.index, cut.index_upto)] { |
| 191 | let rel = res!(p.strip_prefix(&old).map_err(|e| err!(e, "{:?} is not under {:?}.", p, old; Test))); |
| 192 | for root in [&old, &new] { |
| 193 | let mut f = res!(fs::OpenOptions::new().append(true).open(root.join(rel))); |
| 194 | res!(f.write_all(&[0xa5u8; 64])); |
| 195 | } |
| 196 | tails.push((rel.to_path_buf(), upto, len(&old.join(rel)))); |
| 197 | } |
| 198 | } |
| 199 | assert!(res!(Refusal::clear(&new))); |
| 200 | let db = res!(open(&new, &c, false)); |
| 201 | assert_eq!(res!(reads(&db, "kept")), Some(val(1)), "the durable value was lost in the move"); |
| 202 | assert_eq!(res!(reads(&db, "lost")), None, "the moved store replayed the unconfirmed value"); |
| 203 | res!(db.close()); |
| 204 | for (rel, upto, old_len) in &tails { |
| 205 | assert!(len(&new.join(rel)) <= *upto, "{:?} of the moved store was not cut to {}", rel, upto); |
| 206 | assert_eq!(len(&old.join(rel)), *old_len, "{:?} at the old path was touched", rel); |
| 207 | } |
| 208 | assert!(old.join(constant::FAULT_FILENAME).exists(), "the old path's record was used up"); |
| 209 | assert!(!new.join(constant::FAULT_FILENAME).exists(), "the moved store kept its record"); |
| 210 | let _ = fs::remove_dir_all(&old); |
| 211 | let _ = fs::remove_dir_all(&new); |
| 212 | Ok(()) |
| 213 | } |
| 214 | |
| 215 | /// A file in doubt that is missing refuses the open, and the record is kept for the next try. |
| 216 | #[test] |
| 217 | fn a_missing_file_in_doubt_refuses_the_open() -> Outcome<()> { |
| 218 | let _g = lock(); |
| 219 | let (root, c, report) = res!(stopped("./test_db_fault_stop_missing")); |
| 220 | assert!(res!(Refusal::clear(&root))); |
| 221 | let gone = report.cuts[0].data.clone(); |
| 222 | res!(fs::remove_file(&gone)); |
| 223 | match open(&root, &c, false) { |
| 224 | Ok(db) => { |
| 225 | let _ = db.close(); |
| 226 | return Err(err!("The store opened though its file in doubt {:?} is missing.", gone; Test)); |
| 227 | }, |
| 228 | Err(e) => assert!(fmt!("{}", e).contains("is missing"), |
| 229 | "the refusal does not say the file is missing: {}", e), |
| 230 | } |
| 231 | assert!(root.join(constant::FAULT_FILENAME).exists(), "the refused open removed the record"); |
| 232 | let _ = fs::remove_dir_all(&root); |
| 233 | Ok(()) |
| 234 | } |
| 235 | |
| 236 | /// The writer rolls over while the failing barrier is held, so the file it rolls over to was |
| 237 | /// not in doubt when the stop named its files. It joins the record, and is cut (F3). |
| 238 | #[test] |
| 239 | fn the_file_rolled_over_to_is_cut() -> Outcome<()> { |
| 240 | let _g = lock(); |
| 241 | let root = res!(fresh("./test_db_fault_stop_rollover")); |
| 242 | let c = res!(cfg(true, 2_000)); |
| 243 | let db = Arc::new(res!(open(&root, &c, true))); |
| 244 | for i in 0..5 { |
| 245 | res!(put(&db, &fmt!("before {}", i), i as u8 + 1)); |
| 246 | } |
| 247 | res!(db.api().sync_now(wait())); |
| 248 | let zdir = c.zone_root(&root).join("zone_001"); |
| 249 | let dat = |f: u32| zdir.join(ZoneDir::relative_file_path(&FileType::Data, f)); |
| 250 | let first = res!((1..10_000u32).rev().find(|f| dat(*f).exists()) |
| 251 | .ok_or_else(|| err!("No data file is live."; Test))); |
| 252 | res!(put(&db, "dirty", 9)); // so the next sync_now has a barrier to run |
| 253 | hooks::set_barrier_delay(Duration::from_millis(800)); |
| 254 | hooks::set_barrier_failure(true); |
| 255 | let db2 = db.clone(); |
| 256 | let sync = std::thread::spawn(move || db2.api().sync_now(wait())); |
| 257 | std::thread::sleep(Duration::from_millis(100)); |
| 258 | // A put waits on its release, which the held barrier holds back, so the puts that must fill |
| 259 | // the live file during it run side by side rather than one after another. |
| 260 | let puts: Vec<_> = (0..40).map(|i| { |
| 261 | let db3 = db.clone(); |
| 262 | std::thread::spawn(move || put(&db3, &fmt!("during {}", i), 7)) |
| 263 | }).collect(); |
| 264 | let synced = sync.join(); |
| 265 | for p in puts { |
| 266 | let _ = p.join(); |
| 267 | } |
| 268 | hooks::set_barrier_failure(false); |
| 269 | hooks::set_barrier_delay(Duration::ZERO); |
| 270 | assert!(matches!(synced, Ok(Err(_))), "the barrier armed to fail did not fail"); |
| 271 | let last = res!((1..10_000u32).rev().find(|f| dat(*f).exists()) |
| 272 | .ok_or_else(|| err!("No data file is live."; Test))); |
| 273 | assert!(last > first, "the writer did not roll over during the held barrier ({} to {})", first, last); |
| 274 | assert!(wait_until(Duration::from_secs(10), || |
| 275 | db.api().fault().map(|r| r.cuts.iter().any(|cut| cut.fnum == last)).unwrap_or(false)), |
| 276 | "the file {} rolled over to is not in the fault's cuts: {:?}", last, |
| 277 | db.api().fault().map(|r| r.cuts.iter().map(|cut| cut.fnum).collect::<Vec<_>>())); |
| 278 | let report = res!(db.api().fault().ok_or_else(|| err!("No fault."; Test))); |
| 279 | match Arc::try_unwrap(db) { |
| 280 | Ok(db) => { let _ = db.close(); }, |
| 281 | Err(_) => return Err(err!("The store is still shared."; Test)), |
| 282 | } |
| 283 | let record = res!(fs::read_to_string(root.join(constant::FAULT_FILENAME))); |
| 284 | assert!(record.lines().any(|l| l.starts_with(&fmt!("cut 0 {} ", last))), |
| 285 | "the record does not cut file {}: {}", last, record); |
| 286 | assert!(res!(Refusal::clear(&root))); |
| 287 | let db = res!(open(&root, &c, false)); |
| 288 | for i in 0..5 { |
| 289 | assert_eq!(res!(reads(&db, &fmt!("before {}", i))), Some(val(i as u8 + 1)), "a durable value was lost"); |
| 290 | } |
| 291 | res!(db.close()); |
| 292 | for cut in &report.cuts { |
| 293 | assert!(len(&cut.data) <= cut.data_upto && len(&cut.index) <= cut.index_upto, |
| 294 | "file {} was not cut to its mark ({}, {}): ({}, {})", cut.fnum, cut.data_upto, |
| 295 | cut.index_upto, len(&cut.data), len(&cut.index)); |
| 296 | } |
| 297 | let _ = fs::remove_dir_all(&root); |
| 298 | Ok(()) |
| 299 | } |