oxedyne/fe2o3/fe2o3_o3db_sync/tests/fault_kill9.rs
8.1 KiB, 1 run
created by r1870400018:70852, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! A process killed with SIGKILL at each step of a barrier and of the stop after a failed one |
| 2 | //! (2026-10-10, QA arm d). The child writes under the every-write policy, then stops at a |
| 3 | //! `hooks::Halt`, says so through a flag file, and waits to be killed. On the reopen every |
| 4 | //! confirmed value reads back, the unconfirmed one is never served once the fault record |
| 5 | //! existed, a marker left before the kill refuses the open until it is cleared, and every file |
| 6 | //! the record names ends at or below its mark. |
| 7 | |
| 8 | mod gc_pair; |
| 9 | |
| 10 | use gc_pair::*; |
| 11 | |
| 12 | use oxedyne_fe2o3_core::prelude::*; |
| 13 | use oxedyne_fe2o3_iop_db::api::Database; |
| 14 | use oxedyne_fe2o3_jdat::prelude::*; |
| 15 | use oxedyne_fe2o3_o3db_sync::{ |
| 16 | base::{ |
| 17 | cfg::OzoneConfig, |
| 18 | constant, |
| 19 | }, |
| 20 | file::{ |
| 21 | core::FileType, |
| 22 | refusal::Refusal, |
| 23 | zdir::ZoneDir, |
| 24 | }, |
| 25 | test::{ |
| 26 | hooks::{ |
| 27 | self, |
| 28 | Halt, |
| 29 | }, |
| 30 | setup::{ |
| 31 | self, |
| 32 | Uid, |
| 33 | }, |
| 34 | }, |
| 35 | }; |
| 36 | |
| 37 | use std::{ |
| 38 | collections::BTreeMap, |
| 39 | fs, |
| 40 | path::{ |
| 41 | Path, |
| 42 | PathBuf, |
| 43 | }, |
| 44 | process::{ |
| 45 | Command, |
| 46 | Stdio, |
| 47 | }, |
| 48 | thread, |
| 49 | time::Duration, |
| 50 | }; |
| 51 | |
| 52 | const ROOT: &str = "FAULT_KILL9_ROOT"; |
| 53 | const HALT: &str = "FAULT_KILL9_HALT"; |
| 54 | const FLAG: &str = "FAULT_KILL9_FLAG"; |
| 55 | const NKEPT: usize = 10; |
| 56 | |
| 57 | const HALTS: [Halt; 5] = [ |
| 58 | Halt::Synced, |
| 59 | Halt::RecordSynced, |
| 60 | Halt::Recorded, |
| 61 | Halt::Quarantined, |
| 62 | Halt::DataCut, |
| 63 | ]; |
| 64 | |
| 65 | fn cfg() -> Outcome<OzoneConfig> { |
| 66 | let mut c = res!(setup::default_cfg()); |
| 67 | c.num_zones = 1; |
| 68 | c.num_wbots_per_zone = 1; |
| 69 | c.zone_overrides = BTreeMap::new(); |
| 70 | c.sync_on_write = true; |
| 71 | c.sync_every_n_writes = 0; |
| 72 | c.sync_interval_ms = 0; |
| 73 | Ok(c) |
| 74 | } |
| 75 | |
| 76 | fn val(seed: u8) -> Dat { |
| 77 | Dat::BU32((0..100).map(|j| seed ^ (j as u8)).collect()) |
| 78 | } |
| 79 | |
| 80 | fn kept(i: usize) -> Dat { |
| 81 | dat!(fmt!("kill9 kept {}", i)) |
| 82 | } |
| 83 | |
| 84 | fn halt_named(name: &str) -> Outcome<Halt> { |
| 85 | match HALTS.iter().find(|h| fmt!("{:?}", h) == name) { |
| 86 | Some(h) => Ok(*h), |
| 87 | None => Err(err!("No halt is named {:?}.", name; Test, Invalid)), |
| 88 | } |
| 89 | } |
| 90 | |
| 91 | // Writes the kept values, each confirmed durable, then one more that stops at `halt`: under |
| 92 | // `Synced` its barrier succeeds, otherwise it fails. |
| 93 | fn child(root: PathBuf, halt: Halt, flag: PathBuf) -> Outcome<()> { |
| 94 | log_set_level!("error"); |
| 95 | let db: TestDb = res!(setup::start_db(root, Some(res!(cfg())), schemes(), None, false, true)); |
| 96 | for i in 0..NKEPT { |
| 97 | res!(db.insert(kept(i), val(i as u8 + 1), Uid::default(), None)); |
| 98 | } |
| 99 | hooks::set_halt(Some((halt, flag))); |
| 100 | if halt != Halt::Synced { |
| 101 | hooks::set_barrier_failure(true); |
| 102 | } |
| 103 | let _ = db.insert(dat!("kill9 last"), val(99), Uid::default(), None); |
| 104 | // A child that is not killed is a failure of the parent, and ends itself. |
| 105 | thread::sleep(Duration::from_secs(120)); |
| 106 | std::process::exit(3); |
| 107 | } |
| 108 | |
| 109 | // The cuts a fault record names, as (file number, data mark, index mark). |
| 110 | fn record_cuts(root: &Path) -> Outcome<Option<Vec<(u32, u64, u64)>>> { |
| 111 | let text = match fs::read_to_string(root.join(constant::FAULT_FILENAME)) { |
| 112 | Ok(text) => text, |
| 113 | Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), |
| 114 | Err(e) => return Err(err!(e, "Cannot read the fault record."; Test, IO)), |
| 115 | }; |
| 116 | let mut cuts = Vec::new(); |
| 117 | for line in text.lines() { |
| 118 | if let Some(rest) = line.strip_prefix("cut ") { |
| 119 | let n: Vec<u64> = rest.split(' ').filter_map(|f| f.parse().ok()).collect(); |
| 120 | if n.len() < 4 { |
| 121 | return Err(err!("A cut line is short: {:?}", line; Test, Invalid)); |
| 122 | } |
| 123 | cuts.push((n[1] as u32, n[2], n[3])); |
| 124 | } |
| 125 | } |
| 126 | Ok(Some(cuts)) |
| 127 | } |
| 128 | |
| 129 | fn kill_at(halt: Halt) -> Outcome<()> { |
| 130 | let label = fmt!("{:?}", halt); |
| 131 | let dir = fmt!("./test_db_fault_kill9_{}", label.to_lowercase()); |
| 132 | let _ = fs::remove_dir_all(&dir); |
| 133 | res!(fs::create_dir_all(&dir)); |
| 134 | let root = res!(Path::new(&dir).canonicalize()); |
| 135 | let flag = root.with_extension("halted"); |
| 136 | let log = root.with_extension("log"); |
| 137 | let _ = fs::remove_file(&flag); |
| 138 | |
| 139 | let out = res!(fs::File::create(&log)); |
| 140 | let err = res!(out.try_clone()); |
| 141 | let mut kid = res!(Command::new(res!(std::env::current_exe())) |
| 142 | .args(["main", "--exact", "--nocapture", "--test-threads=1"]) |
| 143 | .env(ROOT, &root) |
| 144 | .env(HALT, &label) |
| 145 | .env(FLAG, &flag) |
| 146 | .stdout(Stdio::from(out)) |
| 147 | .stderr(Stdio::from(err)) |
| 148 | .spawn()); |
| 149 | let halted = wait_until(Duration::from_secs(120), || flag.exists()); |
| 150 | res!(kid.kill()); |
| 151 | let _ = kid.wait(); |
| 152 | assert!(halted, "{}: the child never reached its halt; see {:?}", label, log); |
| 153 | |
| 154 | let recorded = halt != Halt::Synced && halt != Halt::RecordSynced; |
| 155 | let marked = halt == Halt::Quarantined || halt == Halt::DataCut; |
| 156 | let cuts = res!(record_cuts(&root)); |
| 157 | assert_eq!(cuts.is_some(), recorded, "{}: the fault record is {} after the kill", label, |
| 158 | if cuts.is_some() { "there" } else { "missing" }); |
| 159 | assert_eq!(Refusal::path(&root).exists(), marked, "{}: the refusal marker is {} after the kill", |
| 160 | label, if marked { "missing" } else { "there" }); |
| 161 | |
| 162 | let c = res!(cfg()); |
| 163 | let open = || -> Outcome<TestDb> { |
| 164 | log_set_level!("error"); |
| 165 | setup::start_db(root.clone(), Some(c.clone()), schemes(), None, false, false) |
| 166 | }; |
| 167 | if marked { |
| 168 | match open() { |
| 169 | Ok(db) => { |
| 170 | let _ = db.close(); |
| 171 | return Err(err!("{}: the store opened over its refusal marker.", label; Test)); |
| 172 | }, |
| 173 | Err(e) => assert!(fmt!("{}", e).contains(constant::REFUSAL_FILENAME), |
| 174 | "{}: the refusal does not name its marker: {}", label, e), |
| 175 | } |
| 176 | assert!(res!(Refusal::clear(&root))); |
| 177 | } |
| 178 | let db = res!(open()); |
| 179 | for i in 0..NKEPT { |
| 180 | let got = res!(db.get(&kept(i), None)).map(|(v, _)| v); |
| 181 | assert_eq!(got, Some(val(i as u8 + 1)), "{}: confirmed key {} did not read back", label, i); |
| 182 | } |
| 183 | let last = res!(db.get(&dat!("kill9 last"), None)).map(|(v, _)| v); |
| 184 | if recorded { |
| 185 | assert_eq!(last, None, "{}: the unconfirmed value was served after its fault was recorded", label); |
| 186 | } else { |
| 187 | assert!(last.is_none() || last == Some(val(99)), "{}: the last key read torn", label); |
| 188 | } |
| 189 | res!(db.close()); |
| 190 | assert!(!root.join(constant::FAULT_FILENAME).exists(), "{}: the record outlived the cut", label); |
| 191 | let zdir = c.zone_root(&root).join("zone_001"); |
| 192 | for (f, d, i) in cuts.unwrap_or_default() { |
| 193 | let dl = fs::metadata(zdir.join(ZoneDir::relative_file_path(&FileType::Data, f))).map(|m| m.len()).unwrap_or(0); |
| 194 | let il = fs::metadata(zdir.join(ZoneDir::relative_file_path(&FileType::Index, f))).map(|m| m.len()).unwrap_or(0); |
| 195 | assert!(dl <= d && il <= i, "{}: file {} is ({}, {}), past its mark ({}, {})", label, f, dl, il, d, i); |
| 196 | } |
| 197 | // QA r2b: the store keeps working after the cut, and a second reopen agrees with the first. |
| 198 | let db = res!(open()); |
| 199 | res!(db.insert(dat!("kill9 after"), val(77), Uid::default(), None)); |
| 200 | res!(db.close()); |
| 201 | let db = res!(open()); |
| 202 | for i in 0..NKEPT { |
| 203 | let got = res!(db.get(&kept(i), None)).map(|(v, _)| v); |
| 204 | assert_eq!(got, Some(val(i as u8 + 1)), "{}: confirmed key {} lost at the second reopen", label, i); |
| 205 | } |
| 206 | let last2 = res!(db.get(&dat!("kill9 last"), None)).map(|(v, _)| v); |
| 207 | assert_eq!(last2, last, "{}: the last key changed between reopens", label); |
| 208 | let after = res!(db.get(&dat!("kill9 after"), None)).map(|(v, _)| v); |
| 209 | assert_eq!(after, Some(val(77)), "{}: the write after the cut was lost", label); |
| 210 | res!(db.close()); |
| 211 | let _ = fs::remove_dir_all(&root); |
| 212 | let _ = fs::remove_file(&flag); |
| 213 | let _ = fs::remove_file(&log); |
| 214 | Ok(()) |
| 215 | } |
| 216 | |
| 217 | #[test] |
| 218 | fn main() -> Outcome<()> { |
| 219 | match (std::env::var(ROOT), std::env::var(HALT), std::env::var(FLAG)) { |
| 220 | (Ok(root), Ok(halt), Ok(flag)) => |
| 221 | child(PathBuf::from(root), res!(halt_named(&halt)), PathBuf::from(flag)), |
| 222 | _ => { |
| 223 | for halt in HALTS { |
| 224 | res!(kill_at(halt)); |
| 225 | } |
| 226 | Ok(()) |
| 227 | }, |
| 228 | } |
| 229 | } |