oxedyne/fe2o3/fe2o3_o3db_sync/tests/append_rewind.rs
9.7 KiB, 1 run
created by r1870400018:70846, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! An append to a live file that fails part-way (2026-10-11, QA round 1 F4). The live files are |
| 2 | //! opened for appending, so a seek back has no effect on where the next write lands: the torn |
| 3 | //! bytes stayed in the file, the next record went in after them, and the writer's idea of the |
| 4 | //! file's length, from which the next record's location is taken, fell behind the file. The torn |
| 5 | //! append is now cut back off, both files of the pair return to their last good lengths, and a |
| 6 | //! cut that fails stops the store through its fault. The hooks are process-wide, so the tests |
| 7 | //! run one at a time. |
| 8 | |
| 9 | mod gc_pair; |
| 10 | |
| 11 | use gc_pair::*; |
| 12 | |
| 13 | use oxedyne_fe2o3_core::prelude::*; |
| 14 | use oxedyne_fe2o3_iop_db::api::Database; |
| 15 | use oxedyne_fe2o3_jdat::prelude::*; |
| 16 | use oxedyne_fe2o3_o3db_sync::{ |
| 17 | base::cfg::OzoneConfig, |
| 18 | file::{ |
| 19 | core::FileType, |
| 20 | refusal::Refusal, |
| 21 | }, |
| 22 | test::{ |
| 23 | hooks, |
| 24 | setup::{ |
| 25 | self, |
| 26 | Uid, |
| 27 | }, |
| 28 | }, |
| 29 | }; |
| 30 | |
| 31 | use std::{ |
| 32 | collections::BTreeMap, |
| 33 | fs, |
| 34 | path::{ |
| 35 | Path, |
| 36 | PathBuf, |
| 37 | }, |
| 38 | sync::Mutex, |
| 39 | }; |
| 40 | |
| 41 | static SERIAL: Mutex<()> = Mutex::new(()); |
| 42 | |
| 43 | fn lock() -> std::sync::MutexGuard<'static, ()> { |
| 44 | let g = match SERIAL.lock() { Ok(g) => g, Err(p) => p.into_inner() }; |
| 45 | hooks::set_write_tear(FileType::Data, None); |
| 46 | hooks::set_write_tear(FileType::Index, None); |
| 47 | hooks::set_cut_back_failure(false); |
| 48 | g |
| 49 | } |
| 50 | |
| 51 | // One zone of one writer, syncing on every write. |
| 52 | fn cfg() -> Outcome<OzoneConfig> { |
| 53 | let mut c = res!(setup::default_cfg()); |
| 54 | c.num_zones = 1; |
| 55 | c.num_wbots_per_zone = 1; |
| 56 | c.zone_overrides = BTreeMap::new(); |
| 57 | c.sync_on_write = true; |
| 58 | c.sync_every_n_writes = 0; |
| 59 | c.sync_interval_ms = 0; |
| 60 | c.data_file_max_bytes = 1_000_000; |
| 61 | Ok(c) |
| 62 | } |
| 63 | |
| 64 | fn fresh(dir: &str) -> Outcome<PathBuf> { |
| 65 | let _ = fs::remove_dir_all(dir); |
| 66 | res!(fs::create_dir_all(dir)); |
| 67 | Ok(res!(Path::new(dir).canonicalize())) |
| 68 | } |
| 69 | |
| 70 | fn open(root: &Path, c: &OzoneConfig, wipe: bool) -> Outcome<TestDb> { |
| 71 | log_set_level!("error"); |
| 72 | setup::start_db(root.to_path_buf(), Some(c.clone()), schemes(), None, false, wipe) |
| 73 | } |
| 74 | |
| 75 | fn val(seed: u8) -> Dat { |
| 76 | Dat::BU32((0..100).map(|j| seed ^ (j as u8)).collect()) |
| 77 | } |
| 78 | |
| 79 | fn put(db: &TestDb, k: &str, seed: u8) -> Outcome<()> { |
| 80 | db.insert(dat!(k), val(seed), Uid::default(), None).map(|_| ()) |
| 81 | } |
| 82 | |
| 83 | fn reads(db: &TestDb, k: &str) -> Outcome<Option<Dat>> { |
| 84 | Ok(res!(db.get(&dat!(k), None)).map(|(v, _)| v)) |
| 85 | } |
| 86 | |
| 87 | // The live pair, the highest numbered data file and its index. |
| 88 | fn live(root: &Path, c: &OzoneConfig) -> Outcome<(PathBuf, PathBuf)> { |
| 89 | let n = res!((1..10_000u32).rev().find(|n| file(root, c, FileType::Data, *n).exists()) |
| 90 | .ok_or_else(|| err!("No data file is live."; Test))); |
| 91 | Ok((file(root, c, FileType::Data, n), file(root, c, FileType::Index, n))) |
| 92 | } |
| 93 | |
| 94 | // Two good records, "k0" and "k1", and the pair's lengths before and after the second. |
| 95 | struct Start { |
| 96 | root: PathBuf, |
| 97 | c: OzoneConfig, |
| 98 | db: TestDb, |
| 99 | dat: PathBuf, |
| 100 | ind: PathBuf, |
| 101 | rec: u64, // data bytes in one record |
| 102 | dlen: u64, // data length after "k1" |
| 103 | ilen: u64, // index length after "k1" |
| 104 | } |
| 105 | |
| 106 | fn start(dir: &str) -> Outcome<Start> { |
| 107 | let root = res!(fresh(dir)); |
| 108 | let c = res!(cfg()); |
| 109 | let db = res!(open(&root, &c, true)); |
| 110 | res!(put(&db, "k0", 1)); |
| 111 | let (dat, ind) = res!(live(&root, &c)); |
| 112 | let before = size(&dat); |
| 113 | res!(put(&db, "k1", 2)); |
| 114 | let dlen = size(&dat); |
| 115 | let ilen = size(&ind); |
| 116 | Ok(Start { root, c, db, dat, ind, rec: dlen - before, dlen, ilen }) |
| 117 | } |
| 118 | |
| 119 | // After the next record, and again after a reopen, every key reads its value and the failed one |
| 120 | // is absent. |
| 121 | fn check_after(s: Start, failed: &str) -> Outcome<()> { |
| 122 | let Start { root, c, db, dat, ind, rec, dlen, ilen } = s; |
| 123 | res!(put(&db, "k3", 4)); |
| 124 | assert_eq!(size(&dat), dlen + rec, |
| 125 | "the record after the failed one did not land at the last good length {}", dlen); |
| 126 | assert!(size(&ind) > ilen, "the record after the failed one is not in the index"); |
| 127 | let want = [("k0", Some(val(1))), ("k1", Some(val(2))), (failed, None), ("k3", Some(val(4)))]; |
| 128 | for (k, v) in &want { |
| 129 | assert_eq!(&res!(reads(&db, k)), v, "{} reads wrong before the reopen", k); |
| 130 | } |
| 131 | res!(db.close()); |
| 132 | let db = res!(open(&root, &c, false)); |
| 133 | for (k, v) in &want { |
| 134 | assert_eq!(&res!(reads(&db, k)), v, "{} reads wrong after the reopen", k); |
| 135 | } |
| 136 | res!(db.close()); |
| 137 | let _ = fs::remove_dir_all(&root); |
| 138 | Ok(()) |
| 139 | } |
| 140 | |
| 141 | /// A data append torn part-way is cut back off, and the next record lands where it began. |
| 142 | #[test] |
| 143 | fn a_torn_data_append_is_cut_back() -> Outcome<()> { |
| 144 | let _g = lock(); |
| 145 | let s = res!(start("./test_db_append_rewind_data")); |
| 146 | let torn = hooks::writes_torn(); |
| 147 | hooks::set_write_tear(FileType::Data, Some(s.rec / 2)); |
| 148 | let failed = put(&s.db, "k2", 3); |
| 149 | assert_eq!(hooks::writes_torn(), torn + 1, "the tear was not reached"); |
| 150 | assert!(failed.is_err(), "a torn write was confirmed"); |
| 151 | assert_eq!(size(&s.dat), s.dlen, "the torn bytes were left in the data file"); |
| 152 | assert_eq!(size(&s.ind), s.ilen, "the index changed for a record that failed"); |
| 153 | assert!(s.db.api().fault().is_none(), "a torn write cut back cleanly stopped the store"); |
| 154 | check_after(s, "k2") |
| 155 | } |
| 156 | |
| 157 | /// An index append torn part-way withdraws the whole record, its data as well, so a write |
| 158 | /// reported failed does not come back at the next open. |
| 159 | #[test] |
| 160 | fn a_torn_index_append_withdraws_the_record() -> Outcome<()> { |
| 161 | let _g = lock(); |
| 162 | let s = res!(start("./test_db_append_rewind_index")); |
| 163 | let torn = hooks::writes_torn(); |
| 164 | hooks::set_write_tear(FileType::Index, Some(3)); |
| 165 | let failed = put(&s.db, "k2", 3); |
| 166 | assert_eq!(hooks::writes_torn(), torn + 1, "the tear was not reached"); |
| 167 | assert!(failed.is_err(), "a torn write was confirmed"); |
| 168 | assert_eq!(size(&s.ind), s.ilen, "the torn bytes were left in the index file"); |
| 169 | assert_eq!(size(&s.dat), s.dlen, "the data of a record whose index failed was left"); |
| 170 | assert!(s.db.api().fault().is_none(), "a torn write cut back cleanly stopped the store"); |
| 171 | check_after(s, "k2") |
| 172 | } |
| 173 | |
| 174 | /// A torn append that cannot be cut back off stops the store through its fault, which names the |
| 175 | /// last good lengths, and the next open cuts the pair back to them. |
| 176 | #[test] |
| 177 | fn a_failed_cut_back_stops_the_store() -> Outcome<()> { |
| 178 | let _g = lock(); |
| 179 | let Start { root, c, db, dat, ind, rec, dlen, ilen } = |
| 180 | res!(start("./test_db_append_rewind_stop")); |
| 181 | hooks::set_cut_back_failure(true); |
| 182 | hooks::set_write_tear(FileType::Data, Some(rec / 2)); |
| 183 | let failed = put(&db, "k2", 3); |
| 184 | hooks::set_cut_back_failure(false); |
| 185 | assert!(failed.is_err(), "a torn write was confirmed"); |
| 186 | let report = res!(db.api().fault().ok_or_else(|| err!( |
| 187 | "A torn append that could not be cut back did not stop the store."; Test))); |
| 188 | assert!(report.recorded(), "the stop was not written down"); |
| 189 | assert!(report.cuts.iter().any(|cut| cut.data == dat && cut.data_upto == dlen |
| 190 | && cut.index == ind && cut.index_upto == ilen), |
| 191 | "the fault does not cut the pair to ({}, {}): {:?}", dlen, ilen, report.cuts); |
| 192 | assert!(put(&db, "k3", 4).is_err(), "a stopped store took a write"); |
| 193 | let _ = db.close(); |
| 194 | assert!(res!(Refusal::clear(&root))); |
| 195 | let db = res!(open(&root, &c, false)); |
| 196 | assert_eq!(size(&dat), dlen, "the open did not cut the torn bytes off the data file"); |
| 197 | assert_eq!(res!(reads(&db, "k0")), Some(val(1))); |
| 198 | assert_eq!(res!(reads(&db, "k1")), Some(val(2))); |
| 199 | assert_eq!(res!(reads(&db, "k2")), None, "the torn record was served"); |
| 200 | assert_eq!(res!(reads(&db, "k3")), None, "the refused record was served"); |
| 201 | res!(db.close()); |
| 202 | let _ = fs::remove_dir_all(&root); |
| 203 | Ok(()) |
| 204 | } |
| 205 | |
| 206 | /// A failed cut back whose fault record cannot be written is still recorded, by the refusal |
| 207 | /// marker, which carries the cut; clearing the marker folds it into the record, and the next open |
| 208 | /// cuts the pair back (2026-10-11, rebased onto durable-mark re-QA round 2 F1, F2). |
| 209 | #[test] |
| 210 | fn a_failed_cut_back_is_held_by_the_marker_alone() -> Outcome<()> { |
| 211 | let _g = lock(); |
| 212 | let Start { root, c, db, dat, ind, rec, dlen, ilen } = |
| 213 | res!(start("./test_db_append_rewind_marker")); |
| 214 | hooks::set_fault_record_fails_at(1); |
| 215 | hooks::set_cut_back_failure(true); |
| 216 | hooks::set_write_tear(FileType::Data, Some(rec / 2)); |
| 217 | let failed = put(&db, "k2", 3); |
| 218 | hooks::set_cut_back_failure(false); |
| 219 | hooks::set_fault_record_fails_at(0); |
| 220 | assert!(failed.is_err(), "a torn write was confirmed"); |
| 221 | let report = res!(db.api().fault().ok_or_else(|| err!( |
| 222 | "A torn append that could not be cut back did not stop the store."; Test))); |
| 223 | assert!(report.record.is_none(), "a record that failed to be written is named"); |
| 224 | assert!(report.refusal.is_some(), "the refusal marker was not written"); |
| 225 | assert!(report.recorded(), "a stop held by the marker says it is not recorded"); |
| 226 | let cut = res!(report.cuts.iter().find(|cut| cut.data == dat && cut.data_upto == dlen |
| 227 | && cut.index == ind && cut.index_upto == ilen).ok_or_else(|| err!( |
| 228 | "The fault does not cut the pair to ({}, {}): {:?}", dlen, ilen, report.cuts; Test))); |
| 229 | let marker = res!(fs::read_to_string(Refusal::path(&root))); |
| 230 | assert!(marker.lines().any(|l| l.starts_with(&cut.line())), |
| 231 | "the refusal marker does not carry the cut {:?}: {:?}", cut.line(), marker); |
| 232 | let _ = db.close(); |
| 233 | assert!(res!(Refusal::clear(&root))); |
| 234 | let db = res!(open(&root, &c, false)); |
| 235 | assert_eq!(size(&dat), dlen, "the open did not cut the torn bytes off the data file"); |
| 236 | assert_eq!(res!(reads(&db, "k0")), Some(val(1))); |
| 237 | assert_eq!(res!(reads(&db, "k1")), Some(val(2))); |
| 238 | assert_eq!(res!(reads(&db, "k2")), None, "the torn record was served"); |
| 239 | res!(db.close()); |
| 240 | let _ = fs::remove_dir_all(&root); |
| 241 | Ok(()) |
| 242 | } |