oxedyne/fe2o3/fe2o3_o3db_sync/tests/chunk_surplus.rs
12.4 KiB, 11 runs
created by r1870400018:68517, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! An overwrite that changes a chunked value's geometry retires the chunks it leaves (A3 R4, |
| 2 | //! 2026-10-08). A chunk is addressed by `(set_id, index, data length, count, size)`, so a value of |
| 3 | //! another length shares no chunk key with its predecessor: nothing supersedes the old chunks, and |
| 4 | //! until now they waited for the orphan sweep, which `compact_now` does not run. Each erase test |
| 5 | //! finds the old chunks in the data files first (the positive control), overwrites, runs |
| 6 | //! `compact_now`, and counts the bytes of the files from outside the store. The hooks used are |
| 7 | //! process-wide, so every test takes one lock and puts each hook back. |
| 8 | |
| 9 | mod gc_pair; |
| 10 | |
| 11 | use gc_pair::*; |
| 12 | |
| 13 | use oxedyne_fe2o3_core::prelude::*; |
| 14 | use oxedyne_fe2o3_iop_db::api::Database; |
| 15 | use oxedyne_fe2o3_jdat::{ |
| 16 | prelude::*, |
| 17 | chunk::PartKey, |
| 18 | }; |
| 19 | use oxedyne_fe2o3_o3db_sync::{ |
| 20 | base::cfg::OzoneConfig, |
| 21 | test::{ |
| 22 | hooks, |
| 23 | setup::{ |
| 24 | self, |
| 25 | Uid, |
| 26 | }, |
| 27 | }, |
| 28 | }; |
| 29 | |
| 30 | use std::{ |
| 31 | collections::BTreeMap, |
| 32 | path::{ |
| 33 | Path, |
| 34 | PathBuf, |
| 35 | }, |
| 36 | sync::{ |
| 37 | Arc, |
| 38 | Mutex, |
| 39 | MutexGuard, |
| 40 | atomic::{ |
| 41 | AtomicBool, |
| 42 | Ordering, |
| 43 | }, |
| 44 | }, |
| 45 | thread, |
| 46 | time::Duration, |
| 47 | }; |
| 48 | |
| 49 | static HOOKS: Mutex<()> = Mutex::new(()); |
| 50 | |
| 51 | fn lock() -> MutexGuard<'static, ()> { |
| 52 | match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() } |
| 53 | } |
| 54 | |
| 55 | const CSIZE: usize = 600; // bytes in a chunk |
| 56 | const FIVE: usize = 2_900; // a value of five chunks |
| 57 | const FIVE_B: usize = 2_800; // another value of five |
| 58 | const TWO: usize = 1_100; // a value of two |
| 59 | const PLAIN: usize = 500; // below the threshold, so not chunked |
| 60 | |
| 61 | fn chunk_cfg() -> Outcome<OzoneConfig> { |
| 62 | let mut c = res!(setup::default_cfg()); |
| 63 | c.num_zones = 1; |
| 64 | c.num_cbots_per_zone = 4; |
| 65 | c.num_fbots_per_zone = 1; |
| 66 | c.num_wbots_per_zone = 2; |
| 67 | c.zone_overrides = BTreeMap::new(); |
| 68 | c.rest_chunk_threshold = 1_000; |
| 69 | c.rest_chunk_bytes = CSIZE as u64; |
| 70 | c.sync_on_write = true; |
| 71 | Ok(c) |
| 72 | } |
| 73 | |
| 74 | // Bytes that do not repeat inside a window, so that a window is found by its own record only. |
| 75 | fn fill_of(seed: u8, len: usize) -> Vec<u8> { |
| 76 | let mut x = 0x9E37_79B9_7F4A_7C15u64 ^ ((seed as u64) << 32 | seed as u64); |
| 77 | (0..len).map(|_| { |
| 78 | x = x.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); |
| 79 | (x >> 56) as u8 |
| 80 | }).collect() |
| 81 | } |
| 82 | |
| 83 | // A hundred bytes from the middle of each chunk, so that a chunk is found by its own bytes. The |
| 84 | // encoded value is five bytes longer than the bytes in it, so a window of the middle is inside |
| 85 | // one chunk. |
| 86 | fn windows(seed: u8, len: usize) -> Vec<Vec<u8>> { |
| 87 | let v = fill_of(seed, len); |
| 88 | let n = (len + 5 + CSIZE - 1) / CSIZE; |
| 89 | (0..n).map(|i| v[i * CSIZE + 250..i * CSIZE + 350].to_vec()).collect() |
| 90 | } |
| 91 | |
| 92 | struct Store { |
| 93 | dir: String, |
| 94 | root: PathBuf, |
| 95 | cfg: OzoneConfig, |
| 96 | db: TestDb, |
| 97 | } |
| 98 | |
| 99 | fn store(name: &str) -> Outcome<Store> { |
| 100 | log_set_level!("error"); |
| 101 | let dir = fmt!("./test_db_surplus_{}", name); |
| 102 | let _ = std::fs::remove_dir_all(&dir); |
| 103 | res!(std::fs::create_dir_all(&dir)); |
| 104 | let root = res!(Path::new(&dir).canonicalize()); |
| 105 | let cfg = res!(chunk_cfg()); |
| 106 | // Collection off, so that only the order to compact collects. |
| 107 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, false, true)); |
| 108 | Ok(Store { dir, root, cfg, db }) |
| 109 | } |
| 110 | |
| 111 | impl Store { |
| 112 | fn holds(&self, seed: u8, len: usize) -> Vec<usize> { |
| 113 | windows(seed, len).iter().map(|w| files_holding(&self.root, &self.cfg, w).len()).collect() |
| 114 | } |
| 115 | |
| 116 | fn num_chunks(&self, k: &Dat) -> Outcome<u64> { |
| 117 | let api = self.db.api(); |
| 118 | let resp = res!(api.fetch_using_schemes(k, None)); |
| 119 | match res!(resp.recv_daticle(api.schemes().encrypter(), None)) { |
| 120 | (Some((Dat::Tup5u64(tup), _)), _) => Ok(PartKey(tup).num_parts()), |
| 121 | other => Err(err!("Key {:?} holds {:?}, not a bunch key.", k, other; Test, Invalid)), |
| 122 | } |
| 123 | } |
| 124 | |
| 125 | fn put(&self, k: &Dat, seed: u8, len: usize) -> Outcome<()> { |
| 126 | let _ = res!(self.db.insert(k.clone(), Dat::BU32(fill_of(seed, len)), Uid::default(), None)); |
| 127 | Ok(()) |
| 128 | } |
| 129 | |
| 130 | // Five chunks written and found in the files, with a value beside it that stays. |
| 131 | fn five(&self, k: &Dat, seed: u8) -> Outcome<()> { |
| 132 | res!(self.db.insert(key(0), value(0, 9), Uid::default(), None)); |
| 133 | res!(self.put(k, seed, FIVE)); |
| 134 | assert_eq!(res!(self.num_chunks(k)), 5, "the value did not split into five chunks"); |
| 135 | res!(self.db.api().settle_for_test(Duration::from_secs(10))); |
| 136 | assert!(self.holds(seed, FIVE).iter().all(|n| *n >= 1), |
| 137 | "positive control: every chunk is in a data file: {:?}", self.holds(seed, FIVE)); |
| 138 | Ok(()) |
| 139 | } |
| 140 | |
| 141 | fn compact_and_check(&self, k: &Dat, old: u8, new: u8, len: usize) -> Outcome<()> { |
| 142 | res!(self.db.api().settle_for_test(Duration::from_secs(10))); |
| 143 | let report = res!(self.db.compact_now(Duration::from_secs(30))); |
| 144 | msg!("{:?}", report); |
| 145 | assert_eq!(self.holds(old, FIVE), vec![0; 5], "old chunks still in data files after compact_now"); |
| 146 | match res!(self.db.get(k, None)) { |
| 147 | Some((v, _)) => assert_eq!(v, Dat::BU32(fill_of(new, len)), "the new value reads back changed"), |
| 148 | None => return Err(err!("The new value is gone."; Test, Missing)), |
| 149 | } |
| 150 | match res!(self.db.get(&key(0), None)) { |
| 151 | Some((v, _)) => assert_eq!(v, value(0, 9), "the value beside it changed"), |
| 152 | None => return Err(err!("The value beside it is gone."; Test, Missing)), |
| 153 | } |
| 154 | Ok(()) |
| 155 | } |
| 156 | |
| 157 | fn end(self) -> Outcome<()> { |
| 158 | res!(self.db.close()); |
| 159 | let _ = std::fs::remove_dir_all(&self.dir); |
| 160 | Ok(()) |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | #[test] |
| 165 | fn fewer_chunks_leave_no_old_chunk() -> Outcome<()> { |
| 166 | let _lock = lock(); |
| 167 | let s = res!(store("fewer")); |
| 168 | let k = dat!("shrinks to two"); |
| 169 | res!(s.five(&k, 5)); |
| 170 | res!(s.put(&k, 7, TWO)); |
| 171 | assert_eq!(res!(s.num_chunks(&k)), 2, "the overwrite did not split into two chunks"); |
| 172 | res!(s.compact_and_check(&k, 5, 7, TWO)); |
| 173 | assert!(s.holds(7, TWO).iter().all(|n| *n >= 1), "the new value's own chunks were erased: {:?}", s.holds(7, TWO)); |
| 174 | s.end() |
| 175 | } |
| 176 | |
| 177 | #[test] |
| 178 | fn plain_value_leaves_no_old_chunk() -> Outcome<()> { |
| 179 | let _lock = lock(); |
| 180 | let s = res!(store("plain")); |
| 181 | let k = dat!("shrinks to plain"); |
| 182 | res!(s.five(&k, 5)); |
| 183 | res!(s.put(&k, 8, PLAIN)); |
| 184 | match res!(s.db.api().fetch_using_schemes(&k, None)).recv_daticle(s.db.api().schemes().encrypter(), None) { |
| 185 | Ok((Some((Dat::Tup5u64(_), _)), _)) => return Err(err!("The plain value is still chunked."; Test, Invalid)), |
| 186 | _ => (), |
| 187 | } |
| 188 | res!(s.compact_and_check(&k, 5, 8, PLAIN)); |
| 189 | s.end() |
| 190 | } |
| 191 | |
| 192 | // The same length reuses every chunk key, which the ordinary supersession retires: the retiring of |
| 193 | // a surplus must keep keys the new value holds, or it erases the value it has just written. |
| 194 | #[test] |
| 195 | fn same_geometry_keeps_the_new_chunks() -> Outcome<()> { |
| 196 | let _lock = lock(); |
| 197 | let s = res!(store("same")); |
| 198 | let k = dat!("same size"); |
| 199 | res!(s.five(&k, 5)); |
| 200 | res!(s.put(&k, 6, FIVE)); |
| 201 | res!(s.compact_and_check(&k, 5, 6, FIVE)); |
| 202 | assert!(s.holds(6, FIVE).iter().all(|n| *n >= 1), "the new value's own chunks were erased: {:?}", s.holds(6, FIVE)); |
| 203 | s.end() |
| 204 | } |
| 205 | |
| 206 | // The same count of chunks of another length shares no chunk key with the old value, because the |
| 207 | // length is part of the key, so all five old chunks are left and all five must go (QA A1-3). |
| 208 | #[test] |
| 209 | fn same_count_other_length_leaves_no_old_chunk() -> Outcome<()> { |
| 210 | let _lock = lock(); |
| 211 | let s = res!(store("count")); |
| 212 | let k = dat!("five over five"); |
| 213 | res!(s.five(&k, 5)); |
| 214 | res!(s.put(&k, 6, FIVE_B)); |
| 215 | assert_eq!(res!(s.num_chunks(&k)), 5, "the overwrite did not split into five chunks"); |
| 216 | res!(s.compact_and_check(&k, 5, 6, FIVE_B)); |
| 217 | assert!(s.holds(6, FIVE_B).iter().all(|n| *n >= 1), "the new value's own chunks were erased: {:?}", s.holds(6, FIVE_B)); |
| 218 | s.end() |
| 219 | } |
| 220 | |
| 221 | // The new value is durable before the old chunks go: a reader that comes while the old chunks are |
| 222 | // being retired finds the old value or the new one, never a bunch key with chunks missing. |
| 223 | #[test] |
| 224 | fn reader_during_the_retire_finds_a_whole_value() -> Outcome<()> { |
| 225 | let _lock = lock(); |
| 226 | let s = res!(store("reader")); |
| 227 | let k = dat!("read during retire"); |
| 228 | res!(s.five(&k, 5)); |
| 229 | let old = Dat::BU32(fill_of(5, FIVE)); |
| 230 | let new = Dat::BU32(fill_of(9, TWO)); |
| 231 | |
| 232 | let done = Arc::new(AtomicBool::new(false)); |
| 233 | let seen = { |
| 234 | let (db, k, old, new, done) = (s.db.clone(), k.clone(), old.clone(), new.clone(), done.clone()); |
| 235 | thread::spawn(move || { |
| 236 | let (mut n_old, mut n_new, mut bad) = (0usize, 0usize, 0usize); |
| 237 | while !done.load(Ordering::Relaxed) { |
| 238 | match db.get(&k, None) { |
| 239 | Ok(Some((v, _))) if v == old => n_old += 1, |
| 240 | Ok(Some((v, _))) if v == new => n_new += 1, |
| 241 | other => { |
| 242 | bad += 1; |
| 243 | let s: String = fmt!("{:?}", other).chars().take(700).collect(); |
| 244 | msg!("a reader found neither value: {}", s); |
| 245 | }, |
| 246 | } |
| 247 | thread::sleep(Duration::from_millis(15)); |
| 248 | } |
| 249 | (n_old, n_new, bad) |
| 250 | }) |
| 251 | }; |
| 252 | let prior = match res!(s.db.api().get_head_wait(&k, None)) { |
| 253 | Some((Dat::Tup5u64(t), _)) => t, |
| 254 | other => return Err(err!("No chunked head before the overwrite: {:?}", other; Test, Missing)), |
| 255 | }; |
| 256 | hooks::set_chunk_tombstone_delay(Duration::from_millis(400)); |
| 257 | let put = s.put(&k, 9, TWO); |
| 258 | // The old chunks are retired by the cache bot after the put returns, so the reader goes on |
| 259 | // until they are gone. |
| 260 | let retired = wait_until(Duration::from_secs(20), |
| 261 | || s.db.api().fetch_chunks(&Dat::Tup5u64(prior), None).is_err()); |
| 262 | hooks::set_chunk_tombstone_delay(Duration::ZERO); |
| 263 | done.store(true, Ordering::Relaxed); |
| 264 | assert!(retired, "the old chunks were never retired"); |
| 265 | let (n_old, n_new, bad) = match seen.join() { |
| 266 | Ok(t) => t, |
| 267 | Err(_) => return Err(err!("The reader thread panicked."; Test, Invalid)), |
| 268 | }; |
| 269 | res!(put); |
| 270 | msg!("reader saw the old value {} times, the new {} times, {} broken.", n_old, n_new, bad); |
| 271 | assert_eq!(bad, 0, "a reader found a value that was neither the old one nor the new one"); |
| 272 | assert!(n_new >= 1, "the reader never saw the new value, so it did not read while the old chunks were retired"); |
| 273 | res!(s.compact_and_check(&k, 5, 9, TWO)); |
| 274 | s.end() |
| 275 | } |
| 276 | |
| 277 | // The retire is stamped with the write that causes it. Stamped when it is sent, a store of the |
| 278 | // old geometry begun while the first is still becoming durable (its chunks stamped before the |
| 279 | // retire, its bunch key newer than the first store's) has its chunks superseded by the retire of |
| 280 | // the chunks it shares, and its bunch key names tombstones (QA A2-6). |
| 281 | #[test] |
| 282 | fn retire_does_not_erase_a_newer_value() -> Outcome<()> { |
| 283 | let _lock = lock(); |
| 284 | let s = res!(store("newer")); |
| 285 | let k = dat!("newer than the retire"); |
| 286 | res!(s.put(&k, 5, TWO)); |
| 287 | assert_eq!(res!(s.num_chunks(&k)), 2, "the value did not split into two chunks"); |
| 288 | let newer = Dat::BU32(fill_of(9, TWO)); |
| 289 | |
| 290 | // The first store (plain, so it must retire the old two chunks) is held at its barrier, and |
| 291 | // the second (two chunks again, the old keys) begins inside that hold. |
| 292 | hooks::set_barrier_delay(Duration::from_millis(800)); |
| 293 | let first = { |
| 294 | let (db, k) = (s.db.clone(), k.clone()); |
| 295 | thread::spawn(move || db.insert(k, Dat::BU32(fill_of(7, PLAIN)), Uid::default(), None).map(|_| ())) |
| 296 | }; |
| 297 | thread::sleep(Duration::from_millis(300)); |
| 298 | let second = { |
| 299 | let (db, k, v) = (s.db.clone(), k.clone(), newer.clone()); |
| 300 | thread::spawn(move || db.insert(k, v, Uid::default(), None).map(|_| ())) |
| 301 | }; |
| 302 | let first = first.join(); |
| 303 | let second = second.join(); |
| 304 | hooks::set_barrier_delay(Duration::ZERO); |
| 305 | match (first, second) { |
| 306 | (Ok(a), Ok(b)) => { res!(a); res!(b); }, |
| 307 | _ => return Err(err!("A storing thread panicked."; Test, Invalid)), |
| 308 | } |
| 309 | res!(s.db.api().settle_for_test(Duration::from_secs(10))); |
| 310 | match s.db.get(&k, None) { |
| 311 | Ok(Some((v, _))) => assert_eq!(v, newer, "the newer value reads back changed"), |
| 312 | other => { |
| 313 | let t: String = fmt!("{:?}", other).chars().take(300).collect(); |
| 314 | panic!("the newer value does not read back whole: {}", t); |
| 315 | }, |
| 316 | } |
| 317 | s.end() |
| 318 | } |