oxedyne/fe2o3/fe2o3_o3db_sync/tests/chunk_hold.rs
14.4 KiB, 30 runs
created by r1870400018:70271, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! A store's chunk set stays held until its head is in the cache, or for the life of the process |
| 2 | //! once the head is in the files but could not be confirmed (run 2, 2026-10-10). The hooks are |
| 3 | //! process-wide, so every test takes one lock. |
| 4 | |
| 5 | mod gc_pair; |
| 6 | |
| 7 | use gc_pair::*; |
| 8 | |
| 9 | use oxedyne_fe2o3_core::prelude::*; |
| 10 | use oxedyne_fe2o3_data::time::Timestamp; |
| 11 | use oxedyne_fe2o3_iop_db::api::{ |
| 12 | Database, |
| 13 | Meta, |
| 14 | }; |
| 15 | use oxedyne_fe2o3_jdat::prelude::*; |
| 16 | use oxedyne_fe2o3_o3db_sync::{ |
| 17 | base::{ |
| 18 | cfg::OzoneConfig, |
| 19 | constant, |
| 20 | }, |
| 21 | comm::response::Wait, |
| 22 | data::cache::{ |
| 23 | Cache, |
| 24 | Prior, |
| 25 | Superseded, |
| 26 | ValueOrLocation, |
| 27 | }, |
| 28 | file::floc::FileLocation, |
| 29 | sweep, |
| 30 | test::{ |
| 31 | hooks, |
| 32 | setup::{ |
| 33 | self, |
| 34 | Uid, |
| 35 | UID_LEN, |
| 36 | }, |
| 37 | }, |
| 38 | }; |
| 39 | |
| 40 | use std::{ |
| 41 | collections::BTreeMap, |
| 42 | path::Path, |
| 43 | sync::{ |
| 44 | Mutex, |
| 45 | MutexGuard, |
| 46 | }, |
| 47 | thread, |
| 48 | time::{ |
| 49 | Duration, |
| 50 | Instant, |
| 51 | }, |
| 52 | }; |
| 53 | |
| 54 | static HOOKS: Mutex<()> = Mutex::new(()); |
| 55 | |
| 56 | fn lock() -> MutexGuard<'static, ()> { |
| 57 | match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() } |
| 58 | } |
| 59 | |
| 60 | fn cfg() -> Outcome<OzoneConfig> { |
| 61 | let mut c = res!(setup::default_cfg()); |
| 62 | c.num_zones = 1; |
| 63 | c.num_cbots_per_zone = 4; |
| 64 | c.num_fbots_per_zone = 1; |
| 65 | c.num_wbots_per_zone = 2; |
| 66 | c.zone_overrides = BTreeMap::new(); |
| 67 | c.rest_chunk_threshold = 1_000; |
| 68 | c.rest_chunk_bytes = 400; |
| 69 | c.sync_on_write = true; |
| 70 | Ok(c) |
| 71 | } |
| 72 | |
| 73 | fn val(len: usize, seed: u8) -> Dat { |
| 74 | Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect()) |
| 75 | } |
| 76 | |
| 77 | fn start(dir: &str, wipe: bool) -> Outcome<TestDb> { |
| 78 | log_set_level!("error"); |
| 79 | if wipe { |
| 80 | let _ = std::fs::remove_dir_all(dir); |
| 81 | } |
| 82 | res!(std::fs::create_dir_all(dir)); |
| 83 | let root = res!(Path::new(dir).canonicalize()); |
| 84 | setup::start_db(root, Some(res!(cfg())), schemes(), None, false, wipe) |
| 85 | } |
| 86 | |
| 87 | fn scan_wait() -> Wait { |
| 88 | Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL } |
| 89 | } |
| 90 | |
| 91 | // "1100/<seed>" for a whole value, "none", "TORN" or "ERR:<text>". |
| 92 | fn describe(r: &Outcome<Option<(Dat, Meta<{ UID_LEN }, Uid>)>>) -> String { |
| 93 | match r { |
| 94 | Ok(None) => fmt!("none"), |
| 95 | Ok(Some((Dat::BU32(b), _))) if !b.is_empty() |
| 96 | && b.iter().enumerate().all(|(j, x)| *x == b[0] ^ (j as u8)) => fmt!("{}/{}", b.len(), b[0]), |
| 97 | Ok(Some(_)) => fmt!("TORN"), |
| 98 | Err(e) => fmt!("ERR:{}", fmt!("{}", e).chars().take(160).collect::<String>()), |
| 99 | } |
| 100 | } |
| 101 | |
| 102 | fn pending_empty(db: &TestDb) -> Outcome<bool> { |
| 103 | let held = lock_mutex!(db.api().chans().pending_sets()); |
| 104 | Ok(held.is_empty()) |
| 105 | } |
| 106 | |
| 107 | // A put whose head failed after its record reached the files: the store reports failure, the |
| 108 | // sweep spares the put's chunks, and after a restart the key reads whole. |
| 109 | fn failed_head_keeps_hold(dir: &str, arm: fn(bool)) -> Outcome<Vec<String>> { |
| 110 | let mut bad = Vec::new(); |
| 111 | let k = dat!("failed head"); |
| 112 | let db = res!(start(dir, true)); |
| 113 | res!(db.insert(k.clone(), val(1_100, 1), Uid::default(), None)); |
| 114 | hooks::set_durability_timeout(Some(Duration::from_secs(2))); |
| 115 | arm(true); |
| 116 | let put = db.insert(k.clone(), val(1_100, 2), Uid::default(), None); |
| 117 | arm(false); |
| 118 | hooks::set_durability_timeout(None); |
| 119 | if put.is_ok() { bad.push(fmt!("the put whose head failed was confirmed")); } |
| 120 | let live = describe(&db.get(&k, None)); |
| 121 | if live != "1100/1" { bad.push(fmt!("live read {}", live)); } |
| 122 | let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait())); |
| 123 | if report.orphans_found > 0 { bad.push(fmt!("the sweep took {} chunks of the held set", report.orphans_found)); } |
| 124 | if report.skipped_pending == 0 { bad.push(fmt!("the sweep saw none of the put's chunks, so it proved nothing")); } |
| 125 | res!(db.close()); |
| 126 | let db = res!(start(dir, false)); |
| 127 | let after = describe(&db.get(&k, None)); |
| 128 | if !(after == "1100/1" || after == "1100/2") { bad.push(fmt!("restart read {}", after)); } |
| 129 | res!(db.close()); |
| 130 | let _ = std::fs::remove_dir_all(dir); |
| 131 | msg!("failed head: put {:?} live {} sweep {}/{} restart {}", put.map(|_| ()).map_err(|e| e.tags().to_vec()), |
| 132 | live, report.orphans_found, report.skipped_pending, after); |
| 133 | Ok(bad) |
| 134 | } |
| 135 | |
| 136 | // Change 3: the head is in the data file, but the writer cannot hand it on. |
| 137 | #[test] |
| 138 | fn writer_failure_after_append_keeps_hold() -> Outcome<()> { |
| 139 | let _lock = lock(); |
| 140 | let bad = res!(failed_head_keeps_hold("./test_db_chunk_hold_writer", hooks::set_head_hand_failure)); |
| 141 | assert!(bad.is_empty(), "{:#?}", bad); |
| 142 | Ok(()) |
| 143 | } |
| 144 | |
| 145 | // Change 4: the head is durable, but the cache bot cannot enter it. |
| 146 | #[test] |
| 147 | fn cbot_insert_failure_keeps_hold() -> Outcome<()> { |
| 148 | let _lock = lock(); |
| 149 | let bad = res!(failed_head_keeps_hold("./test_db_chunk_hold_cbot", hooks::set_head_insert_failure)); |
| 150 | assert!(bad.is_empty(), "{:#?}", bad); |
| 151 | Ok(()) |
| 152 | } |
| 153 | |
| 154 | // Q1-4: a head that times out at the store keeps its set held until it lands, so a sweep in the |
| 155 | // meantime takes none of its chunks, and the key then reads whole. Staged on the head hold, where |
| 156 | // the store has its chunks in and is about to send its head, so the barrier delay catches the head |
| 157 | // and never a chunk; every wait is on a condition against a deadline, never a fixed sleep (the |
| 158 | // sleeps it was staged with failed it under disk load, 2026-10-10). |
| 159 | #[test] |
| 160 | fn timed_out_head_keeps_its_set_until_it_lands() -> Outcome<()> { |
| 161 | let _lock = lock(); |
| 162 | let dir = "./test_db_chunk_hold_timeout"; |
| 163 | let db = res!(start(dir, true)); |
| 164 | let mut bad = Vec::new(); |
| 165 | let k = dat!("timed out head"); |
| 166 | res!(db.insert(k.clone(), val(1_100, 1), Uid::default(), None)); |
| 167 | hooks::set_durability_timeout(Some(Duration::from_secs(2))); |
| 168 | let held = hooks::heads_held(); |
| 169 | hooks::set_head_hold(true); |
| 170 | let put = { let (db, k) = (db.clone(), k.clone()); |
| 171 | thread::spawn(move || db.insert(k, val(1_100, 2), Uid::default(), None).map(|_| ())) }; |
| 172 | let reached = wait_until(Duration::from_secs(30), || Ok(hooks::heads_held() > held)); |
| 173 | hooks::set_barrier_delay(Duration::from_secs(5)); |
| 174 | hooks::set_head_hold(false); |
| 175 | let put = match put.join() { Ok(r) => r, Err(_) => Err(err!("The put panicked."; Test)) }; |
| 176 | let staged = match (res!(reached), put) { |
| 177 | (false, _) => { bad.push(fmt!("the store never reached the head hold")); false }, |
| 178 | (true, Ok(())) => { bad.push(fmt!("the timed-out put was confirmed")); false }, |
| 179 | (true, Err(e)) if !e.tags().contains(&ErrTag::Unconfirmed) => { |
| 180 | bad.push(fmt!("the head was not written before the put timed out: {}", e)); false }, |
| 181 | (true, Err(_)) => true, |
| 182 | }; |
| 183 | if staged { |
| 184 | let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait())); |
| 185 | if report.orphans_found > 0 { bad.push(fmt!("the sweep took {} chunks of a head in flight", report.orphans_found)); } |
| 186 | if report.skipped_pending == 0 { bad.push(fmt!("the sweep saw none of the put's chunks, so it proved nothing")); } |
| 187 | } |
| 188 | hooks::set_barrier_delay(Duration::ZERO); |
| 189 | hooks::set_durability_timeout(None); |
| 190 | if staged { |
| 191 | let landed = res!(wait_until(Duration::from_secs(20), || Ok(describe(&db.get(&k, None)) == "1100/2"))); |
| 192 | if !landed { bad.push(fmt!("the head did not land whole: {}", describe(&db.get(&k, None)))); } |
| 193 | // The cache bot lets the set go once the head it carries is in; wait on that, not a clock. |
| 194 | if !res!(wait_until(Duration::from_secs(20), || pending_empty(&db))) { |
| 195 | bad.push(fmt!("a set is still held after the head landed")); |
| 196 | } |
| 197 | } |
| 198 | res!(db.close()); |
| 199 | if staged { |
| 200 | let db = res!(start(dir, false)); |
| 201 | let after = describe(&db.get(&k, None)); |
| 202 | if after != "1100/2" { bad.push(fmt!("restart read {}", after)); } |
| 203 | res!(db.close()); |
| 204 | } |
| 205 | let _ = std::fs::remove_dir_all(dir); |
| 206 | assert!(bad.is_empty(), "{:#?}", bad); |
| 207 | Ok(()) |
| 208 | } |
| 209 | |
| 210 | // Polls `cond` until it holds or `limit` passes. |
| 211 | fn wait_until(limit: Duration, mut cond: impl FnMut() -> Outcome<bool>) -> Outcome<bool> { |
| 212 | let deadline = Instant::now() + limit; |
| 213 | loop { |
| 214 | if res!(cond()) { |
| 215 | return Ok(true); |
| 216 | } |
| 217 | if Instant::now() >= deadline { |
| 218 | return Ok(false); |
| 219 | } |
| 220 | thread::sleep(Duration::from_millis(10)); |
| 221 | } |
| 222 | } |
| 223 | |
| 224 | // --------------------------------------------------------------------------------------------- |
| 225 | // The cache, driven directly. |
| 226 | |
| 227 | type C = Cache<{ UID_LEN }, Uid>; |
| 228 | |
| 229 | fn meta(s: u64) -> Meta<{ UID_LEN }, Uid> { |
| 230 | Meta { time: Timestamp::new(s, 0), user: Uid::default() } |
| 231 | } |
| 232 | |
| 233 | fn floc(f: u32) -> FileLocation { |
| 234 | FileLocation { fnum: f, start: 0, klen: 10, vlen: 10 } |
| 235 | } |
| 236 | |
| 237 | // Change 8: the write tracker returns to where it began after a head, a small value, a |
| 238 | // tombstone and a head overwrite one key, and a head never enters it. |
| 239 | #[test] |
| 240 | fn write_tracker_follows_head_transitions() -> Outcome<()> { |
| 241 | let mut cache = C::new(None); |
| 242 | cache.set_lim(1 << 30); |
| 243 | let k = b"tracked key".to_vec(); |
| 244 | let t0 = cache.tracker_size(); |
| 245 | let size0 = cache.get_size(); |
| 246 | let steps: [(Option<usize>, &[u8], bool); 6] = [ |
| 247 | (Some(0), b"head one", false), // a head enters no tracker entry |
| 248 | (None, b"small", true), // head to non-head with bytes |
| 249 | (None, b"tombstone", true), // non-head to non-head |
| 250 | (Some(0), b"head two", false), // non-head to head |
| 251 | (Some(0), b"head three", false), // head to head |
| 252 | (None, b"small again", true), |
| 253 | ]; |
| 254 | for (i, (cind, v, tracked)) in steps.iter().enumerate() { |
| 255 | let s = i as u64 + 1; |
| 256 | let sup = res!(cache.insert(k.clone(), Some(v.to_vec()), *cind, floc(s as u32), meta(s))); |
| 257 | let grown = cache.tracker_size() > t0; |
| 258 | assert_eq!(grown, *tracked, "step {}: the tracker is {} from {}", i, cache.tracker_size(), t0); |
| 259 | if i == 4 { |
| 260 | match sup { |
| 261 | Superseded::Cached { prior: Prior::Head(b), .. } => |
| 262 | assert_eq!(b, b"head two".to_vec(), "the displaced head's bytes"), |
| 263 | other => panic!("a head over a head reported {:?}", other), |
| 264 | } |
| 265 | } |
| 266 | } |
| 267 | assert!(res!(cache.remove(&k, &meta(6))), "the entry was not removed"); |
| 268 | assert_eq!(cache.tracker_size(), t0, "the tracker did not return to its start"); |
| 269 | assert_eq!(cache.get_size(), size0, "the cache size did not return to its start"); |
| 270 | Ok(()) |
| 271 | } |
| 272 | |
| 273 | // A newer record that arrives with no value, as a replayed one does, takes the old value out with |
| 274 | // the old location: a reader is sent to the new record, never given the old bytes under the new |
| 275 | // record's metadata. An older record with no value leaves the cache as it was. |
| 276 | #[test] |
| 277 | fn record_without_value_drops_the_old_value() -> Outcome<()> { |
| 278 | for cind in [None, Some(0)] { |
| 279 | let mut cache = C::new(None); |
| 280 | cache.set_lim(1 << 30); |
| 281 | let k = b"replayed key".to_vec(); |
| 282 | let t0 = cache.tracker_size(); |
| 283 | res!(cache.insert(k.clone(), Some(b"old value".to_vec()), cind, floc(1), meta(2))); |
| 284 | // An older record, offered late: it loses, and the cached value stands. |
| 285 | match res!(cache.insert(k.clone(), None, cind, floc(0), meta(1))) { |
| 286 | Superseded::Offered { .. } => (), |
| 287 | other => panic!("{:?}: an older record reported {:?}", cind, other), |
| 288 | } |
| 289 | match res!(cache.get(&k)) { |
| 290 | Some(ValueOrLocation::Value(v, m)) => { |
| 291 | assert_eq!(v, b"old value".to_vec(), "{:?}: the cached value changed", cind); |
| 292 | assert_eq!(m, meta(2), "{:?}: the cached metadata changed", cind); |
| 293 | }, |
| 294 | other => panic!("{:?}: after an older record the cache gave {:?}", cind, other), |
| 295 | } |
| 296 | // A newer record with no value. |
| 297 | let sup = res!(cache.insert(k.clone(), None, cind, floc(2), meta(3))); |
| 298 | match (&sup, cind) { |
| 299 | (Superseded::Cached { prior: Prior::Head(b), .. }, Some(0)) => |
| 300 | assert_eq!(b, &b"old value".to_vec(), "the displaced head's bytes"), |
| 301 | (Superseded::Cached { prior: Prior::NotHead, .. }, None) => (), |
| 302 | other => panic!("a newer record reported {:?}", other), |
| 303 | } |
| 304 | match res!(cache.get(&k)) { |
| 305 | Some(ValueOrLocation::Location(mloc)) => { |
| 306 | assert_eq!(*mloc.file_location(), floc(2), "{:?}: the reader is not sent to the new record", cind); |
| 307 | assert_eq!(*mloc.meta(), meta(3), "{:?}: the new record's metadata", cind); |
| 308 | }, |
| 309 | other => panic!("{:?}: a newer record without a value left {:?}", cind, other), |
| 310 | } |
| 311 | assert_eq!(cache.tracker_size(), t0, "{:?}: the old value is still tracked", cind); |
| 312 | } |
| 313 | Ok(()) |
| 314 | } |
| 315 | |
| 316 | // The pinned-heads gauge follows every transition of a key: a head pins its bytes, a head over a |
| 317 | // head re-pins at the new size, a head displaced by a non-head or removed unpins, and an older |
| 318 | // head offered late pins nothing. |
| 319 | #[test] |
| 320 | fn pinned_heads_follow_head_transitions() -> Outcome<()> { |
| 321 | let mut cache = C::new(None); |
| 322 | cache.set_lim(1 << 30); |
| 323 | let k = b"pinned key".to_vec(); |
| 324 | let none = cache.pinned_heads(); |
| 325 | assert_eq!((none.count, none.bytes), (0, 0), "a new cache pins nothing"); |
| 326 | res!(cache.insert(k.clone(), Some(vec![1u8; 4]), Some(0), floc(2), meta(2))); |
| 327 | let one = cache.pinned_heads(); |
| 328 | assert_eq!(one.count, 1, "a head pinned {:?}", one); |
| 329 | assert!(one.bytes >= 4, "a head of 4 bytes pinned {:?}", one); |
| 330 | // An older head, offered late, loses and pins nothing. |
| 331 | res!(cache.insert(k.clone(), Some(vec![9u8; 64]), Some(0), floc(1), meta(1))); |
| 332 | assert_eq!(cache.pinned_heads(), one, "an older head offered late changed the pins"); |
| 333 | // A head over a head: still one pinned, at the new size. |
| 334 | res!(cache.insert(k.clone(), Some(vec![2u8; 12]), Some(0), floc(3), meta(3))); |
| 335 | let bigger = cache.pinned_heads(); |
| 336 | assert_eq!(bigger.count, 1, "a head over a head pinned {:?}", bigger); |
| 337 | assert_eq!(bigger.bytes, one.bytes + 8, "a head of 12 bytes over one of 4 pinned {:?}", bigger); |
| 338 | // Displaced by a non-head: nothing pinned. |
| 339 | res!(cache.insert(k.clone(), Some(b"small".to_vec()), None, floc(4), meta(4))); |
| 340 | assert_eq!(cache.pinned_heads(), none, "a head displaced by a non-head stayed pinned"); |
| 341 | // A non-head displaced by a head pins again, and a remove unpins it. |
| 342 | res!(cache.insert(k.clone(), Some(vec![3u8; 4]), Some(0), floc(5), meta(5))); |
| 343 | assert_eq!(cache.pinned_heads(), one, "a head over a non-head pinned"); |
| 344 | assert!(res!(cache.remove(&k, &meta(5))), "the entry was not removed"); |
| 345 | assert_eq!(cache.pinned_heads(), none, "a removed head stayed pinned"); |
| 346 | Ok(()) |
| 347 | } |