oxedyne/fe2o3/fe2o3_o3db_sync/tests/sweep_clockless.rs
7.9 KiB, 1 run
created by r1870400018:70429, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! The orphan sweep reads no clock (Ozone R1-1, 2026-10-10). After a restart the stamp floor can |
| 2 | //! put every new record past the wall clock, and a sweep that kept "recent" chunks by comparing |
| 3 | //! their stamps with the wall clock kept such an orphan until the clock caught up. A store in |
| 4 | //! progress is spared by the set registry instead, which a sweep watches for as long as it runs. |
| 5 | |
| 6 | mod gc_pair; |
| 7 | |
| 8 | use gc_pair::*; |
| 9 | |
| 10 | use oxedyne_fe2o3_core::prelude::*; |
| 11 | use oxedyne_fe2o3_data::time::Timestamp; |
| 12 | use oxedyne_fe2o3_hash::hash::HashScheme; |
| 13 | use oxedyne_fe2o3_iop_db::api::{ |
| 14 | Database, |
| 15 | RestSchemesOverride, |
| 16 | ScanOpts, |
| 17 | }; |
| 18 | use oxedyne_fe2o3_jdat::prelude::*; |
| 19 | use oxedyne_fe2o3_o3db_sync::{ |
| 20 | base::constant, |
| 21 | comm::{ |
| 22 | channels::SetRegistry, |
| 23 | msg::OzoneMsg, |
| 24 | response::Wait, |
| 25 | }, |
| 26 | sweep, |
| 27 | test::{ |
| 28 | hooks, |
| 29 | setup::{ |
| 30 | self, |
| 31 | Uid, |
| 32 | }, |
| 33 | }, |
| 34 | }; |
| 35 | |
| 36 | use std::{ |
| 37 | sync::{ |
| 38 | Mutex, |
| 39 | MutexGuard, |
| 40 | }, |
| 41 | thread, |
| 42 | time::{ |
| 43 | Duration, |
| 44 | Instant, |
| 45 | }, |
| 46 | }; |
| 47 | |
| 48 | const DAY: Duration = Duration::from_secs(86_400); |
| 49 | |
| 50 | // The hooks are process-wide, so every test that starts a store takes this lock. |
| 51 | static SERIAL: Mutex<()> = Mutex::new(()); |
| 52 | |
| 53 | fn lock() -> MutexGuard<'static, ()> { |
| 54 | SERIAL.lock().unwrap_or_else(|p| p.into_inner()) |
| 55 | } |
| 56 | |
| 57 | fn scan_wait() -> Wait { |
| 58 | Wait { |
| 59 | max_wait: Duration::from_secs(60), |
| 60 | check_interval: constant::CHECK_INTERVAL, |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | fn value_of(seed: u8, len: usize) -> Dat { |
| 65 | Dat::BU32((0..len).map(|i| seed.wrapping_add((i % 251) as u8)).collect()) |
| 66 | } |
| 67 | |
| 68 | // An overwrite that leaves its predecessor's chunks behind, as a crash between a head and its |
| 69 | // retire does. |
| 70 | fn store_without_reclaim(db: &TestDb, k: Dat, v: Dat) -> Outcome<()> { |
| 71 | let resp = db.api().responder(); |
| 72 | let msgs = res!(db.api().prepare_write_dat(k, v, Uid::default(), |
| 73 | None::<&RestSchemesOverride<(), HashScheme>>, resp.clone(), None)); |
| 74 | res!(resp.send(OzoneMsg::Chunks(msgs.len()))); |
| 75 | res!(db.api().store_bytes(msgs)); |
| 76 | res!(resp.recv_store_ack()); |
| 77 | Ok(()) |
| 78 | } |
| 79 | |
| 80 | #[test] |
| 81 | fn an_orphan_stamped_past_the_wall_clock_is_reclaimed() -> Outcome<()> { |
| 82 | let _serial = lock(); |
| 83 | log_set_level!("error"); |
| 84 | let dir = "./test_db_sweep_clockless"; |
| 85 | let _ = std::fs::remove_dir_all(dir); |
| 86 | res!(std::fs::create_dir_all(dir)); |
| 87 | let root = res!(std::path::Path::new(dir).canonicalize()); |
| 88 | let mut cfg = res!(config(16_000)); |
| 89 | cfg.rest_chunk_threshold = 3_000; |
| 90 | cfg.rest_chunk_bytes = 1_000; |
| 91 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg), schemes(), None, true, true)); |
| 92 | |
| 93 | // The jump: one record stamped a day ahead, as one replayed after the clock stepped back is. |
| 94 | // Every stamp after it is later still. |
| 95 | let ahead = *res!(Timestamp::now()) + DAY; |
| 96 | hooks::set_fixed_stamp(Some(ahead)); |
| 97 | let jumped = db.insert(dat!("jump"), dat!("ahead"), Uid::default(), None); |
| 98 | hooks::set_fixed_stamp(None); |
| 99 | res!(jumped); |
| 100 | |
| 101 | let k = dat!("chunked"); |
| 102 | res!(db.insert(k.clone(), value_of(1, 6_000), Uid::default(), None)); |
| 103 | res!(store_without_reclaim(&db, k.clone(), value_of(2, 3_200))); |
| 104 | res!(db.api().settle_for_test(Duration::from_secs(10))); |
| 105 | |
| 106 | // Positive control: the orphan's chunks are stamped past the wall clock. |
| 107 | let now = res!(Timestamp::now()); |
| 108 | let chunks = res!(db.api().scan_with_wait(&ScanOpts::all().chunk_data_only(true), |
| 109 | None::<&RestSchemesOverride<(), HashScheme>>, scan_wait())); |
| 110 | let ahead_of_clock = chunks.iter().filter(|(_, _, m)| m.time > now).count(); |
| 111 | |
| 112 | let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait())); |
| 113 | let got = res!(db.get(&k, None)); |
| 114 | res!(db.close()); |
| 115 | log_finish_wait!(); |
| 116 | let _ = std::fs::remove_dir_all(dir); |
| 117 | |
| 118 | msg!("{}", report.summary().replace('\n', " | ")); |
| 119 | assert!(chunks.len() > 3, "the values were not chunked: {} chunk records", chunks.len()); |
| 120 | assert_eq!(ahead_of_clock, chunks.len(), "not every chunk is stamped past the wall clock"); |
| 121 | assert!(report.orphans_found >= 6, "the sweep kept the orphan stamped past the wall clock: {:?}", report); |
| 122 | assert_eq!(report.orphans_retired, report.orphans_found, "a tombstone did not land: {:?}", report); |
| 123 | assert!(matches!(&got, Some((g, _)) if *g == value_of(2, 3_200)), "the live value does not read whole"); |
| 124 | Ok(()) |
| 125 | } |
| 126 | |
| 127 | // A chunked store that starts and finishes while a sweep is between its two scans is in neither |
| 128 | // the live set, read before its head landed, nor the held sets, let go before the chunks were |
| 129 | // classified. Only the sweep's watch of the registry spares it. |
| 130 | #[test] |
| 131 | fn a_store_made_between_the_sweeps_scans_is_spared() -> Outcome<()> { |
| 132 | let _serial = lock(); |
| 133 | log_set_level!("error"); |
| 134 | let dir = "./test_db_sweep_between_scans"; |
| 135 | let _ = std::fs::remove_dir_all(dir); |
| 136 | res!(std::fs::create_dir_all(dir)); |
| 137 | let root = res!(std::path::Path::new(dir).canonicalize()); |
| 138 | let mut cfg = res!(config(16_000)); |
| 139 | cfg.rest_chunk_threshold = 3_000; |
| 140 | cfg.rest_chunk_bytes = 1_000; |
| 141 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg), schemes(), None, true, true)); |
| 142 | res!(db.insert(dat!("before"), value_of(3, 6_000), Uid::default(), None)); |
| 143 | res!(db.api().settle_for_test(Duration::from_secs(10))); |
| 144 | |
| 145 | hooks::set_sweep_hold(true); |
| 146 | let held_before = hooks::sweeps_held(); |
| 147 | let sweeper = { |
| 148 | let db = db.clone(); |
| 149 | thread::spawn(move || sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait()) |
| 150 | .map_err(|e| fmt!("{}", e))) |
| 151 | }; |
| 152 | let deadline = Instant::now() + Duration::from_secs(60); |
| 153 | while hooks::sweeps_held() == held_before && Instant::now() < deadline { |
| 154 | thread::sleep(Duration::from_millis(5)); |
| 155 | } |
| 156 | let reached = hooks::sweeps_held() > held_before; |
| 157 | let k = dat!("between"); |
| 158 | let put = db.insert(k.clone(), value_of(4, 6_000), Uid::default(), None); |
| 159 | let settled = db.api().settle_for_test(Duration::from_secs(10)); |
| 160 | hooks::set_sweep_hold(false); |
| 161 | let report = match sweeper.join() { |
| 162 | Ok(r) => r, |
| 163 | Err(_) => Err(fmt!("the sweep panicked")), |
| 164 | }; |
| 165 | let got = db.get(&k, None); |
| 166 | let pending_empty = lock_mutex!(db.api().chans().pending_sets()).is_empty(); |
| 167 | res!(db.close()); |
| 168 | log_finish_wait!(); |
| 169 | let _ = std::fs::remove_dir_all(dir); |
| 170 | |
| 171 | assert!(reached, "the sweep never reached its hold between the scans"); |
| 172 | res!(put); |
| 173 | res!(settled); |
| 174 | assert!(pending_empty, "the store's set was still held, so the watch was not what spared it"); |
| 175 | let report = match report { |
| 176 | Ok(r) => r, |
| 177 | Err(e) => return Err(err!("The sweep failed: {}", e; Test)), |
| 178 | }; |
| 179 | msg!("{}", report.summary().replace('\n', " | ")); |
| 180 | assert_eq!(report.orphans_found, 0, "the sweep took the chunks of a store made between its scans: {:?}", report); |
| 181 | assert!(report.skipped_pending >= 6, "the store's chunks were not spared as watched: {:?}", report); |
| 182 | assert!(matches!(&got, Ok(Some((g, _))) if *g == value_of(4, 6_000)), |
| 183 | "the value stored between the scans does not read whole"); |
| 184 | Ok(()) |
| 185 | } |
| 186 | |
| 187 | // A set let go while a sweep watches is still reported to it, until the last watch ends. |
| 188 | #[test] |
| 189 | fn a_set_released_during_a_watch_stays_watched() { |
| 190 | let mut reg = SetRegistry::default(); |
| 191 | reg.hold(1); |
| 192 | reg.release(&1); |
| 193 | assert!(!reg.watched().contains(&1), "a set released before any watch is still watched"); |
| 194 | reg.watch(); |
| 195 | reg.hold(2); |
| 196 | reg.hold(3); |
| 197 | reg.release(&2); |
| 198 | let seen = reg.watched(); |
| 199 | assert!(seen.contains(&2), "a set released during the watch was not reported: {:?}", seen); |
| 200 | assert!(seen.contains(&3), "a held set was not reported: {:?}", seen); |
| 201 | assert!(!reg.is_held(&2), "a released set is still held"); |
| 202 | reg.watch(); |
| 203 | reg.unwatch(); |
| 204 | assert!(reg.watched().contains(&2), "one watch ending forgot the other's released set"); |
| 205 | reg.unwatch(); |
| 206 | assert_eq!(reg.watched().into_iter().collect::<Vec<_>>(), vec![3], |
| 207 | "the last watch ending did not forget the released sets"); |
| 208 | } |