oxedyne/fe2o3/fe2o3_o3db_sync/tests/gc_torn_pair.rs
7.6 KiB, 1 run
created by r1870400018:62339, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! A collection whose cache bot is slow to answer must still leave its data and index files a |
| 2 | //! matching pair. Until 2026-09-24 the collector renamed the rebuilt index over the old one, |
| 3 | //! waited five seconds for the cache, gave up, and never renamed the data file: the next start read |
| 4 | //! the new index beside the old data file and filled the cache with offsets from the one for the |
| 5 | //! other. Before o3i that returned other keys' values for a whole session, and after it the same |
| 6 | //! reads failed. Found by QA of lane o3i (D-C, 2026-09-24). `test::hooks` holds the one cache |
| 7 | //! bot and the collection, and the hooks are process-wide, which is why this is a test binary of |
| 8 | //! its own with a single test. |
| 9 | |
| 10 | use oxedyne_fe2o3_core::prelude::*; |
| 11 | use oxedyne_fe2o3_hash::{ |
| 12 | csum::ChecksumScheme, |
| 13 | hash::HashScheme, |
| 14 | }; |
| 15 | use oxedyne_fe2o3_iop_db::api::Database; |
| 16 | use oxedyne_fe2o3_jdat::prelude::*; |
| 17 | use oxedyne_fe2o3_o3db_sync::{ |
| 18 | O3db, |
| 19 | base::cfg::OzoneConfig, |
| 20 | data::core::RestSchemesInput, |
| 21 | file::{ |
| 22 | core::FileType, |
| 23 | zdir::ZoneDir, |
| 24 | }, |
| 25 | test::{ |
| 26 | hooks, |
| 27 | setup::{ |
| 28 | self, |
| 29 | Uid, |
| 30 | UID_LEN, |
| 31 | }, |
| 32 | }, |
| 33 | }; |
| 34 | |
| 35 | use std::{ |
| 36 | collections::BTreeMap, |
| 37 | path::{ |
| 38 | Path, |
| 39 | PathBuf, |
| 40 | }, |
| 41 | thread, |
| 42 | time::Duration, |
| 43 | }; |
| 44 | |
| 45 | type TestDb = O3db< |
| 46 | { UID_LEN }, |
| 47 | Uid, |
| 48 | (), |
| 49 | HashScheme, |
| 50 | HashScheme, |
| 51 | ChecksumScheme, |
| 52 | >; |
| 53 | |
| 54 | const NKEYS: usize = 12; |
| 55 | |
| 56 | fn config(max: u64) -> Outcome<OzoneConfig> { |
| 57 | let mut cfg = res!(setup::default_cfg()); |
| 58 | cfg.num_zones = 1; |
| 59 | cfg.num_cbots_per_zone = 1; |
| 60 | cfg.num_fbots_per_zone = 1; |
| 61 | cfg.num_igbots_per_zone = 1; |
| 62 | cfg.num_rbots_per_zone = 1; |
| 63 | cfg.num_wbots_per_zone = 1; |
| 64 | cfg.data_file_max_bytes = max; |
| 65 | cfg.rest_chunk_threshold = max * 7 / 10; |
| 66 | cfg.zone_overrides = BTreeMap::new(); |
| 67 | cfg.sync_on_write = true; |
| 68 | Ok(cfg) |
| 69 | } |
| 70 | |
| 71 | fn schemes() -> RestSchemesInput<(), HashScheme, HashScheme, ChecksumScheme> { |
| 72 | RestSchemesInput::new( |
| 73 | None::<()>, |
| 74 | None::<HashScheme>, |
| 75 | None::<HashScheme>, |
| 76 | Some(ChecksumScheme::new_crc32()), |
| 77 | ) |
| 78 | } |
| 79 | |
| 80 | fn key(i: usize) -> Dat { dat!(fmt!("torn pair key {:03}", i)) } |
| 81 | |
| 82 | /// Every value is 100 bytes and carries its key's number and version, so a value read back names |
| 83 | /// the record it came from. |
| 84 | fn value(i: usize, ver: u8) -> Dat { |
| 85 | let mut v = vec![0u8; 100]; |
| 86 | v[0] = i as u8; |
| 87 | v[1] = ver; |
| 88 | for j in 2..100 { |
| 89 | v[j] = (i as u8) ^ ver ^ (j as u8); |
| 90 | } |
| 91 | Dat::BU32(v) |
| 92 | } |
| 93 | |
| 94 | fn size(p: &Path) -> u64 { |
| 95 | match std::fs::metadata(p) { Ok(m) => m.len(), Err(_) => 0 } |
| 96 | } |
| 97 | |
| 98 | fn file(root: &Path, cfg: &OzoneConfig, typ: FileType, n: u32) -> PathBuf { |
| 99 | cfg.zone_root(root).join("zone_001").join(ZoneDir::relative_file_path(&typ, n)) |
| 100 | } |
| 101 | |
| 102 | /// Reads every key and counts the answers that are not the version last written. |
| 103 | fn judge(db: &TestDb, want: &[u8], label: &str) -> usize { |
| 104 | let mut bad = 0; |
| 105 | for i in 0..NKEYS { |
| 106 | match db.get(&key(i), None) { |
| 107 | Ok(Some((v, _))) if v == value(i, want[i]) => (), |
| 108 | Ok(Some((v, _))) => { |
| 109 | bad += 1; |
| 110 | let head = match v.bytes_ref() { |
| 111 | Some(b) if b.len() >= 2 => fmt!("key {} version {}", b[0], b[1]), |
| 112 | _ => fmt!("{:?}", v), |
| 113 | }; |
| 114 | msg!("{}: key {} read back {}, not version {}.", label, i, head, want[i]); |
| 115 | }, |
| 116 | Ok(None) => { |
| 117 | bad += 1; |
| 118 | msg!("{}: key {} is missing.", label, i); |
| 119 | }, |
| 120 | Err(e) => { |
| 121 | bad += 1; |
| 122 | let s: String = fmt!("{}", e).chars().take(300).collect(); |
| 123 | msg!("{}: key {} could not be read: {}", label, i, s); |
| 124 | }, |
| 125 | } |
| 126 | } |
| 127 | msg!("{}: {} of {} keys wrong.", label, bad, NKEYS); |
| 128 | bad |
| 129 | } |
| 130 | |
| 131 | #[test] |
| 132 | fn main() -> Outcome<()> { |
| 133 | log_set_level!("error"); |
| 134 | let dir = "./test_db_gc_torn_pair"; |
| 135 | let _ = std::fs::remove_dir_all(dir); |
| 136 | res!(std::fs::create_dir_all(dir)); |
| 137 | let root = res!(Path::new(dir).canonicalize()); |
| 138 | |
| 139 | // The length of one record, all being one length. |
| 140 | let pdir = "./test_db_gc_torn_pair_probe"; |
| 141 | let _ = std::fs::remove_dir_all(pdir); |
| 142 | res!(std::fs::create_dir_all(pdir)); |
| 143 | let probe = res!(Path::new(pdir).canonicalize()); |
| 144 | let pcfg = res!(config(4_000)); |
| 145 | let pdb: TestDb = res!(setup::start_db(probe.clone(), Some(pcfg.clone()), schemes(), None, false, true)); |
| 146 | res!(pdb.insert(key(0), value(0, 1), Uid::default(), None)); |
| 147 | res!(pdb.close()); |
| 148 | let len = size(&file(&probe, &pcfg, FileType::Data, 1)); |
| 149 | assert!(len > 100, "the probe record is {} bytes, shorter than its own value", len); |
| 150 | |
| 151 | // Five records to a file, so that superseding two of file 1's crosses the collection trigger. |
| 152 | let cfg = res!(config(5 * len + len / 2)); |
| 153 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, true, true)); |
| 154 | let mut want = vec![1u8; NKEYS]; |
| 155 | for i in 0..NKEYS { |
| 156 | res!(db.insert(key(i), value(i, 1), Uid::default(), None)); |
| 157 | } |
| 158 | let d1 = file(&root, &cfg, FileType::Data, 1); |
| 159 | let full = size(&d1); |
| 160 | assert_eq!(full, 5 * len, "file 1 does not hold the five records it was sized for"); |
| 161 | |
| 162 | // The collection of file 1 starts a second after keys 0 and 1 are superseded, and its cache |
| 163 | // update then waits behind an insert held eight seconds, past the five a collector used to |
| 164 | // wait for it. |
| 165 | hooks::set_collect_delay(Duration::from_millis(1_000)); |
| 166 | res!(db.insert(key(0), value(0, 2), Uid::default(), None)); |
| 167 | want[0] = 2; |
| 168 | res!(db.insert(key(1), value(1, 2), Uid::default(), None)); |
| 169 | want[1] = 2; |
| 170 | hooks::set_insert_delay(Duration::from_millis(8_000)); |
| 171 | let writing = db.clone(); |
| 172 | let slow = res!(thread::Builder::new().name(fmt!("slow insert")).spawn(move || { |
| 173 | let _ = writing.insert(key(NKEYS - 1), value(NKEYS - 1, 3), Uid::default(), None); |
| 174 | })); |
| 175 | want[NKEYS - 1] = 3; |
| 176 | thread::sleep(Duration::from_millis(9_000)); |
| 177 | hooks::set_insert_delay(Duration::ZERO); |
| 178 | hooks::set_collect_delay(Duration::ZERO); |
| 179 | let _ = slow.join(); |
| 180 | let collected = size(&d1); |
| 181 | let same = judge(&db, &want, "Same process"); |
| 182 | res!(db.close()); |
| 183 | |
| 184 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, true, false)); |
| 185 | let reopened = judge(&db, &want, "After reopening"); |
| 186 | |
| 187 | // Churn so that file 1 is collected again with whatever state the start built. |
| 188 | for round in 4..7u8 { |
| 189 | for i in 2..6 { |
| 190 | res!(db.insert(key(i), value(i, round), Uid::default(), None)); |
| 191 | want[i] = round; |
| 192 | } |
| 193 | } |
| 194 | thread::sleep(Duration::from_secs(3)); |
| 195 | let churned = judge(&db, &want, "After more collection"); |
| 196 | res!(db.close()); |
| 197 | |
| 198 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, true, false)); |
| 199 | let second = judge(&db, &want, "After a second reopening"); |
| 200 | res!(db.close()); |
| 201 | log_finish_wait!(); |
| 202 | |
| 203 | assert_eq!(same, 0, "{} keys read wrong in the process that ran the collection", same); |
| 204 | assert_eq!(reopened, 0, "{} keys read wrong after the first reopening", reopened); |
| 205 | assert_eq!(churned, 0, "{} keys read wrong after file 1 was collected again", churned); |
| 206 | assert_eq!(second, 0, "{} keys read wrong after the second reopening", second); |
| 207 | // The collection has to have happened for any of this to mean anything. |
| 208 | assert_eq!(collected, 3 * len, |
| 209 | "file 1 should have lost the two superseded records to its collection, and is {} bytes \ |
| 210 | where it was {}", collected, full); |
| 211 | let _ = std::fs::remove_dir_all(dir); |
| 212 | let _ = std::fs::remove_dir_all(pdir); |
| 213 | Ok(()) |
| 214 | } |