oxedyne/fe2o3/fe2o3_o3db_sync/tests/chunk_publish.rs
11.8 KiB, 1 run
created by r1870400018:69617, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! A chunked value's bunch key is published after its chunks (A3 round 2a, 2026-10-09). The |
| 2 | //! records of a store go to writers that make them readable one at a time, each when its own disk |
| 3 | //! barrier completes, so a bunch key sent beside its chunks was readable for as long as the |
| 4 | //! slowest chunk's fsync took: a reader found a key whose chunks were "not found". The chunks go |
| 5 | //! first now and the bunch key follows once every chunk is written and readable. The hooks used |
| 6 | //! are process-wide, so every test takes one lock and puts each hook back. |
| 7 | |
| 8 | mod gc_pair; |
| 9 | |
| 10 | use gc_pair::*; |
| 11 | |
| 12 | use oxedyne_fe2o3_core::{ |
| 13 | channels::Recv, |
| 14 | prelude::*, |
| 15 | }; |
| 16 | use oxedyne_fe2o3_iop_db::api::Database; |
| 17 | use oxedyne_fe2o3_jdat::prelude::*; |
| 18 | use oxedyne_fe2o3_o3db_sync::{ |
| 19 | base::cfg::OzoneConfig, |
| 20 | comm::msg::OzoneMsg, |
| 21 | test::{ |
| 22 | hooks, |
| 23 | setup::{ |
| 24 | self, |
| 25 | Uid, |
| 26 | }, |
| 27 | }, |
| 28 | }; |
| 29 | |
| 30 | use std::{ |
| 31 | collections::BTreeMap, |
| 32 | path::Path, |
| 33 | sync::{ |
| 34 | Arc, |
| 35 | Mutex, |
| 36 | MutexGuard, |
| 37 | atomic::{ |
| 38 | AtomicBool, |
| 39 | Ordering, |
| 40 | }, |
| 41 | }, |
| 42 | thread, |
| 43 | time::{ |
| 44 | Duration, |
| 45 | Instant, |
| 46 | }, |
| 47 | }; |
| 48 | |
| 49 | static HOOKS: Mutex<()> = Mutex::new(()); |
| 50 | |
| 51 | fn lock() -> MutexGuard<'static, ()> { |
| 52 | match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() } |
| 53 | } |
| 54 | |
| 55 | const HOLD: Duration = Duration::from_millis(250); // a slow chunk |
| 56 | const VALUES: usize = 6; // stored in turn |
| 57 | |
| 58 | fn cfg(cbots: u16) -> Outcome<OzoneConfig> { |
| 59 | let mut c = res!(setup::default_cfg()); |
| 60 | c.num_zones = 1; |
| 61 | c.num_cbots_per_zone = cbots; // so that a bunch key often has a cache bot to itself |
| 62 | c.num_fbots_per_zone = 1; |
| 63 | c.num_wbots_per_zone = 2; |
| 64 | c.zone_overrides = BTreeMap::new(); |
| 65 | c.rest_chunk_threshold = 1_000; |
| 66 | c.rest_chunk_bytes = 400; |
| 67 | c.sync_on_write = true; |
| 68 | Ok(c) |
| 69 | } |
| 70 | |
| 71 | fn big(seed: u8) -> Dat { |
| 72 | Dat::BU32((0..1_100usize).map(|j| seed ^ (j as u8)).collect()) // three chunks |
| 73 | } |
| 74 | |
| 75 | fn open(dir: &str) -> Outcome<TestDb> { |
| 76 | open_with(dir, 8) |
| 77 | } |
| 78 | |
| 79 | fn open_with(dir: &str, cbots: u16) -> Outcome<TestDb> { |
| 80 | log_set_level!("error"); |
| 81 | let _ = std::fs::remove_dir_all(dir); |
| 82 | res!(std::fs::create_dir_all(dir)); |
| 83 | let root = res!(Path::new(dir).canonicalize()); |
| 84 | setup::start_db(root, Some(res!(cfg(cbots))), schemes(), None, false, true) |
| 85 | } |
| 86 | |
| 87 | // A reader that comes while the chunks of a new value are slow to become readable finds no value |
| 88 | // yet, or the whole one, and never a bunch key over chunks that are not there. A cache bot |
| 89 | // enters its records in turn, so a bunch key queued behind a held chunk at the same cache bot is |
| 90 | // held with it; several values are stored over many cache bots so that some bunch key has one to |
| 91 | // itself. They are stored in turn here; `concurrent_chunked_puts_do_not_queue` stores them at once. |
| 92 | #[test] |
| 93 | fn bunch_key_is_not_readable_before_its_chunks() -> Outcome<()> { |
| 94 | let _lock = lock(); |
| 95 | let dir = "./test_db_chunk_publish_order"; |
| 96 | let db = res!(open(dir)); |
| 97 | let keys: Vec<Dat> = (0..VALUES).map(|i| dat!(fmt!("chunks first {}", i))).collect(); |
| 98 | |
| 99 | let done = Arc::new(AtomicBool::new(false)); |
| 100 | let seen = { |
| 101 | let (db, keys, done) = (db.clone(), keys.clone(), done.clone()); |
| 102 | thread::spawn(move || { |
| 103 | let (mut n_none, mut n_whole, mut bad) = (0usize, 0usize, 0usize); |
| 104 | while !done.load(Ordering::Relaxed) { |
| 105 | for (i, k) in keys.iter().enumerate() { |
| 106 | match db.get(k, None) { |
| 107 | Ok(None) => n_none += 1, |
| 108 | Ok(Some((v, _))) if v == big(i as u8) => n_whole += 1, |
| 109 | other => { |
| 110 | bad += 1; |
| 111 | let s: String = fmt!("{:?}", other).chars().take(300).collect(); |
| 112 | msg!("a reader found neither no value nor the whole one: {}", s); |
| 113 | }, |
| 114 | } |
| 115 | } |
| 116 | thread::sleep(Duration::from_millis(2)); |
| 117 | } |
| 118 | (n_none, n_whole, bad) |
| 119 | }) |
| 120 | }; |
| 121 | hooks::set_chunk_insert_delay(HOLD); |
| 122 | let begun = Instant::now(); |
| 123 | let mut put = Ok(()); |
| 124 | for (i, k) in keys.iter().enumerate() { |
| 125 | if let Err(e) = db.insert(k.clone(), big(i as u8), Uid::default(), None) { |
| 126 | put = Err(e); |
| 127 | } |
| 128 | } |
| 129 | let took = begun.elapsed(); |
| 130 | hooks::set_chunk_insert_delay(Duration::ZERO); |
| 131 | done.store(true, Ordering::Relaxed); |
| 132 | let (n_none, n_whole, bad) = match seen.join() { |
| 133 | Ok(t) => t, |
| 134 | Err(_) => return Err(err!("The reader thread panicked."; Test, Invalid)), |
| 135 | }; |
| 136 | res!(put); |
| 137 | msg!("reader saw no value {} times, a whole one {} times, {} broken; the stores took {:?}.", |
| 138 | n_none, n_whole, bad, took); |
| 139 | // Positive control: the chunks were held, so a bunch key sent beside them had a window. |
| 140 | assert!(took >= HOLD, "the hook did not hold the chunks: the stores took {:?}", took); |
| 141 | assert_eq!(bad, 0, "a reader found a bunch key whose chunks were not readable"); |
| 142 | for (i, k) in keys.iter().enumerate() { |
| 143 | match res!(db.get(k, None)) { |
| 144 | Some((v, _)) => assert_eq!(v, big(i as u8), "the stored value reads back changed"), |
| 145 | None => return Err(err!("A stored value is gone."; Test, Missing)), |
| 146 | } |
| 147 | } |
| 148 | res!(db.close()); |
| 149 | let _ = std::fs::remove_dir_all(dir); |
| 150 | Ok(()) |
| 151 | } |
| 152 | |
| 153 | // Chunks that are not confirmed durable are not published: the caller is told the write failed, |
| 154 | // untagged `Unconfirmed` because nothing of the value is readable, and the key holds no value |
| 155 | // rather than a bunch key over chunks of doubtful standing. |
| 156 | #[test] |
| 157 | fn unconfirmed_chunks_publish_no_bunch_key() -> Outcome<()> { |
| 158 | let _lock = lock(); |
| 159 | let dir = "./test_db_chunk_publish_failed"; |
| 160 | let db = res!(open(dir)); |
| 161 | let k = dat!("chunks that fail"); |
| 162 | // A value beside it that stays, so the check is of this key and not of a dead store. |
| 163 | res!(db.insert(key(0), value(0, 9), Uid::default(), None)); |
| 164 | |
| 165 | hooks::set_barrier_failure(true); |
| 166 | let told = db.insert(k.clone(), big(3), Uid::default(), None); |
| 167 | hooks::set_barrier_failure(false); |
| 168 | match told { |
| 169 | Ok(_) => return Err(err!( |
| 170 | "A chunked store whose chunks failed their barrier was confirmed."; |
| 171 | Test, Unexpected)), |
| 172 | Err(e) => { |
| 173 | if e.tags().contains(&ErrTag::Unconfirmed) { |
| 174 | return Err(err!( |
| 175 | "A chunked store that published nothing was told it was written but \ |
| 176 | unconfirmed: {}", e; |
| 177 | Test, Mismatch)); |
| 178 | } |
| 179 | }, |
| 180 | } |
| 181 | match db.get(&k, None) { |
| 182 | Ok(None) => (), |
| 183 | other => return Err(err!( |
| 184 | "A chunked store whose chunks failed left {:?} at its key, where it should hold no \ |
| 185 | value.", other; |
| 186 | Test, Mismatch)), |
| 187 | } |
| 188 | match res!(db.get(&key(0), None)) { |
| 189 | Some((v, _)) => assert_eq!(v, value(0, 9), "the value beside it changed"), |
| 190 | None => return Err(err!("The value beside it is gone."; Test, Missing)), |
| 191 | } |
| 192 | res!(db.close()); |
| 193 | let _ = std::fs::remove_dir_all(dir); |
| 194 | Ok(()) |
| 195 | } |
| 196 | |
| 197 | // A caller is told its records are written without waiting for the chunks to become readable. The |
| 198 | // answers of a write are counted against deadlines of silence, 6 s for each record to be written |
| 199 | // and far longer for the disk to make them durable. The chunks' answers, and the bunch key's |
| 200 | // that waits behind them, were first reported only once the chunks were readable, so a disk |
| 201 | // slower than 6 s (an fsync on a loaded machine) failed a put with "0 of 8 records were confirmed |
| 202 | // written" although nothing was wrong (A3 round 2a, `chunk_leak` under load). |
| 203 | #[test] |
| 204 | fn chunks_are_reported_written_while_they_are_held() -> Outcome<()> { |
| 205 | let _lock = lock(); |
| 206 | let dir = "./test_db_chunk_publish_told"; |
| 207 | let db = res!(open(dir)); |
| 208 | let k = dat!("chunks told early"); |
| 209 | let hold = Duration::from_millis(2_500); |
| 210 | |
| 211 | hooks::set_chunk_insert_delay(hold); |
| 212 | let begun = Instant::now(); |
| 213 | let resp = res!(db.api().put(k.clone(), big(5), Uid::default(), None)); |
| 214 | let chan = match resp.channel() { |
| 215 | Some(chan) => chan, |
| 216 | None => return Err(err!("A put returned a responder without a channel."; Test, Missing)), |
| 217 | }; |
| 218 | let (mut records, mut written, mut told) = (0usize, 0usize, None); |
| 219 | while told.is_none() && begun.elapsed() < Duration::from_secs(10) { |
| 220 | match chan.recv_timeout(Duration::from_millis(50)) { |
| 221 | Recv::Result(Ok(OzoneMsg::Chunks(n))) => records = n, |
| 222 | Recv::Result(Ok(OzoneMsg::Written)) => { |
| 223 | written += 1; |
| 224 | if records > 0 && written == records { // every record, the bunch key included |
| 225 | told = Some(begun.elapsed()); |
| 226 | } |
| 227 | }, |
| 228 | _ => (), |
| 229 | } |
| 230 | } |
| 231 | hooks::set_chunk_insert_delay(Duration::ZERO); |
| 232 | let told = match told { |
| 233 | Some(t) => t, |
| 234 | None => return Err(err!( |
| 235 | "The caller of a put of {} records was told {} written in 10 s.", records, written; |
| 236 | Test, Missing)), |
| 237 | }; |
| 238 | msg!("the caller was told {} records written after {:?}; the hook holds each chunk for {:?}.", |
| 239 | written, told, hold); |
| 240 | // The chunks are held for 2.5 s each, so a caller told only when they are readable hears |
| 241 | // nothing for at least that long. |
| 242 | assert!(told < hold, "the caller was told its records were written only after {:?}, when the \ |
| 243 | chunks became readable", told); |
| 244 | // The put goes on to complete. |
| 245 | let end = Instant::now(); |
| 246 | loop { |
| 247 | match res!(db.get(&k, None)) { |
| 248 | Some((v, _)) => { assert_eq!(v, big(5), "the stored value reads back changed"); break; }, |
| 249 | None if end.elapsed() < Duration::from_secs(30) => thread::sleep(Duration::from_millis(50)), |
| 250 | None => return Err(err!("The put never completed."; Test, Missing)), |
| 251 | } |
| 252 | } |
| 253 | res!(db.close()); |
| 254 | let _ = std::fs::remove_dir_all(dir); |
| 255 | Ok(()) |
| 256 | } |
| 257 | |
| 258 | // Chunked puts do not queue behind one another for the disk. A server bot that waited through |
| 259 | // its put's chunks held one put each, so twelve at once against the two server bots made the |
| 260 | // last of a queue wait, with chunks held for 1.5 s, past the 6 s a request is given to be |
| 261 | // counted (A3 round 2a). Twelve over two bots leave one with six, whatever the choice: at |
| 262 | // 1.5 s a put, the last would be reached after 7.5 s. |
| 263 | #[test] |
| 264 | fn concurrent_chunked_puts_do_not_queue() -> Outcome<()> { |
| 265 | let _lock = lock(); |
| 266 | let dir = "./test_db_chunk_publish_many"; |
| 267 | let db = res!(open_with(dir, 32)); |
| 268 | let nputs = 12usize; |
| 269 | let hold = Duration::from_millis(1_500); |
| 270 | |
| 271 | hooks::set_chunk_insert_delay(hold); |
| 272 | let begun = Instant::now(); |
| 273 | let workers: Vec<_> = (0..nputs).map(|i| { |
| 274 | let db = db.clone(); |
| 275 | thread::spawn(move || { |
| 276 | db.insert(dat!(fmt!("many {}", i)), big(i as u8), Uid::default(), None) |
| 277 | .map(|_| ()).map_err(|e| fmt!("{}", e)) |
| 278 | }) |
| 279 | }).collect(); |
| 280 | let mut failed = Vec::new(); |
| 281 | for (i, w) in workers.into_iter().enumerate() { |
| 282 | match w.join() { |
| 283 | Ok(Ok(())) => (), |
| 284 | Ok(Err(e)) => failed.push((i, e.chars().take(200).collect::<String>())), |
| 285 | Err(_) => failed.push((i, fmt!("the thread panicked"))), |
| 286 | } |
| 287 | } |
| 288 | let took = begun.elapsed(); |
| 289 | hooks::set_chunk_insert_delay(Duration::ZERO); |
| 290 | msg!("{} chunked puts at once took {:?}; {} failed.", nputs, took, failed.len()); |
| 291 | assert!(failed.is_empty(), "concurrent chunked puts failed: {:?}", failed); |
| 292 | for i in 0..nputs { |
| 293 | match res!(db.get(&dat!(fmt!("many {}", i)), None)) { |
| 294 | Some((v, _)) => assert_eq!(v, big(i as u8), "a stored value reads back changed"), |
| 295 | None => return Err(err!("Chunked put {} reported success but its value is gone.", i; |
| 296 | Test, Missing)), |
| 297 | } |
| 298 | } |
| 299 | res!(db.close()); |
| 300 | let _ = std::fs::remove_dir_all(dir); |
| 301 | Ok(()) |
| 302 | } |