oxedyne/fe2o3/fe2o3_o3db_sync/tests/unconfirmed_winner.rs
9.0 KiB, 58 runs
created by r1870400018:70431, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! No record is ever beaten or displaced by an unconfirmed winner (durable mark, commit A). |
| 2 | //! Before it, a record U whose sync failed could stand in the cache over an older chunked put P, |
| 3 | //! and P had to be held from retiring, collection and the sweep until a restart (R2-1). Now the |
| 4 | //! failed sync stops the store: U is refused, a store in flight is refused, a read of what no |
| 5 | //! mark covers is refused, compaction and the sweep are refused, and the restart cuts U from the |
| 6 | //! disk. Two orders, as before: P's head waits while U fails (P would be beaten), and U fails |
| 7 | //! after P (P would be displaced). The hooks are process-wide, so the two run one at a time. |
| 8 | |
| 9 | mod gc_pair; |
| 10 | |
| 11 | use gc_pair::*; |
| 12 | |
| 13 | use oxedyne_fe2o3_core::prelude::*; |
| 14 | use oxedyne_fe2o3_iop_db::api::{ |
| 15 | Database, |
| 16 | Meta, |
| 17 | }; |
| 18 | use oxedyne_fe2o3_jdat::prelude::*; |
| 19 | use oxedyne_fe2o3_o3db_sync::{ |
| 20 | base::{ |
| 21 | cfg::OzoneConfig, |
| 22 | constant, |
| 23 | }, |
| 24 | comm::response::Wait, |
| 25 | file::{ |
| 26 | core::FileType, |
| 27 | refusal::Refusal, |
| 28 | }, |
| 29 | sweep, |
| 30 | test::{ |
| 31 | hooks, |
| 32 | setup::{ |
| 33 | self, |
| 34 | Uid, |
| 35 | UID_LEN, |
| 36 | }, |
| 37 | }, |
| 38 | verify, |
| 39 | }; |
| 40 | |
| 41 | use std::{ |
| 42 | collections::BTreeMap, |
| 43 | path::{ |
| 44 | Path, |
| 45 | PathBuf, |
| 46 | }, |
| 47 | sync::Mutex, |
| 48 | thread, |
| 49 | time::Duration, |
| 50 | }; |
| 51 | |
| 52 | fn cfg() -> Outcome<OzoneConfig> { |
| 53 | let mut c = res!(setup::default_cfg()); |
| 54 | c.num_zones = 1; |
| 55 | c.num_cbots_per_zone = 4; |
| 56 | c.num_fbots_per_zone = 1; |
| 57 | c.num_wbots_per_zone = 1; |
| 58 | c.zone_overrides = BTreeMap::new(); |
| 59 | c.rest_chunk_threshold = 1_000; |
| 60 | c.rest_chunk_bytes = 400; |
| 61 | c.sync_on_write = true; |
| 62 | Ok(c) |
| 63 | } |
| 64 | |
| 65 | fn val(len: usize, seed: u8) -> Dat { |
| 66 | Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect()) |
| 67 | } |
| 68 | |
| 69 | fn start(root: &Path, wipe: bool) -> Outcome<TestDb> { |
| 70 | log_set_level!("error"); |
| 71 | setup::start_db(root.to_path_buf(), Some(res!(cfg())), schemes(), None, false, wipe) |
| 72 | } |
| 73 | |
| 74 | fn scan_wait() -> Wait { |
| 75 | Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL } |
| 76 | } |
| 77 | |
| 78 | // "L/S" for a whole value of length L and seed S as `val` writes it, else what was read. |
| 79 | fn describe(r: &Outcome<Option<(Dat, Meta<{ UID_LEN }, Uid>)>>) -> String { |
| 80 | match r { |
| 81 | Ok(None) => fmt!("none"), |
| 82 | Ok(Some((Dat::BU32(b), _))) | Ok(Some((Dat::BU8(b), _))) | Ok(Some((Dat::BU16(b), _))) | |
| 83 | Ok(Some((Dat::BU64(b), _))) if !b.is_empty() |
| 84 | && b.iter().enumerate().all(|(j, x)| *x == b[0] ^ (j as u8)) => fmt!("{}/{}", b.len(), b[0]), |
| 85 | Ok(Some((d, _))) => fmt!("TORN:{:?}", d).chars().take(100).collect(), |
| 86 | Err(e) => fmt!("ERR:{}", fmt!("{}", e).chars().take(200).collect::<String>()), |
| 87 | } |
| 88 | } |
| 89 | |
| 90 | static SERIAL: Mutex<()> = Mutex::new(()); |
| 91 | |
| 92 | // The highest numbered data file, the live one just after `new_live_files`. |
| 93 | fn live_fnum(root: &Path, c: &OzoneConfig) -> u32 { |
| 94 | let mut n = 1; |
| 95 | while file(root, c, FileType::Data, n + 1).exists() { |
| 96 | n += 1; |
| 97 | } |
| 98 | n |
| 99 | } |
| 100 | |
| 101 | // Writes U, unchunked, into a live file of its own whose barrier fails. The roll after it is |
| 102 | // refused once the store has stopped, and that is not what is tested here. |
| 103 | fn write_unconfirmed(db: &TestDb, root: &Path, k: &Dat) -> Outcome<Outcome<()>> { |
| 104 | let c = res!(cfg()); |
| 105 | res!(db.api().new_live_files()); |
| 106 | hooks::set_barrier_failure_for(Some(live_fnum(root, &c))); |
| 107 | let u = db.insert(k.clone(), val(50, 3), Uid::default(), None).map(|_| ()); |
| 108 | let _ = db.api().new_live_files(); |
| 109 | hooks::set_barrier_failure_for(None); |
| 110 | Ok(u) |
| 111 | } |
| 112 | |
| 113 | // What the stopped store still does: a read of the key, compaction and the sweep. |
| 114 | fn stopped(db: &TestDb, k: &Dat) -> Outcome<(bool, String, Outcome<()>, Outcome<sweep::SweepReport>)> { |
| 115 | res!(db.api().settle_for_test(Duration::from_secs(10))); |
| 116 | let faulted = db.api().fault().is_some(); |
| 117 | let live = describe(&db.get(k, None)); |
| 118 | let compact = db.compact_now(Duration::from_secs(60)).map(|_| ()); |
| 119 | let swept = sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait()); |
| 120 | Ok((faulted, live, compact, swept)) |
| 121 | } |
| 122 | |
| 123 | // The restart cuts U. Returns the key's value, whether the fault is gone, and whether the store |
| 124 | // verifies clean after a sweep. |
| 125 | fn restart(root: &Path, k: &Dat) -> Outcome<(String, bool, bool)> { |
| 126 | let db = res!(start(root, false)); |
| 127 | let after = describe(&db.get(k, None)); |
| 128 | let cleared = db.api().fault().is_none(); |
| 129 | res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait())); |
| 130 | let v = res!(verify::verify_live_set(db.api(), None, scan_wait())); |
| 131 | res!(db.close()); |
| 132 | Ok((after, cleared, v.clean())) |
| 133 | } |
| 134 | |
| 135 | fn fresh(dir: &str) -> Outcome<PathBuf> { |
| 136 | let _ = std::fs::remove_dir_all(dir); |
| 137 | res!(std::fs::create_dir_all(dir)); |
| 138 | Ok(res!(Path::new(dir).canonicalize())) |
| 139 | } |
| 140 | |
| 141 | fn assert_stopped( |
| 142 | faulted: bool, |
| 143 | live: &str, |
| 144 | compact: &Outcome<()>, |
| 145 | swept: &Outcome<sweep::SweepReport>, |
| 146 | ) |
| 147 | -> Outcome<()> |
| 148 | { |
| 149 | assert!(faulted, "a failed sync did not stop the store"); |
| 150 | assert!(live.starts_with("ERR:"), "a read of U, which no mark covers, was served: {}", live); |
| 151 | assert!(compact.is_err(), "compaction ran on a stopped store"); |
| 152 | match swept { |
| 153 | Ok(r) => return Err(err!("the sweep ran on a stopped store: {:?}", r; Test)), |
| 154 | Err(e) => assert!(fmt!("{}", e).contains("Restart the store"), |
| 155 | "the sweep's refusal does not say what to do: {}", e), |
| 156 | } |
| 157 | Ok(()) |
| 158 | } |
| 159 | |
| 160 | /// P's head waits while U's sync fails. The store stops, so P's head is refused rather than |
| 161 | /// beaten, and after the restart neither U nor P is the key's value. |
| 162 | #[test] |
| 163 | fn a_store_in_flight_when_a_sync_fails_is_refused() -> Outcome<()> { |
| 164 | let _serial = SERIAL.lock(); |
| 165 | let dir = "./test_db_unconfirmed_winner_beaten"; |
| 166 | let root = res!(fresh(dir)); |
| 167 | let db = res!(start(&root, true)); |
| 168 | let k = dat!("unconfirmed winner beaten"); |
| 169 | // P: chunked, held with its chunks in until U is in. Stored through the API, its store |
| 170 | // runs in this thread rather than a server bot. |
| 171 | hooks::set_head_hold(true); |
| 172 | let p = { let (db, k) = (db.clone(), k.clone()); |
| 173 | thread::spawn(move || -> Outcome<()> { |
| 174 | let resp = db.api().responder(); |
| 175 | res!(db.api().store_dat_using_responder(k, val(1_500, 2), Uid::default(), None, resp.clone())); |
| 176 | resp.recv_store_ack().map(|_| ()) |
| 177 | }) }; |
| 178 | if !wait_until(Duration::from_secs(30), || hooks::heads_held() >= 1) { |
| 179 | hooks::set_head_hold(false); |
| 180 | return Err(err!("P's store never reached its head"; Test)); |
| 181 | } |
| 182 | let u = res!(write_unconfirmed(&db, &root, &k)); |
| 183 | hooks::set_head_hold(false); |
| 184 | let pr = match p.join() { Ok(r) => r, Err(_) => Err(err!("P panicked"; Test)) }; |
| 185 | let (faulted, live, compact, swept) = res!(stopped(&db, &k)); |
| 186 | let _ = db.close(); |
| 187 | println!("beaten: p={:?} u={:?} faulted={} live={} compact={:?} swept={:?}", |
| 188 | pr.as_ref().map_err(|e| fmt!("{}", e)), u.as_ref().map_err(|e| fmt!("{}", e)), |
| 189 | faulted, live, compact.as_ref().map_err(|e| fmt!("{}", e)), |
| 190 | swept.as_ref().map_err(|e| fmt!("{}", e))); |
| 191 | assert!(u.is_err(), "U's store succeeded although its sync was failed"); |
| 192 | assert!(pr.is_err(), "P's head was written on a stopped store"); |
| 193 | res!(assert_stopped(faulted, &live, &compact, &swept)); |
| 194 | res!(Refusal::clear(&root)); // as the operator would |
| 195 | let (after, cleared, clean) = res!(restart(&root, &k)); |
| 196 | let _ = std::fs::remove_dir_all(dir); |
| 197 | assert_eq!(after, "none", "after the restart's cut the key holds neither U nor P"); |
| 198 | assert!(cleared, "the restart left the fault standing"); |
| 199 | assert!(clean, "the store does not verify clean after the restart and a sweep"); |
| 200 | Ok(()) |
| 201 | } |
| 202 | |
| 203 | /// U's sync fails after P has landed. U never displaces P: the store stops, U is not served, and |
| 204 | /// the restart's cut leaves P as the key's value. |
| 205 | #[test] |
| 206 | fn a_failed_sync_never_displaces_a_durable_head() -> Outcome<()> { |
| 207 | let _serial = SERIAL.lock(); |
| 208 | let dir = "./test_db_unconfirmed_winner_displaced"; |
| 209 | let root = res!(fresh(dir)); |
| 210 | let db = res!(start(&root, true)); |
| 211 | let k = dat!("unconfirmed winner displaced"); |
| 212 | res!(db.insert(k.clone(), val(1_500, 2), Uid::default(), None)); |
| 213 | res!(db.api().sync_now(scan_wait())); |
| 214 | let u = res!(write_unconfirmed(&db, &root, &k)); |
| 215 | let (faulted, live, compact, swept) = res!(stopped(&db, &k)); |
| 216 | let _ = db.close(); |
| 217 | println!("displaced: u={:?} faulted={} live={} compact={:?} swept={:?}", |
| 218 | u.as_ref().map_err(|e| e.tags().to_vec()), faulted, live, |
| 219 | compact.as_ref().map_err(|e| fmt!("{}", e)), swept.as_ref().map_err(|e| fmt!("{}", e))); |
| 220 | assert!(u.is_err(), "U's store succeeded although its sync was failed"); |
| 221 | res!(assert_stopped(faulted, &live, &compact, &swept)); |
| 222 | res!(Refusal::clear(&root)); // as the operator would |
| 223 | let (after, cleared, clean) = res!(restart(&root, &k)); |
| 224 | let _ = std::fs::remove_dir_all(dir); |
| 225 | assert_eq!(after, "1500/2", "after the restart's cut P is the key's value"); |
| 226 | assert!(cleared, "the restart left the fault standing"); |
| 227 | assert!(clean, "the store does not verify clean after the restart and a sweep"); |
| 228 | Ok(()) |
| 229 | } |