oxedyne/fe2o3/fe2o3_o3db_sync/tests/write_stall.rs
4.3 KiB, 1 run
created by r1870400018:75223, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! A writer stuck in its append for longer than the caller waits (WSTALL, 2026-10-16: a gateway |
| 2 | //! on a disk that stopped completing for 39 s). The caller is told the write's outcome is not |
| 3 | //! known, and the record lands once the writer is let go: "reported failed, actually written", |
| 4 | //! pinned down so that a caller can be built to survive it. `test::hooks` is process-wide, which |
| 5 | //! is why this is a test binary of its own with a single test. |
| 6 | |
| 7 | use oxedyne_fe2o3_core::prelude::*; |
| 8 | use oxedyne_fe2o3_hash::{ |
| 9 | csum::ChecksumScheme, |
| 10 | hash::HashScheme, |
| 11 | }; |
| 12 | use oxedyne_fe2o3_iop_db::api::Database; |
| 13 | use oxedyne_fe2o3_jdat::prelude::*; |
| 14 | use oxedyne_fe2o3_o3db_sync::{ |
| 15 | O3db, |
| 16 | base::{ |
| 17 | cfg::OzoneConfig, |
| 18 | constant, |
| 19 | }, |
| 20 | comm::response::outcome_unknown, |
| 21 | data::core::RestSchemesInput, |
| 22 | test::{ |
| 23 | hooks, |
| 24 | setup::{ |
| 25 | self, |
| 26 | Uid, |
| 27 | UID_LEN, |
| 28 | }, |
| 29 | }, |
| 30 | }; |
| 31 | |
| 32 | use std::{ |
| 33 | collections::BTreeMap, |
| 34 | path::{ |
| 35 | Path, |
| 36 | PathBuf, |
| 37 | }, |
| 38 | thread, |
| 39 | time::{ |
| 40 | Duration, |
| 41 | Instant, |
| 42 | }, |
| 43 | }; |
| 44 | |
| 45 | type TestDb = O3db< |
| 46 | { UID_LEN }, |
| 47 | Uid, |
| 48 | (), |
| 49 | HashScheme, |
| 50 | HashScheme, |
| 51 | ChecksumScheme, |
| 52 | >; |
| 53 | |
| 54 | #[test] |
| 55 | fn main() -> Outcome<()> { |
| 56 | log_set_level!("error"); |
| 57 | let outcome = a_stalled_append_is_not_known_and_then_lands(); |
| 58 | hooks::set_write_hold(0, Duration::ZERO); |
| 59 | log_finish_wait!(); |
| 60 | outcome |
| 61 | } |
| 62 | |
| 63 | fn a_stalled_append_is_not_known_and_then_lands() -> Outcome<()> { |
| 64 | let hold = constant::USER_REQUEST_TIMEOUT + Duration::from_secs(2); |
| 65 | let root = res!(fresh("./test_db_write_stall")); |
| 66 | let mut cfg = res!(setup::default_cfg()); |
| 67 | cfg.zone_overrides = BTreeMap::new(); |
| 68 | let db = res!(open(&root, cfg)); |
| 69 | let k = dat!("write stall key"); |
| 70 | |
| 71 | hooks::set_write_hold(0, hold); |
| 72 | let begun = Instant::now(); |
| 73 | let outcome = db.insert(k.clone(), dat!(42u8), Uid::default(), None); |
| 74 | let took = begun.elapsed(); |
| 75 | if hooks::write_holds_taken() != 1 { |
| 76 | return Err(err!("The write hold was taken {} times, not once.", |
| 77 | hooks::write_holds_taken(); Test, Mismatch)); |
| 78 | } |
| 79 | match outcome { |
| 80 | Ok(r) => return Err(err!( |
| 81 | "A write held for {:?} was confirmed in {:?}: {:?}.", hold, took, r; |
| 82 | Test, Unexpected)), |
| 83 | Err(e) => { |
| 84 | if !outcome_unknown(&e) { |
| 85 | return Err(err!( |
| 86 | "A write whose writer said nothing in time is tagged {:?}, which \ |
| 87 | outcome_unknown does not read as a write that may yet land: {}", |
| 88 | e.tags(), e; |
| 89 | Test, Mismatch)); |
| 90 | } |
| 91 | if !fmt!("{}", e).contains("not") || !fmt!("{}", e).contains("known") { |
| 92 | return Err(err!("The error does not say the outcome is not known: {}", e; |
| 93 | Test, Mismatch)); |
| 94 | } |
| 95 | if took >= hold { |
| 96 | return Err(err!("The caller waited {:?}, the whole of the {:?} hold.", took, hold; |
| 97 | Test, Timeout)); |
| 98 | } |
| 99 | }, |
| 100 | } |
| 101 | |
| 102 | // Reported failed, and written all the same once the disk returns. |
| 103 | thread::sleep(hold.saturating_sub(begun.elapsed()) + Duration::from_millis(500)); |
| 104 | let mut found = None; |
| 105 | let until = Instant::now() + Duration::from_secs(5); |
| 106 | while Instant::now() < until { |
| 107 | if let Some((v, _)) = res!(db.get(&k, None)) { |
| 108 | found = Some(v); |
| 109 | break; |
| 110 | } |
| 111 | thread::sleep(Duration::from_millis(100)); |
| 112 | } |
| 113 | match found { |
| 114 | Some(v) => req!(v, dat!(42u8), "The held write landed with another value."), |
| 115 | None => return Err(err!( |
| 116 | "A write reported as of unknown outcome never landed after its hold."; |
| 117 | Test, Missing)), |
| 118 | } |
| 119 | res!(db.close()); |
| 120 | Ok(()) |
| 121 | } |
| 122 | |
| 123 | /// An empty directory of this test's own. |
| 124 | fn fresh(dir: &str) -> Outcome<PathBuf> { |
| 125 | let _ = std::fs::remove_dir_all(dir); |
| 126 | res!(std::fs::create_dir_all(dir)); |
| 127 | Ok(res!(Path::new(dir).canonicalize())) |
| 128 | } |
| 129 | |
| 130 | fn open(root: &Path, cfg: OzoneConfig) -> Outcome<TestDb> { |
| 131 | let schms = RestSchemesInput::new( |
| 132 | None::<()>, |
| 133 | None::<HashScheme>, |
| 134 | None::<HashScheme>, |
| 135 | Some(ChecksumScheme::new_crc32()), |
| 136 | ); |
| 137 | let mut db = res!(TestDb::new(root.to_path_buf(), Some(cfg), schms, Uid::default())); |
| 138 | res!(db.start("test")); |
| 139 | Ok(db) |
| 140 | } |