oxedyne/fe2o3/fe2o3_o3db_sync/tests/verify.rs
10.0 KiB, 19 runs
created by r1870400018:69893, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! `verify::verify_live_set` over an encrypted store opened as the gateway's tools open it: a |
| 2 | //! clean store reads back whole with the right chunked count and is left byte-identical, and a |
| 3 | //! value with one chunk of another write is named by its key. `verify::verify_key` on an open |
| 4 | //! store tells a record whose bytes were damaged on disk from a whole one, where a presence lookup |
| 5 | //! reports both held. |
| 6 | |
| 7 | use oxedyne_fe2o3_core::prelude::*; |
| 8 | use oxedyne_fe2o3_iop_db::api::Database; |
| 9 | use oxedyne_fe2o3_jdat::prelude::*; |
| 10 | use oxedyne_fe2o3_o3db_sync::{ |
| 11 | base::{ |
| 12 | cfg::OzoneConfig, |
| 13 | constant, |
| 14 | }, |
| 15 | comm::response::Wait, |
| 16 | gateway::{ |
| 17 | self, |
| 18 | Uid, |
| 19 | }, |
| 20 | test::setup, |
| 21 | verify::{ |
| 22 | self, |
| 23 | KeyCheck, |
| 24 | VerifyReport, |
| 25 | }, |
| 26 | }; |
| 27 | |
| 28 | use std::{ |
| 29 | collections::BTreeMap, |
| 30 | fs, |
| 31 | path::{ |
| 32 | Path, |
| 33 | PathBuf, |
| 34 | }, |
| 35 | time::Duration, |
| 36 | }; |
| 37 | |
| 38 | // A throwaway store's key; a real one is only ever read from a file at runtime. |
| 39 | const KEY: [u8; 32] = [0x3cu8; 32]; |
| 40 | |
| 41 | fn cfg() -> Outcome<OzoneConfig> { |
| 42 | let mut c = res!(setup::default_cfg()); |
| 43 | c.num_zones = 2; |
| 44 | c.num_cbots_per_zone = 1; |
| 45 | c.num_fbots_per_zone = 1; |
| 46 | c.num_wbots_per_zone = 1; |
| 47 | c.num_igbots_per_zone = 1; |
| 48 | c.rest_chunk_threshold = 500; |
| 49 | c.rest_chunk_bytes = 128; |
| 50 | c.zone_overrides = BTreeMap::new(); |
| 51 | Ok(c) |
| 52 | } |
| 53 | |
| 54 | fn wait() -> Wait { |
| 55 | Wait { max_wait: Duration::from_secs(120), check_interval: constant::CHECK_INTERVAL } |
| 56 | } |
| 57 | |
| 58 | fn val(len: usize, seed: u8) -> Dat { |
| 59 | Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect()) |
| 60 | } |
| 61 | |
| 62 | fn snapshot(root: &Path) -> Outcome<BTreeMap<PathBuf, Vec<u8>>> { |
| 63 | let mut out = BTreeMap::new(); |
| 64 | let mut stack = vec![root.to_path_buf()]; |
| 65 | while let Some(dir) = stack.pop() { |
| 66 | for entry in res!(fs::read_dir(&dir)) { |
| 67 | let entry = res!(entry); |
| 68 | let path = entry.path(); |
| 69 | let meta = res!(entry.metadata()); |
| 70 | if meta.is_dir() { |
| 71 | stack.push(path); |
| 72 | } else if meta.is_file() { |
| 73 | out.insert(path.clone(), res!(fs::read(&path))); |
| 74 | } |
| 75 | } |
| 76 | } |
| 77 | Ok(out) |
| 78 | } |
| 79 | |
| 80 | // Opens the store at `root` as `o3db_verify` does, checks it and closes it. |
| 81 | fn check(root: &Path) -> Outcome<VerifyReport> { |
| 82 | let (db, n) = res!(gateway::open_store(root, None, &KEY, false, "verify-test")); |
| 83 | res!(gateway::require_answer(n, "verify-test", root)); |
| 84 | let rep = verify::verify_live_set(db.api(), None, wait()); |
| 85 | res!(db.shutdown()); |
| 86 | rep |
| 87 | } |
| 88 | |
| 89 | #[test] |
| 90 | fn verify_finds_a_torn_value_and_writes_nothing() -> Outcome<()> { |
| 91 | log_set_level!("error"); |
| 92 | let dir = "./test_db_verify"; |
| 93 | let _ = fs::remove_dir_all(dir); |
| 94 | res!(fs::create_dir_all(dir)); |
| 95 | let root = res!(Path::new(dir).canonicalize()); |
| 96 | let torn = dat!("chunked 1"); |
| 97 | { |
| 98 | let (db, _) = res!(gateway::open_store(&root, Some(res!(cfg())), &KEY, false, "verify-fill")); |
| 99 | for i in 0..5u8 { |
| 100 | res!(db.insert(dat!(fmt!("small {}", i)), val(40, i), Uid::default(), None)); |
| 101 | } |
| 102 | for i in 0..3u8 { |
| 103 | res!(db.insert(dat!(fmt!("chunked {}", i)), val(700, i), Uid::default(), None)); |
| 104 | } |
| 105 | res!(db.insert(dat!("gone"), val(40, 9), Uid::default(), None)); |
| 106 | res!(db.delete(&dat!("gone"), Uid::default(), None)); |
| 107 | res!(db.shutdown()); |
| 108 | } |
| 109 | |
| 110 | // Clean: every key reads, the chunked ones are counted, and not a byte changes. |
| 111 | let before = res!(snapshot(&root)); |
| 112 | let rep = res!(check(&root)); |
| 113 | let after = res!(snapshot(&root)); |
| 114 | msg!("clean store: {}", rep.summary(80)); |
| 115 | assert!(rep.clean(), "a clean store reported failures: {:?}", rep.failures); |
| 116 | // The deleted key is not listed by the scan, so it is neither counted nor read. |
| 117 | assert_eq!((rep.keys, rep.present, rep.absent, rep.chunked), (8, 8, 0, 3)); |
| 118 | assert_eq!(before.keys().collect::<Vec<_>>(), after.keys().collect::<Vec<_>>(), |
| 119 | "the check added or removed a file"); |
| 120 | for (path, bytes) in &before { |
| 121 | assert!(after.get(path) == Some(bytes), "the check changed {:?}", path); |
| 122 | } |
| 123 | |
| 124 | // Torn: the first chunk of another value of the same length lands under the same chunk key, |
| 125 | // as a crash or race between two writes of one key left it while every write of a key used |
| 126 | // one chunk set (A3 QA B1, B2). Writes have sets of their own now, so the set is forced. |
| 127 | { |
| 128 | let (db, _) = res!(gateway::open_store(&root, None, &KEY, false, "verify-tear")); |
| 129 | let api = db.api(); |
| 130 | let set = match res!(api.get_head_wait(&torn, None)) { |
| 131 | Some((Dat::Tup5u64(t), _)) => t[0], |
| 132 | other => return Err(err!("{:?} is not chunked: {:?}", torn, other; Test, Invalid)), |
| 133 | }; |
| 134 | let resp = api.responder(); |
| 135 | let msgs = res!(api.prepare_write_dat( |
| 136 | torn.clone(), val(700, 77), Uid::default(), None, resp.clone(), Some(set))); |
| 137 | assert!(msgs.len() > 2, "the value was not chunked"); |
| 138 | let chunk1 = msgs.into_iter().nth(1).into_iter().collect::<Vec<_>>(); |
| 139 | res!(api.store_bytes(chunk1)); |
| 140 | let _ = resp.recv_timeout(Duration::from_secs(10)); // written, then shut down after it |
| 141 | res!(db.shutdown()); |
| 142 | } |
| 143 | let rep = res!(check(&root)); |
| 144 | msg!("torn store: {}", rep.summary(80)); |
| 145 | let _ = fs::remove_dir_all(dir); |
| 146 | assert_eq!(rep.failures.len(), 1, "expected one failure: {:?}", rep.failures); |
| 147 | assert_eq!(rep.failures[0].key, torn); |
| 148 | assert!(rep.failures[0].chunked); |
| 149 | assert_eq!((rep.keys, rep.present, rep.chunked), (8, 7, 3)); |
| 150 | Ok(()) |
| 151 | } |
| 152 | |
| 153 | #[test] |
| 154 | fn verify_key_finds_a_flipped_byte_that_presence_reports_held() -> Outcome<()> { |
| 155 | log_set_level!("error"); |
| 156 | let dir = "./test_db_verify_key"; |
| 157 | let _ = fs::remove_dir_all(dir); |
| 158 | res!(fs::create_dir_all(dir)); |
| 159 | let root = res!(Path::new(dir).canonicalize()); |
| 160 | let victim = dat!("victim"); |
| 161 | { |
| 162 | let (db, _) = res!(gateway::open_store(&root, Some(res!(cfg())), &KEY, false, "vk-fill")); |
| 163 | for i in 0..3u8 { |
| 164 | res!(db.insert(dat!(fmt!("small {}", i)), val(300, i), Uid::default(), None)); |
| 165 | } |
| 166 | res!(db.shutdown()); |
| 167 | } |
| 168 | let before = res!(snapshot(&root)); |
| 169 | { |
| 170 | let (db, _) = res!(gateway::open_store(&root, None, &KEY, false, "vk-victim")); |
| 171 | res!(db.insert(victim.clone(), val(300, 42), Uid::default(), None)); |
| 172 | res!(db.shutdown()); |
| 173 | } |
| 174 | // The victim's record is the tail the last write appended to the file that grew most; one |
| 175 | // byte well inside it is flipped, as a failing disk would leave it. |
| 176 | let after = res!(snapshot(&root)); |
| 177 | let mut grown: Option<(PathBuf, usize, usize)> = None; |
| 178 | for (path, bytes) in &after { |
| 179 | let old = before.get(path).map(|b| b.len()).unwrap_or(0); |
| 180 | if bytes.len() > old && grown.as_ref().map_or(true, |(_, o, n)| bytes.len() - old > n - o) { |
| 181 | grown = Some((path.clone(), old, bytes.len())); |
| 182 | } |
| 183 | } |
| 184 | let (path, old, new) = res!(grown.ok_or_else(|| err!("No file grew with the victim's write."; Missing))); |
| 185 | assert!(new - old > 300, "the victim's record is {} bytes, not a whole value", new - old); |
| 186 | let mut bytes = res!(fs::read(&path)); |
| 187 | let at = new - (new - old) / 4; |
| 188 | bytes[at] ^= 0xff; |
| 189 | res!(fs::write(&path, &bytes)); |
| 190 | |
| 191 | let (db, n) = res!(gateway::open_store(&root, None, &KEY, false, "vk-check")); |
| 192 | res!(gateway::require_answer(n, "vk-check", &root)); |
| 193 | let held = res!(db.contains_many(&[victim.clone(), dat!("small 1"), dat!("never")], None)); |
| 194 | let damaged = verify::verify_key(db.api(), &victim, None); |
| 195 | let whole = verify::verify_key(db.api(), &dat!("small 1"), None); |
| 196 | let never = verify::verify_key(db.api(), &dat!("never"), None); |
| 197 | // A record that does not read can still be deleted, which is what a caller rids itself of |
| 198 | // it with: the delete must not first need to read it. |
| 199 | let deleted = db.delete(&victim, Uid::default(), None); |
| 200 | let gone = res!(db.contains_many(&[victim.clone()], None)); |
| 201 | let after = verify::verify_key(db.api(), &victim, None); |
| 202 | res!(db.shutdown()); |
| 203 | let _ = fs::remove_dir_all(dir); |
| 204 | assert_eq!(held, vec![true, true, false], "presence of the victim, a whole key and no key"); |
| 205 | assert!(deleted.is_ok(), "the unreadable record could not be deleted: {:?}", deleted); |
| 206 | assert_eq!(gone, vec![false], "the deleted record still reads present"); |
| 207 | assert_eq!(after, KeyCheck::Absent); |
| 208 | assert!(matches!(damaged, KeyCheck::Damaged { chunked: false, .. }), |
| 209 | "the flipped record read as {:?}", damaged); |
| 210 | assert_eq!(whole, KeyCheck::Whole { chunked: false }); |
| 211 | assert_eq!(never, KeyCheck::Absent); |
| 212 | Ok(()) |
| 213 | } |
| 214 | |
| 215 | // A read the store did not answer says nothing about the bytes, so it must never classify as |
| 216 | // damage, however it is wrapped; the bytes failing must, however deep (CHUNK-VERIFY D1). |
| 217 | #[test] |
| 218 | fn a_timeout_is_unavailable_and_only_bad_bytes_are_damaged() -> Outcome<()> { |
| 219 | let timeout: Error<ErrTag> = err!( |
| 220 | "Failed to receive a message via responder within {:.2} [s].", |
| 221 | constant::USER_REQUEST_TIMEOUT.as_secs_f32(); |
| 222 | Missing, Data); |
| 223 | let wrapped: Error<ErrTag> = err!(timeout.clone(), "While reading a head."; Read); |
| 224 | let channel: Error<ErrTag> = err!("Could not read from responder channel"; Channel, Read); |
| 225 | let busy: Error<ErrTag> = err!("Not known."; Missing); |
| 226 | for e in [&timeout, &wrapped, &channel, &busy] { |
| 227 | assert!(matches!(KeyCheck::of_error(false, e), KeyCheck::Unavailable { chunked: false, .. }), |
| 228 | "{} classified as damage", e); |
| 229 | } |
| 230 | let checksum: Error<ErrTag> = err!("Mismatch detected"; Input, Mismatch); |
| 231 | let reader: Error<ErrTag> = err!(checksum.clone(), "While reading Data file 1."; Data, Mismatch); |
| 232 | let deep: Error<ErrTag> = err!(reader.clone(), "Fetching chunks."; Read); |
| 233 | let decrypt: Error<ErrTag> = err!("While decrypting."; Decrypt); |
| 234 | let decode: Error<ErrTag> = err!("Could not form a Dat."; Decode, Bytes); |
| 235 | for e in [&checksum, &reader, &deep, &decrypt, &decode] { |
| 236 | assert!(matches!(KeyCheck::of_error(true, e), KeyCheck::Damaged { chunked: true, .. }), |
| 237 | "{} not classified as damage", e); |
| 238 | } |
| 239 | Ok(()) |
| 240 | } |