Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/verify.rs

10.0 KiB, 19 runs

created by r1870400018:69893, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! `verify::verify_live_set` over an encrypted store opened as the gateway's tools open it: a
2//! clean store reads back whole with the right chunked count and is left byte-identical, and a
3//! value with one chunk of another write is named by its key. `verify::verify_key` on an open
4//! store tells a record whose bytes were damaged on disk from a whole one, where a presence lookup
5//! reports both held.
6
7use oxedyne_fe2o3_core::prelude::*;
8use oxedyne_fe2o3_iop_db::api::Database;
9use oxedyne_fe2o3_jdat::prelude::*;
10use oxedyne_fe2o3_o3db_sync::{
11 base::{
12 cfg::OzoneConfig,
13 constant,
14 },
15 comm::response::Wait,
16 gateway::{
17 self,
18 Uid,
19 },
20 test::setup,
21 verify::{
22 self,
23 KeyCheck,
24 VerifyReport,
25 },
26};
27
28use std::{
29 collections::BTreeMap,
30 fs,
31 path::{
32 Path,
33 PathBuf,
34 },
35 time::Duration,
36};
37
38// A throwaway store's key; a real one is only ever read from a file at runtime.
39const KEY: [u8; 32] = [0x3cu8; 32];
40
41fn cfg() -> Outcome<OzoneConfig> {
42 let mut c = res!(setup::default_cfg());
43 c.num_zones = 2;
44 c.num_cbots_per_zone = 1;
45 c.num_fbots_per_zone = 1;
46 c.num_wbots_per_zone = 1;
47 c.num_igbots_per_zone = 1;
48 c.rest_chunk_threshold = 500;
49 c.rest_chunk_bytes = 128;
50 c.zone_overrides = BTreeMap::new();
51 Ok(c)
52}
53
54fn wait() -> Wait {
55 Wait { max_wait: Duration::from_secs(120), check_interval: constant::CHECK_INTERVAL }
56}
57
58fn val(len: usize, seed: u8) -> Dat {
59 Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect())
60}
61
62fn snapshot(root: &Path) -> Outcome<BTreeMap<PathBuf, Vec<u8>>> {
63 let mut out = BTreeMap::new();
64 let mut stack = vec![root.to_path_buf()];
65 while let Some(dir) = stack.pop() {
66 for entry in res!(fs::read_dir(&dir)) {
67 let entry = res!(entry);
68 let path = entry.path();
69 let meta = res!(entry.metadata());
70 if meta.is_dir() {
71 stack.push(path);
72 } else if meta.is_file() {
73 out.insert(path.clone(), res!(fs::read(&path)));
74 }
75 }
76 }
77 Ok(out)
78}
79
80// Opens the store at `root` as `o3db_verify` does, checks it and closes it.
81fn check(root: &Path) -> Outcome<VerifyReport> {
82 let (db, n) = res!(gateway::open_store(root, None, &KEY, false, "verify-test"));
83 res!(gateway::require_answer(n, "verify-test", root));
84 let rep = verify::verify_live_set(db.api(), None, wait());
85 res!(db.shutdown());
86 rep
87}
88
89#[test]
90fn verify_finds_a_torn_value_and_writes_nothing() -> Outcome<()> {
91 log_set_level!("error");
92 let dir = "./test_db_verify";
93 let _ = fs::remove_dir_all(dir);
94 res!(fs::create_dir_all(dir));
95 let root = res!(Path::new(dir).canonicalize());
96 let torn = dat!("chunked 1");
97 {
98 let (db, _) = res!(gateway::open_store(&root, Some(res!(cfg())), &KEY, false, "verify-fill"));
99 for i in 0..5u8 {
100 res!(db.insert(dat!(fmt!("small {}", i)), val(40, i), Uid::default(), None));
101 }
102 for i in 0..3u8 {
103 res!(db.insert(dat!(fmt!("chunked {}", i)), val(700, i), Uid::default(), None));
104 }
105 res!(db.insert(dat!("gone"), val(40, 9), Uid::default(), None));
106 res!(db.delete(&dat!("gone"), Uid::default(), None));
107 res!(db.shutdown());
108 }
109
110 // Clean: every key reads, the chunked ones are counted, and not a byte changes.
111 let before = res!(snapshot(&root));
112 let rep = res!(check(&root));
113 let after = res!(snapshot(&root));
114 msg!("clean store: {}", rep.summary(80));
115 assert!(rep.clean(), "a clean store reported failures: {:?}", rep.failures);
116 // The deleted key is not listed by the scan, so it is neither counted nor read.
117 assert_eq!((rep.keys, rep.present, rep.absent, rep.chunked), (8, 8, 0, 3));
118 assert_eq!(before.keys().collect::<Vec<_>>(), after.keys().collect::<Vec<_>>(),
119 "the check added or removed a file");
120 for (path, bytes) in &before {
121 assert!(after.get(path) == Some(bytes), "the check changed {:?}", path);
122 }
123
124 // Torn: the first chunk of another value of the same length lands under the same chunk key,
125 // as a crash or race between two writes of one key left it while every write of a key used
126 // one chunk set (A3 QA B1, B2). Writes have sets of their own now, so the set is forced.
127 {
128 let (db, _) = res!(gateway::open_store(&root, None, &KEY, false, "verify-tear"));
129 let api = db.api();
130 let set = match res!(api.get_head_wait(&torn, None)) {
131 Some((Dat::Tup5u64(t), _)) => t[0],
132 other => return Err(err!("{:?} is not chunked: {:?}", torn, other; Test, Invalid)),
133 };
134 let resp = api.responder();
135 let msgs = res!(api.prepare_write_dat(
136 torn.clone(), val(700, 77), Uid::default(), None, resp.clone(), Some(set)));
137 assert!(msgs.len() > 2, "the value was not chunked");
138 let chunk1 = msgs.into_iter().nth(1).into_iter().collect::<Vec<_>>();
139 res!(api.store_bytes(chunk1));
140 let _ = resp.recv_timeout(Duration::from_secs(10)); // written, then shut down after it
141 res!(db.shutdown());
142 }
143 let rep = res!(check(&root));
144 msg!("torn store: {}", rep.summary(80));
145 let _ = fs::remove_dir_all(dir);
146 assert_eq!(rep.failures.len(), 1, "expected one failure: {:?}", rep.failures);
147 assert_eq!(rep.failures[0].key, torn);
148 assert!(rep.failures[0].chunked);
149 assert_eq!((rep.keys, rep.present, rep.chunked), (8, 7, 3));
150 Ok(())
151}
152
153#[test]
154fn verify_key_finds_a_flipped_byte_that_presence_reports_held() -> Outcome<()> {
155 log_set_level!("error");
156 let dir = "./test_db_verify_key";
157 let _ = fs::remove_dir_all(dir);
158 res!(fs::create_dir_all(dir));
159 let root = res!(Path::new(dir).canonicalize());
160 let victim = dat!("victim");
161 {
162 let (db, _) = res!(gateway::open_store(&root, Some(res!(cfg())), &KEY, false, "vk-fill"));
163 for i in 0..3u8 {
164 res!(db.insert(dat!(fmt!("small {}", i)), val(300, i), Uid::default(), None));
165 }
166 res!(db.shutdown());
167 }
168 let before = res!(snapshot(&root));
169 {
170 let (db, _) = res!(gateway::open_store(&root, None, &KEY, false, "vk-victim"));
171 res!(db.insert(victim.clone(), val(300, 42), Uid::default(), None));
172 res!(db.shutdown());
173 }
174 // The victim's record is the tail the last write appended to the file that grew most; one
175 // byte well inside it is flipped, as a failing disk would leave it.
176 let after = res!(snapshot(&root));
177 let mut grown: Option<(PathBuf, usize, usize)> = None;
178 for (path, bytes) in &after {
179 let old = before.get(path).map(|b| b.len()).unwrap_or(0);
180 if bytes.len() > old && grown.as_ref().map_or(true, |(_, o, n)| bytes.len() - old > n - o) {
181 grown = Some((path.clone(), old, bytes.len()));
182 }
183 }
184 let (path, old, new) = res!(grown.ok_or_else(|| err!("No file grew with the victim's write."; Missing)));
185 assert!(new - old > 300, "the victim's record is {} bytes, not a whole value", new - old);
186 let mut bytes = res!(fs::read(&path));
187 let at = new - (new - old) / 4;
188 bytes[at] ^= 0xff;
189 res!(fs::write(&path, &bytes));
190
191 let (db, n) = res!(gateway::open_store(&root, None, &KEY, false, "vk-check"));
192 res!(gateway::require_answer(n, "vk-check", &root));
193 let held = res!(db.contains_many(&[victim.clone(), dat!("small 1"), dat!("never")], None));
194 let damaged = verify::verify_key(db.api(), &victim, None);
195 let whole = verify::verify_key(db.api(), &dat!("small 1"), None);
196 let never = verify::verify_key(db.api(), &dat!("never"), None);
197 // A record that does not read can still be deleted, which is what a caller rids itself of
198 // it with: the delete must not first need to read it.
199 let deleted = db.delete(&victim, Uid::default(), None);
200 let gone = res!(db.contains_many(&[victim.clone()], None));
201 let after = verify::verify_key(db.api(), &victim, None);
202 res!(db.shutdown());
203 let _ = fs::remove_dir_all(dir);
204 assert_eq!(held, vec![true, true, false], "presence of the victim, a whole key and no key");
205 assert!(deleted.is_ok(), "the unreadable record could not be deleted: {:?}", deleted);
206 assert_eq!(gone, vec![false], "the deleted record still reads present");
207 assert_eq!(after, KeyCheck::Absent);
208 assert!(matches!(damaged, KeyCheck::Damaged { chunked: false, .. }),
209 "the flipped record read as {:?}", damaged);
210 assert_eq!(whole, KeyCheck::Whole { chunked: false });
211 assert_eq!(never, KeyCheck::Absent);
212 Ok(())
213}
214
215// A read the store did not answer says nothing about the bytes, so it must never classify as
216// damage, however it is wrapped; the bytes failing must, however deep (CHUNK-VERIFY D1).
217#[test]
218fn a_timeout_is_unavailable_and_only_bad_bytes_are_damaged() -> Outcome<()> {
219 let timeout: Error<ErrTag> = err!(
220 "Failed to receive a message via responder within {:.2} [s].",
221 constant::USER_REQUEST_TIMEOUT.as_secs_f32();
222 Missing, Data);
223 let wrapped: Error<ErrTag> = err!(timeout.clone(), "While reading a head."; Read);
224 let channel: Error<ErrTag> = err!("Could not read from responder channel"; Channel, Read);
225 let busy: Error<ErrTag> = err!("Not known."; Missing);
226 for e in [&timeout, &wrapped, &channel, &busy] {
227 assert!(matches!(KeyCheck::of_error(false, e), KeyCheck::Unavailable { chunked: false, .. }),
228 "{} classified as damage", e);
229 }
230 let checksum: Error<ErrTag> = err!("Mismatch detected"; Input, Mismatch);
231 let reader: Error<ErrTag> = err!(checksum.clone(), "While reading Data file 1."; Data, Mismatch);
232 let deep: Error<ErrTag> = err!(reader.clone(), "Fetching chunks."; Read);
233 let decrypt: Error<ErrTag> = err!("While decrypting."; Decrypt);
234 let decode: Error<ErrTag> = err!("Could not form a Dat."; Decode, Bytes);
235 for e in [&checksum, &reader, &deep, &decrypt, &decode] {
236 assert!(matches!(KeyCheck::of_error(true, e), KeyCheck::Damaged { chunked: true, .. }),
237 "{} not classified as damage", e);
238 }
239 Ok(())
240}