Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/qa_r2a.rs

18.3 KiB, 11 runs

created by r1870400018:70858, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! QA round 2, agent A (2026-10-11): sticky refusal, `FaultReport::recorded()`, and a missing
2//! file in doubt. Adversarial probes only; the hooks are process-wide, so one at a time.
3
4mod gc_pair;
5
6use gc_pair::*;
7
8use oxedyne_fe2o3_core::prelude::*;
9use oxedyne_fe2o3_iop_db::api::Database;
10use oxedyne_fe2o3_jdat::prelude::*;
11use oxedyne_fe2o3_o3db_sync::{
12 base::{
13 cfg::OzoneConfig,
14 constant,
15 },
16 comm::channels::{
17 Cut,
18 Fault,
19 FaultCause,
20 FaultReport,
21 },
22 db::O3db,
23 file::refusal::Refusal,
24 test::{
25 hooks,
26 setup::{
27 self,
28 Uid,
29 },
30 },
31};
32
33use std::{
34 collections::BTreeMap,
35 fs,
36 io::Write,
37 path::{
38 Path,
39 PathBuf,
40 },
41 sync::Mutex,
42 time::Duration,
43};
44
45static SERIAL: Mutex<()> = Mutex::new(());
46
47fn lock() -> std::sync::MutexGuard<'static, ()> {
48 let g = match SERIAL.lock() { Ok(g) => g, Err(p) => p.into_inner() };
49 hooks::set_barrier_failure(false);
50 hooks::set_barrier_failure_for(None);
51 hooks::set_fault_record_fails(false);
52 hooks::set_fault_record_fails_at(0);
53 hooks::set_replay_sync_failure(false);
54 hooks::set_barrier_delay(Duration::ZERO);
55 g
56}
57
58fn cfg(zones: usize) -> Outcome<OzoneConfig> {
59 let mut c = res!(setup::default_cfg());
60 c.num_zones = zones as u16;
61 c.num_wbots_per_zone = 1;
62 c.zone_overrides = BTreeMap::new();
63 c.sync_on_write = true;
64 c.sync_every_n_writes = 0;
65 c.sync_interval_ms = 0;
66 c.data_file_max_bytes = 1_000_000;
67 Ok(c)
68}
69
70fn fresh(dir: &str) -> Outcome<PathBuf> {
71 let _ = fs::remove_dir_all(dir);
72 res!(fs::create_dir_all(dir));
73 Ok(res!(Path::new(dir).canonicalize()))
74}
75
76fn open(root: &Path, c: &OzoneConfig, wipe: bool) -> Outcome<TestDb> {
77 log_set_level!("error");
78 setup::start_db(root.to_path_buf(), Some(c.clone()), schemes(), None, false, wipe)
79}
80
81fn val(seed: u8) -> Dat {
82 Dat::BU32((0..100).map(|j| seed ^ (j as u8)).collect())
83}
84
85fn put(db: &TestDb, k: &str, seed: u8) -> Outcome<()> {
86 db.insert(dat!(k), val(seed), Uid::default(), None).map(|_| ())
87}
88
89fn reads(db: &TestDb, k: &str) -> Outcome<Option<Dat>> {
90 Ok(res!(db.get(&dat!(k), None)).map(|(v, _)| v))
91}
92
93fn refused(r: Outcome<TestDb>, what: &str) -> Outcome<String> {
94 match r {
95 Ok(db) => {
96 let _ = db.close();
97 Err(err!("{}: the store opened.", what; Test))
98 },
99 Err(e) => Ok(fmt!("{}", e)),
100 }
101}
102
103// A store with durable "kept 0..n" in every zone, stopped by a failed barrier over "lost".
104// Returns the store still open when `keep_open`.
105fn stopped(dir: &str, zones: usize, keep_open: bool)
106 -> Outcome<(PathBuf, OzoneConfig, FaultReport, Option<TestDb>)>
107{
108 let root = res!(fresh(dir));
109 let c = res!(cfg(zones));
110 let db = res!(open(&root, &c, true));
111 for i in 0..20u8 {
112 res!(put(&db, &fmt!("kept {}", i), i + 1));
113 }
114 hooks::set_barrier_failure(true);
115 let lost = put(&db, "lost", 99);
116 hooks::set_barrier_failure(false);
117 assert!(lost.is_err(), "a write whose barrier failed was confirmed");
118 let report = res!(db.api().fault().ok_or_else(|| err!("No fault raised."; Test)));
119 assert!(!report.cuts.is_empty(), "the fault names no file in doubt");
120 if keep_open {
121 return Ok((root, c, report, Some(db)));
122 }
123 let _ = db.close();
124 Ok((root, c, report, None))
125}
126
127fn cut(fnum: u32) -> Cut {
128 Cut {
129 zone: 0,
130 fnum,
131 data: PathBuf::from(fmt!("/nonexistent/{}.dat", fnum)),
132 data_upto: 10,
133 index: PathBuf::from(fmt!("/nonexistent/{}.ind", fnum)),
134 index_upto: 10,
135 }
136}
137
138/// P1 (surface 2). A second raise whose record rewrite fails keeps `record` pointing at the old
139/// record, which lacks the new cut, so `recorded()` says yes while the new cut is durable nowhere.
140#[test]
141fn p1_stale_record_counts_as_recorded() -> Outcome<()> {
142 let _g = lock();
143 let root = res!(fresh("./test_db_qa_r2a_p1"));
144 // The marker's temporary path is a directory, so the marker cannot be written; the record can.
145 res!(fs::create_dir_all(root.join(fmt!("{}.tmp", constant::REFUSAL_FILENAME))));
146 let f = Fault::default();
147 let e = err!("injected barrier failure"; Test);
148 let r1 = f.raise(&root, FaultCause::Barrier, vec![cut(1)], &e);
149 println!("P1 first raise: record {:?} refusal {:?}", r1.record, r1.refusal);
150 assert!(r1.record.is_some() && r1.refusal.is_none(), "setup: record only");
151 hooks::set_fault_record_fails(true);
152 let r2 = f.raise(&root, FaultCause::Barrier, vec![cut(2)], &e);
153 hooks::set_fault_record_fails(false);
154 let text = res!(fs::read_to_string(root.join(constant::FAULT_FILENAME)));
155 let marker = Refusal::path(&root).exists();
156 println!("P1 second raise: record {:?} refusal {:?} recorded() {}", r2.record, r2.refusal, r2.recorded());
157 println!("P1 record on disk:\n{}P1 marker exists: {}", text, marker);
158 let cut2_durable = text.lines().any(|l| l.starts_with("cut 0 2 ")) || marker;
159 let _ = fs::remove_dir_all(&root);
160 assert!(!r2.recorded() || cut2_durable,
161 "FINDING: recorded() is true but cut 2 is in neither the record nor a marker");
162 Ok(())
163}
164
165/// P2 (surfaces 1, 2). The marker stands but the record does not (its write failed, or it was
166/// lost). Clearing then opening replays the unconfirmed tail: the marker names no file.
167#[test]
168fn p2_marker_without_record_replays_after_clear() -> Outcome<()> {
169 let _g = lock();
170 let (root, c, report, _) = res!(stopped("./test_db_qa_r2a_p2", 1, false));
171 let marker = res!(fs::read_to_string(Refusal::path(&root)));
172 println!("P2 marker text: {:?}", marker);
173 // Put back what the in-process cut removed, as a crash before that cut would leave it.
174 let zdir = c.zone_root(&root).join("zone_001");
175 let qdir = zdir.join(constant::QUARANTINE_DIR);
176 let mut restored = 0;
177 if let Ok(rd) = fs::read_dir(&qdir) {
178 for ent in rd {
179 let q = res!(ent).path();
180 let name = q.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
181 let base = match name.rsplit_once('.') { Some((b, _)) => b.to_string(), None => continue };
182 let tail = res!(fs::read(&q));
183 let mut f = res!(fs::OpenOptions::new().append(true).open(zdir.join(&base)));
184 res!(f.write_all(&tail));
185 restored += tail.len();
186 }
187 }
188 println!("P2 tail bytes restored from quarantine: {}; cuts {:?}", restored,
189 report.cuts.iter().map(|c| (c.fnum, c.data_upto, c.index_upto)).collect::<Vec<_>>());
190 res!(fs::remove_file(root.join(constant::FAULT_FILENAME)));
191 assert!(res!(Refusal::clear(&root)));
192 let db = res!(open(&root, &c, false));
193 let lost = res!(reads(&db, "lost"));
194 let kept = res!(reads(&db, "kept 0"));
195 res!(db.close());
196 let _ = fs::remove_dir_all(&root);
197 println!("P2 after clear with no record: lost = {:?}", lost.is_some());
198 assert_eq!(kept, Some(val(1)));
199 assert!(lost.is_none() || !marker.contains("cut"),
200 "unreachable: the marker carried cuts");
201 assert!(lost.is_none(), "FINDING: the unconfirmed value was served after clear, and the marker named no file to check: {:?}", marker);
202 Ok(())
203}
204
205/// P3 (surface 1). `Refusal::clear` while the faulted store is still open in a live process.
206#[test]
207fn p3_clear_while_open_elsewhere() -> Outcome<()> {
208 let _g = lock();
209 let (root, c, _, db) = res!(stopped("./test_db_qa_r2a_p3", 1, true));
210 let db = res!(db.ok_or_else(|| err!("no db"; Test)));
211 assert!(Refusal::path(&root).exists());
212 let cleared = Refusal::clear(&root);
213 let second = refused(open(&root, &c, false), "second open while held");
214 println!("P3 clear while held: {:?}; a second open: {:?}", cleared.as_ref().map(|b| *b).map_err(|e| fmt!("{}", e)),
215 second.as_ref().map(|s| s.chars().take(160).collect::<String>()).map_err(|e| fmt!("{}", e)));
216 let _ = db.close();
217 let _ = fs::remove_dir_all(&root);
218 assert!(matches!(second, Ok(ref s) if s.contains("held by another open")), "setup: the lock was not held");
219 assert!(!matches!(cleared, Ok(true)),
220 "FINDING: the refusal was cleared while the faulted store was open (lock held)");
221 Ok(())
222}
223
224/// P4 (surface 3). The index file in doubt is missing: the open refuses, names the index file,
225/// keeps the record, and refuses again on a second try.
226#[test]
227fn p4_missing_index_in_doubt() -> Outcome<()> {
228 let _g = lock();
229 let (root, c, report, _) = res!(stopped("./test_db_qa_r2a_p4", 1, false));
230 assert!(res!(Refusal::clear(&root)));
231 let gone = report.cuts[0].index.clone();
232 res!(fs::remove_file(&gone));
233 let name = gone.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
234 let e1 = res!(refused(open(&root, &c, false), "missing index"));
235 let e2 = res!(refused(open(&root, &c, false), "missing index, second try"));
236 let rec = root.join(constant::FAULT_FILENAME).exists();
237 let mk = Refusal::path(&root).exists();
238 println!("P4 error: {}", e1.chars().take(400).collect::<String>());
239 println!("P4 record kept {} marker {}", rec, mk);
240 let _ = fs::remove_dir_all(&root);
241 assert!(e1.contains("is missing") && e1.contains(&name), "the refusal does not name {}: {}", name, e1);
242 assert!(e2.contains("is missing"));
243 assert!(rec, "the record was removed");
244 Ok(())
245}
246
247/// P5 (surface 3). The data file in doubt is missing: the message names that file.
248#[test]
249fn p5_missing_data_names_the_file() -> Outcome<()> {
250 let _g = lock();
251 let (root, c, report, _) = res!(stopped("./test_db_qa_r2a_p5", 1, false));
252 assert!(res!(Refusal::clear(&root)));
253 let gone = report.cuts[0].data.clone();
254 res!(fs::remove_file(&gone));
255 let name = gone.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
256 let e = res!(refused(open(&root, &c, false), "missing data"));
257 let _ = fs::remove_dir_all(&root);
258 assert!(e.contains("is missing") && e.contains(&name), "the refusal does not name {}: {}", name, e);
259 Ok(())
260}
261
262/// P6 (surface 1). A failed replay flush whose marker cannot be written either: the error says
263/// so; the next open is then not refused (expected; documented in the error).
264#[test]
265fn p6_flush_fails_and_marker_fails() -> Outcome<()> {
266 let _g = lock();
267 let root = res!(fresh("./test_db_qa_r2a_p6"));
268 let c = res!(cfg(1));
269 let db = res!(open(&root, &c, true));
270 res!(put(&db, "kept 0", 1));
271 res!(db.close());
272 hooks::set_replay_sync_failure(true);
273 hooks::set_fault_record_fails(true);
274 let r = refused(open(&root, &c, false), "flush failed");
275 hooks::set_replay_sync_failure(false);
276 hooks::set_fault_record_fails(false);
277 let e = res!(r);
278 let mk = Refusal::path(&root).exists();
279 let next = open(&root, &c, false);
280 let next_opened = next.is_ok();
281 if let Ok(db) = next { let _ = db.close(); }
282 println!("P6 error has CRITICAL: {}; marker {}; next open opened: {}", e.contains("CRITICAL"), mk, next_opened);
283 let _ = fs::remove_dir_all(&root);
284 assert!(e.contains("CRITICAL"), "the double failure is not said: {}", e);
285 Ok(())
286}
287
288/// P7 (surface 3). A data file missing from a zone the record does not name.
289#[test]
290fn p7_missing_file_in_unnamed_zone() -> Outcome<()> {
291 let _g = lock();
292 let (root, c, report, _) = res!(stopped("./test_db_qa_r2a_p7", 2, false));
293 let named: Vec<usize> = report.cuts.iter().map(|c| c.zone).collect();
294 let other = if named.contains(&0) { 1 } else { 0 };
295 let zdir = c.zone_root(&root).join(fmt!("zone_{:03}", other + 1));
296 let mut victim = None;
297 for ent in res!(fs::read_dir(&zdir)) {
298 let p = res!(ent).path();
299 if p.is_file() && p.extension().map(|e| e == "dat" || e == "data").unwrap_or(false) {
300 victim = Some(p);
301 }
302 }
303 let listing: Vec<String> = res!(fs::read_dir(&zdir)).filter_map(|e| e.ok())
304 .map(|e| e.file_name().to_string_lossy().to_string()).collect();
305 println!("P7 cuts in zones {:?}; other zone {} holds {:?}", named, other + 1, listing);
306 let victim = res!(victim.ok_or_else(|| err!("no data file in {:?}", zdir; Test)));
307 res!(fs::remove_file(&victim));
308 assert!(res!(Refusal::clear(&root)));
309 let r = open(&root, &c, false);
310 let name = victim.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
311 match r {
312 Ok(db) => {
313 let mut missing = Vec::new();
314 for i in 0..20u8 {
315 match reads(&db, &fmt!("kept {}", i)) {
316 Ok(Some(v)) if v == val(i + 1) => (),
317 other => missing.push((i, fmt!("{:?}", other.map(|o| o.is_some())))),
318 }
319 }
320 let _ = db.close();
321 let _ = fs::remove_dir_all(&root);
322 Err(err!("FINDING: the store opened with {:?} removed; keys not read back: {:?}",
323 victim, missing; Test))
324 },
325 Err(e) => {
326 let e = fmt!("{}", e);
327 println!("P7 refused: {}", e.chars().take(300).collect::<String>());
328 let _ = fs::remove_dir_all(&root);
329 assert!(e.contains("is missing") && e.contains(&name),
330 "the refusal does not name {}: {}", name, e);
331 Ok(())
332 },
333 }
334}
335
336/// P8 (surface 3). A record naming a zone the store does not have refuses, and is kept.
337#[test]
338fn p8_record_names_absent_zone() -> Outcome<()> {
339 let _g = lock();
340 let (root, c, _, _) = res!(stopped("./test_db_qa_r2a_p8", 1, false));
341 assert!(res!(Refusal::clear(&root)));
342 let rp = root.join(constant::FAULT_FILENAME);
343 let mut text = res!(fs::read_to_string(&rp));
344 text.push_str("cut 4 1 0 0\n");
345 res!(fs::write(&rp, text));
346 let e = res!(refused(open(&root, &c, false), "absent zone"));
347 let kept = rp.exists();
348 let _ = fs::remove_dir_all(&root);
349 println!("P8 error: {}", e.chars().take(300).collect::<String>());
350 assert!(e.contains("zone 5") && kept);
351 Ok(())
352}
353
354// Does `text` hold a cut line for every cut in `cuts`?
355fn holds(text: &str, cuts: &[Cut]) -> bool {
356 cuts.iter().all(|c| text.lines().any(|l|
357 l == fmt!("cut {} {} {} {}", c.zone, c.fnum, c.data_upto, c.index_upto)))
358}
359
360/// B's F-B1. `record` is set only while the record on disk holds every cut in the report, and
361/// `refusal` only while the marker does, so `recorded()` never vouches for a cut written nowhere.
362#[test]
363fn b1_recorded_matches_the_record_on_disk() -> Outcome<()> {
364 let _g = lock();
365 let root = res!(fresh("./test_db_qa_r2a_b1"));
366 let f = Fault::default();
367 let e = err!("injected barrier failure"; Test);
368 let r1 = f.raise(&root, FaultCause::Barrier, vec![cut(1)], &e);
369 assert!(r1.record.is_some() && r1.refusal.is_some(), "setup: both written");
370 hooks::set_fault_record_fails_at(1);
371 let r2 = f.raise(&root, FaultCause::Barrier, vec![cut(2)], &e);
372 hooks::set_fault_record_fails_at(0);
373 let rec = res!(fs::read_to_string(root.join(constant::FAULT_FILENAME)));
374 let mk = fs::read_to_string(Refusal::path(&root)).unwrap_or_default();
375 let _ = fs::remove_dir_all(&root);
376 assert_eq!(r2.record.is_some(), holds(&rec, &r2.cuts),
377 "record {:?} but the record on disk is {:?}", r2.record, rec);
378 assert!(r2.refusal.is_none() || holds(&mk, &r2.cuts),
379 "refusal {:?} but the marker on disk is {:?}", r2.refusal, mk);
380 assert!(r2.recorded(), "the marker holds every cut, yet recorded() is false");
381 Ok(())
382}
383
384/// B's I-B2. A record left beside a store restored whole says what it is, where it is, and when
385/// removing it is right.
386#[test]
387fn b2_a_left_over_record_names_itself() -> Outcome<()> {
388 let _g = lock();
389 let (root, c, report, _) = res!(stopped("./test_db_qa_r2a_b2", 1, false));
390 assert!(res!(Refusal::clear(&root)));
391 res!(fs::remove_file(&report.cuts[0].data));
392 let e = res!(refused(open(&root, &c, false), "left-over record"));
393 let rp = root.join(constant::FAULT_FILENAME);
394 let _ = fs::remove_dir_all(&root);
395 assert!(e.contains(&fmt!("{:?}", rp)), "the error does not name {:?}: {}", rp, e);
396 assert!(e.contains("left over from a stop"), "{}", e);
397 assert!(e.contains("only if the store was restored whole from a copy taken before the stop"), "{}", e);
398 Ok(())
399}
400
401/// F2. With the record gone, the marker's cuts still cut the files in doubt.
402#[test]
403fn f2_cut_after_reads_the_marker() -> Outcome<()> {
404 let _g = lock();
405 let (root, c, report, _) = res!(stopped("./test_db_qa_r2a_f2", 1, false));
406 let cut0 = report.cuts[0].clone();
407 {
408 let mut f = res!(fs::OpenOptions::new().append(true).open(&cut0.data));
409 res!(f.write_all(&[7u8; 64]));
410 }
411 res!(fs::remove_file(root.join(constant::FAULT_FILENAME)));
412 let zdirs = vec![c.zone_root(&root).join("zone_001")];
413 res!(Fault::cut_after(&root, &zdirs));
414 let n = res!(fs::metadata(&cut0.data)).len();
415 let marker = Refusal::path(&root).exists();
416 let _ = fs::remove_dir_all(&root);
417 assert_eq!(n, cut0.data_upto, "the marker's cut was not applied");
418 assert!(marker, "cut_after removed the operator's marker");
419 Ok(())
420}
421
422/// F5. The marker and the record are plain text, without terminal colour escapes.
423#[test]
424fn f5_the_marker_and_record_are_plain() -> Outcome<()> {
425 let _g = lock();
426 let (root, _, _, _) = res!(stopped("./test_db_qa_r2a_f5", 1, false));
427 let mk = res!(fs::read_to_string(Refusal::path(&root)));
428 let rec = res!(fs::read_to_string(root.join(constant::FAULT_FILENAME)));
429 let _ = fs::remove_dir_all(&root);
430 assert!(!mk.contains('\x1b') && !rec.contains('\x1b'), "escapes in {:?} or {:?}", mk, rec);
431 assert!(mk.lines().any(|l| l.starts_with("why ") && l.len() > 10), "the marker does not say why: {:?}", mk);
432 Ok(())
433}
434
435/// F3. A start refused by the marker lets go of the store's lock, so the operator can clear it
436/// while the refused process lives on.
437#[test]
438fn f3_a_refused_start_lets_go_of_the_lock() -> Outcome<()> {
439 let _g = lock();
440 let (root, c, _, _) = res!(stopped("./test_db_qa_r2a_f3", 1, false));
441 let mut db = res!(O3db::new(root.clone(), Some(c.clone()), schemes(), Uid::default()));
442 let started = db.start("qa_r2a");
443 let cleared = Refusal::clear(&root);
444 drop(db);
445 let _ = fs::remove_dir_all(&root);
446 assert!(started.is_err(), "the start was not refused");
447 assert!(matches!(cleared, Ok(true)), "the clear was refused: {:?}", cleared.map_err(|e| fmt!("{}", e)));
448 Ok(())
449}