Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/fault_stop.rs

11.7 KiB, 1 run

created by r1870400018:70854, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! The stop after a failed barrier (2026-10-10, QA F1, F2 and F3). The fault record names each
2//! file in doubt by zone and number, so a store moved or restored elsewhere is cut where it now
3//! stands and a copy left at the old path is not touched; a file in doubt that is missing refuses
4//! the open; the file a writer rolled over to while the failing barrier was held is cut too; and
5//! the stop is written down, as the record and the refusal marker, before a process may exit on
6//! it. The hooks are process-wide, so the tests run one at a time.
7
8mod gc_pair;
9
10use gc_pair::*;
11
12use oxedyne_fe2o3_core::prelude::*;
13use oxedyne_fe2o3_iop_db::api::Database;
14use oxedyne_fe2o3_jdat::prelude::*;
15use oxedyne_fe2o3_o3db_sync::{
16 base::{
17 cfg::OzoneConfig,
18 constant,
19 },
20 comm::{
21 channels::FaultReport,
22 response::Wait,
23 },
24 file::{
25 core::FileType,
26 refusal::Refusal,
27 zdir::ZoneDir,
28 },
29 test::{
30 hooks,
31 setup::{
32 self,
33 Uid,
34 },
35 },
36};
37
38use std::{
39 collections::BTreeMap,
40 fs,
41 io::Write,
42 path::{
43 Path,
44 PathBuf,
45 },
46 sync::{
47 Arc,
48 Mutex,
49 },
50 time::Duration,
51};
52
53static SERIAL: Mutex<()> = Mutex::new(());
54
55fn lock() -> std::sync::MutexGuard<'static, ()> {
56 let g = match SERIAL.lock() { Ok(g) => g, Err(p) => p.into_inner() };
57 hooks::set_barrier_failure(false);
58 hooks::set_barrier_failure_for(None);
59 hooks::set_fault_record_fails(false);
60 hooks::set_barrier_delay(Duration::ZERO);
61 g
62}
63
64// One zone of one writer, syncing on every write unless `never`.
65fn cfg(never: bool, max: u64) -> Outcome<OzoneConfig> {
66 let mut c = res!(setup::default_cfg());
67 c.num_zones = 1;
68 c.num_wbots_per_zone = 1;
69 c.zone_overrides = BTreeMap::new();
70 c.sync_on_write = !never;
71 c.sync_every_n_writes = 0;
72 c.sync_interval_ms = 0;
73 c.data_file_max_bytes = max;
74 Ok(c)
75}
76
77fn fresh(dir: &str) -> Outcome<PathBuf> {
78 let _ = fs::remove_dir_all(dir);
79 res!(fs::create_dir_all(dir));
80 Ok(res!(Path::new(dir).canonicalize()))
81}
82
83fn open(root: &Path, c: &OzoneConfig, wipe: bool) -> Outcome<TestDb> {
84 log_set_level!("error");
85 setup::start_db(root.to_path_buf(), Some(c.clone()), schemes(), None, false, wipe)
86}
87
88fn wait() -> Wait {
89 Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL }
90}
91
92fn val(seed: u8) -> Dat {
93 Dat::BU32((0..100).map(|j| seed ^ (j as u8)).collect())
94}
95
96fn put(db: &TestDb, k: &str, seed: u8) -> Outcome<()> {
97 db.insert(dat!(k), val(seed), Uid::default(), None).map(|_| ())
98}
99
100fn reads(db: &TestDb, k: &str) -> Outcome<Option<Dat>> {
101 Ok(res!(db.get(&dat!(k), None)).map(|(v, _)| v))
102}
103
104fn len(p: &Path) -> u64 {
105 fs::metadata(p).map(|m| m.len()).unwrap_or(0)
106}
107
108fn copy_tree(from: &Path, to: &Path) -> Outcome<()> {
109 res!(fs::create_dir_all(to));
110 for e in res!(fs::read_dir(from)) {
111 let p = res!(e).path();
112 let name = res!(p.file_name().ok_or_else(|| err!("{:?} has no name.", p; Test)));
113 match p.is_dir() {
114 true => res!(copy_tree(&p, &to.join(name))),
115 false => { res!(fs::copy(&p, to.join(name))); },
116 }
117 }
118 Ok(())
119}
120
121// A store at `dir` holding a durable "kept", stopped by a failed barrier over "lost".
122fn stopped(dir: &str) -> Outcome<(PathBuf, OzoneConfig, FaultReport)> {
123 let root = res!(fresh(dir));
124 let c = res!(cfg(false, 1_000_000));
125 let db = res!(open(&root, &c, true));
126 res!(put(&db, "kept", 1));
127 hooks::set_barrier_failure(true);
128 let lost = put(&db, "lost", 2);
129 hooks::set_barrier_failure(false);
130 assert!(lost.is_err(), "a write whose barrier failed was confirmed");
131 let report = res!(db.api().fault().ok_or_else(|| err!("The failed barrier raised no fault."; Test)));
132 let _ = db.close();
133 assert!(!report.cuts.is_empty(), "the fault names no file in doubt");
134 Ok((root, c, report))
135}
136
137/// The stop is written down as the record and the marker, so `recorded()` lets a gateway exit,
138/// and the store refuses to open until the marker is cleared.
139#[test]
140fn a_stop_is_recorded_and_refuses_the_open() -> Outcome<()> {
141 let _g = lock();
142 let (root, c, report) = res!(stopped("./test_db_fault_stop_recorded"));
143 assert!(report.recorded(), "a stop written down says it is not");
144 assert_eq!(report.record, Some(root.join(constant::FAULT_FILENAME)));
145 assert_eq!(report.refusal, Some(Refusal::path(&root)));
146 assert!(Refusal::path(&root).exists(), "the stop left no refusal marker");
147 match open(&root, &c, false) {
148 Ok(db) => {
149 let _ = db.close();
150 return Err(err!("The store opened over its refusal marker."; Test));
151 },
152 Err(e) => assert!(fmt!("{}", e).contains(constant::REFUSAL_FILENAME),
153 "the refusal does not name the marker: {}", e),
154 }
155 assert!(res!(Refusal::clear(&root)));
156 let db = res!(open(&root, &c, false));
157 assert_eq!(res!(reads(&db, "kept")), Some(val(1)), "the durable value was lost");
158 assert_eq!(res!(reads(&db, "lost")), None, "the unconfirmed value was replayed");
159 res!(db.close());
160 let _ = fs::remove_dir_all(&root);
161 Ok(())
162}
163
164/// A stop that could not be written down says so, and a gateway must then stay up.
165#[test]
166fn a_stop_not_written_down_is_not_recorded() -> Outcome<()> {
167 let _g = lock();
168 hooks::set_fault_record_fails(true);
169 let stopped = stopped("./test_db_fault_stop_unrecorded");
170 hooks::set_fault_record_fails(false);
171 let (root, _, report) = res!(stopped);
172 assert!(!report.recorded(), "a stop written nowhere says it is recorded");
173 assert!(report.record.is_none() && report.refusal.is_none());
174 assert!(!root.join(constant::FAULT_FILENAME).exists() && !Refusal::path(&root).exists());
175 let _ = fs::remove_dir_all(&root);
176 Ok(())
177}
178
179/// A stopped store moved elsewhere is cut where it now stands, and the copy left at the old path
180/// is not touched (F2). Each file in doubt is given a tail past its mark, as a crash before the
181/// in-process cut would leave it.
182#[test]
183fn a_moved_store_is_cut_where_it_stands() -> Outcome<()> {
184 let _g = lock();
185 let (old, c, report) = res!(stopped("./test_db_fault_stop_old"));
186 let new = res!(fresh("./test_db_fault_stop_new"));
187 res!(copy_tree(&old, &new));
188 let mut tails = Vec::new();
189 for cut in &report.cuts {
190 for (p, upto) in [(&cut.data, cut.data_upto), (&cut.index, cut.index_upto)] {
191 let rel = res!(p.strip_prefix(&old).map_err(|e| err!(e, "{:?} is not under {:?}.", p, old; Test)));
192 for root in [&old, &new] {
193 let mut f = res!(fs::OpenOptions::new().append(true).open(root.join(rel)));
194 res!(f.write_all(&[0xa5u8; 64]));
195 }
196 tails.push((rel.to_path_buf(), upto, len(&old.join(rel))));
197 }
198 }
199 assert!(res!(Refusal::clear(&new)));
200 let db = res!(open(&new, &c, false));
201 assert_eq!(res!(reads(&db, "kept")), Some(val(1)), "the durable value was lost in the move");
202 assert_eq!(res!(reads(&db, "lost")), None, "the moved store replayed the unconfirmed value");
203 res!(db.close());
204 for (rel, upto, old_len) in &tails {
205 assert!(len(&new.join(rel)) <= *upto, "{:?} of the moved store was not cut to {}", rel, upto);
206 assert_eq!(len(&old.join(rel)), *old_len, "{:?} at the old path was touched", rel);
207 }
208 assert!(old.join(constant::FAULT_FILENAME).exists(), "the old path's record was used up");
209 assert!(!new.join(constant::FAULT_FILENAME).exists(), "the moved store kept its record");
210 let _ = fs::remove_dir_all(&old);
211 let _ = fs::remove_dir_all(&new);
212 Ok(())
213}
214
215/// A file in doubt that is missing refuses the open, and the record is kept for the next try.
216#[test]
217fn a_missing_file_in_doubt_refuses_the_open() -> Outcome<()> {
218 let _g = lock();
219 let (root, c, report) = res!(stopped("./test_db_fault_stop_missing"));
220 assert!(res!(Refusal::clear(&root)));
221 let gone = report.cuts[0].data.clone();
222 res!(fs::remove_file(&gone));
223 match open(&root, &c, false) {
224 Ok(db) => {
225 let _ = db.close();
226 return Err(err!("The store opened though its file in doubt {:?} is missing.", gone; Test));
227 },
228 Err(e) => assert!(fmt!("{}", e).contains("is missing"),
229 "the refusal does not say the file is missing: {}", e),
230 }
231 assert!(root.join(constant::FAULT_FILENAME).exists(), "the refused open removed the record");
232 let _ = fs::remove_dir_all(&root);
233 Ok(())
234}
235
236/// The writer rolls over while the failing barrier is held, so the file it rolls over to was
237/// not in doubt when the stop named its files. It joins the record, and is cut (F3).
238#[test]
239fn the_file_rolled_over_to_is_cut() -> Outcome<()> {
240 let _g = lock();
241 let root = res!(fresh("./test_db_fault_stop_rollover"));
242 let c = res!(cfg(true, 2_000));
243 let db = Arc::new(res!(open(&root, &c, true)));
244 for i in 0..5 {
245 res!(put(&db, &fmt!("before {}", i), i as u8 + 1));
246 }
247 res!(db.api().sync_now(wait()));
248 let zdir = c.zone_root(&root).join("zone_001");
249 let dat = |f: u32| zdir.join(ZoneDir::relative_file_path(&FileType::Data, f));
250 let first = res!((1..10_000u32).rev().find(|f| dat(*f).exists())
251 .ok_or_else(|| err!("No data file is live."; Test)));
252 res!(put(&db, "dirty", 9)); // so the next sync_now has a barrier to run
253 hooks::set_barrier_delay(Duration::from_millis(800));
254 hooks::set_barrier_failure(true);
255 let db2 = db.clone();
256 let sync = std::thread::spawn(move || db2.api().sync_now(wait()));
257 std::thread::sleep(Duration::from_millis(100));
258 // A put waits on its release, which the held barrier holds back, so the puts that must fill
259 // the live file during it run side by side rather than one after another.
260 let puts: Vec<_> = (0..40).map(|i| {
261 let db3 = db.clone();
262 std::thread::spawn(move || put(&db3, &fmt!("during {}", i), 7))
263 }).collect();
264 let synced = sync.join();
265 for p in puts {
266 let _ = p.join();
267 }
268 hooks::set_barrier_failure(false);
269 hooks::set_barrier_delay(Duration::ZERO);
270 assert!(matches!(synced, Ok(Err(_))), "the barrier armed to fail did not fail");
271 let last = res!((1..10_000u32).rev().find(|f| dat(*f).exists())
272 .ok_or_else(|| err!("No data file is live."; Test)));
273 assert!(last > first, "the writer did not roll over during the held barrier ({} to {})", first, last);
274 assert!(wait_until(Duration::from_secs(10), ||
275 db.api().fault().map(|r| r.cuts.iter().any(|cut| cut.fnum == last)).unwrap_or(false)),
276 "the file {} rolled over to is not in the fault's cuts: {:?}", last,
277 db.api().fault().map(|r| r.cuts.iter().map(|cut| cut.fnum).collect::<Vec<_>>()));
278 let report = res!(db.api().fault().ok_or_else(|| err!("No fault."; Test)));
279 match Arc::try_unwrap(db) {
280 Ok(db) => { let _ = db.close(); },
281 Err(_) => return Err(err!("The store is still shared."; Test)),
282 }
283 let record = res!(fs::read_to_string(root.join(constant::FAULT_FILENAME)));
284 assert!(record.lines().any(|l| l.starts_with(&fmt!("cut 0 {} ", last))),
285 "the record does not cut file {}: {}", last, record);
286 assert!(res!(Refusal::clear(&root)));
287 let db = res!(open(&root, &c, false));
288 for i in 0..5 {
289 assert_eq!(res!(reads(&db, &fmt!("before {}", i))), Some(val(i as u8 + 1)), "a durable value was lost");
290 }
291 res!(db.close());
292 for cut in &report.cuts {
293 assert!(len(&cut.data) <= cut.data_upto && len(&cut.index) <= cut.index_upto,
294 "file {} was not cut to its mark ({}, {}): ({}, {})", cut.fnum, cut.data_upto,
295 cut.index_upto, len(&cut.data), len(&cut.index));
296 }
297 let _ = fs::remove_dir_all(&root);
298 Ok(())
299}