Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/qa_appendseek.rs

27.9 KiB, 1 run

created by r1870400018:71730, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! The APPENDSEEK QA probes (rc-b, 2026-10-11, written against ac09ec89), adopted with the
2//! fix-forward of findings F1 and F3: a stop after a failed cut back is named as such, never as a
3//! failed sync, and a tear hook that an append outran is not counted as a tear. Each test gathers
4//! every deviation into `bad` and prints its observations, so one run shows the whole picture.
5//! The hooks are process-wide, so the tests run one at a time.
6
7mod gc_pair;
8
9use gc_pair::*;
10
11use oxedyne_fe2o3_core::prelude::*;
12use oxedyne_fe2o3_iop_db::api::Database;
13use oxedyne_fe2o3_jdat::prelude::*;
14use oxedyne_fe2o3_o3db_sync::{
15 base::{
16 cfg::OzoneConfig,
17 constant,
18 },
19 comm::response::Wait,
20 file::{
21 core::FileType,
22 refusal::Refusal,
23 },
24 sweep,
25 test::{
26 hooks,
27 setup::{
28 self,
29 Uid,
30 },
31 },
32};
33
34use std::{
35 collections::BTreeMap,
36 fs,
37 path::{
38 Path,
39 PathBuf,
40 },
41 sync::{
42 Arc,
43 Mutex,
44 atomic::{
45 AtomicBool,
46 Ordering,
47 },
48 },
49 time::Duration,
50};
51
52static SERIAL: Mutex<()> = Mutex::new(());
53
54fn lock() -> std::sync::MutexGuard<'static, ()> {
55 let g = match SERIAL.lock() { Ok(g) => g, Err(p) => p.into_inner() };
56 disarm();
57 g
58}
59
60fn disarm() {
61 hooks::set_write_tear(FileType::Data, None);
62 hooks::set_write_tear(FileType::Index, None);
63 hooks::set_cut_back_failure(false);
64 hooks::set_barrier_failure(false);
65 hooks::set_fault_record_fails_at(0);
66 hooks::set_owed_drop(false);
67}
68
69// One zone, one writer. `held` holds the sync back for an hour, else every write syncs.
70fn cfg(held: bool, chunked: bool, max: u64) -> Outcome<OzoneConfig> {
71 let mut c = res!(setup::default_cfg());
72 c.num_zones = 1;
73 c.num_wbots_per_zone = 1;
74 c.num_cbots_per_zone = 4;
75 c.num_fbots_per_zone = 1;
76 c.zone_overrides = BTreeMap::new();
77 c.sync_on_write = !held;
78 c.sync_every_n_writes = 0;
79 c.sync_interval_ms = if held { 3_600_000 } else { 0 };
80 c.data_file_max_bytes = max;
81 if chunked {
82 c.rest_chunk_threshold = 1_000;
83 c.rest_chunk_bytes = 400;
84 }
85 Ok(c)
86}
87
88fn fresh(dir: &str) -> Outcome<PathBuf> {
89 let _ = fs::remove_dir_all(dir);
90 res!(fs::create_dir_all(dir));
91 Ok(res!(Path::new(dir).canonicalize()))
92}
93
94fn open(root: &Path, c: &OzoneConfig, wipe: bool) -> Outcome<TestDb> {
95 log_set_level!("error");
96 setup::start_db(root.to_path_buf(), Some(c.clone()), schemes(), None, false, wipe)
97}
98
99fn wait() -> Wait {
100 Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL }
101}
102
103fn val(len: usize, seed: u8) -> Dat {
104 Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect())
105}
106
107fn put(db: &TestDb, k: &str, len: usize, seed: u8) -> Outcome<()> {
108 db.insert(dat!(k), val(len, seed), Uid::default(), None).map(|_| ())
109}
110
111// "L/S" for a whole value, "none", "TORN" or "ERR:...".
112fn read(db: &TestDb, k: &str) -> String {
113 match db.get(&dat!(k), None) {
114 Ok(None) => fmt!("none"),
115 Ok(Some((Dat::BU32(b), _))) if !b.is_empty()
116 && b.iter().enumerate().all(|(j, x)| *x == b[0] ^ (j as u8)) => fmt!("{}/{}", b.len(), b[0]),
117 Ok(Some(_)) => fmt!("TORN"),
118 Err(e) => fmt!("ERR:{}", plain(&fmt!("{}", e))),
119 }
120}
121
122// Text without colour codes.
123fn bare(s: &str) -> String {
124 let mut out = String::new();
125 let mut esc = false;
126 for ch in s.chars() {
127 if ch == '\u{1b}' { esc = true; continue; }
128 if esc { if ch == 'm' { esc = false; } continue; }
129 out.push(ch);
130 }
131 out
132}
133
134// The innermost message of an error, without colour codes.
135fn plain(s: &str) -> String {
136 let out = bare(s);
137 match out.rfind("LocalErr") {
138 Some(i) => out[i..].chars().take(400).collect(),
139 None => out.chars().take(400).collect(),
140 }
141}
142
143fn live(root: &Path, c: &OzoneConfig) -> Outcome<(u32, PathBuf, PathBuf)> {
144 let n = res!((1..10_000u32).rev().find(|n| file(root, c, FileType::Data, *n).exists())
145 .ok_or_else(|| err!("No data file is live."; Test)));
146 Ok((n, file(root, c, FileType::Data, n), file(root, c, FileType::Index, n)))
147}
148
149fn marker(root: &Path) -> String {
150 fs::read_to_string(Refusal::path(root)).unwrap_or_else(|_| fmt!("<no marker>"))
151}
152
153fn record(root: &Path) -> String {
154 fs::read_to_string(root.join(constant::FAULT_FILENAME)).unwrap_or_else(|_| fmt!("<no record>"))
155}
156
157fn check(bad: &mut Vec<String>, db: &TestDb, want: &[(&str, &str)], when: &str) {
158 for (k, v) in want {
159 let got = read(db, k);
160 if got != *v {
161 bad.push(fmt!("{}: {} reads {} not {}", when, k, got, v));
162 }
163 }
164}
165
166fn finish(name: &str, bad: Vec<String>) -> Outcome<()> {
167 eprintln!("QA[{}] {} deviation(s)", name, bad.len());
168 for b in &bad {
169 eprintln!("QA[{}] BAD {}", name, b);
170 }
171 if !bad.is_empty() {
172 return Err(err!("{}: {:?}", name, bad; Test));
173 }
174 Ok(())
175}
176
177// F1: a refusal on a store stopped by a failed cut back names the cut back, not a sync.
178fn names_cut_back(bad: &mut Vec<String>, what: &str, r: &Outcome<()>) {
179 match r {
180 Ok(()) => bad.push(fmt!("{} succeeded on a stopped store", what)),
181 Err(e) => {
182 let t = bare(&fmt!("{}", e));
183 if !t.contains("stopped writing after a failed cut back of a torn append") {
184 bad.push(fmt!("{} does not name the failed cut back: {}", what, plain(&t)));
185 }
186 if t.contains("failed sync") {
187 bad.push(fmt!("{} names a failed sync: {}", what, plain(&t)));
188 }
189 },
190 }
191}
192
193// F1: the marker and the record left by a failed cut back carry its cause.
194fn marks_cut_back(bad: &mut Vec<String>, root: &Path) {
195 let mk = marker(root);
196 if !mk.lines().any(|l| l.starts_with("why a torn append could not be cut back at run time: ")) {
197 bad.push(fmt!("the marker's why line does not name the cut back:\n{}", mk));
198 }
199 if !mk.lines().any(|l| l == "cause cut-back") {
200 bad.push(fmt!("the marker names no cut-back cause:\n{}", mk));
201 }
202 let rc = record(root);
203 if !rc.lines().any(|l| l == "cause cut-back") {
204 bad.push(fmt!("the record names no cut-back cause:\n{}", rc));
205 }
206}
207
208// Surface 1: sync held back, k1 k2 acknowledged past the mark, k3 torn on `typ`, cut fails.
209fn torn_under_held_sync(name: &str, typ: FileType, crash_to: Option<i64>) -> Outcome<()> {
210 let mut bad = Vec::new();
211 let root = res!(fresh(&fmt!("./test_db_qa_seek_{}", name)));
212 let c = res!(cfg(true, false, 1_000_000));
213 let db = res!(open(&root, &c, true));
214 res!(put(&db, "k0", 100, 1));
215 res!(db.api().sync_now(wait()));
216 let (_, dat, ind) = res!(live(&root, &c));
217 let mark = res!(db.api().chans().marks().files(0));
218 let m0 = size(&dat);
219 for (k, s) in [("k1", 2u8), ("k2", 3)] {
220 if let Err(e) = put(&db, k, 100, s) {
221 bad.push(fmt!("{} under a held sync was refused: {}", k, e));
222 }
223 }
224 let (dlen, ilen) = (size(&dat), size(&ind));
225 let rec = dlen - m0;
226 hooks::set_cut_back_failure(true);
227 match typ {
228 FileType::Data => hooks::set_write_tear(FileType::Data, Some(rec / 4)),
229 FileType::Index => hooks::set_write_tear(FileType::Index, Some(3)),
230 }
231 let failed = put(&db, "k3", 100, 4);
232 disarm();
233 if failed.is_ok() { bad.push(fmt!("the torn k3 was confirmed")); }
234 eprintln!("QA[{}] marks {:?} m0 {} dlen {} ilen {} now ({}, {})",
235 name, mark, m0, dlen, ilen, size(&dat), size(&ind));
236 let report = db.api().fault();
237 match &report {
238 None => bad.push(fmt!("no fault after a failed cut back")),
239 Some(r) => {
240 eprintln!("QA[{}] cuts {:?}", name, r.cuts);
241 if !r.cuts.iter().any(|x| x.data_upto == dlen && x.index_upto == ilen) {
242 bad.push(fmt!("fault cuts {:?} do not name ({}, {})", r.cuts, dlen, ilen));
243 }
244 },
245 }
246 // Observation only: the durable-mark design refuses, in process, reads past the mark once
247 // the store has stopped.
248 for k in ["k0", "k1", "k2", "k3"] {
249 eprintln!("QA[{}] OBS in process after the fault {} = {}", name, k, read(&db, k));
250 }
251 names_cut_back(&mut bad, "k4's put", &put(&db, "k4", 100, 5));
252 names_cut_back(&mut bad, "sync_now", &db.api().sync_now(wait()));
253 let _ = db.close();
254 eprintln!("QA[{}] after close ({}, {}) marker:\n{}record:\n{}", name, size(&dat), size(&ind),
255 marker(&root), record(&root));
256 marks_cut_back(&mut bad, &root);
257 // A power cut leaves the files somewhere between the mark and their lengths.
258 if let Some(to) = crash_to {
259 let upto = (m0 as i64 + to) as u64;
260 let f = res!(fs::OpenOptions::new().write(true).open(&dat));
261 res!(f.set_len(upto));
262 let ilen0 = mark.iter().map(|(_, m)| m.1).max().unwrap_or(0);
263 let f = res!(fs::OpenOptions::new().write(true).open(&ind));
264 res!(f.set_len(ilen0));
265 eprintln!("QA[{}] crash: data to {}, index to {}", name, upto, ilen0);
266 }
267 // The marker refuses the open until cleared.
268 match open(&root, &c, false) {
269 Ok(db) => { bad.push(fmt!("the store opened over its refusal marker")); let _ = db.close(); },
270 Err(e) => eprintln!("QA[{}] refused as wanted: {}", name, fmt!("{}", e).chars().take(120).collect::<String>()),
271 }
272 match Refusal::clear(&root) {
273 Ok(true) => (),
274 other => bad.push(fmt!("clear returned {:?}", other)),
275 }
276 let db = res!(open(&root, &c, false));
277 eprintln!("QA[{}] after open ({}, {})", name, size(&dat), size(&ind));
278 let want: Vec<(&str, &str)> = match crash_to {
279 None => vec![("k0", "100/1"), ("k1", "100/2"), ("k2", "100/3"), ("k3", "none")],
280 // k1, k2 were never synced: lost or whole, never torn; k0 was synced.
281 Some(_) => vec![("k0", "100/1"), ("k3", "none")],
282 };
283 check(&mut bad, &db, &want, "after the cleared open");
284 if crash_to.is_none() && size(&dat) != dlen {
285 bad.push(fmt!("data file {} after open, not the cut {}", size(&dat), dlen));
286 }
287 if crash_to.is_some() {
288 for k in ["k1", "k2"] {
289 let r = read(&db, k);
290 if r != "none" && !r.starts_with("100/") { bad.push(fmt!("after crash {} reads {}", k, r)); }
291 eprintln!("QA[{}] after crash {} = {}", name, k, r);
292 }
293 }
294 if let Err(e) = put(&db, "k5", 100, 6) { bad.push(fmt!("k5 after the open failed: {}", e)); }
295 if Refusal::path(&root).exists() { bad.push(fmt!("a marker stands after a clean open")); }
296 res!(db.close());
297 let db = res!(open(&root, &c, false));
298 check(&mut bad, &db, &[("k0", "100/1"), ("k3", "none"), ("k5", "100/6")], "after a second open");
299 res!(db.close());
300 let _ = fs::remove_dir_all(&root);
301 finish(name, bad)
302}
303
304#[test]
305fn s1_torn_data_failed_cut_under_held_sync() -> Outcome<()> {
306 let _g = lock();
307 torn_under_held_sync("s1d", FileType::Data, None)
308}
309
310#[test]
311fn s1_torn_index_failed_cut_under_held_sync() -> Outcome<()> {
312 let _g = lock();
313 torn_under_held_sync("s1i", FileType::Index, None)
314}
315
316// The power is cut part-way through k1 before the restart.
317#[test]
318fn s1_torn_data_failed_cut_then_power_cut_mid_record() -> Outcome<()> {
319 let _g = lock();
320 torn_under_held_sync("s1c", FileType::Data, Some(37))
321}
322
323// Surface 1 at rollover: the torn append starts a new live pair, and its cut fails.
324#[test]
325fn s1_torn_first_append_of_a_new_pair_failed_cut() -> Outcome<()> {
326 let _g = lock();
327 let mut bad = Vec::new();
328 let root = res!(fresh("./test_db_qa_seek_s1r"));
329 let c = res!(cfg(false, false, 2_000));
330 let db = res!(open(&root, &c, true));
331 res!(put(&db, "k0", 100, 1));
332 res!(put(&db, "k1", 100, 2));
333 let (n0, d0, _) = res!(live(&root, &c));
334 let rec = size(&d0) / 2 + 1;
335 let mut k = 2u8;
336 // Fill until the next put cannot fit, so that it rolls over.
337 loop {
338 let before = size(&d0);
339 if before + rec > 2_000 { break; }
340 res!(put(&db, &fmt!("k{}", k), 100, k + 1));
341 k += 1;
342 }
343 hooks::set_cut_back_failure(true);
344 hooks::set_write_tear(FileType::Data, Some(20));
345 let failed = put(&db, "kr", 100, 99);
346 disarm();
347 if failed.is_ok() { bad.push(fmt!("the torn kr was confirmed")); }
348 let (n1, d1, i1) = res!(live(&root, &c));
349 eprintln!("QA[s1r] pair {} -> {}; new data {} index {}; cuts {:?}", n0, n1,
350 size(&d1), size(&i1), db.api().fault().map(|r| r.cuts));
351 if n1 == n0 { bad.push(fmt!("positive control: no rollover happened")); }
352 let _ = db.close();
353 match Refusal::clear(&root) {
354 Ok(true) => (),
355 other => bad.push(fmt!("clear returned {:?}", other)),
356 }
357 let db = res!(open(&root, &c, false));
358 eprintln!("QA[s1r] after open new data {} index {}", size(&d1), size(&i1));
359 let mut want = vec![("kr".to_string(), "none".to_string())];
360 for j in 0..k { want.push((fmt!("k{}", j), fmt!("100/{}", j + 1))); }
361 let want: Vec<(&str, &str)> = want.iter().map(|(a, b)| (a.as_str(), b.as_str())).collect();
362 check(&mut bad, &db, &want, "after the cleared open");
363 if let Err(e) = put(&db, "kz", 100, 50) { bad.push(fmt!("kz after the open failed: {}", e)); }
364 res!(db.close());
365 let db = res!(open(&root, &c, false));
366 check(&mut bad, &db, &[("kz", "100/50"), ("kr", "none")], "after a second open");
367 res!(db.close());
368 let _ = fs::remove_dir_all(&root);
369 finish("s1r", bad)
370}
371
372// Surface 2: the writer's cut above the mark, then a barrier fails, so the syncer cuts to the
373// mark as well. The two must compose to the lower; the marker must carry both.
374#[test]
375fn s2_writer_cut_above_mark_with_syncer_cut_to_mark() -> Outcome<()> {
376 let _g = lock();
377 let mut bad = Vec::new();
378 let root = res!(fresh("./test_db_qa_seek_s2"));
379 let c = res!(cfg(true, false, 1_000_000));
380 let db = res!(open(&root, &c, true));
381 res!(put(&db, "k0", 100, 1));
382 res!(db.api().sync_now(wait()));
383 let (_, dat, ind) = res!(live(&root, &c));
384 let mark = res!(db.api().chans().marks().files(0));
385 res!(put(&db, "k1", 100, 2));
386 let (dlen, ilen) = (size(&dat), size(&ind));
387 hooks::set_cut_back_failure(true);
388 hooks::set_write_tear(FileType::Data, Some(30));
389 let _ = put(&db, "k2", 100, 3);
390 disarm();
391 // The syncer's barrier now fails. sync_now refuses on a stopped store, so the barrier comes
392 // from the close; report what each does.
393 hooks::set_barrier_failure(true);
394 let sn = db.api().sync_now(wait());
395 let cl = db.close();
396 hooks::set_barrier_failure(false);
397 eprintln!("QA[s2] mark {:?} writer cut ({}, {}); sync_now {:?}; close ok {}",
398 mark, dlen, ilen, sn.as_ref().map_err(|e| fmt!("{}", e).chars().take(80).collect::<String>()), cl.is_ok());
399 let mk = marker(&root);
400 let rc = record(&root);
401 eprintln!("QA[s2] marker:\n{}record:\n{}", mk, rc);
402 let cuts = |t: &str| t.lines().filter(|l| l.starts_with("cut ")).count();
403 if cuts(&mk) != cuts(&rc) { bad.push(fmt!("marker and record disagree on cuts")); }
404 if open(&root, &c, false).map(|db| { let _ = db.close(); }).is_ok() {
405 bad.push(fmt!("the store opened over its refusal marker"));
406 }
407 let _ = Refusal::clear(&root);
408 let db = res!(open(&root, &c, false));
409 let low = mark.iter().map(|(_, m)| m.0).max().unwrap_or(0);
410 eprintln!("QA[s2] after open data {} (mark {}, writer cut {})", size(&dat), low, dlen);
411 if size(&dat) > dlen { bad.push(fmt!("data {} above the writer's cut {}", size(&dat), dlen)); }
412 let k1 = read(&db, "k1");
413 eprintln!("QA[s2] k1 = {}", k1);
414 if k1 != "none" && k1 != "100/2" { bad.push(fmt!("k1 reads {}", k1)); }
415 check(&mut bad, &db, &[("k0", "100/1"), ("k2", "none")], "after the cleared open");
416 res!(db.close());
417 let _ = fs::remove_dir_all(&root);
418 finish("s2", bad)
419}
420
421// Surface 2, "nothing more": a torn append cut back cleanly leaves no marker, no record, and the
422// store opens without a clear.
423#[test]
424fn s2_clean_cut_back_leaves_no_refusal() -> Outcome<()> {
425 let _g = lock();
426 let mut bad = Vec::new();
427 let root = res!(fresh("./test_db_qa_seek_s2n"));
428 let c = res!(cfg(true, false, 1_000_000));
429 let db = res!(open(&root, &c, true));
430 res!(put(&db, "k0", 100, 1));
431 hooks::set_write_tear(FileType::Data, Some(30));
432 let _ = put(&db, "k1", 100, 2);
433 hooks::set_write_tear(FileType::Index, Some(3));
434 let _ = put(&db, "k2", 100, 3);
435 disarm();
436 res!(put(&db, "k3", 100, 4));
437 if db.api().fault().is_some() { bad.push(fmt!("a clean cut back stopped the store")); }
438 if let Err(e) = db.api().sync_now(wait()) { bad.push(fmt!("sync_now after clean cuts: {}", e)); }
439 res!(db.close());
440 if Refusal::path(&root).exists() { bad.push(fmt!("a clean cut back wrote a marker")); }
441 if root.join(constant::FAULT_FILENAME).exists() { bad.push(fmt!("a clean cut back wrote a record")); }
442 let db = res!(open(&root, &c, false));
443 check(&mut bad, &db, &[("k0", "100/1"), ("k1", "none"), ("k2", "none"), ("k3", "100/4")], "after reopen");
444 res!(db.close());
445 let _ = fs::remove_dir_all(&root);
446 finish("s2n", bad)
447}
448
449// Surface 3: a chunked put whose middle chunk is torn and whose cut fails. The set stays held in
450// the stopped process; after the clear and the open, the old value reads whole and the sweep
451// retires what the failed put left. The tear is armed by a thread watching the file grow, which
452// under load can arm it only after the put has finished; such a try proves nothing and is run
453// again on a fresh store.
454fn chunk_case(name: &str, fail_cut: bool, arm_after: u64, tear: u64) -> Outcome<()> {
455 for attempt in 1..=5 {
456 match res!(chunk_try(name, fail_cut, arm_after, tear)) {
457 Some(bad) => return finish(name, bad),
458 None => eprintln!("QA[{}] try {}: the tear missed the put, trying again",
459 name, attempt),
460 }
461 }
462 Err(err!("{}: the tear missed the put in every try.", name; Test))
463}
464
465// One try of `chunk_case`: `None` when the tear did not fire within the put.
466fn chunk_try(name: &str, fail_cut: bool, arm_after: u64, tear: u64)
467 -> Outcome<Option<Vec<String>>>
468{
469 let mut bad = Vec::new();
470 let root = res!(fresh(&fmt!("./test_db_qa_seek_{}", name)));
471 let c = res!(cfg(false, true, 1_000_000));
472 let db = res!(open(&root, &c, true));
473 res!(put(&db, "K", 3_000, 1));
474 res!(put(&db, "S", 50, 9));
475 let (_, dat, _) = res!(live(&root, &c));
476 let base = size(&dat);
477 // Arm the tear once the put's first chunk is in, so a middle chunk tears.
478 let stop = Arc::new(AtomicBool::new(false));
479 let armer = {
480 let stop = stop.clone();
481 let dat = dat.clone();
482 std::thread::spawn(move || {
483 while !stop.load(Ordering::SeqCst) {
484 if size(&dat) > base + arm_after {
485 hooks::set_write_tear(FileType::Data, Some(tear));
486 return true;
487 }
488 std::hint::spin_loop();
489 }
490 false
491 })
492 };
493 if fail_cut { hooks::set_cut_back_failure(true); }
494 let torn0 = hooks::writes_torn();
495 let failed = put(&db, "K", 3_000, 2);
496 eprintln!("QA[{}] data grew {} bytes", name, size(&dat) - base);
497 stop.store(true, Ordering::SeqCst);
498 let armed = armer.join().unwrap_or(false);
499 let torn = hooks::writes_torn() - torn0;
500 disarm();
501 eprintln!("QA[{}] armed {} torn {} put {:?}", name, armed, torn,
502 failed.as_ref().map_err(|e| fmt!("{}", e).chars().take(100).collect::<String>()));
503 if torn == 0 {
504 let _ = db.close();
505 let _ = fs::remove_dir_all(&root);
506 return Ok(None);
507 }
508 if failed.is_ok() { bad.push(fmt!("a chunked put with a torn chunk was confirmed")); }
509 let held = |db: &TestDb| -> bool {
510 match db.api().chans().pending_sets().lock() { Ok(g) => !g.is_empty(), Err(p) => !p.into_inner().is_empty() }
511 };
512 std::thread::sleep(Duration::from_millis(500));
513 let h1 = held(&db);
514 eprintln!("QA[{}] fault {} held {}", name, db.api().fault().is_some(), h1);
515 let head = arm_after > 3_000;
516 if fail_cut && head && !h1 { bad.push(fmt!("the head's set was released after a failed cut")); }
517 if !fail_cut && h1 { bad.push(fmt!("the set stays held after a clean cut")); }
518 check(&mut bad, &db, &[("K", "3000/1"), ("S", "50/9")], "in process after the failed put");
519 if fail_cut {
520 match sweep::sweep_orphans(db.api(), Uid::default(), None, wait()) {
521 Ok(r) => bad.push(fmt!("the sweep ran on a stopped store: {:?}", r.orphans_found)),
522 Err(_) => (),
523 }
524 if head && !held(&db) { bad.push(fmt!("the hold went before the clear")); }
525 let _ = db.close();
526 if open(&root, &c, false).map(|db| { let _ = db.close(); }).is_ok() {
527 bad.push(fmt!("the store opened over its refusal marker"));
528 }
529 match Refusal::clear(&root) {
530 Ok(true) => (),
531 other => bad.push(fmt!("clear returned {:?}", other)),
532 }
533 } else {
534 res!(db.close());
535 }
536 let db = res!(open(&root, &c, false));
537 check(&mut bad, &db, &[("K", "3000/1"), ("S", "50/9")], "after the open");
538 match sweep::sweep_orphans(db.api(), Uid::default(), None, wait()) {
539 Ok(r) => eprintln!("QA[{}] sweep after open: orphans {} skipped_pending {}", name,
540 r.orphans_found, r.skipped_pending),
541 Err(e) => bad.push(fmt!("sweep after open failed: {}", e)),
542 }
543 if held(&db) { bad.push(fmt!("a set is held in the new process")); }
544 check(&mut bad, &db, &[("K", "3000/1"), ("S", "50/9")], "after the sweep");
545 if let Err(e) = put(&db, "K", 3_000, 3) { bad.push(fmt!("a new chunked put failed: {}", e)); }
546 res!(db.close());
547 let db = res!(open(&root, &c, false));
548 check(&mut bad, &db, &[("K", "3000/3"), ("S", "50/9")], "after the second open");
549 match sweep::sweep_orphans(db.api(), Uid::default(), None, wait()) {
550 Ok(r) => if r.orphans_found > 0 { bad.push(fmt!("{} orphans after the second open", r.orphans_found)); },
551 Err(e) => bad.push(fmt!("second sweep failed: {}", e)),
552 }
553 check(&mut bad, &db, &[("K", "3000/3"), ("S", "50/9")], "after the second sweep");
554 res!(db.close());
555 let _ = fs::remove_dir_all(&root);
556 Ok(Some(bad))
557}
558
559#[test]
560fn s3_chunk_torn_failed_cut_holds_then_clears() -> Outcome<()> {
561 let _g = lock();
562 chunk_case("s3f", true, 400, 100)
563}
564
565#[test]
566fn s3_chunk_torn_clean_cut_releases() -> Outcome<()> {
567 let _g = lock();
568 chunk_case("s3c", false, 400, 100)
569}
570
571// The head, the last append of the put, torn: the writer keeps its set's hold when its cut fails.
572#[test]
573fn s3_head_torn_failed_cut_holds_then_clears() -> Outcome<()> {
574 let _g = lock();
575 chunk_case("s3hf", true, 3_951, 10)
576}
577
578#[test]
579fn s3_head_torn_clean_cut_releases() -> Outcome<()> {
580 let _g = lock();
581 chunk_case("s3hc", false, 3_951, 10)
582}
583
584// Surface 1, pipelined: under sync_on_write, k1's barrier is held back (delayed) when k2 tears and
585// its cut fails. k1's caller's answer must agree with what the restart serves: confirmed and
586// present, or failed and absent.
587#[test]
588fn s1_pipelined_barrier_held_when_cut_fails() -> Outcome<()> {
589 let _g = lock();
590 let mut bad = Vec::new();
591 let root = res!(fresh("./test_db_qa_seek_s1p"));
592 let c = res!(cfg(false, false, 1_000_000));
593 let db = Arc::new(res!(open(&root, &c, true)));
594 res!(put(&db, "k0", 100, 1));
595 let (_, dat, ind) = res!(live(&root, &c));
596 let d0 = size(&dat);
597 hooks::set_barrier_delay(Duration::from_millis(1_500));
598 let t = {
599 let db = db.clone();
600 std::thread::spawn(move || put(&db, "k1", 100, 2).map_err(|e| plain(&fmt!("{}", e))))
601 };
602 // Wait for k1 to reach the files, its barrier still to come.
603 let landed = wait_until(Duration::from_secs(5), || size(&dat) > d0);
604 let (dlen, ilen) = (size(&dat), size(&ind));
605 hooks::set_cut_back_failure(true);
606 hooks::set_write_tear(FileType::Data, Some(30));
607 let t0 = std::time::Instant::now();
608 let failed = put(&db, "k2", 100, 3);
609 let k2_ms = t0.elapsed().as_millis();
610 let k1_done = t.is_finished();
611 hooks::set_cut_back_failure(false);
612 hooks::set_write_tear(FileType::Data, None);
613 eprintln!("QA[s1p] k2 answered in {} ms; k1 answered before k2: {}", k2_ms, k1_done);
614 if k1_done { bad.push(fmt!("positive control: k1's barrier was not pending when k2 tore")); }
615 let k1 = t.join().unwrap_or_else(|_| Err(fmt!("panicked")));
616 hooks::set_barrier_delay(Duration::from_millis(0));
617 eprintln!("QA[s1p] landed {} k1 answer {:?} k2 failed {} cuts {:?}", landed, k1, failed.is_err(),
618 db.api().fault().map(|r| r.cuts.iter().map(|x| x.line()).collect::<Vec<_>>()));
619 if !landed { bad.push(fmt!("positive control: k1 never reached the file")); }
620 if failed.is_ok() { bad.push(fmt!("the torn k2 was confirmed")); }
621 eprintln!("QA[s1p] OBS in process k1 = {}", read(&db, "k1"));
622 let db = match Arc::try_unwrap(db) { Ok(db) => db, Err(_) => return Err(err!("db still shared"; Test)) };
623 let _ = db.close();
624 let _ = Refusal::clear(&root);
625 let db = res!(open(&root, &c, false));
626 let k1r = read(&db, "k1");
627 eprintln!("QA[s1p] after open k1 = {} data {} (cut {}) index {} (cut {})", k1r, size(&dat), dlen, size(&ind), ilen);
628 match (&k1, k1r.as_str()) {
629 (Ok(()), "100/2") | (Err(_), "none") => (),
630 (Ok(()), r) => bad.push(fmt!("k1 confirmed but reads {} after the restart", r)),
631 (Err(e), r) => bad.push(fmt!("k1 reported failed ({}) but reads {} after the restart", e, r)),
632 }
633 check(&mut bad, &db, &[("k0", "100/1"), ("k2", "none")], "after the cleared open");
634 res!(db.close());
635 let _ = fs::remove_dir_all(&root);
636 finish("s1p", bad)
637}
638
639// F1, on its own: the stop after a failed cut back is reported as one by every refusal, by the
640// marker and by the record, under a sync on every write.
641#[test]
642fn f1_failed_cut_back_is_named_as_a_cut_back() -> Outcome<()> {
643 let _g = lock();
644 let mut bad = Vec::new();
645 let root = res!(fresh("./test_db_qa_seek_f1"));
646 let c = res!(cfg(false, false, 1_000_000));
647 let db = res!(open(&root, &c, true));
648 res!(put(&db, "k0", 100, 1));
649 hooks::set_cut_back_failure(true);
650 hooks::set_write_tear(FileType::Data, Some(30));
651 let failed = put(&db, "k1", 100, 2);
652 disarm();
653 if failed.is_ok() { bad.push(fmt!("the torn k1 was confirmed")); }
654 names_cut_back(&mut bad, "k2's put", &put(&db, "k2", 100, 3));
655 names_cut_back(&mut bad, "sync_now", &db.api().sync_now(wait()));
656 names_cut_back(&mut bad, "the sweep",
657 &sweep::sweep_orphans(db.api(), Uid::default(), None, wait()).map(|_| ()));
658 let _ = db.close();
659 marks_cut_back(&mut bad, &root);
660 let _ = Refusal::clear(&root);
661 let db = res!(open(&root, &c, false));
662 check(&mut bad, &db, &[("k0", "100/1"), ("k1", "none"), ("k2", "none")], "after the cleared open");
663 res!(db.close());
664 let _ = fs::remove_dir_all(&root);
665 finish("f1", bad)
666}
667
668// F3: a tear armed past the end of the append writes it whole, so it is not counted as torn.
669#[test]
670fn f3_an_append_shorter_than_the_tear_is_not_counted_torn() -> Outcome<()> {
671 let _g = lock();
672 let mut bad = Vec::new();
673 let root = res!(fresh("./test_db_qa_seek_f3"));
674 let c = res!(cfg(false, false, 1_000_000));
675 let db = res!(open(&root, &c, true));
676 let before = hooks::writes_torn();
677 hooks::set_write_tear(FileType::Data, Some(100_000));
678 if let Err(e) = put(&db, "k0", 100, 1) { bad.push(fmt!("k0 under an outrun tear failed: {}", e)); }
679 disarm();
680 if hooks::writes_torn() != before {
681 bad.push(fmt!("an append the tear never reached was counted torn ({} -> {})",
682 before, hooks::writes_torn()));
683 }
684 // Positive control: a tear the append reaches is counted.
685 hooks::set_write_tear(FileType::Data, Some(10));
686 if put(&db, "k1", 100, 2).is_ok() { bad.push(fmt!("the torn k1 was confirmed")); }
687 disarm();
688 if hooks::writes_torn() != before + 1 {
689 bad.push(fmt!("a real tear was not counted once ({} -> {})", before, hooks::writes_torn()));
690 }
691 check(&mut bad, &db, &[("k0", "100/1"), ("k1", "none")], "in process");
692 res!(db.close());
693 let _ = fs::remove_dir_all(&root);
694 finish("f3", bad)
695}