Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/qa_r2b.rs

20.7 KiB, 1 run

created by r1870400018:70860, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! QA round 2, agent B (2026-10-11): zone+fnum cut on a restored store and across zones whose
2//! file numbers collide; the F3 rollover cut twice in one window, under a collection, and at the
3//! seal. Every test compares what was acknowledged against what reads back after the reopen.
4mod gc_pair;
5
6use gc_pair::*;
7
8use oxedyne_fe2o3_core::prelude::*;
9use oxedyne_fe2o3_iop_db::api::Database;
10use oxedyne_fe2o3_jdat::prelude::*;
11use oxedyne_fe2o3_o3db_sync::{
12 base::{
13 cfg::OzoneConfig,
14 constant,
15 },
16 comm::response::Wait,
17 file::{
18 core::FileType,
19 refusal::Refusal,
20 zdir::ZoneDir,
21 },
22 test::{
23 hooks,
24 setup::{
25 self,
26 Uid,
27 },
28 },
29 verify,
30};
31
32use std::{
33 collections::BTreeMap,
34 fs,
35 path::{
36 Path,
37 PathBuf,
38 },
39 sync::{
40 Arc,
41 Mutex,
42 },
43 time::Duration,
44};
45
46static SERIAL: Mutex<()> = Mutex::new(());
47
48fn lock() -> std::sync::MutexGuard<'static, ()> {
49 let g = match SERIAL.lock() { Ok(g) => g, Err(p) => p.into_inner() };
50 reset();
51 g
52}
53
54fn reset() {
55 hooks::set_barrier_failure(false);
56 hooks::set_barrier_failure_for(None);
57 hooks::set_fault_record_fails(false);
58 hooks::set_barrier_delay(Duration::ZERO);
59 hooks::set_collect_delay(Duration::ZERO);
60}
61
62#[derive(Clone, Copy)]
63enum Pol { Every, N(u32), Never }
64
65fn cfg(zones: u16, p: Pol, max: u64) -> Outcome<OzoneConfig> {
66 let mut c = res!(setup::default_cfg());
67 c.num_zones = zones;
68 c.num_wbots_per_zone = 1;
69 c.zone_overrides = BTreeMap::new();
70 c.sync_on_write = matches!(p, Pol::Every);
71 c.sync_every_n_writes = match p { Pol::N(n) => n, _ => 0 };
72 c.sync_interval_ms = 0;
73 c.data_file_max_bytes = max;
74 Ok(c)
75}
76
77fn fresh(dir: &str) -> Outcome<PathBuf> {
78 let _ = fs::remove_dir_all(dir);
79 res!(fs::create_dir_all(dir));
80 Ok(res!(Path::new(dir).canonicalize()))
81}
82
83fn open(root: &Path, c: &OzoneConfig, wipe: bool) -> Outcome<TestDb> {
84 log_set_level!("error");
85 setup::start_db(root.to_path_buf(), Some(c.clone()), schemes(), None, false, wipe)
86}
87
88fn wait() -> Wait {
89 Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL }
90}
91
92fn val(seed: u8) -> Dat {
93 Dat::BU32((0..100).map(|j| seed ^ (j as u8)).collect())
94}
95
96fn put(db: &TestDb, k: &str, seed: u8) -> Outcome<()> {
97 db.insert(dat!(k), val(seed), Uid::default(), None).map(|_| ())
98}
99
100fn reads(db: &TestDb, k: &str) -> Outcome<Option<Dat>> {
101 Ok(res!(db.get(&dat!(k), None)).map(|(v, _)| v))
102}
103
104fn len(p: &Path) -> u64 {
105 fs::metadata(p).map(|m| m.len()).unwrap_or(0)
106}
107
108fn zdir(root: &Path, c: &OzoneConfig, z: usize) -> PathBuf {
109 c.zone_root(root).join(fmt!("zone_{:03}", z))
110}
111
112fn dat(zd: &Path, f: u32) -> PathBuf { zd.join(ZoneDir::relative_file_path(&FileType::Data, f)) }
113
114// Every ozone file of a zone directory with its length.
115fn lens(zd: &Path) -> Outcome<BTreeMap<String, u64>> {
116 let mut m = BTreeMap::new();
117 for e in res!(fs::read_dir(zd)) {
118 let p = res!(e).path();
119 if p.is_file() {
120 m.insert(p.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(), len(&p));
121 }
122 }
123 Ok(m)
124}
125
126fn data_total(zd: &Path) -> u64 {
127 match lens(zd) {
128 Ok(m) => m.iter().filter(|(n, _)| n.ends_with(".dat")).map(|(_, l)| *l).sum(),
129 Err(_) => 0,
130 }
131}
132
133fn last_fnum(zd: &Path) -> Option<u32> {
134 (1..10_000u32).rev().find(|f| dat(zd, *f).exists())
135}
136
137fn copy_tree(from: &Path, to: &Path) -> Outcome<()> {
138 res!(fs::create_dir_all(to));
139 for e in res!(fs::read_dir(from)) {
140 let p = res!(e).path();
141 let name = res!(p.file_name().ok_or_else(|| err!("{:?} has no name.", p; Test)));
142 match p.is_dir() {
143 true => res!(copy_tree(&p, &to.join(name))),
144 false => { res!(fs::copy(&p, to.join(name))); },
145 }
146 }
147 Ok(())
148}
149
150// What each key was told: durable (proven by a barrier), acknowledged but not proven (under a
151// lazy policy), and refused. After the reopen a key reads its durable value or an acknowledged
152// one (under EveryWrite, an acknowledged one if it has any), never only a refused one.
153#[derive(Default)]
154struct Book {
155 every: bool,
156 durable: BTreeMap<String, u8>,
157 acked: BTreeMap<String, Vec<u8>>,
158 failed: BTreeMap<String, Vec<u8>>,
159}
160
161impl Book {
162 fn durable(&mut self, k: &str, s: u8) { self.durable.insert(k.to_string(), s); self.acked.remove(k); }
163 fn acked(&mut self, k: &str, s: u8) {
164 if self.every { self.durable(k, s); } else { self.acked.entry(k.to_string()).or_default().push(s); }
165 }
166 fn failed(&mut self, k: &str, s: u8) { self.failed.entry(k.to_string()).or_default().push(s); }
167 // Everything acknowledged so far is now durable (a sync_now that succeeded).
168 fn synced(&mut self) {
169 let acked = std::mem::take(&mut self.acked);
170 for (k, v) in acked { if let Some(s) = v.last() { self.durable.insert(k, *s); } }
171 }
172}
173
174// (lost, resurrected, torn)
175fn check(db: &TestDb, label: &str, b: &Book) -> Outcome<(usize, usize, usize)> {
176 let mut keys: Vec<&String> = b.durable.keys().chain(b.acked.keys()).chain(b.failed.keys()).collect();
177 keys.sort();
178 keys.dedup();
179 let (mut lost, mut risen, mut torn) = (0, 0, 0);
180 for k in keys {
181 let got = res!(reads(db, k));
182 let acked = b.acked.get(k).cloned().unwrap_or_default();
183 let allowed: Vec<u8> = b.durable.get(k).cloned().into_iter().chain(acked.iter().cloned()).collect();
184 match got {
185 None => if b.durable.contains_key(k) {
186 msg!("{}: LOST {} (durable seed {:?})", label, k, b.durable.get(k));
187 lost += 1;
188 },
189 Some(g) => if allowed.iter().any(|s| val(*s) == g) {
190 // fine
191 } else if b.failed.get(k).map(|f| f.iter().any(|s| val(*s) == g)).unwrap_or(false) {
192 msg!("{}: RESURRECTED {} (refused value served; allowed {:?})", label, k, allowed);
193 risen += 1;
194 } else {
195 msg!("{}: TORN {} (allowed {:?})", label, k, allowed);
196 torn += 1;
197 },
198 }
199 }
200 Ok((lost, risen, torn))
201}
202
203// --- Surface 1 ---------------------------------------------------------------------------------
204
205/// Two zones each hold a file 1. The fault is in zone 2 only; zone 1's file 1, longer than zone
206/// 2's mark, must come through the open untouched.
207#[test]
208fn colliding_file_numbers_cut_only_the_named_zone() -> Outcome<()> {
209 let _g = lock();
210 let c = res!(cfg(2, Pol::Every, 1_000_000));
211 // Which zone each key lands in, learned from a throwaway store.
212 let probe = res!(fresh("./test_db_qa_r2b_probe"));
213 let mut z1 = Vec::new();
214 let mut z2 = Vec::new();
215 {
216 let db = res!(open(&probe, &c, true));
217 for i in 0..80 {
218 let k = fmt!("collide {}", i);
219 let (a, b) = (data_total(&zdir(&probe, &c, 1)), data_total(&zdir(&probe, &c, 2)));
220 res!(put(&db, &k, 1));
221 let (a2, b2) = (data_total(&zdir(&probe, &c, 1)), data_total(&zdir(&probe, &c, 2)));
222 if a2 > a { z1.push(k); } else if b2 > b { z2.push(k); }
223 }
224 let _ = db.close();
225 }
226 let _ = fs::remove_dir_all(&probe);
227 msg!("collide: {} keys in zone 1, {} in zone 2", z1.len(), z2.len());
228 assert!(z1.len() > 10 && z2.len() > 3, "the probe did not split the keys");
229
230 let root = res!(fresh("./test_db_qa_r2b_collide"));
231 let db = res!(open(&root, &c, true));
232 let mut bk = Book { every: true, ..Default::default() };
233 for (i, k) in z1.iter().enumerate() {
234 res!(put(&db, k, 10 + i as u8));
235 bk.durable(k, 10 + i as u8);
236 }
237 for k in &z2[0..2] {
238 res!(put(&db, k, 3));
239 bk.durable(k, 3);
240 }
241 hooks::set_barrier_failure(true);
242 let lost = put(&db, &z2[2], 4);
243 hooks::set_barrier_failure(false);
244 assert!(lost.is_err(), "the failed write was confirmed");
245 bk.failed(&z2[2], 4);
246 let report = res!(db.api().fault().ok_or_else(|| err!("No fault."; Test)));
247 let _ = db.close();
248 msg!("collide cuts: {:?}", report.cuts.iter().map(|c| (c.zone, c.fnum, c.data_upto, c.index_upto)).collect::<Vec<_>>());
249 assert!(report.cuts.iter().all(|c| c.zone == 1), "a cut names zone 1: {:?}", report.cuts);
250 let z1dir = zdir(&root, &c, 1);
251 for cut in &report.cuts {
252 assert!(len(&dat(&z1dir, cut.fnum)) > cut.data_upto,
253 "precondition: zone 1's file {} is not longer than zone 2's mark", cut.fnum);
254 }
255 let before = res!(lens(&z1dir));
256 assert!(res!(Refusal::clear(&root)));
257 let db = res!(open(&root, &c, false));
258 let t = res!(check(&db, "collide", &bk));
259 res!(db.close());
260 assert_eq!(t, (0, 0, 0), "collide: (lost, resurrected, torn) {:?}", t);
261 let after = res!(lens(&z1dir));
262 for (n, b) in &before {
263 if n.ends_with(".dat") || n.ends_with(".ind") {
264 assert_eq!(after.get(n), Some(b), "zone 1's {} changed in the open: {} to {:?}", n, b, after.get(n));
265 }
266 }
267 let z2dir = zdir(&root, &c, 2);
268 for cut in &report.cuts {
269 assert!(len(&dat(&z2dir, cut.fnum)) <= cut.data_upto, "zone 2's file {} was not cut", cut.fnum);
270 }
271 let _ = fs::remove_dir_all(&root);
272 Ok(())
273}
274
275/// A store restored from an older backup under its newer record: (a) the file in doubt is in
276/// the backup, shorter than its mark, and the open is a no-op cut; (b) the backup predates the
277/// file in doubt, and the open refuses. Then the operator restores the whole backup over the
278/// root: does the store open?
279#[test]
280fn a_restore_from_an_older_backup() -> Outcome<()> {
281 let _g = lock();
282 let c = res!(cfg(1, Pol::Every, 1_000_000));
283 let root = res!(fresh("./test_db_qa_r2b_restore"));
284 let early = res!(fresh("./test_db_qa_r2b_restore_early"));
285 let late = res!(fresh("./test_db_qa_r2b_restore_late"));
286 let zd = zdir(&root, &c, 1);
287 let mut db = res!(open(&root, &c, true));
288 let mut bk = Book { every: true, ..Default::default() };
289 for i in 0..5 { res!(put(&db, &fmt!("a {}", i), i + 1)); bk.durable(&fmt!("a {}", i), i + 1); }
290 let _ = db.close();
291 res!(copy_tree(&root, &early)); // before the file in doubt existed
292 db = res!(open(&root, &c, false));
293 res!(db.api().new_live_files());
294 for i in 0..5 { res!(put(&db, &fmt!("b {}", i), i + 20)); bk.durable(&fmt!("b {}", i), i + 20); }
295 let _ = db.close();
296 res!(copy_tree(&root, &late)); // the file in doubt there, shorter than its mark
297 db = res!(open(&root, &c, false));
298 for i in 0..5 { res!(put(&db, &fmt!("c {}", i), i + 40)); }
299 hooks::set_barrier_failure(true);
300 assert!(put(&db, "lost", 99).is_err());
301 hooks::set_barrier_failure(false);
302 let report = res!(db.api().fault().ok_or_else(|| err!("No fault."; Test)));
303 let _ = db.close();
304 msg!("restore cuts {:?}, files now {:?}", report.cuts.iter().map(|c| (c.fnum, c.data_upto)).collect::<Vec<_>>(), res!(lens(&zd)));
305 let record = res!(fs::read_to_string(root.join(constant::FAULT_FILENAME)));
306
307 // (a) late backup under the record.
308 let a = res!(fresh("./test_db_qa_r2b_restore_a"));
309 res!(copy_tree(&late, &a));
310 res!(fs::write(a.join(constant::FAULT_FILENAME), &record));
311 let alens = res!(lens(&zdir(&a, &c, 1)));
312 let db = res!(open(&a, &c, false));
313 bk.failed("lost", 99);
314 let t = res!(check(&db, "restore a", &bk));
315 for i in 0..5 { assert_eq!(res!(reads(&db, &fmt!("c {}", i))), None, "a key newer than the backup appeared"); }
316 res!(db.close());
317 assert_eq!(t, (0, 0, 0), "restore a: (lost, resurrected, torn) {:?}", t);
318 let alens2 = res!(lens(&zdir(&a, &c, 1)));
319 for (n, b) in &alens {
320 if n.ends_with(".dat") || n.ends_with(".ind") {
321 assert_eq!(alens2.get(n), Some(b), "restore a: {} changed", n);
322 }
323 }
324
325 // (b) early backup under the record and the marker.
326 let b = res!(fresh("./test_db_qa_r2b_restore_b"));
327 res!(copy_tree(&early, &b));
328 res!(fs::write(b.join(constant::FAULT_FILENAME), &record));
329 match open(&b, &c, false) {
330 Ok(db) => { let _ = db.close(); msg!("restore b: OPENED over a record naming a file it lacks"); },
331 Err(e) => msg!("restore b: refused: {}", fmt!("{}", e).chars().take(600).collect::<String>()),
332 }
333 let _ = fs::remove_dir_all(&root);
334 let _ = fs::remove_dir_all(&early);
335 let _ = fs::remove_dir_all(&late);
336 let _ = fs::remove_dir_all(&a);
337 let _ = fs::remove_dir_all(&b);
338 Ok(())
339}
340
341// --- Surface 2 ---------------------------------------------------------------------------------
342
343// Fills the live file during a held, failing barrier, so the writer rolls over, possibly more
344// than once. Returns the store's root, its config, the first and last live files, the ack map,
345// the failed map and the fault.
346fn rollover_window(dir: &str, p: Pol, max: u64, nputs: usize, overwrite: bool, collect: bool)
347 -> Outcome<()>
348{
349 let label = dir.to_string();
350 let root = res!(fresh(dir));
351 let c = res!(cfg(1, p, max));
352 let zd = zdir(&root, &c, 1);
353 let db = Arc::new(res!(open(&root, &c, true)));
354 let mut bk = Book { every: matches!(p, Pol::Every), ..Default::default() };
355 // Several files, with old versions for a collection to compact.
356 for round in 0..3u8 {
357 for i in 0..10 {
358 res!(put(&db, &fmt!("k {}", i), round * 10 + i as u8 + 1));
359 bk.acked(&fmt!("k {}", i), round * 10 + i as u8 + 1);
360 }
361 res!(db.api().sync_now(wait()));
362 bk.synced();
363 }
364 let first = res!(last_fnum(&zd).ok_or_else(|| err!("No file."; Test)));
365 res!(put(&db, "dirty", 9));
366 bk.acked("dirty", 9);
367 let compactor = if collect {
368 hooks::set_collect_delay(Duration::from_millis(600));
369 let dbc = db.clone();
370 let h = std::thread::spawn(move || dbc.compact_now(Duration::from_secs(30)).map(|r| fmt!("{:?}", r)));
371 std::thread::sleep(Duration::from_millis(50));
372 Some(h)
373 } else { None };
374 hooks::set_barrier_delay(Duration::from_millis(800));
375 hooks::set_barrier_failure(true);
376 let db2 = db.clone();
377 let sync = std::thread::spawn(move || db2.api().sync_now(wait()));
378 std::thread::sleep(Duration::from_millis(100));
379 let puts: Vec<_> = (0..nputs).map(|i| {
380 let db3 = db.clone();
381 let k = if overwrite && i % 2 == 0 { fmt!("k {}", (i / 2) % 10) } else { fmt!("during {}", i) };
382 let s = 100 + i as u8;
383 std::thread::spawn(move || (k.clone(), s, put(&db3, &k, s)))
384 }).collect();
385 let synced = sync.join();
386 let mut acked_during = 0;
387 let mut nfailed = 0;
388 for p in puts {
389 if let Ok((k, s, r)) = p.join() {
390 match r {
391 Ok(()) => { bk.acked(&k, s); acked_during += 1; },
392 Err(_) => { bk.failed(&k, s); nfailed += 1; },
393 }
394 }
395 }
396 reset();
397 let gc = compactor.map(|h| h.join());
398 msg!("{}: sync {:?}, {} during acked, {} failed, compaction {:?}", label,
399 synced.as_ref().map(|r| r.is_ok()), acked_during, nfailed, gc.map(|g| g.map(|r| r.is_ok())));
400 let last = res!(last_fnum(&zd).ok_or_else(|| err!("No file."; Test)));
401 let report = res!(db.api().fault().ok_or_else(|| err!("No fault."; Test)));
402 // Let any late raise land.
403 let _ = wait_until(Duration::from_secs(5), || db.api().fault()
404 .map(|r| (first + 1..=last).all(|f| r.cuts.iter().any(|c| c.fnum == f))).unwrap_or(false));
405 let report2 = res!(db.api().fault().ok_or_else(|| err!("No fault."; Test)));
406 msg!("{}: files {}..{}, cuts {:?} (first look {})", label, first, last,
407 report2.cuts.iter().map(|c| (c.fnum, c.data_upto, c.index_upto)).collect::<Vec<_>>(), report.cuts.len());
408 match Arc::try_unwrap(db) {
409 Ok(db) => { let _ = db.close(); },
410 Err(_) => return Err(err!("{}: the store is still shared.", label; Test)),
411 }
412 let record = res!(fs::read_to_string(root.join(constant::FAULT_FILENAME)));
413 let pre = res!(lens(&zd));
414 msg!("{}: record:\n{}files before reopen {:?}", label, record, pre);
415 let mut missing = Vec::new();
416 for f in (first + 1)..=last {
417 if dat(&zd, f).exists() && !record.lines().any(|l| l.starts_with(&fmt!("cut 0 {} ", f))) {
418 missing.push(f);
419 }
420 }
421 assert!(res!(Refusal::clear(&root)));
422 let db = res!(open(&root, &c, false));
423 let t1 = res!(check(&db, &label, &bk));
424 let v = res!(verify::verify_live_set(db.api(), None, wait()));
425 res!(db.close());
426 for cut in &report2.cuts {
427 assert!(len(&cut.data) <= cut.data_upto && len(&cut.index) <= cut.index_upto,
428 "{}: file {} past its mark ({}, {}): ({}, {})", label, cut.fnum, cut.data_upto,
429 cut.index_upto, len(&cut.data), len(&cut.index));
430 }
431 // Round trip: a write after the cut, a second reopen.
432 let db = res!(open(&root, &c, false));
433 res!(put(&db, "after", 77));
434 res!(db.close());
435 let db = res!(open(&root, &c, false));
436 let t2 = res!(check(&db, &label, &bk));
437 let after = res!(reads(&db, "after"));
438 res!(db.close());
439 assert!(missing.is_empty(), "{}: files rolled over to but not in the record: {:?}", label, missing);
440 assert!(v.clean(), "{}: {}", label, v.summary(4));
441 assert_eq!((t1, t2), ((0, 0, 0), (0, 0, 0)), "{}: (lost, resurrected, torn) first reopen {:?}, second {:?}", label, t1, t2);
442 assert_eq!(after, Some(val(77)), "{}: the write after the cut was lost", label);
443 let _ = fs::remove_dir_all(&root);
444 Ok(())
445}
446
447#[test]
448fn rollover_twice_never() -> Outcome<()> {
449 let _g = lock();
450 let r = rollover_window("./test_db_qa_r2b_roll2_never", Pol::Never, 2_000, 60, false, false);
451 reset();
452 r
453}
454
455#[test]
456fn rollover_twice_every_write_overwrites() -> Outcome<()> {
457 let _g = lock();
458 let r = rollover_window("./test_db_qa_r2b_roll2_every", Pol::Every, 2_000, 60, true, false);
459 reset();
460 r
461}
462
463#[test]
464fn rollover_under_collection_every_write() -> Outcome<()> {
465 let _g = lock();
466 let r = rollover_window("./test_db_qa_r2b_rollgc_every", Pol::Every, 2_000, 60, true, true);
467 reset();
468 r
469}
470
471#[test]
472fn rollover_under_collection_every_n() -> Outcome<()> {
473 let _g = lock();
474 let r = rollover_window("./test_db_qa_r2b_rollgc_n", Pol::N(1000), 2_000, 60, true, true);
475 reset();
476 r
477}
478
479/// The first barrier to fail is the seal's: EveryN with a large N runs none per record, so the
480/// rollover's own barrier over the old file is the one that fails.
481#[test]
482fn a_fault_exactly_at_the_seal() -> Outcome<()> {
483 let _g = lock();
484 let root = res!(fresh("./test_db_qa_r2b_seal"));
485 let c = res!(cfg(1, Pol::N(100_000), 2_000));
486 let zd = zdir(&root, &c, 1);
487 let db = res!(open(&root, &c, true));
488 let mut bk = Book::default();
489 for i in 0..5 { res!(put(&db, &fmt!("s {}", i), i + 1)); bk.acked(&fmt!("s {}", i), i + 1); }
490 res!(db.api().sync_now(wait()));
491 bk.synced();
492 let n = res!(last_fnum(&zd).ok_or_else(|| err!("No file."; Test)));
493 let mark = len(&dat(&zd, n));
494 hooks::set_barrier_failure_for(Some(n));
495 let mut acked_unsynced = BTreeMap::new();
496 let mut nfailed = 0;
497 for i in 0..60u8 {
498 let k = fmt!("t {}", i);
499 match put(&db, &k, 50 + i) {
500 Ok(()) => { acked_unsynced.insert(k.clone(), 50 + i); bk.acked(&k, 50 + i); },
501 Err(_) => { bk.failed(&k, 50 + i); nfailed += 1; },
502 }
503 }
504 hooks::set_barrier_failure_for(None);
505 let report = res!(db.api().fault().ok_or_else(|| err!("The seal's failed barrier raised no fault."; Test)));
506 let last = res!(last_fnum(&zd).ok_or_else(|| err!("No file."; Test)));
507 let _ = db.close();
508 let record = res!(fs::read_to_string(root.join(constant::FAULT_FILENAME)));
509 msg!("seal: file {} mark {}, last {}, {} acked unsynced, {} failed, cuts {:?}\nrecord:\n{}", n, mark, last,
510 acked_unsynced.len(), nfailed,
511 report.cuts.iter().map(|c| (c.fnum, c.data_upto, c.index_upto)).collect::<Vec<_>>(), record);
512 assert!(last > n, "no rollover happened");
513 assert!(record.lines().any(|l| l.starts_with(&fmt!("cut 0 {} ", n))), "the sealed file is not in the record");
514 for f in (n + 1)..=last {
515 assert!(record.lines().any(|l| l.starts_with(&fmt!("cut 0 {} ", f))), "file {} is not in the record", f);
516 }
517 assert!(res!(Refusal::clear(&root)));
518 let db = res!(open(&root, &c, false));
519 let t = res!(check(&db, "seal", &bk));
520 let mut survived = 0;
521 for (k, s) in &acked_unsynced {
522 if res!(reads(&db, k)) == Some(val(*s)) { survived += 1; }
523 }
524 res!(db.close());
525 msg!("seal: {} of {} acked-but-unsynced (EveryN) survived the cut", survived, acked_unsynced.len());
526 assert_eq!(t, (0, 0, 0), "seal: (lost, resurrected, torn) {:?}", t);
527 assert!(len(&dat(&zd, n)) <= mark.max(report.cuts.iter().find(|c| c.fnum == n).map(|c| c.data_upto).unwrap_or(0)),
528 "file {} is past its mark", n);
529 let _ = fs::remove_dir_all(&root);
530 Ok(())
531}