Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/chunk_set.rs

44.7 KiB, 33 runs

created by r1870400018:69929, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! Chunk sets (B1, B2, M1, 2026-10-09): every chunked write has a set of its own, and the
2//! store retires the set its key no longer names. The hooks are process-wide, so every test
3//! takes one lock.
4
5mod gc_pair;
6
7use gc_pair::*;
8
9use oxedyne_fe2o3_core::prelude::*;
10use oxedyne_fe2o3_data::time::Timestamp;
11use oxedyne_fe2o3_hash::{
12 csum::ChecksumScheme,
13 hash::HashScheme,
14};
15use oxedyne_fe2o3_iop_db::api::{
16 Database,
17 Meta,
18};
19use oxedyne_fe2o3_jdat::prelude::*;
20use oxedyne_fe2o3_o3db_sync::{
21 api::OzoneApi,
22 base::{
23 cfg::OzoneConfig,
24 constant,
25 },
26 api::CompactReport,
27 comm::response::Wait,
28 data::cache::{
29 Cache,
30 CacheEntry,
31 is_chunk_key,
32 },
33 file::{
34 floc::FileLocation,
35 state::DataState,
36 stored::RecordDigest,
37 },
38 sweep,
39 test::{
40 hooks,
41 setup::{
42 self,
43 Uid,
44 UID_LEN,
45 },
46 },
47 verify,
48};
49
50use std::{
51 collections::{
52 BTreeMap,
53 BTreeSet,
54 },
55 path::Path,
56 sync::{
57 Mutex,
58 MutexGuard,
59 },
60 thread,
61 time::Duration,
62};
63
64static HOOKS: Mutex<()> = Mutex::new(());
65
66fn lock() -> MutexGuard<'static, ()> {
67 match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() }
68}
69
70fn cfg() -> Outcome<OzoneConfig> {
71 let mut c = res!(setup::default_cfg());
72 c.num_zones = 1;
73 c.num_cbots_per_zone = 4;
74 c.num_fbots_per_zone = 1;
75 c.num_wbots_per_zone = 2;
76 c.zone_overrides = BTreeMap::new();
77 c.rest_chunk_threshold = 1_000;
78 c.rest_chunk_bytes = 400;
79 c.sync_on_write = true;
80 Ok(c)
81}
82
83// Every byte is `seed ^ j`, so a value assembled from two values' chunks is told by `whole`.
84fn val(len: usize, seed: u8) -> Dat {
85 Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect())
86}
87
88fn whole(v: &Dat, lens: &[usize]) -> bool {
89 match v {
90 Dat::BU32(b) | Dat::BU64(b) | Dat::BU8(b) | Dat::BU16(b) => {
91 if !lens.contains(&b.len()) || b.is_empty() { return false; }
92 let seed = b[0];
93 b.iter().enumerate().all(|(j, x)| *x == seed ^ (j as u8))
94 },
95 _ => false,
96 }
97}
98
99fn start(dir: &str, c: OzoneConfig, wipe: bool) -> Outcome<TestDb> {
100 log_set_level!("error");
101 if wipe {
102 let _ = std::fs::remove_dir_all(dir);
103 }
104 res!(std::fs::create_dir_all(dir));
105 let root = res!(Path::new(dir).canonicalize());
106 setup::start_db(root, Some(c), schemes(), None, false, wipe)
107}
108
109fn scan_wait() -> Wait {
110 Wait {
111 max_wait: Duration::from_secs(60),
112 check_interval: constant::CHECK_INTERVAL,
113 }
114}
115
116fn head(db: &TestDb, k: &Dat) -> Outcome<Option<[u64; 5]>> {
117 match res!(db.api().get_head_wait(k, None)) {
118 Some((Dat::Tup5u64(t), _)) => Ok(Some(t)),
119 _ => Ok(None),
120 }
121}
122
123fn pause(d: Duration) {
124 thread::sleep(d);
125}
126
127// A value written by a build that derived its set from the key alone reads whole after an
128// overwrite of the same length, which moves it to a set of its own and retires the old one.
129#[test]
130fn legacy_key_derived_value_overwritten_same_length_reads_whole_and_retires() -> Outcome<()> {
131 let _lock = lock();
132 let dir = "./test_db_chunk_set_legacy";
133 let db = res!(start(dir, res!(cfg()), true));
134 let k = dat!("legacy key");
135 let (kbuf, _, _) = res!(db.api().ozone_key_dat(&k, None));
136 let legacy = OzoneApi::<{ UID_LEN }, Uid, (), HashScheme, HashScheme, ChecksumScheme>::legacy_chunk_set_id(&kbuf);
137 let resp = db.api().responder();
138 res!(db.api().store_dat_using_responder_forcing_set_id(
139 k.clone(), val(1_100, 1), Uid::default(), None, resp.clone(), legacy));
140 res!(resp.recv_store_ack());
141 let old = res!(res!(head(&db, &k)).ok_or_else(|| err!("No legacy head."; Test, Missing)));
142 assert_eq!(old[0], legacy, "the forced value is not under the legacy set");
143
144 res!(db.insert(k.clone(), val(1_100, 2), Uid::default(), None));
145 let got = res!(db.get(&k, None));
146 let new = res!(res!(head(&db, &k)).ok_or_else(|| err!("No new head."; Test, Missing)));
147 // Retired by the cache bot once the new head is durable, after the put has returned.
148 let gone = wait_until(Duration::from_secs(10), || db.api().fetch_chunks(&Dat::Tup5u64(old), None).is_err());
149 res!(db.close());
150 let _ = std::fs::remove_dir_all(dir);
151 assert!(matches!(&got, Some((v, _)) if *v == val(1_100, 2)), "the overwrite did not read back whole");
152 assert_ne!(new[0], legacy, "the overwrite reused the legacy set");
153 assert!(gone, "the legacy chunks were not retired");
154 Ok(())
155}
156
157// The orphan sweep, run while a chunked put is between its chunks and its bunch key, leaves the
158// put's chunks alone: they are named by no key yet, and only the pending sets say whose they are.
159#[test]
160fn sweep_spares_chunks_of_a_put_in_flight() -> Outcome<()> {
161 let _lock = lock();
162 let dir = "./test_db_chunk_set_sweep";
163 let db = res!(start(dir, res!(cfg()), true));
164 let k = dat!("in flight");
165 let v = val(3_000, 7); // 8 chunks
166 hooks::set_chunk_insert_delay(Duration::from_millis(300));
167 let put = {
168 let (db, k, v) = (db.clone(), k.clone(), v.clone());
169 thread::spawn(move || db.insert(k, v, Uid::default(), None).map(|_| ()).map_err(|e| fmt!("{}", e)))
170 };
171 pause(Duration::from_millis(400));
172 let report = sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait());
173 let put = match put.join() { Ok(r) => r, Err(_) => Err(fmt!("panicked")) };
174 hooks::set_chunk_insert_delay(Duration::ZERO);
175 let got = db.get(&k, None);
176 res!(db.close());
177 let _ = std::fs::remove_dir_all(dir);
178 let report = res!(report);
179 msg!("sweep mid-put: {}", report.summary().replace('\n', " | "));
180 assert!(put.is_ok(), "the put failed: {:?}", put);
181 assert!(report.skipped_pending > 0, "the sweep saw none of the put's chunks, so it proved nothing");
182 assert_eq!(report.orphans_found, 0, "the sweep took the chunks of a put in flight for orphans");
183 assert!(matches!(&got, Ok(Some((g, _))) if *g == v), "the put does not read back whole: {:?}",
184 got.map(|o| o.map(|(d, _)| fmt!("{:?}", d).chars().take(120).collect::<String>())));
185 Ok(())
186}
187
188// A put whose bunch key loses to a newer value already there (its stamp is older) retires its
189// own chunks, so the sweep finds nothing, and the newer value survives a restart.
190#[test]
191fn losing_concurrent_put_retires_its_own_chunks() -> Outcome<()> {
192 let _lock = lock();
193 let dir = "./test_db_chunk_set_losing";
194 let k = dat!("losing put");
195 let v0 = val(1_100, 3);
196 let (report, before, after) = {
197 let db = res!(start(dir, res!(cfg()), true));
198 res!(db.insert(k.clone(), v0.clone(), Uid::default(), None));
199 let t0 = match res!(db.get(&k, None)) {
200 Some((_, meta)) => *meta.time,
201 None => return Err(err!("v0 is missing."; Test, Missing)),
202 };
203 hooks::set_stale_stamp(Some(t0.saturating_sub(Duration::from_secs(1))));
204 let lost = db.insert(k.clone(), val(1_500, 4), Uid::default(), None);
205 hooks::set_stale_stamp(None);
206 res!(lost);
207 let before = res!(db.get(&k, None)).map(|(v, _)| v);
208 let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait()));
209 res!(db.close());
210 let db = res!(start(dir, res!(cfg()), false));
211 let after = res!(db.get(&k, None)).map(|(v, _)| v);
212 res!(db.close());
213 (report, before, after)
214 };
215 let _ = std::fs::remove_dir_all(dir);
216 msg!("losing put: {}", report.summary().replace('\n', " | "));
217 assert!(before == Some(v0.clone()), "the older put displaced the newer value");
218 assert_eq!(report.orphans_found, 0, "the losing put left its chunks for the sweep");
219 assert!(after == Some(v0), "the newer value did not survive a restart");
220 Ok(())
221}
222
223// Two puts of one key at once, of one length, under one clock reading, or one stamp outright.
224// Returns, per key, whether it read one value whole live and after a restart, and the failures.
225fn two_puts_per_key(dir: &str, stale: bool) -> Outcome<(Vec<bool>, Vec<bool>, Vec<String>)> {
226 let lens = [1_100usize];
227 let keys: Vec<Dat> = (0..6u8).map(|i| dat!(fmt!("tick {}", i))).collect();
228 let db = res!(start(dir, res!(cfg()), true));
229 let now = *res!(Timestamp::now());
230 if stale { hooks::set_stale_stamp(Some(now)); } else { hooks::set_fixed_stamp(Some(now)); }
231 hooks::set_chunk_insert_delay(Duration::from_millis(5));
232 let ws: Vec<_> = keys.iter().enumerate().flat_map(|(i, k)| {
233 (0..2u8).map(|j| {
234 let (db, k) = (db.clone(), k.clone());
235 let seed = (i as u8).wrapping_mul(16).wrapping_add(j);
236 thread::spawn(move || db.insert(k, val(1_100, seed), Uid::default(), None)
237 .map(|_| ()).map_err(|e| fmt!("{}", e)))
238 }).collect::<Vec<_>>()
239 }).collect();
240 let mut failed = Vec::new();
241 for w in ws {
242 match w.join() {
243 Ok(Ok(())) => (),
244 Ok(Err(e)) => failed.push(e.chars().take(200).collect::<String>()),
245 Err(_) => failed.push(fmt!("panicked")),
246 }
247 }
248 hooks::set_chunk_insert_delay(Duration::ZERO);
249 hooks::set_fixed_stamp(None);
250 hooks::set_stale_stamp(None);
251 let mut live = Vec::new();
252 for k in &keys {
253 live.push(matches!(db.get(k, None), Ok(Some((v, _))) if whole(&v, &lens)));
254 }
255 res!(db.close());
256 let db = res!(start(dir, res!(cfg()), false));
257 let mut restarted = Vec::new();
258 for k in &keys {
259 restarted.push(matches!(db.get(k, None), Ok(Some((v, _))) if whole(&v, &lens)));
260 }
261 res!(db.close());
262 let _ = std::fs::remove_dir_all(dir);
263 Ok((live, restarted, failed))
264}
265
266// Two puts of one key with one stamp get distinct chunk sets, so neither writes over the other's
267// chunks and every key reads one value whole. Which value a restart keeps is not asked: two
268// records of a key with one stamp have no order, which is why a process never stamps two alike.
269#[test]
270fn two_puts_one_stamp_get_distinct_sets() -> Outcome<()> {
271 let _lock = lock();
272 let (live, _, failed) = res!(two_puts_per_key("./test_db_chunk_set_stamp", true));
273 assert!(failed.is_empty(), "puts failed: {:?}", failed);
274 assert!(live.iter().all(|b| *b), "a key read torn or failed: {:?}", live);
275 Ok(())
276}
277
278// Two puts of one key in one clock tick are stamped apart, so the value a key reads is the one it
279// reads after a restart, and its chunks were not retired by the put that lost.
280#[test]
281fn two_puts_one_clock_tick_settle_alike_across_a_restart() -> Outcome<()> {
282 let _lock = lock();
283 let (live, restarted, failed) = res!(two_puts_per_key("./test_db_chunk_set_tick", false));
284 assert!(failed.is_empty(), "puts failed: {:?}", failed);
285 assert!(live.iter().all(|b| *b), "a key read torn or failed: {:?}", live);
286 assert!(restarted.iter().all(|b| *b), "after a restart a key read torn or failed: {:?}", restarted);
287 Ok(())
288}
289
290// Two records of a key with one stamp: the cache, replay and the scan pick the same one, by place
291// (`supersedes`), so a key reads the same whole value live and after a restart, the live set
292// verifies, and the put that lost left nothing for the sweep. Each key's two puts are made in
293// turn from a thread of its own, so the two writers interleave across the keys.
294#[test]
295fn equal_stamps_pick_one_record_live_and_after_restart() -> Outcome<()> {
296 let _lock = lock();
297 let dir = "./test_db_chunk_set_equal";
298 let lens = [1_100usize];
299 let keys: Vec<Dat> = (0..12u8).map(|i| dat!(fmt!("equal {}", i))).collect();
300 let mut bad = Vec::new();
301 for round in 0..4u8 {
302 let db = res!(start(dir, res!(cfg()), true));
303 hooks::set_stale_stamp(Some(*res!(Timestamp::now())));
304 let ws: Vec<_> = keys.iter().enumerate().map(|(i, k)| {
305 let (db, k) = (db.clone(), k.clone());
306 let seed = round.wrapping_mul(64).wrapping_add((i as u8).wrapping_mul(2));
307 thread::spawn(move || -> Vec<String> {
308 let mut errs = Vec::new();
309 for j in 0..2u8 {
310 if let Err(e) = db.insert(k.clone(), val(1_100, seed.wrapping_add(j)), Uid::default(), None) {
311 errs.push(fmt!("{}", e).chars().take(200).collect::<String>());
312 }
313 }
314 errs
315 })
316 }).collect();
317 for w in ws {
318 match w.join() {
319 Ok(errs) => for e in errs { bad.push(fmt!("round {} put: {}", round, e)); },
320 Err(_) => bad.push(fmt!("round {} put panicked", round)),
321 }
322 }
323 hooks::set_stale_stamp(None);
324 let mut live = Vec::new();
325 for k in &keys {
326 let got = res!(db.get(k, None)).map(|(v, _)| v);
327 if !matches!(&got, Some(v) if whole(v, &lens)) {
328 bad.push(fmt!("round {} live {:?}: not whole", round, k));
329 }
330 live.push(got);
331 }
332 let v = res!(verify::verify_live_set(db.api(), None, scan_wait()));
333 if !v.clean() { bad.push(fmt!("round {} live verify: {}", round, v.summary(4))); }
334 res!(db.close());
335 let db = res!(start(dir, res!(cfg()), false));
336 for (k, want) in keys.iter().zip(&live) {
337 match db.get(k, None) {
338 Ok(got) => if got.map(|(v, _)| v) != *want {
339 bad.push(fmt!("round {} {:?}: the restart reads another value than live", round, k));
340 },
341 Err(e) => bad.push(fmt!("round {} {:?}: after the restart: {}", round, k, e)),
342 }
343 }
344 let v = res!(verify::verify_live_set(db.api(), None, scan_wait()));
345 if !v.clean() { bad.push(fmt!("round {} restarted verify: {}", round, v.summary(4))); }
346 let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait()));
347 if report.orphans_found != 0 {
348 bad.push(fmt!("round {} sweep: {}", round, report.summary().replace('\n', " | ")));
349 }
350 res!(db.close());
351 }
352 let _ = std::fs::remove_dir_all(dir);
353 assert!(bad.is_empty(), "equal stamps did not settle on one record:\n{}", bad.join("\n"));
354 Ok(())
355}
356
357// With one writer a zone's records lie in write order, so of two puts of a key with one stamp the
358// later is the later place and must be the value read, live and after a restart. The interleaved
359// test above catches a split between the paths only when replay happens to read in another order;
360// this one catches a tie broken by anything but place every time.
361#[test]
362fn equal_stamps_one_writer_later_put_wins() -> Outcome<()> {
363 let _lock = lock();
364 let dir = "./test_db_chunk_set_equal_one";
365 let mut c = res!(cfg());
366 c.num_wbots_per_zone = 1;
367 let keys: Vec<Dat> = (0..6u8).map(|i| dat!(fmt!("equal one {}", i))).collect();
368 let db = res!(start(dir, c.clone(), true));
369 hooks::set_stale_stamp(Some(*res!(Timestamp::now())));
370 for (i, k) in keys.iter().enumerate() {
371 for j in 0..2u8 {
372 res!(db.insert(k.clone(), val(1_100, (i as u8) * 2 + j), Uid::default(), None));
373 }
374 }
375 hooks::set_stale_stamp(None);
376 let mut bad = Vec::new();
377 for (i, k) in keys.iter().enumerate() {
378 if res!(db.get(k, None)).map(|(v, _)| v) != Some(val(1_100, (i as u8) * 2 + 1)) {
379 bad.push(fmt!("live {:?}: not the later put", k));
380 }
381 }
382 res!(db.close());
383 let db = res!(start(dir, c, false));
384 for (i, k) in keys.iter().enumerate() {
385 if res!(db.get(k, None)).map(|(v, _)| v) != Some(val(1_100, (i as u8) * 2 + 1)) {
386 bad.push(fmt!("restarted {:?}: not the later put", k));
387 }
388 }
389 res!(db.close());
390 let _ = std::fs::remove_dir_all(dir);
391 assert!(bad.is_empty(), "a tie of stamps was not broken by place:\n{}", bad.join("\n"));
392 Ok(())
393}
394
395// The riskiest surface: a store reads its key back once its bunch key is durable and retires its
396// own set if the key names another. Under many concurrent chunked overwrites of a few keys, with
397// slow file reads, no store may retire the set its key ends up naming: every key's current head
398// fetches whole, the live set verifies clean, before and after a restart and after a sweep.
399#[test]
400fn store_never_retires_its_own_live_chunks_under_load() -> Outcome<()> {
401 let _lock = lock();
402 let dir = "./test_db_chunk_set_load";
403 let lens = vec![1_100usize, 1_500, 2_300];
404 let keys: Vec<Dat> = (0..4u8).map(|i| dat!(fmt!("load {}", i))).collect();
405 let mut c = res!(cfg());
406 c.cache_size_limit_bytes = 1;
407 let mut bad = Vec::new();
408 let mut failed = Vec::new();
409 {
410 let db = res!(start(dir, c.clone(), true));
411 hooks::set_read_delay(Duration::from_millis(3));
412 hooks::set_chunk_insert_delay(Duration::from_millis(2));
413 let ws: Vec<_> = (0..12usize).map(|t| {
414 let (db, keys, lens) = (db.clone(), keys.clone(), lens.clone());
415 thread::spawn(move || -> Vec<String> {
416 let mut errs = Vec::new();
417 for n in 0..10usize {
418 let k = keys[(t + n) % keys.len()].clone();
419 let seed = (t as u8).wrapping_mul(37).wrapping_add(n as u8);
420 if let Err(e) = db.insert(k, val(lens[(t * 7 + n) % 3], seed), Uid::default(), None) {
421 errs.push(fmt!("{}", e).chars().take(200).collect::<String>());
422 }
423 }
424 errs
425 })
426 }).collect();
427 for w in ws {
428 match w.join() {
429 Ok(errs) => failed.extend(errs),
430 Err(_) => failed.push(fmt!("panicked")),
431 }
432 }
433 hooks::set_read_delay(Duration::ZERO);
434 hooks::set_chunk_insert_delay(Duration::ZERO);
435 res!(check(&db, &keys, &lens, "live", &mut bad));
436 res!(db.close());
437 let db = res!(start(dir, c.clone(), false));
438 res!(check(&db, &keys, &lens, "restarted", &mut bad));
439 let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait()));
440 msg!("load: {}", report.summary().replace('\n', " | "));
441 res!(check(&db, &keys, &lens, "swept", &mut bad));
442 res!(db.close());
443 }
444 let _ = std::fs::remove_dir_all(dir);
445 assert!(failed.is_empty(), "puts failed: {:?}", failed);
446 assert!(bad.is_empty(), "a store retired chunks its key still names: {:?}", bad);
447 Ok(())
448}
449
450fn check(db: &TestDb, keys: &[Dat], lens: &[usize], phase: &str, bad: &mut Vec<String>) -> Outcome<()> {
451 for k in keys {
452 match res!(head(db, k)) {
453 Some(t) => if let Err(e) = db.api().fetch_chunks(&Dat::Tup5u64(t), None) {
454 bad.push(fmt!("{} {:?}: head {:?}: {}", phase, k, t, e).chars().take(300).collect());
455 },
456 None => bad.push(fmt!("{} {:?}: no head", phase, k)),
457 }
458 match db.get(k, None) {
459 Ok(Some((v, _))) if whole(&v, lens) => (),
460 other => bad.push(fmt!("{} {:?}: read {:?}", phase, k,
461 other.map(|o| o.map(|(d, _)| fmt!("{:?}", d).chars().take(80).collect::<String>())))),
462 }
463 }
464 let report = res!(verify::verify_live_set(db.api(), None, scan_wait()));
465 if !report.clean() {
466 bad.push(fmt!("{} verify: {}", phase, report.summary(4)));
467 }
468 Ok(())
469}
470
471// A delete is a store with no chunks: its head tombstone is durable before the chunks of the value
472// it deletes are retired, so a read made between the two finds the value whole or gone, never a
473// current bunch key missing a chunk (fix A, 2026-10-09). Since the durable mark the cache bot
474// retires them after the delete has returned, and `set_chunk_tombstone_delay` holds the chunk
475// tombstones so that the read lands between the two. `marker` deletes by storing the deleted marker, as the distributed adapters
476// erase.
477fn read_during_delete(dir: &str, marker: bool) -> Outcome<()> {
478 let _lock = lock();
479 let db = res!(start(dir, res!(cfg()), true));
480 let k = dat!("deleted while read");
481 let v = val(1_100, 5); // 3 chunks
482 let tomb = Dat::Usr(oxedyne_fe2o3_o3db_sync::base::id::usr_kind_id_deleted(), Some(Box::new(Dat::Empty)));
483 res!(db.insert(k.clone(), v.clone(), Uid::default(), None));
484 let old = res!(res!(head(&db, &k)).ok_or_else(|| err!("No head for the value."; Test, Missing)));
485 hooks::set_chunk_tombstone_delay(Duration::from_millis(300));
486 let del = {
487 let (db, k, tomb) = (db.clone(), k.clone(), tomb.clone());
488 thread::spawn(move || {
489 let r = if marker {
490 db.api().store(k, tomb, Uid::default()).and_then(|resp| resp.recv_store_ack().map(|_| ()))
491 } else {
492 db.delete(&k, Uid::default(), None).map(|_| ())
493 };
494 r.map_err(|e| fmt!("{}", e))
495 })
496 };
497 pause(Duration::from_millis(100));
498 let mid = db.get(&k, None);
499 let del = match del.join() { Ok(r) => r, Err(_) => Err(fmt!("panicked")) };
500 let after = db.get(&k, None);
501 let gone = wait_until(Duration::from_secs(10), || db.api().fetch_chunks(&Dat::Tup5u64(old), None).is_err());
502 hooks::set_chunk_tombstone_delay(Duration::ZERO);
503 res!(db.close());
504 let _ = std::fs::remove_dir_all(dir);
505 assert!(del.is_ok(), "the delete failed: {:?}", del);
506 match &mid {
507 Ok(None) => (),
508 Ok(Some((g, _))) if *g == v || *g == tomb => (),
509 other => panic!("a read during the delete found neither the value nor its absence: {}",
510 fmt!("{:?}", other).chars().take(300).collect::<String>()),
511 }
512 match &after {
513 Ok(None) => (),
514 Ok(Some((g, _))) if *g == tomb => (),
515 other => panic!("the key does not read as deleted after the delete: {}",
516 fmt!("{:?}", other).chars().take(300).collect::<String>()),
517 }
518 assert!(gone, "the deleted value's chunks were not retired");
519 Ok(())
520}
521
522#[test]
523fn read_during_delete_of_chunked_value_is_whole_or_none() -> Outcome<()> {
524 read_during_delete("./test_db_chunk_set_read_delete", false)
525}
526
527#[test]
528fn read_during_marker_store_delete_is_whole_or_none() -> Outcome<()> {
529 read_during_delete("./test_db_chunk_set_read_marker", true)
530}
531
532// A delete stamped before the value at its key loses to it, and must leave that value whole: the
533// delete retires the set it read only when the key no longer names it.
534#[test]
535fn stale_delete_keeps_the_newer_value() -> Outcome<()> {
536 let _lock = lock();
537 let dir = "./test_db_chunk_set_stale_delete";
538 let db = res!(start(dir, res!(cfg()), true));
539 let k = dat!("stale delete");
540 let v0 = val(1_100, 6);
541 res!(db.insert(k.clone(), v0.clone(), Uid::default(), None));
542 let t0 = match res!(db.get(&k, None)) {
543 Some((_, meta)) => *meta.time,
544 None => return Err(err!("v0 is missing."; Test, Missing)),
545 };
546 hooks::set_stale_stamp(Some(t0.saturating_sub(Duration::from_secs(1))));
547 let del = db.delete(&k, Uid::default(), None);
548 hooks::set_stale_stamp(None);
549 let got = db.get(&k, None);
550 res!(db.close());
551 let _ = std::fs::remove_dir_all(dir);
552 res!(del);
553 assert!(matches!(&got, Ok(Some((g, _))) if *g == v0), "a stale delete took the newer value: {}",
554 fmt!("{:?}", got).chars().take(300).collect::<String>());
555 Ok(())
556}
557
558// Fix B (2026-10-09): a chunk tombstone is forgotten once every older record of its key is durably
559// gone, so a store whose chunked values are overwritten does not grow its caches for ever.
560
561const VLEN: usize = 1_100; // three chunks of 400
562
563fn cfg_b() -> Outcome<OzoneConfig> {
564 let mut c = res!(cfg());
565 c.data_file_max_bytes = 4_000;
566 c.num_wbots_per_zone = 1; // each writer has a live file of its own, so one puts a value in one file
567 Ok(c)
568}
569
570fn start_gc(dir: &str, wipe: bool) -> Outcome<TestDb> {
571 log_set_level!("error");
572 if wipe {
573 let _ = std::fs::remove_dir_all(dir);
574 }
575 res!(std::fs::create_dir_all(dir));
576 let root = res!(Path::new(dir).canonicalize());
577 let db = res!(setup::start_db(root, Some(res!(cfg_b())), schemes(), None, true, wipe));
578 // Only `compact_now` collects, so a test places its tombstones and holds before any collection,
579 // and the reports count every collection and deletion.
580 res!(db.api().activate_gc(false));
581 Ok(db)
582}
583
584#[derive(Debug, Default)]
585struct Census {
586 chunks: usize, // chunk-key cache entries
587 tombs: BTreeMap<Vec<u8>, FileLocation>, // chunk tombstones, where they lie
588 files: BTreeSet<u32>, // files of the live chunk entries
589 entries: usize, // all cache entries
590 anc: usize, // ancillary bytes of every cache
591}
592
593fn census(db: &TestDb) -> Outcome<Census> {
594 res!(db.api().settle_for_test(Duration::from_secs(20)));
595 let mut c = Census::default();
596 for (_, cache) in res!(db.api().collect_caches(scan_wait())) {
597 c.entries += cache.map().len();
598 c.anc += cache.get_ancillary_size();
599 let tombs = cache.tomb_tracker().tombs();
600 for (k, e) in cache.map() {
601 if !is_chunk_key(k) { continue; }
602 c.chunks += 1;
603 let CacheEntry::LocatedValue(mloc, _) = e;
604 match tombs.contains_key(k) {
605 true => { c.tombs.insert(k.clone(), *mloc.file_location()); },
606 false => { c.files.insert(mloc.file_number()); },
607 }
608 }
609 }
610 Ok(c)
611}
612
613// Every cache entry is one current record in the file states, and nothing waits on a record.
614fn accounted(db: &TestDb, c: &Census, label: &str) -> Outcome<()> {
615 let mut cur = 0;
616 for (_, fmap) in res!(db.api().collect_file_states(scan_wait())) {
617 for (fnum, fstat) in fmap.map() {
618 assert!(fstat.pending_old_empty(), "{}: file {} has parked supersessions", label, fnum);
619 cur += fstat.data_map().values().filter(|d| **d == DataState::Cur).count();
620 }
621 }
622 assert_eq!(cur, c.entries, "{}: current records against cache entries", label);
623 Ok(())
624}
625
626// Compacts until quiet: a dropped tombstone is flagged old only after its shadowed records go.
627fn quiet(db: &TestDb) -> Outcome<CompactReport> {
628 let mut sum = CompactReport::default();
629 for _ in 0..3 {
630 res!(db.api().settle_for_test(Duration::from_secs(20)));
631 let r = res!(db.api().compact_now(Duration::from_secs(60)));
632 sum.files_collected += r.files_collected;
633 sum.files_deleted += r.files_deleted;
634 pause(Duration::from_millis(300));
635 }
636 Ok(sum)
637}
638
639// As `quiet`, with a file held, which keeps its old bytes and so cannot settle.
640fn quiet_held(db: &TestDb) {
641 for _ in 0..2 {
642 let _ = db.api().settle_for_test(Duration::from_secs(20));
643 let _ = db.api().compact_now(Duration::from_secs(3));
644 pause(Duration::from_millis(300));
645 }
646}
647
648fn filler(db: &TestDb, tag: &str, n: usize, seed: u8) -> Outcome<()> {
649 for i in 0..n {
650 res!(db.insert(dat!(fmt!("filler {} {:03}", tag, i)), val(100, seed), Uid::default(), None));
651 }
652 Ok(())
653}
654
655fn reads(db: &TestDb, k: &Dat, want: &Dat) -> Outcome<bool> {
656 Ok(matches!(res!(db.get(k, None)), Some((v, _)) if v == *want))
657}
658
659fn accumulate(dir: &str, n: usize) -> Outcome<(usize, Census, CompactReport)> {
660 let db = res!(start_gc(dir, true));
661 let k = dat!("accumulate");
662 res!(db.insert(k.clone(), val(VLEN, 0), Uid::default(), None));
663 let base = res!(census(&db)).chunks;
664 // Each compaction seals the value just written in a file of its own, which goes all-old once
665 // the next overwrite lands, so its deletion, and not only a collection, reports the records
666 // a tombstone shadows gone.
667 let mut rep = CompactReport::default();
668 for i in 1..=n {
669 res!(db.insert(k.clone(), val(VLEN, i as u8), Uid::default(), None));
670 let r = res!(db.api().compact_now(Duration::from_secs(60)));
671 rep.files_collected += r.files_collected;
672 rep.files_deleted += r.files_deleted;
673 }
674 let r = res!(quiet(&db));
675 rep.files_collected += r.files_collected;
676 rep.files_deleted += r.files_deleted;
677 let c = res!(census(&db));
678 let acc = accounted(&db, &c, "accumulate");
679 let ok = res!(reads(&db, &k, &val(VLEN, n as u8)));
680 res!(db.close());
681 let _ = std::fs::remove_dir_all(dir);
682 res!(acc);
683 assert!(ok, "the last value does not read back");
684 Ok((base, c, rep))
685}
686
687#[test]
688fn chunk_tombstones_do_not_accumulate() -> Outcome<()> {
689 let _lock = lock();
690 let (base1, c1, r1) = res!(accumulate("./test_db_chunk_set_accumulate_a", 140));
691 let (base2, c2, r2) = res!(accumulate("./test_db_chunk_set_accumulate_b", 280));
692 assert!(base1 >= 2, "the value was not chunked");
693 assert_eq!(c1.chunks, base1, "140 overwrites leave {} tombstones", c1.tombs.len());
694 assert_eq!(c2.chunks, base2, "280 overwrites leave {} tombstones", c2.tombs.len());
695 assert!(c2.anc <= c1.anc + 256, "the ancillary size grows with overwrites: {} then {}", c1.anc, c2.anc);
696 assert!(r1.files_collected + r2.files_collected > 0, "no file was collected");
697 assert!(r1.files_deleted + r2.files_deleted > 0, "no all-old file was deleted");
698 Ok(())
699}
700
701// A value, sealed in a file of its own, overwritten with its tombstones in later files.
702fn overwritten(db: &TestDb, k: &Dat) -> Outcome<(usize, BTreeSet<u32>)> {
703 res!(db.insert(k.clone(), val(VLEN, 1), Uid::default(), None));
704 let c0 = res!(census(db));
705 res!(filler(db, "a", 40, 1));
706 res!(db.insert(k.clone(), val(VLEN, 2), Uid::default(), None));
707 res!(filler(db, "b", 40, 1));
708 Ok((c0.chunks, c0.files))
709}
710
711#[test]
712fn failed_dir_sync_keeps_tombstone() -> Outcome<()> {
713 let _lock = lock();
714 let dir = "./test_db_chunk_set_dir_sync";
715 let db = res!(start_gc(dir, true));
716 let k = dat!("dir sync");
717 let (base, _) = res!(overwritten(&db, &k));
718 hooks::set_dir_sync_failure(true);
719 quiet_held(&db);
720 let c = census(&db);
721 let acc = match &c { Ok(c) => accounted(&db, c, "dir sync"), Err(_) => Ok(()) };
722 hooks::set_dir_sync_failure(false);
723 let ok = reads(&db, &k, &val(VLEN, 2));
724 res!(db.close());
725 let _ = std::fs::remove_dir_all(dir);
726 let c = res!(c);
727 res!(acc);
728 assert!(res!(ok), "the value does not read back");
729 assert!(c.tombs.len() >= base, "{} tombstones kept after a failed directory sync, {} wanted",
730 c.tombs.len(), base);
731 Ok(())
732}
733
734#[test]
735fn lone_chunk_tombstones_drop_at_restart() -> Outcome<()> {
736 let _lock = lock();
737 let dir = "./test_db_chunk_set_lone";
738 let db = res!(start_gc(dir, true));
739 let k = dat!("lone");
740 let (base, _) = res!(overwritten(&db, &k));
741 hooks::set_dir_sync_failure(true);
742 quiet_held(&db);
743 let before = census(&db);
744 hooks::set_dir_sync_failure(false);
745 res!(db.close());
746 let before = res!(before);
747 let db = res!(start_gc(dir, false));
748 let after = census(&db);
749 let rep = quiet(&db);
750 let fin = census(&db);
751 let acc = match &fin { Ok(c) => accounted(&db, c, "lone"), Err(_) => Ok(()) };
752 let ok = reads(&db, &k, &val(VLEN, 2));
753 res!(db.close());
754 let _ = std::fs::remove_dir_all(dir);
755 let (after, fin) = (res!(after), res!(fin));
756 res!(rep);
757 res!(acc);
758 assert!(res!(ok), "the value does not read back");
759 assert!(before.tombs.len() >= base, "the setup kept no tombstones");
760 assert_eq!(after.chunks, base, "lone tombstones survived the restart: {}", after.tombs.len());
761 assert_eq!(fin.chunks, base, "tombstones after compaction: {}", fin.tombs.len());
762 Ok(())
763}
764
765#[test]
766fn tombstone_outlives_uncollected_record() -> Outcome<()> {
767 let _lock = lock();
768 let dir = "./test_db_chunk_set_outlives";
769 let db = res!(start_gc(dir, true));
770 let k = dat!("outlives");
771 let (base, files) = res!(overwritten(&db, &k));
772 assert_eq!(files.len(), 1, "the first value is not in one file: {:?}", files);
773 let f = files.iter().next().copied();
774 hooks::set_collect_fails_for(f);
775 quiet_held(&db);
776 let held = census(&db);
777 res!(db.close());
778 let db = res!(start_gc(dir, false));
779 let restarted = census(&db);
780 hooks::set_collect_fails_for(None);
781 let rep = quiet(&db);
782 let fin = census(&db);
783 let acc = match &fin { Ok(c) => accounted(&db, c, "outlives"), Err(_) => Ok(()) };
784 let ok = reads(&db, &k, &val(VLEN, 2));
785 res!(db.close());
786 let _ = std::fs::remove_dir_all(dir);
787 let (held, restarted, fin) = (res!(held), res!(restarted), res!(fin));
788 res!(rep);
789 res!(acc);
790 assert!(res!(ok), "the value does not read back");
791 assert!(held.tombs.len() >= base, "a tombstone dropped while its record is on disk: {}", held.tombs.len());
792 assert!(restarted.tombs.len() >= base, "a tombstone dropped at a restart while its record is on disk: {}",
793 restarted.tombs.len());
794 assert_eq!(fin.chunks, base, "tombstones after the hold: {}", fin.tombs.len());
795 Ok(())
796}
797
798#[test]
799fn reanchored_tombstone_still_drops() -> Outcome<()> {
800 let _lock = lock();
801 let dir = "./test_db_chunk_set_reanchored";
802 let db = res!(start_gc(dir, true));
803 let k = dat!("reanchored");
804 res!(db.insert(k.clone(), val(VLEN, 1), Uid::default(), None));
805 let c0 = res!(census(&db));
806 assert_eq!(c0.files.len(), 1, "the first value is not in one file: {:?}", c0.files);
807 res!(filler(&db, "a", 40, 1));
808 // Records ahead of the tombstones in their file, then made old, so that collecting that file
809 // carries the tombstones to new offsets.
810 res!(filler(&db, "b", 8, 1));
811 res!(db.insert(k.clone(), val(VLEN, 2), Uid::default(), None));
812 res!(filler(&db, "b", 8, 2));
813 res!(filler(&db, "c", 40, 1));
814 let placed = res!(census(&db));
815 hooks::set_collect_fails_for(c0.files.iter().next().copied());
816 quiet_held(&db);
817 let held = census(&db);
818 hooks::set_collect_fails_for(None);
819 let rep = quiet(&db);
820 let fin = census(&db);
821 let acc = match &fin { Ok(c) => accounted(&db, c, "reanchored"), Err(_) => Ok(()) };
822 let ok = reads(&db, &k, &val(VLEN, 2));
823 res!(db.close());
824 let _ = std::fs::remove_dir_all(dir);
825 let (held, fin) = (res!(held), res!(fin));
826 res!(rep);
827 res!(acc);
828 assert!(res!(ok), "the value does not read back");
829 let moved = placed.tombs.iter().filter(|(t, floc)| match held.tombs.get(*t) {
830 Some(now) => now.start != floc.start,
831 None => false,
832 }).count();
833 assert!(moved > 0, "no tombstone was carried by a collection: {:?} then {:?}", placed.tombs, held.tombs);
834 assert_eq!(fin.chunks, c0.chunks, "re-anchored tombstones did not drop: {}", fin.tombs.len());
835 Ok(())
836}
837
838#[test]
839fn legacy_tombstone_waits_for_every_older_record() -> Outcome<()> {
840 let _lock = lock();
841 let dir = "./test_db_chunk_set_legacy_wait";
842 let db = res!(start_gc(dir, true));
843 let k = dat!("legacy wait");
844 let (kbuf, _, _) = res!(db.api().ozone_key_dat(&k, None));
845 let legacy = OzoneApi::<{ UID_LEN }, Uid, (), HashScheme, HashScheme, ChecksumScheme>::legacy_chunk_set_id(&kbuf);
846 let mut first = BTreeSet::new();
847 for (seed, tag) in [(1u8, "a"), (2u8, "b")] {
848 let resp = db.api().responder();
849 res!(db.api().store_dat_using_responder_forcing_set_id(
850 k.clone(), val(VLEN, seed), Uid::default(), None, resp.clone(), legacy));
851 res!(resp.recv_store_ack());
852 if seed == 1 {
853 first = res!(census(&db)).files;
854 }
855 res!(filler(&db, tag, 40, seed));
856 }
857 let old = res!(res!(head(&db, &k)).ok_or_else(|| err!("No legacy head."; Test, Missing)));
858 let c0 = res!(census(&db));
859 assert_eq!(first.len(), 1, "the first legacy value is not in one file: {:?}", first);
860 assert!(c0.files.is_disjoint(&first), "the two legacy values share a file");
861 let f1 = first.iter().next().copied();
862 res!(db.insert(k.clone(), val(VLEN, 3), Uid::default(), None));
863 res!(filler(&db, "c", 40, 3));
864 hooks::set_collect_fails_for(f1);
865 quiet_held(&db);
866 let held = census(&db);
867 res!(db.close());
868 let db = res!(start_gc(dir, false));
869 let restarted = census(&db);
870 let report = sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait());
871 let gone = db.api().fetch_chunks(&Dat::Tup5u64(old), None).is_err();
872 hooks::set_collect_fails_for(None);
873 let rep = quiet(&db);
874 let fin = census(&db);
875 let ok = reads(&db, &k, &val(VLEN, 3));
876 res!(db.close());
877 let _ = std::fs::remove_dir_all(dir);
878 let (held, restarted, fin, report) = (res!(held), res!(restarted), res!(fin), res!(report));
879 res!(rep);
880 assert!(res!(ok), "the value does not read back");
881 assert!(held.tombs.len() >= c0.chunks, "a legacy tombstone dropped with an older record on disk: {}",
882 held.tombs.len());
883 assert!(restarted.tombs.len() >= c0.chunks, "a legacy tombstone dropped at the restart: {}",
884 restarted.tombs.len());
885 assert_eq!(report.orphans_found, 0, "a retired legacy chunk came back");
886 assert!(gone, "the legacy value reads back after its retirement");
887 assert_eq!(fin.chunks, c0.chunks, "tombstones after the hold: {}", fin.tombs.len());
888 Ok(())
889}
890
891// A record reported gone before the replay ends drops no tombstone, since an older record of its
892// key may lie in a file not yet replayed. Driven on a cache directly: the start sequence lets no
893// collection begin before the zone bot sends `ReplayDone`, so no store can reach this order.
894#[test]
895fn no_tombstone_drops_before_replay_done() -> Outcome<()> {
896 type C = Cache<{ UID_LEN }, Uid>;
897 let mut cache = C::new(None);
898 let k = res!(Dat::Tup5u64([1, 2, 3, 4, 5]).to_bytes(Vec::new()));
899 assert!(is_chunk_key(&k), "the key is not a chunk key");
900 let meta = |s: u64| Meta::<{ UID_LEN }, Uid> { time: Timestamp::new(s, 0), user: Uid::default() };
901 let floc = |f: u32| FileLocation { fnum: f, start: 0, klen: k.len() as u64, vlen: 10 };
902 let put = |cache: &mut C, cind: Option<usize>, f: u32, s: u64| -> Outcome<()> {
903 let sup = res!(cache.insert(k.clone(), None, cind, floc(f), meta(s)));
904 cache.note(&k, cind, &floc(f), &meta(s), &sup);
905 Ok(())
906 };
907 let current = |cache: &C| match cache.map().get(&k) {
908 Some(CacheEntry::LocatedValue(mloc, _)) => Some(mloc.meta().time.secs()),
909 _ => None,
910 };
911 // The replay meets a chunk record in file 1, then the tombstone in file 3; file 2 waits.
912 res!(put(&mut cache, Some(0), 1, 1));
913 res!(put(&mut cache, None, 3, 3));
914 let early = res!(cache.records_gone(1, &[res!(RecordDigest::new(&k, &meta(1)))]));
915 assert!(early.is_empty(), "a tombstone dropped before the replay ended: {:?}", early);
916 // File 2's record of the key arrives late, and must not become current.
917 res!(put(&mut cache, Some(0), 2, 2));
918 assert_eq!(current(&cache), Some(3), "a record older than the tombstone became current");
919 let at_end = res!(cache.replay_done());
920 assert!(at_end.is_empty(), "the tombstone dropped with file 2's record unaccounted for");
921 let last = res!(cache.records_gone(2, &[res!(RecordDigest::new(&k, &meta(2)))]));
922 assert_eq!(last.len(), 1, "the tombstone did not drop once its last older record went");
923 assert_eq!(current(&cache), None, "the dropped tombstone is still in the cache");
924 Ok(())
925}
926
927// Run 2 (2026-10-10): a store retires the set of the head it displaced, whatever order
928// overlapping puts and deletes land in, so live chunk records are bounded by the current heads
929// with no sweep.
930
931// Live chunk records, tombstones aside.
932fn live_chunks(db: &TestDb) -> Outcome<usize> {
933 res!(db.api().settle_for_test(Duration::from_secs(20)));
934 let mut n = 0;
935 for (_, cache) in res!(db.api().collect_caches(scan_wait())) {
936 let tombs = cache.tomb_tracker().tombs();
937 for (k, e) in cache.map() {
938 if is_chunk_key(k) && matches!(e, CacheEntry::LocatedValue(..)) && !tombs.contains_key(k) {
939 n += 1;
940 }
941 }
942 }
943 Ok(n)
944}
945
946// Chunks the current heads name.
947fn named(db: &TestDb, keys: &[Dat]) -> Outcome<usize> {
948 let mut n = 0;
949 for k in keys {
950 if let Some(t) = res!(head(db, k)) {
951 n += t[3] as usize;
952 }
953 }
954 Ok(n)
955}
956
957fn whole_or_none(r: &Outcome<Option<(Dat, Meta<{ UID_LEN }, Uid>)>>) -> bool {
958 match r {
959 Ok(None) => true,
960 Ok(Some((Dat::Usr(..), _))) => true, // a tombstone
961 Ok(Some((v, _))) => whole(v, &[VLEN]),
962 Err(_) => false,
963 }
964}
965
966// Puts B, A and C and a delete of each key, staggered and slowed so that they overlap.
967fn overlap_round(db: &TestDb, keys: &[Dat], r: usize, bad: &mut Vec<String>) -> Outcome<()> {
968 hooks::set_chunk_insert_delay(Duration::from_millis(40));
969 let mut ops = Vec::new();
970 for (i, k) in keys.iter().enumerate() {
971 // The delete lands third in even rounds and last in odd ones.
972 let del_at = if r % 2 == 0 { 30 } else { 45 };
973 for (j, (at, put)) in [(0u64, true), (15, true), (del_at, false), (75 - del_at, true)].into_iter().enumerate() {
974 let (db, k) = (db.clone(), k.clone());
975 let seed = ((r * 16 + i * 4 + j) % 251) as u8;
976 ops.push(thread::spawn(move || {
977 thread::sleep(Duration::from_millis(at));
978 let res = match put {
979 true => db.insert(k, val(VLEN, seed), Uid::default(), None).map(|_| ()),
980 false => db.delete(&k, Uid::default(), None).map(|_| ()),
981 };
982 res.map_err(|e| fmt!("{}", e))
983 }));
984 }
985 }
986 for _ in 0..4 {
987 for k in keys {
988 let got = db.get(k, None);
989 if !whole_or_none(&got) {
990 bad.push(fmt!("round {}: a read mid-round was not whole: {:?}", r,
991 got.map(|o| o.map(|(d, _)| fmt!("{:?}", d).chars().take(80).collect::<String>()))));
992 }
993 }
994 pause(Duration::from_millis(20));
995 }
996 for op in ops {
997 match op.join() {
998 Ok(Ok(())) => (),
999 Ok(Err(e)) => bad.push(fmt!("round {}: {}", r, e.chars().take(200).collect::<String>())),
1000 Err(_) => bad.push(fmt!("round {}: an op panicked", r)),
1001 }
1002 }
1003 hooks::set_chunk_insert_delay(Duration::ZERO);
1004 for k in keys {
1005 let got = db.get(k, None);
1006 if !whole_or_none(&got) {
1007 bad.push(fmt!("round {}: {:?} does not read whole after the round", r, k));
1008 }
1009 }
1010 Ok(())
1011}
1012
1013#[test]
1014fn overlapping_writes_leave_no_live_orphan() -> Outcome<()> {
1015 let _lock = lock();
1016 let dir = "./test_db_chunk_set_overlap_bound";
1017 let keys: Vec<Dat> = (0..3).map(|i| dat!(fmt!("overlap bound {}", i))).collect();
1018 let mut bad = Vec::new();
1019 let db = res!(start_gc(dir, true));
1020 for r in 0..20 {
1021 res!(overlap_round(&db, &keys, r, &mut bad));
1022 }
1023 // A round can end in a delete, so every key is given a value before the restart: each then
1024 // replays a head whose bytes are not resident, which the first write over it displaces.
1025 for (i, k) in keys.iter().enumerate() {
1026 res!(db.insert(k.clone(), val(VLEN, 0xA0 + i as u8), Uid::default(), None));
1027 }
1028 res!(quiet(&db));
1029 let (live1, named1) = (res!(live_chunks(&db)), res!(named(&db, &keys)));
1030 res!(db.close());
1031 let db = res!(start_gc(dir, false));
1032 for r in 20..22 {
1033 res!(overlap_round(&db, &keys, r, &mut bad));
1034 }
1035 res!(quiet(&db));
1036 let (live2, named2) = (res!(live_chunks(&db)), res!(named(&db, &keys)));
1037 res!(db.close());
1038 let _ = std::fs::remove_dir_all(dir);
1039 msg!("overlap bound: live {} named {}, after a restart live {} named {}", live1, named1, live2, named2);
1040 assert!(bad.is_empty(), "{:#?}", bad);
1041 assert_eq!(live1, named1, "live chunk records against those the current heads name");
1042 assert_eq!(live2, named2, "after a restart, live chunk records against those the current heads name");
1043 Ok(())
1044}
1045
1046// Two forced values under one legacy set, of one length: the second overwrites the first's
1047// chunks in place, and its head must retire nothing, or it retires its own chunks.
1048#[test]
1049fn forced_legacy_overwrite_retires_nothing() -> Outcome<()> {
1050 let _lock = lock();
1051 let dir = "./test_db_chunk_set_forced_legacy";
1052 let db = res!(start(dir, res!(cfg()), true));
1053 let k = dat!("forced legacy");
1054 let (kbuf, _, _) = res!(db.api().ozone_key_dat(&k, None));
1055 let legacy = OzoneApi::<{ UID_LEN }, Uid, (), HashScheme, HashScheme, ChecksumScheme>::legacy_chunk_set_id(&kbuf);
1056 for seed in [1u8, 2] {
1057 let resp = db.api().responder();
1058 res!(db.api().store_dat_using_responder_forcing_set_id(
1059 k.clone(), val(1_100, seed), Uid::default(), None, resp.clone(), legacy));
1060 res!(resp.recv_store_ack());
1061 }
1062 let live = res!(db.get(&k, None)).map(|(v, _)| v);
1063 res!(db.close());
1064 let db = res!(start(dir, res!(cfg()), false));
1065 let after = res!(db.get(&k, None)).map(|(v, _)| v);
1066 let v = res!(verify::verify_live_set(db.api(), None, scan_wait()));
1067 res!(db.close());
1068 let _ = std::fs::remove_dir_all(dir);
1069 assert!(live == Some(val(1_100, 2)), "the second forced value does not read whole live");
1070 assert!(after == Some(val(1_100, 2)), "the second forced value does not read whole after a restart");
1071 assert!(v.clean(), "verify after restart: {}", v.summary(4));
1072 Ok(())
1073}