Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/chunk_delete.rs

9.4 KiB, 37 runs

created by r1870400018:67595, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! A delete of a chunked value waits for the tombstones of its chunks and fails if one fails
2//! (A3 R2, 2026-10-08). The chunk tombstones were sent with no responder, so `delete` returned on
3//! the bunch key's answer alone, with the chunks possibly unwritten and a failure only logged.
4//! Since fix A (2026-10-09) a delete is a store with no chunks: the chunks are retired only once
5//! the head tombstone is durable, and a chunk tombstone that fails after that is left to the
6//! orphan sweep rather than failing the delete. Since the durable mark (2026-10-10) the cache bot
7//! sends the chunk tombstones once a barrier covers the head tombstone, and the delete does not
8//! wait for them.
9//! The hooks used are process-wide, so the tests here take a lock and put each back.
10
11mod gc_pair;
12
13use gc_pair::*;
14
15use oxedyne_fe2o3_core::prelude::*;
16use oxedyne_fe2o3_iop_db::api::Database;
17use oxedyne_fe2o3_jdat::{
18 prelude::*,
19 chunk::PartKey,
20};
21use oxedyne_fe2o3_o3db_sync::{
22 base::cfg::OzoneConfig,
23 test::{
24 hooks,
25 setup,
26 },
27};
28
29use std::{
30 collections::BTreeMap,
31 path::Path,
32 sync::Mutex,
33 time::{
34 Duration,
35 Instant,
36 },
37};
38
39static HOOKS: Mutex<()> = Mutex::new(());
40
41const CHUNKS: u64 = 3;
42
43fn cfg() -> Outcome<OzoneConfig> {
44 let mut c = res!(setup::default_cfg());
45 c.num_zones = 1;
46 c.num_cbots_per_zone = 4;
47 c.num_fbots_per_zone = 1;
48 c.num_wbots_per_zone = 2;
49 c.zone_overrides = BTreeMap::new();
50 c.rest_chunk_threshold = 1_000;
51 c.rest_chunk_bytes = 400;
52 c.sync_on_write = true;
53 Ok(c)
54}
55
56fn big(seed: u8) -> Dat {
57 Dat::BU32((0..1_100usize).map(|j| seed ^ (j as u8)).collect())
58}
59
60fn open(dir: &str) -> Outcome<TestDb> {
61 let _ = std::fs::remove_dir_all(dir);
62 res!(std::fs::create_dir_all(dir));
63 let root = res!(Path::new(dir).canonicalize());
64 setup::start_db(root, Some(res!(cfg())), schemes(), None, false, true)
65}
66
67/// The chunk keys of the value held under `k`, read from its bunch key.
68fn chunk_keys(db: &TestDb, k: &Dat) -> Outcome<Vec<Dat>> {
69 let resp = res!(db.api().fetch_using_schemes(k, None));
70 let pkey = match res!(resp.recv_daticle(db.api().schemes().encrypter(), None)) {
71 (Some((Dat::Tup5u64(tup), _)), _) => PartKey(tup),
72 other => return Err(err!("Key {:?} holds {:?}, not a bunch key.", k, other; Test, Invalid)),
73 };
74 assert_eq!(pkey.num_parts(), CHUNKS, "the value did not split into {} chunks", CHUNKS);
75 Ok((1..(pkey.num_parts() + 1)).map(|i| Dat::Tup5u64([
76 pkey.set_id(), i, pkey.data_len(), pkey.num_parts(), pkey.part_size(),
77 ])).collect())
78}
79
80/// Does the newest record at this chunk key retire the chunk?
81fn retired(db: &TestDb, ck: &Dat) -> Outcome<bool> {
82 let resp = res!(db.api().fetch_using_schemes(ck, None));
83 let (found, _) = res!(resp.recv_daticle(db.api().schemes().encrypter(), None));
84 Ok(found.is_none())
85}
86
87fn cbot_of(db: &TestDb, k: &Dat) -> Outcome<usize> {
88 let (_, wind, _) = res!(db.api().ozone_key_dat(k, None));
89 Ok(**wind.bpind())
90}
91
92#[test]
93fn chunked_delete_retires_its_chunks_after_it_returns() -> Outcome<()> {
94 log_set_level!("error");
95 let _lock = match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() };
96 let db = res!(open("./test_db_chunk_delete_wait"));
97 let user = setup::Uid::default();
98
99 // A key whose bunch key shares no cache bot with any of its chunks, so that the bunch key's
100 // answer is not queued behind the held chunk tombstones.
101 let mut found = None;
102 for n in 0..200 {
103 let k = dat!(fmt!("chunk delete wait {:03}", n));
104 res!(db.insert(k.clone(), big(n as u8), user, None));
105 let cks = res!(chunk_keys(&db, &k));
106 let bunch = res!(cbot_of(&db, &k));
107 let mut apart = true;
108 for ck in &cks {
109 if res!(cbot_of(&db, ck)) == bunch { apart = false; }
110 }
111 if apart { found = Some((k, cks)); break; }
112 res!(db.delete(&k, user, None));
113 }
114 let (k, cks) = match found {
115 Some(f) => f,
116 None => return Err(err!("No key kept its bunch key apart from its chunks."; Test, Missing)),
117 };
118 for ck in &cks {
119 assert!(!res!(retired(&db, ck)), "positive control: chunk {:?} is not live before the delete", ck);
120 }
121
122 hooks::set_chunk_tombstone_delay(Duration::from_millis(300));
123 let t0 = Instant::now();
124 let existed = res!(db.delete(&k, user, None));
125 let took = t0.elapsed();
126 assert!(existed, "the delete did not find the value");
127 msg!("The delete returned after {:?}.", took);
128 // The head tombstone is answered once it is entered; its chunks' tombstones follow it from
129 // the cache bot, after its barrier, and no one waits on them.
130 assert!(took < Duration::from_millis(300), "delete waited {:?} for its chunk tombstones", took);
131 let gone = wait_until(Duration::from_secs(10), || cks.iter().all(|ck| matches!(retired(&db, ck), Ok(true))));
132 hooks::set_chunk_tombstone_delay(Duration::ZERO);
133 assert!(gone, "the chunks of the deleted value were never retired");
134 res!(db.close());
135 Ok(())
136}
137
138// A delete whose head tombstone is not confirmed retires nothing: the value's chunks stay live,
139// so a bunch key that did not land still reads whole. Every sync fails, so no tombstone is
140// confirmed; the old order sent the chunks' beside the head's and took them all.
141#[test]
142fn head_tombstone_failure_retires_nothing() -> Outcome<()> {
143 log_set_level!("error");
144 let _lock = match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() };
145 let db = res!(open("./test_db_chunk_delete_fail"));
146 let user = setup::Uid::default();
147 let k = dat!("chunk delete fail");
148 res!(db.insert(k.clone(), big(9), user, None));
149 let cks = res!(chunk_keys(&db, &k));
150
151 hooks::set_barrier_failure(true);
152 let result = db.delete(&k, user, None);
153 hooks::set_barrier_failure(false);
154 if let Ok(b) = result {
155 return Err(err!("A delete whose head tombstone failed returned Ok({}).", b; Test, Invalid));
156 }
157 for (i, ck) in cks.iter().enumerate() {
158 assert!(!res!(retired(&db, ck)), "chunk {} was retired although the head tombstone failed", i + 1);
159 }
160 res!(db.close());
161 Ok(())
162}
163
164// A delete removes what existed when it began. Its tombstones are stamped once, at the start:
165// stamped as each is sent, an overwrite of the same length begun while the chunk tombstones are
166// becoming durable has chunks newer than theirs and a bunch key older than the bunch tombstone,
167// so the delete takes the value and leaves its chunks live for the orphan sweep (QA A2-6).
168#[test]
169fn delete_does_not_erase_a_later_overwrite() -> Outcome<()> {
170 log_set_level!("error");
171 let _lock = match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() };
172 let db = res!(open("./test_db_chunk_delete_race"));
173 let user = setup::Uid::default();
174 let k = dat!("chunk delete race");
175 res!(db.insert(k.clone(), big(1), user, None));
176 let _ = res!(chunk_keys(&db, &k));
177
178 hooks::set_barrier_delay(Duration::from_millis(1_000));
179 let del = {
180 let (db, k) = (db.clone(), k.clone());
181 std::thread::spawn(move || db.delete(&k, user, None))
182 };
183 std::thread::sleep(Duration::from_millis(300));
184 let put = {
185 let (db, k) = (db.clone(), k.clone());
186 std::thread::spawn(move || db.insert(k, big(2), user, None).map(|_| ()))
187 };
188 let del = del.join();
189 let put = put.join();
190 hooks::set_barrier_delay(Duration::ZERO);
191 match (del, put) {
192 (Ok(a), Ok(b)) => { res!(a); res!(b); },
193 _ => return Err(err!("A thread panicked."; Test, Invalid)),
194 }
195 match db.get(&k, None) {
196 Ok(Some((v, _))) => assert_eq!(v, big(2), "the overwrite reads back changed"),
197 other => {
198 let t: String = fmt!("{:?}", other).chars().take(300).collect();
199 panic!("a value begun after the delete began does not read back whole: {}", t);
200 },
201 }
202 res!(db.close());
203 Ok(())
204}
205
206// A chunked delete sends its head tombstone, and the cache bot sends the tombstones of the chunks only
207// once a barrier covers it: sent together, the chunks' could land first and a reader find a chunk
208// gone under a current bunch key (fix A). Counted from the order of the steps, not timed.
209#[test]
210fn chunked_delete_retires_after_its_head_is_durable() -> Outcome<()> {
211 log_set_level!("error");
212 let _lock = match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() };
213 let db = res!(open("./test_db_chunk_delete_round"));
214 let user = setup::Uid::default();
215 let k = dat!("chunk delete round");
216 res!(db.insert(k.clone(), big(3), user, None));
217 let cks = res!(chunk_keys(&db, &k));
218
219 hooks::set_trace(true);
220 let existed = res!(db.delete(&k, user, None));
221 let all = wait_until(Duration::from_secs(10),
222 || hooks::trace_steps().iter().filter(|s| **s == hooks::Step::Sent).count() > cks.len());
223 let steps = hooks::trace_steps();
224 hooks::set_trace(false);
225 assert!(existed, "the delete did not find the value");
226 assert!(all, "expected a tombstone for each chunk and the bunch key: {:?}", steps);
227 let mut want = vec![hooks::Step::Sent, hooks::Step::Durable];
228 want.extend(cks.iter().map(|_| hooks::Step::Sent));
229 assert_eq!(steps, want, "the chunks were not retired after the head tombstone was durable");
230 let gone = wait_until(Duration::from_secs(10), || cks.iter().all(|ck| matches!(retired(&db, ck), Ok(true))));
231 assert!(gone, "a chunk of the deleted value is still live");
232 res!(db.close());
233 Ok(())
234}