Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/chunk_hold.rs

14.4 KiB, 30 runs

created by r1870400018:70271, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! A store's chunk set stays held until its head is in the cache, or for the life of the process
2//! once the head is in the files but could not be confirmed (run 2, 2026-10-10). The hooks are
3//! process-wide, so every test takes one lock.
4
5mod gc_pair;
6
7use gc_pair::*;
8
9use oxedyne_fe2o3_core::prelude::*;
10use oxedyne_fe2o3_data::time::Timestamp;
11use oxedyne_fe2o3_iop_db::api::{
12 Database,
13 Meta,
14};
15use oxedyne_fe2o3_jdat::prelude::*;
16use oxedyne_fe2o3_o3db_sync::{
17 base::{
18 cfg::OzoneConfig,
19 constant,
20 },
21 comm::response::Wait,
22 data::cache::{
23 Cache,
24 Prior,
25 Superseded,
26 ValueOrLocation,
27 },
28 file::floc::FileLocation,
29 sweep,
30 test::{
31 hooks,
32 setup::{
33 self,
34 Uid,
35 UID_LEN,
36 },
37 },
38};
39
40use std::{
41 collections::BTreeMap,
42 path::Path,
43 sync::{
44 Mutex,
45 MutexGuard,
46 },
47 thread,
48 time::{
49 Duration,
50 Instant,
51 },
52};
53
54static HOOKS: Mutex<()> = Mutex::new(());
55
56fn lock() -> MutexGuard<'static, ()> {
57 match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() }
58}
59
60fn cfg() -> Outcome<OzoneConfig> {
61 let mut c = res!(setup::default_cfg());
62 c.num_zones = 1;
63 c.num_cbots_per_zone = 4;
64 c.num_fbots_per_zone = 1;
65 c.num_wbots_per_zone = 2;
66 c.zone_overrides = BTreeMap::new();
67 c.rest_chunk_threshold = 1_000;
68 c.rest_chunk_bytes = 400;
69 c.sync_on_write = true;
70 Ok(c)
71}
72
73fn val(len: usize, seed: u8) -> Dat {
74 Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect())
75}
76
77fn start(dir: &str, wipe: bool) -> Outcome<TestDb> {
78 log_set_level!("error");
79 if wipe {
80 let _ = std::fs::remove_dir_all(dir);
81 }
82 res!(std::fs::create_dir_all(dir));
83 let root = res!(Path::new(dir).canonicalize());
84 setup::start_db(root, Some(res!(cfg())), schemes(), None, false, wipe)
85}
86
87fn scan_wait() -> Wait {
88 Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL }
89}
90
91// "1100/<seed>" for a whole value, "none", "TORN" or "ERR:<text>".
92fn describe(r: &Outcome<Option<(Dat, Meta<{ UID_LEN }, Uid>)>>) -> String {
93 match r {
94 Ok(None) => fmt!("none"),
95 Ok(Some((Dat::BU32(b), _))) if !b.is_empty()
96 && b.iter().enumerate().all(|(j, x)| *x == b[0] ^ (j as u8)) => fmt!("{}/{}", b.len(), b[0]),
97 Ok(Some(_)) => fmt!("TORN"),
98 Err(e) => fmt!("ERR:{}", fmt!("{}", e).chars().take(160).collect::<String>()),
99 }
100}
101
102fn pending_empty(db: &TestDb) -> Outcome<bool> {
103 let held = lock_mutex!(db.api().chans().pending_sets());
104 Ok(held.is_empty())
105}
106
107// A put whose head failed after its record reached the files: the store reports failure, the
108// sweep spares the put's chunks, and after a restart the key reads whole.
109fn failed_head_keeps_hold(dir: &str, arm: fn(bool)) -> Outcome<Vec<String>> {
110 let mut bad = Vec::new();
111 let k = dat!("failed head");
112 let db = res!(start(dir, true));
113 res!(db.insert(k.clone(), val(1_100, 1), Uid::default(), None));
114 hooks::set_durability_timeout(Some(Duration::from_secs(2)));
115 arm(true);
116 let put = db.insert(k.clone(), val(1_100, 2), Uid::default(), None);
117 arm(false);
118 hooks::set_durability_timeout(None);
119 if put.is_ok() { bad.push(fmt!("the put whose head failed was confirmed")); }
120 let live = describe(&db.get(&k, None));
121 if live != "1100/1" { bad.push(fmt!("live read {}", live)); }
122 let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait()));
123 if report.orphans_found > 0 { bad.push(fmt!("the sweep took {} chunks of the held set", report.orphans_found)); }
124 if report.skipped_pending == 0 { bad.push(fmt!("the sweep saw none of the put's chunks, so it proved nothing")); }
125 res!(db.close());
126 let db = res!(start(dir, false));
127 let after = describe(&db.get(&k, None));
128 if !(after == "1100/1" || after == "1100/2") { bad.push(fmt!("restart read {}", after)); }
129 res!(db.close());
130 let _ = std::fs::remove_dir_all(dir);
131 msg!("failed head: put {:?} live {} sweep {}/{} restart {}", put.map(|_| ()).map_err(|e| e.tags().to_vec()),
132 live, report.orphans_found, report.skipped_pending, after);
133 Ok(bad)
134}
135
136// Change 3: the head is in the data file, but the writer cannot hand it on.
137#[test]
138fn writer_failure_after_append_keeps_hold() -> Outcome<()> {
139 let _lock = lock();
140 let bad = res!(failed_head_keeps_hold("./test_db_chunk_hold_writer", hooks::set_head_hand_failure));
141 assert!(bad.is_empty(), "{:#?}", bad);
142 Ok(())
143}
144
145// Change 4: the head is durable, but the cache bot cannot enter it.
146#[test]
147fn cbot_insert_failure_keeps_hold() -> Outcome<()> {
148 let _lock = lock();
149 let bad = res!(failed_head_keeps_hold("./test_db_chunk_hold_cbot", hooks::set_head_insert_failure));
150 assert!(bad.is_empty(), "{:#?}", bad);
151 Ok(())
152}
153
154// Q1-4: a head that times out at the store keeps its set held until it lands, so a sweep in the
155// meantime takes none of its chunks, and the key then reads whole. Staged on the head hold, where
156// the store has its chunks in and is about to send its head, so the barrier delay catches the head
157// and never a chunk; every wait is on a condition against a deadline, never a fixed sleep (the
158// sleeps it was staged with failed it under disk load, 2026-10-10).
159#[test]
160fn timed_out_head_keeps_its_set_until_it_lands() -> Outcome<()> {
161 let _lock = lock();
162 let dir = "./test_db_chunk_hold_timeout";
163 let db = res!(start(dir, true));
164 let mut bad = Vec::new();
165 let k = dat!("timed out head");
166 res!(db.insert(k.clone(), val(1_100, 1), Uid::default(), None));
167 hooks::set_durability_timeout(Some(Duration::from_secs(2)));
168 let held = hooks::heads_held();
169 hooks::set_head_hold(true);
170 let put = { let (db, k) = (db.clone(), k.clone());
171 thread::spawn(move || db.insert(k, val(1_100, 2), Uid::default(), None).map(|_| ())) };
172 let reached = wait_until(Duration::from_secs(30), || Ok(hooks::heads_held() > held));
173 hooks::set_barrier_delay(Duration::from_secs(5));
174 hooks::set_head_hold(false);
175 let put = match put.join() { Ok(r) => r, Err(_) => Err(err!("The put panicked."; Test)) };
176 let staged = match (res!(reached), put) {
177 (false, _) => { bad.push(fmt!("the store never reached the head hold")); false },
178 (true, Ok(())) => { bad.push(fmt!("the timed-out put was confirmed")); false },
179 (true, Err(e)) if !e.tags().contains(&ErrTag::Unconfirmed) => {
180 bad.push(fmt!("the head was not written before the put timed out: {}", e)); false },
181 (true, Err(_)) => true,
182 };
183 if staged {
184 let report = res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait()));
185 if report.orphans_found > 0 { bad.push(fmt!("the sweep took {} chunks of a head in flight", report.orphans_found)); }
186 if report.skipped_pending == 0 { bad.push(fmt!("the sweep saw none of the put's chunks, so it proved nothing")); }
187 }
188 hooks::set_barrier_delay(Duration::ZERO);
189 hooks::set_durability_timeout(None);
190 if staged {
191 let landed = res!(wait_until(Duration::from_secs(20), || Ok(describe(&db.get(&k, None)) == "1100/2")));
192 if !landed { bad.push(fmt!("the head did not land whole: {}", describe(&db.get(&k, None)))); }
193 // The cache bot lets the set go once the head it carries is in; wait on that, not a clock.
194 if !res!(wait_until(Duration::from_secs(20), || pending_empty(&db))) {
195 bad.push(fmt!("a set is still held after the head landed"));
196 }
197 }
198 res!(db.close());
199 if staged {
200 let db = res!(start(dir, false));
201 let after = describe(&db.get(&k, None));
202 if after != "1100/2" { bad.push(fmt!("restart read {}", after)); }
203 res!(db.close());
204 }
205 let _ = std::fs::remove_dir_all(dir);
206 assert!(bad.is_empty(), "{:#?}", bad);
207 Ok(())
208}
209
210// Polls `cond` until it holds or `limit` passes.
211fn wait_until(limit: Duration, mut cond: impl FnMut() -> Outcome<bool>) -> Outcome<bool> {
212 let deadline = Instant::now() + limit;
213 loop {
214 if res!(cond()) {
215 return Ok(true);
216 }
217 if Instant::now() >= deadline {
218 return Ok(false);
219 }
220 thread::sleep(Duration::from_millis(10));
221 }
222}
223
224// ---------------------------------------------------------------------------------------------
225// The cache, driven directly.
226
227type C = Cache<{ UID_LEN }, Uid>;
228
229fn meta(s: u64) -> Meta<{ UID_LEN }, Uid> {
230 Meta { time: Timestamp::new(s, 0), user: Uid::default() }
231}
232
233fn floc(f: u32) -> FileLocation {
234 FileLocation { fnum: f, start: 0, klen: 10, vlen: 10 }
235}
236
237// Change 8: the write tracker returns to where it began after a head, a small value, a
238// tombstone and a head overwrite one key, and a head never enters it.
239#[test]
240fn write_tracker_follows_head_transitions() -> Outcome<()> {
241 let mut cache = C::new(None);
242 cache.set_lim(1 << 30);
243 let k = b"tracked key".to_vec();
244 let t0 = cache.tracker_size();
245 let size0 = cache.get_size();
246 let steps: [(Option<usize>, &[u8], bool); 6] = [
247 (Some(0), b"head one", false), // a head enters no tracker entry
248 (None, b"small", true), // head to non-head with bytes
249 (None, b"tombstone", true), // non-head to non-head
250 (Some(0), b"head two", false), // non-head to head
251 (Some(0), b"head three", false), // head to head
252 (None, b"small again", true),
253 ];
254 for (i, (cind, v, tracked)) in steps.iter().enumerate() {
255 let s = i as u64 + 1;
256 let sup = res!(cache.insert(k.clone(), Some(v.to_vec()), *cind, floc(s as u32), meta(s)));
257 let grown = cache.tracker_size() > t0;
258 assert_eq!(grown, *tracked, "step {}: the tracker is {} from {}", i, cache.tracker_size(), t0);
259 if i == 4 {
260 match sup {
261 Superseded::Cached { prior: Prior::Head(b), .. } =>
262 assert_eq!(b, b"head two".to_vec(), "the displaced head's bytes"),
263 other => panic!("a head over a head reported {:?}", other),
264 }
265 }
266 }
267 assert!(res!(cache.remove(&k, &meta(6))), "the entry was not removed");
268 assert_eq!(cache.tracker_size(), t0, "the tracker did not return to its start");
269 assert_eq!(cache.get_size(), size0, "the cache size did not return to its start");
270 Ok(())
271}
272
273// A newer record that arrives with no value, as a replayed one does, takes the old value out with
274// the old location: a reader is sent to the new record, never given the old bytes under the new
275// record's metadata. An older record with no value leaves the cache as it was.
276#[test]
277fn record_without_value_drops_the_old_value() -> Outcome<()> {
278 for cind in [None, Some(0)] {
279 let mut cache = C::new(None);
280 cache.set_lim(1 << 30);
281 let k = b"replayed key".to_vec();
282 let t0 = cache.tracker_size();
283 res!(cache.insert(k.clone(), Some(b"old value".to_vec()), cind, floc(1), meta(2)));
284 // An older record, offered late: it loses, and the cached value stands.
285 match res!(cache.insert(k.clone(), None, cind, floc(0), meta(1))) {
286 Superseded::Offered { .. } => (),
287 other => panic!("{:?}: an older record reported {:?}", cind, other),
288 }
289 match res!(cache.get(&k)) {
290 Some(ValueOrLocation::Value(v, m)) => {
291 assert_eq!(v, b"old value".to_vec(), "{:?}: the cached value changed", cind);
292 assert_eq!(m, meta(2), "{:?}: the cached metadata changed", cind);
293 },
294 other => panic!("{:?}: after an older record the cache gave {:?}", cind, other),
295 }
296 // A newer record with no value.
297 let sup = res!(cache.insert(k.clone(), None, cind, floc(2), meta(3)));
298 match (&sup, cind) {
299 (Superseded::Cached { prior: Prior::Head(b), .. }, Some(0)) =>
300 assert_eq!(b, &b"old value".to_vec(), "the displaced head's bytes"),
301 (Superseded::Cached { prior: Prior::NotHead, .. }, None) => (),
302 other => panic!("a newer record reported {:?}", other),
303 }
304 match res!(cache.get(&k)) {
305 Some(ValueOrLocation::Location(mloc)) => {
306 assert_eq!(*mloc.file_location(), floc(2), "{:?}: the reader is not sent to the new record", cind);
307 assert_eq!(*mloc.meta(), meta(3), "{:?}: the new record's metadata", cind);
308 },
309 other => panic!("{:?}: a newer record without a value left {:?}", cind, other),
310 }
311 assert_eq!(cache.tracker_size(), t0, "{:?}: the old value is still tracked", cind);
312 }
313 Ok(())
314}
315
316// The pinned-heads gauge follows every transition of a key: a head pins its bytes, a head over a
317// head re-pins at the new size, a head displaced by a non-head or removed unpins, and an older
318// head offered late pins nothing.
319#[test]
320fn pinned_heads_follow_head_transitions() -> Outcome<()> {
321 let mut cache = C::new(None);
322 cache.set_lim(1 << 30);
323 let k = b"pinned key".to_vec();
324 let none = cache.pinned_heads();
325 assert_eq!((none.count, none.bytes), (0, 0), "a new cache pins nothing");
326 res!(cache.insert(k.clone(), Some(vec![1u8; 4]), Some(0), floc(2), meta(2)));
327 let one = cache.pinned_heads();
328 assert_eq!(one.count, 1, "a head pinned {:?}", one);
329 assert!(one.bytes >= 4, "a head of 4 bytes pinned {:?}", one);
330 // An older head, offered late, loses and pins nothing.
331 res!(cache.insert(k.clone(), Some(vec![9u8; 64]), Some(0), floc(1), meta(1)));
332 assert_eq!(cache.pinned_heads(), one, "an older head offered late changed the pins");
333 // A head over a head: still one pinned, at the new size.
334 res!(cache.insert(k.clone(), Some(vec![2u8; 12]), Some(0), floc(3), meta(3)));
335 let bigger = cache.pinned_heads();
336 assert_eq!(bigger.count, 1, "a head over a head pinned {:?}", bigger);
337 assert_eq!(bigger.bytes, one.bytes + 8, "a head of 12 bytes over one of 4 pinned {:?}", bigger);
338 // Displaced by a non-head: nothing pinned.
339 res!(cache.insert(k.clone(), Some(b"small".to_vec()), None, floc(4), meta(4)));
340 assert_eq!(cache.pinned_heads(), none, "a head displaced by a non-head stayed pinned");
341 // A non-head displaced by a head pins again, and a remove unpins it.
342 res!(cache.insert(k.clone(), Some(vec![3u8; 4]), Some(0), floc(5), meta(5)));
343 assert_eq!(cache.pinned_heads(), one, "a head over a non-head pinned");
344 assert!(res!(cache.remove(&k, &meta(5))), "the entry was not removed");
345 assert_eq!(cache.pinned_heads(), none, "a removed head stayed pinned");
346 Ok(())
347}