Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/chunk_surplus.rs

12.4 KiB, 11 runs

created by r1870400018:68517, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! An overwrite that changes a chunked value's geometry retires the chunks it leaves (A3 R4,
2//! 2026-10-08). A chunk is addressed by `(set_id, index, data length, count, size)`, so a value of
3//! another length shares no chunk key with its predecessor: nothing supersedes the old chunks, and
4//! until now they waited for the orphan sweep, which `compact_now` does not run. Each erase test
5//! finds the old chunks in the data files first (the positive control), overwrites, runs
6//! `compact_now`, and counts the bytes of the files from outside the store. The hooks used are
7//! process-wide, so every test takes one lock and puts each hook back.
8
9mod gc_pair;
10
11use gc_pair::*;
12
13use oxedyne_fe2o3_core::prelude::*;
14use oxedyne_fe2o3_iop_db::api::Database;
15use oxedyne_fe2o3_jdat::{
16 prelude::*,
17 chunk::PartKey,
18};
19use oxedyne_fe2o3_o3db_sync::{
20 base::cfg::OzoneConfig,
21 test::{
22 hooks,
23 setup::{
24 self,
25 Uid,
26 },
27 },
28};
29
30use std::{
31 collections::BTreeMap,
32 path::{
33 Path,
34 PathBuf,
35 },
36 sync::{
37 Arc,
38 Mutex,
39 MutexGuard,
40 atomic::{
41 AtomicBool,
42 Ordering,
43 },
44 },
45 thread,
46 time::Duration,
47};
48
49static HOOKS: Mutex<()> = Mutex::new(());
50
51fn lock() -> MutexGuard<'static, ()> {
52 match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() }
53}
54
55const CSIZE: usize = 600; // bytes in a chunk
56const FIVE: usize = 2_900; // a value of five chunks
57const FIVE_B: usize = 2_800; // another value of five
58const TWO: usize = 1_100; // a value of two
59const PLAIN: usize = 500; // below the threshold, so not chunked
60
61fn chunk_cfg() -> Outcome<OzoneConfig> {
62 let mut c = res!(setup::default_cfg());
63 c.num_zones = 1;
64 c.num_cbots_per_zone = 4;
65 c.num_fbots_per_zone = 1;
66 c.num_wbots_per_zone = 2;
67 c.zone_overrides = BTreeMap::new();
68 c.rest_chunk_threshold = 1_000;
69 c.rest_chunk_bytes = CSIZE as u64;
70 c.sync_on_write = true;
71 Ok(c)
72}
73
74// Bytes that do not repeat inside a window, so that a window is found by its own record only.
75fn fill_of(seed: u8, len: usize) -> Vec<u8> {
76 let mut x = 0x9E37_79B9_7F4A_7C15u64 ^ ((seed as u64) << 32 | seed as u64);
77 (0..len).map(|_| {
78 x = x.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
79 (x >> 56) as u8
80 }).collect()
81}
82
83// A hundred bytes from the middle of each chunk, so that a chunk is found by its own bytes. The
84// encoded value is five bytes longer than the bytes in it, so a window of the middle is inside
85// one chunk.
86fn windows(seed: u8, len: usize) -> Vec<Vec<u8>> {
87 let v = fill_of(seed, len);
88 let n = (len + 5 + CSIZE - 1) / CSIZE;
89 (0..n).map(|i| v[i * CSIZE + 250..i * CSIZE + 350].to_vec()).collect()
90}
91
92struct Store {
93 dir: String,
94 root: PathBuf,
95 cfg: OzoneConfig,
96 db: TestDb,
97}
98
99fn store(name: &str) -> Outcome<Store> {
100 log_set_level!("error");
101 let dir = fmt!("./test_db_surplus_{}", name);
102 let _ = std::fs::remove_dir_all(&dir);
103 res!(std::fs::create_dir_all(&dir));
104 let root = res!(Path::new(&dir).canonicalize());
105 let cfg = res!(chunk_cfg());
106 // Collection off, so that only the order to compact collects.
107 let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, false, true));
108 Ok(Store { dir, root, cfg, db })
109}
110
111impl Store {
112 fn holds(&self, seed: u8, len: usize) -> Vec<usize> {
113 windows(seed, len).iter().map(|w| files_holding(&self.root, &self.cfg, w).len()).collect()
114 }
115
116 fn num_chunks(&self, k: &Dat) -> Outcome<u64> {
117 let api = self.db.api();
118 let resp = res!(api.fetch_using_schemes(k, None));
119 match res!(resp.recv_daticle(api.schemes().encrypter(), None)) {
120 (Some((Dat::Tup5u64(tup), _)), _) => Ok(PartKey(tup).num_parts()),
121 other => Err(err!("Key {:?} holds {:?}, not a bunch key.", k, other; Test, Invalid)),
122 }
123 }
124
125 fn put(&self, k: &Dat, seed: u8, len: usize) -> Outcome<()> {
126 let _ = res!(self.db.insert(k.clone(), Dat::BU32(fill_of(seed, len)), Uid::default(), None));
127 Ok(())
128 }
129
130 // Five chunks written and found in the files, with a value beside it that stays.
131 fn five(&self, k: &Dat, seed: u8) -> Outcome<()> {
132 res!(self.db.insert(key(0), value(0, 9), Uid::default(), None));
133 res!(self.put(k, seed, FIVE));
134 assert_eq!(res!(self.num_chunks(k)), 5, "the value did not split into five chunks");
135 res!(self.db.api().settle_for_test(Duration::from_secs(10)));
136 assert!(self.holds(seed, FIVE).iter().all(|n| *n >= 1),
137 "positive control: every chunk is in a data file: {:?}", self.holds(seed, FIVE));
138 Ok(())
139 }
140
141 fn compact_and_check(&self, k: &Dat, old: u8, new: u8, len: usize) -> Outcome<()> {
142 res!(self.db.api().settle_for_test(Duration::from_secs(10)));
143 let report = res!(self.db.compact_now(Duration::from_secs(30)));
144 msg!("{:?}", report);
145 assert_eq!(self.holds(old, FIVE), vec![0; 5], "old chunks still in data files after compact_now");
146 match res!(self.db.get(k, None)) {
147 Some((v, _)) => assert_eq!(v, Dat::BU32(fill_of(new, len)), "the new value reads back changed"),
148 None => return Err(err!("The new value is gone."; Test, Missing)),
149 }
150 match res!(self.db.get(&key(0), None)) {
151 Some((v, _)) => assert_eq!(v, value(0, 9), "the value beside it changed"),
152 None => return Err(err!("The value beside it is gone."; Test, Missing)),
153 }
154 Ok(())
155 }
156
157 fn end(self) -> Outcome<()> {
158 res!(self.db.close());
159 let _ = std::fs::remove_dir_all(&self.dir);
160 Ok(())
161 }
162}
163
164#[test]
165fn fewer_chunks_leave_no_old_chunk() -> Outcome<()> {
166 let _lock = lock();
167 let s = res!(store("fewer"));
168 let k = dat!("shrinks to two");
169 res!(s.five(&k, 5));
170 res!(s.put(&k, 7, TWO));
171 assert_eq!(res!(s.num_chunks(&k)), 2, "the overwrite did not split into two chunks");
172 res!(s.compact_and_check(&k, 5, 7, TWO));
173 assert!(s.holds(7, TWO).iter().all(|n| *n >= 1), "the new value's own chunks were erased: {:?}", s.holds(7, TWO));
174 s.end()
175}
176
177#[test]
178fn plain_value_leaves_no_old_chunk() -> Outcome<()> {
179 let _lock = lock();
180 let s = res!(store("plain"));
181 let k = dat!("shrinks to plain");
182 res!(s.five(&k, 5));
183 res!(s.put(&k, 8, PLAIN));
184 match res!(s.db.api().fetch_using_schemes(&k, None)).recv_daticle(s.db.api().schemes().encrypter(), None) {
185 Ok((Some((Dat::Tup5u64(_), _)), _)) => return Err(err!("The plain value is still chunked."; Test, Invalid)),
186 _ => (),
187 }
188 res!(s.compact_and_check(&k, 5, 8, PLAIN));
189 s.end()
190}
191
192// The same length reuses every chunk key, which the ordinary supersession retires: the retiring of
193// a surplus must keep keys the new value holds, or it erases the value it has just written.
194#[test]
195fn same_geometry_keeps_the_new_chunks() -> Outcome<()> {
196 let _lock = lock();
197 let s = res!(store("same"));
198 let k = dat!("same size");
199 res!(s.five(&k, 5));
200 res!(s.put(&k, 6, FIVE));
201 res!(s.compact_and_check(&k, 5, 6, FIVE));
202 assert!(s.holds(6, FIVE).iter().all(|n| *n >= 1), "the new value's own chunks were erased: {:?}", s.holds(6, FIVE));
203 s.end()
204}
205
206// The same count of chunks of another length shares no chunk key with the old value, because the
207// length is part of the key, so all five old chunks are left and all five must go (QA A1-3).
208#[test]
209fn same_count_other_length_leaves_no_old_chunk() -> Outcome<()> {
210 let _lock = lock();
211 let s = res!(store("count"));
212 let k = dat!("five over five");
213 res!(s.five(&k, 5));
214 res!(s.put(&k, 6, FIVE_B));
215 assert_eq!(res!(s.num_chunks(&k)), 5, "the overwrite did not split into five chunks");
216 res!(s.compact_and_check(&k, 5, 6, FIVE_B));
217 assert!(s.holds(6, FIVE_B).iter().all(|n| *n >= 1), "the new value's own chunks were erased: {:?}", s.holds(6, FIVE_B));
218 s.end()
219}
220
221// The new value is durable before the old chunks go: a reader that comes while the old chunks are
222// being retired finds the old value or the new one, never a bunch key with chunks missing.
223#[test]
224fn reader_during_the_retire_finds_a_whole_value() -> Outcome<()> {
225 let _lock = lock();
226 let s = res!(store("reader"));
227 let k = dat!("read during retire");
228 res!(s.five(&k, 5));
229 let old = Dat::BU32(fill_of(5, FIVE));
230 let new = Dat::BU32(fill_of(9, TWO));
231
232 let done = Arc::new(AtomicBool::new(false));
233 let seen = {
234 let (db, k, old, new, done) = (s.db.clone(), k.clone(), old.clone(), new.clone(), done.clone());
235 thread::spawn(move || {
236 let (mut n_old, mut n_new, mut bad) = (0usize, 0usize, 0usize);
237 while !done.load(Ordering::Relaxed) {
238 match db.get(&k, None) {
239 Ok(Some((v, _))) if v == old => n_old += 1,
240 Ok(Some((v, _))) if v == new => n_new += 1,
241 other => {
242 bad += 1;
243 let s: String = fmt!("{:?}", other).chars().take(700).collect();
244 msg!("a reader found neither value: {}", s);
245 },
246 }
247 thread::sleep(Duration::from_millis(15));
248 }
249 (n_old, n_new, bad)
250 })
251 };
252 let prior = match res!(s.db.api().get_head_wait(&k, None)) {
253 Some((Dat::Tup5u64(t), _)) => t,
254 other => return Err(err!("No chunked head before the overwrite: {:?}", other; Test, Missing)),
255 };
256 hooks::set_chunk_tombstone_delay(Duration::from_millis(400));
257 let put = s.put(&k, 9, TWO);
258 // The old chunks are retired by the cache bot after the put returns, so the reader goes on
259 // until they are gone.
260 let retired = wait_until(Duration::from_secs(20),
261 || s.db.api().fetch_chunks(&Dat::Tup5u64(prior), None).is_err());
262 hooks::set_chunk_tombstone_delay(Duration::ZERO);
263 done.store(true, Ordering::Relaxed);
264 assert!(retired, "the old chunks were never retired");
265 let (n_old, n_new, bad) = match seen.join() {
266 Ok(t) => t,
267 Err(_) => return Err(err!("The reader thread panicked."; Test, Invalid)),
268 };
269 res!(put);
270 msg!("reader saw the old value {} times, the new {} times, {} broken.", n_old, n_new, bad);
271 assert_eq!(bad, 0, "a reader found a value that was neither the old one nor the new one");
272 assert!(n_new >= 1, "the reader never saw the new value, so it did not read while the old chunks were retired");
273 res!(s.compact_and_check(&k, 5, 9, TWO));
274 s.end()
275}
276
277// The retire is stamped with the write that causes it. Stamped when it is sent, a store of the
278// old geometry begun while the first is still becoming durable (its chunks stamped before the
279// retire, its bunch key newer than the first store's) has its chunks superseded by the retire of
280// the chunks it shares, and its bunch key names tombstones (QA A2-6).
281#[test]
282fn retire_does_not_erase_a_newer_value() -> Outcome<()> {
283 let _lock = lock();
284 let s = res!(store("newer"));
285 let k = dat!("newer than the retire");
286 res!(s.put(&k, 5, TWO));
287 assert_eq!(res!(s.num_chunks(&k)), 2, "the value did not split into two chunks");
288 let newer = Dat::BU32(fill_of(9, TWO));
289
290 // The first store (plain, so it must retire the old two chunks) is held at its barrier, and
291 // the second (two chunks again, the old keys) begins inside that hold.
292 hooks::set_barrier_delay(Duration::from_millis(800));
293 let first = {
294 let (db, k) = (s.db.clone(), k.clone());
295 thread::spawn(move || db.insert(k, Dat::BU32(fill_of(7, PLAIN)), Uid::default(), None).map(|_| ()))
296 };
297 thread::sleep(Duration::from_millis(300));
298 let second = {
299 let (db, k, v) = (s.db.clone(), k.clone(), newer.clone());
300 thread::spawn(move || db.insert(k, v, Uid::default(), None).map(|_| ()))
301 };
302 let first = first.join();
303 let second = second.join();
304 hooks::set_barrier_delay(Duration::ZERO);
305 match (first, second) {
306 (Ok(a), Ok(b)) => { res!(a); res!(b); },
307 _ => return Err(err!("A storing thread panicked."; Test, Invalid)),
308 }
309 res!(s.db.api().settle_for_test(Duration::from_secs(10)));
310 match s.db.get(&k, None) {
311 Ok(Some((v, _))) => assert_eq!(v, newer, "the newer value reads back changed"),
312 other => {
313 let t: String = fmt!("{:?}", other).chars().take(300).collect();
314 panic!("the newer value does not read back whole: {}", t);
315 },
316 }
317 s.end()
318}