Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/qa_a3r2a_chunks.rs

11.7 KiB, 1 run

created by r1870400018:69931, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! Opus QA agent 2 of A3 round 2a (2026-10-09): chunked values. Each test is a finding's proof
2//! and fails on bebb931d. The hooks are process-wide, so every test takes one lock.
3
4mod gc_pair;
5
6use gc_pair::*;
7
8use oxedyne_fe2o3_core::prelude::*;
9use oxedyne_fe2o3_iop_db::api::Database;
10use oxedyne_fe2o3_jdat::prelude::*;
11use oxedyne_fe2o3_o3db_sync::{
12 base::{
13 cfg::OzoneConfig,
14 constant,
15 },
16 test::{
17 hooks,
18 setup::{
19 self,
20 Uid,
21 },
22 },
23};
24
25use std::{
26 collections::BTreeMap,
27 path::Path,
28 sync::{
29 Arc,
30 Mutex,
31 MutexGuard,
32 atomic::{
33 AtomicBool,
34 Ordering,
35 },
36 },
37 thread,
38 time::Duration,
39};
40
41static HOOKS: Mutex<()> = Mutex::new(());
42
43fn lock() -> MutexGuard<'static, ()> {
44 match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() }
45}
46
47fn cfg(cbots: u16) -> Outcome<OzoneConfig> {
48 let mut c = res!(setup::default_cfg());
49 c.num_zones = 1;
50 c.num_cbots_per_zone = cbots;
51 c.num_fbots_per_zone = 1;
52 c.num_wbots_per_zone = 2;
53 c.zone_overrides = BTreeMap::new();
54 c.rest_chunk_threshold = 1_000;
55 c.rest_chunk_bytes = 400;
56 c.sync_on_write = true;
57 Ok(c)
58}
59
60// A value of `len` bytes whose every byte is `seed ^ j`, so a value assembled from the chunks of
61// two values is told by `whole`.
62fn val(len: usize, seed: u8) -> Dat {
63 Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect())
64}
65
66// Is this one value, whole, of one of the lengths stored?
67fn whole(v: &Dat, lens: &[usize]) -> bool {
68 match v {
69 Dat::BU32(b) | Dat::BU64(b) | Dat::BU8(b) | Dat::BU16(b) => {
70 if !lens.contains(&b.len()) || b.is_empty() { return false; }
71 let seed = b[0];
72 b.iter().enumerate().all(|(j, x)| *x == seed ^ (j as u8))
73 },
74 _ => false,
75 }
76}
77
78fn start(dir: &str, c: OzoneConfig, wipe: bool) -> Outcome<TestDb> {
79 log_set_level!("error");
80 if wipe {
81 let _ = std::fs::remove_dir_all(dir);
82 }
83 res!(std::fs::create_dir_all(dir));
84 let root = res!(Path::new(dir).canonicalize());
85 setup::start_db(root, Some(c), schemes(), None, false, wipe)
86}
87
88struct Seen { none: usize, ok: usize, torn: usize, err: usize, sample: Option<String> }
89
90fn reader(db: TestDb, k: Dat, lens: Vec<usize>, done: Arc<AtomicBool>) -> thread::JoinHandle<Seen> {
91 thread::spawn(move || {
92 let mut s = Seen { none: 0, ok: 0, torn: 0, err: 0, sample: None };
93 while !done.load(Ordering::Relaxed) {
94 match db.get(&k, None) {
95 Ok(None) => s.none += 1,
96 Ok(Some((v, _))) if whole(&v, &lens) => s.ok += 1,
97 Ok(Some((v, _))) => {
98 s.torn += 1;
99 if s.sample.is_none() {
100 s.sample = Some(fmt!("{:?}", v).chars().take(160).collect());
101 }
102 },
103 Err(e) => {
104 s.err += 1;
105 if s.sample.is_none() {
106 s.sample = Some(fmt!("{}", e).chars().take(300).collect());
107 }
108 },
109 }
110 thread::sleep(Duration::from_millis(1));
111 }
112 s
113 })
114}
115
116fn join(h: thread::JoinHandle<Seen>) -> Outcome<Seen> {
117 match h.join() {
118 Ok(s) => Ok(s),
119 Err(_) => Err(err!("A reader thread panicked."; Test, Invalid)),
120 }
121}
122
123// F1. A value overwritten by one of the same length is written over its own chunk records in
124// place: the chunk set identifier comes from the key, and a chunk key carries only the set, the
125// index, the length, the count and the size. The new bunch key is the old one byte for byte, so
126// publishing it last protects nothing, and a reader finds the old bunch key over a mixture of old
127// and new chunks for as long as the chunks take to land.
128#[test]
129fn f1_same_length_overwrite_is_never_read_torn() -> Outcome<()> {
130 let _lock = lock();
131 let dir = "./test_db_qa2_f1";
132 // No hook: the chunks go to two writers at once, and each writer's barrier releases its own.
133 let db = res!(start(dir, res!(cfg(8)), true));
134 let k = dat!("same length overwrite");
135 let len = 40_000usize; // 100 chunks
136 let lens = vec![len];
137 res!(db.insert(k.clone(), val(len, 0), Uid::default(), None));
138
139 let done = Arc::new(AtomicBool::new(false));
140 let rs: Vec<_> = (0..3).map(|_| reader(db.clone(), k.clone(), lens.clone(), done.clone())).collect();
141 let mut put = Ok(());
142 for seed in 1..60u8 {
143 if let Err(e) = db.insert(k.clone(), val(len, seed), Uid::default(), None) {
144 put = Err(e);
145 }
146 }
147 done.store(true, Ordering::Relaxed);
148 let (mut torn, mut errs, mut oks, mut sample) = (0, 0, 0, None);
149 for r in rs {
150 let s = res!(join(r));
151 torn += s.torn; errs += s.err; oks += s.ok;
152 if sample.is_none() { sample = s.sample; }
153 }
154 res!(put);
155 msg!("F1: whole {} torn {} err {}; first bad: {:?}", oks, torn, errs, sample);
156 res!(db.close());
157 let _ = std::fs::remove_dir_all(dir);
158 assert_eq!(torn + errs, 0, "a reader found a value assembled from two values' chunks: {:?}", sample);
159 Ok(())
160}
161
162// F2. The same in-place overwrite interrupted after one chunk has landed -- the process stopped,
163// which the new store makes more likely by sending every chunk at once and the bunch key only
164// later -- leaves the old bunch key over one new chunk and the rest old, for good. The control
165// does the same with a value of another length, whose chunks have keys of their own: the old value
166// survives whole.
167#[test]
168fn f2_overwrite_interrupted_after_one_chunk_keeps_a_whole_value() -> Outcome<()> {
169 let _lock = lock();
170 let mut results = Vec::new();
171 for (label, newlen) in [("control, other length", 1_500usize), ("same length", 1_100usize)] {
172 let dir = "./test_db_qa2_f2";
173 let k = dat!("interrupted overwrite");
174 {
175 let db = res!(start(dir, res!(cfg(4)), true));
176 res!(db.insert(k.clone(), val(1_100, 1), Uid::default(), None));
177 // The new value, prepared as a store prepares it; only its first chunk reaches a
178 // writer before the "crash".
179 let resp = db.api().responder();
180 let mut msgs = res!(db.api().prepare_write_dat(
181 k.clone(), val(newlen, 2), Uid::default(), None, resp.clone(), None));
182 assert!(msgs.len() >= 3, "the value was not chunked");
183 let first = msgs.remove(1);
184 res!(db.api().store_bytes(vec![first]));
185 res!(resp.recv_write_acks(1, constant::USER_REQUEST_TIMEOUT, constant::DURABILITY_TIMEOUT));
186 let live = res!(db.get(&k, None));
187 let live_whole = match &live { Some((v, _)) => whole(v, &[1_100, newlen]), None => false };
188 msg!("F2 {}: before the restart a reader gets a whole value: {}", label, live_whole);
189 res!(db.close());
190 }
191 let db = res!(start(dir, res!(cfg(4)), false));
192 let got = db.get(&k, None);
193 let ok = match &got {
194 Ok(Some((v, _))) => *v == val(1_100, 1),
195 _ => false,
196 };
197 msg!("F2 {}: after the restart the old value reads back whole: {}", label, ok);
198 results.push((label, ok));
199 res!(db.close());
200 let _ = std::fs::remove_dir_all(dir);
201 }
202 assert!(results[0].1, "the control failed: an interrupted overwrite of another length lost the old value");
203 assert!(results[1].1, "an overwrite interrupted after one chunk left a value of mixed chunks, read as if whole");
204 Ok(())
205}
206
207// F4. Many puts and deletes at once on one key, of three lengths (one not chunked). Afterwards
208// the key holds nothing or one value whole, and every read during the race gets the same.
209#[test]
210fn f4_concurrent_puts_and_deletes_on_one_key_settle_whole() -> Outcome<()> {
211 let _lock = lock();
212 let dir = "./test_db_qa2_f4";
213 let db = res!(start(dir, res!(cfg(8)), true));
214 let k = dat!("contended");
215 let lens = vec![300usize, 1_100, 1_500];
216 let mut bad_final = Vec::new();
217 let mut failed = Vec::new();
218 let (mut torn, mut errs, mut sample) = (0, 0, None);
219 hooks::set_chunk_insert_delay(Duration::from_millis(20));
220 for round in 0..6u8 {
221 let done = Arc::new(AtomicBool::new(false));
222 let r = reader(db.clone(), k.clone(), lens.clone(), done.clone());
223 let ws: Vec<_> = (0..16usize).map(|i| {
224 let (db, k, lens) = (db.clone(), k.clone(), lens.clone());
225 thread::spawn(move || -> Result<(), String> {
226 if i % 5 == 4 {
227 db.delete(&k, Uid::default(), None).map(|_| ()).map_err(|e| fmt!("{}", e))
228 } else {
229 let seed = round.wrapping_mul(31).wrapping_add(i as u8);
230 db.insert(k, val(lens[i % 3], seed), Uid::default(), None)
231 .map(|_| ()).map_err(|e| fmt!("{}", e))
232 }
233 })
234 }).collect();
235 for w in ws {
236 match w.join() {
237 Ok(Ok(())) => (),
238 Ok(Err(e)) => failed.push(e.chars().take(200).collect::<String>()),
239 Err(_) => failed.push(fmt!("panicked")),
240 }
241 }
242 done.store(true, Ordering::Relaxed);
243 let s = res!(join(r));
244 torn += s.torn; errs += s.err;
245 if sample.is_none() { sample = s.sample; }
246 match db.get(&k, None) {
247 Ok(None) => (),
248 Ok(Some((v, _))) if whole(&v, &lens) => (),
249 other => bad_final.push(fmt!("{:?}", other).chars().take(200).collect::<String>()),
250 }
251 }
252 hooks::set_chunk_insert_delay(Duration::ZERO);
253 msg!("F4: torn {} err {} during; finals bad {:?}; failed calls {:?}; first bad read {:?}",
254 torn, errs, bad_final, failed, sample);
255 res!(db.close());
256 let _ = std::fs::remove_dir_all(dir);
257 assert!(bad_final.is_empty(), "the key settled on a value that is not whole: {:?}", bad_final);
258 assert!(failed.is_empty(), "calls failed: {:?}", failed);
259 assert_eq!(torn + errs, 0, "reads during the race were torn or failed: {:?}", sample);
260 Ok(())
261}
262
263// F3b. F3 with reads that go to the files and take their time: the cache holds no values, and
264// each file read is held 15 ms, as a loaded disk would hold it.
265#[test]
266fn f3b_slow_reader_never_errs_while_a_key_is_overwritten_with_a_new_length() -> Outcome<()> {
267 let _lock = lock();
268 let dir = "./test_db_qa2_f3b";
269 let mut c = res!(cfg(8));
270 c.cache_size_limit_bytes = 1;
271 let db = res!(start(dir, c, true));
272 let k = dat!("changing length, slow reader");
273 let lens = vec![1_100usize, 1_500];
274 res!(db.insert(k.clone(), val(lens[0], 0), Uid::default(), None));
275
276 let done = Arc::new(AtomicBool::new(false));
277 hooks::set_read_delay(Duration::from_millis(15));
278 let rs: Vec<_> = (0..3).map(|_| reader(db.clone(), k.clone(), lens.clone(), done.clone())).collect();
279 let mut put = Ok(());
280 for i in 1..40usize {
281 if let Err(e) = db.insert(k.clone(), val(lens[i % 2], i as u8), Uid::default(), None) {
282 put = Err(e);
283 }
284 thread::sleep(Duration::from_millis(20));
285 }
286 done.store(true, Ordering::Relaxed);
287 let (mut torn, mut errs, mut oks, mut nones, mut sample) = (0, 0, 0, 0, None);
288 for r in rs {
289 let s = res!(join(r));
290 torn += s.torn; errs += s.err; oks += s.ok; nones += s.none;
291 if sample.is_none() { sample = s.sample; }
292 }
293 hooks::set_read_delay(Duration::ZERO);
294 res!(put);
295 msg!("F3b: whole {} none {} torn {} err {}; first bad: {:?}", oks, nones, torn, errs, sample);
296 res!(db.close());
297 let _ = std::fs::remove_dir_all(dir);
298 assert_eq!(torn + errs + nones, 0, "a slow reader of a live key failed while it was overwritten: {:?}", sample);
299 Ok(())
300}