Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/lone_index_campaign.rs

13.9 KiB, 1 run

created by r1870400018:72470, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! The review's crash campaign as a regression test (review of 9f64d2fc, 2026-10-11). A child,
2//! this test binary run again, writes a store with collection on, deleting whole files in the
3//! order of builds before 2026-10-11 (data file first), and aborts after the first unlink of its
4//! nth whole-file delete (`test::hooks::set_delete_cut`). The parent opens the image three times
5//! with collection off: each open must complete, every key must read as the child's log of
6//! operations says (the one in flight may land either way), and no live file may take the number
7//! of an index that stood alone. The default runs a few dozen images; O3DB_LONE_CAMPAIGN_CUTS
8//! raises the highest cut tried and O3DB_LONE_CAMPAIGN_SEEDS names the random scenarios' seeds.
9
10mod gc_pair;
11
12use gc_pair::*;
13
14use oxedyne_fe2o3_core::prelude::*;
15use oxedyne_fe2o3_iop_db::api::Database;
16use oxedyne_fe2o3_jdat::prelude::*;
17use oxedyne_fe2o3_o3db_sync::{
18 base::cfg::OzoneConfig,
19 test::{
20 hooks,
21 setup::{
22 self,
23 Uid,
24 },
25 },
26};
27
28use std::{
29 collections::BTreeMap,
30 fs,
31 io::Write,
32 path::{
33 Path,
34 PathBuf,
35 },
36 process::{
37 Command,
38 Stdio,
39 },
40 time::{
41 Duration,
42 Instant,
43 },
44};
45
46const DIR: &str = "O3DB_LONE_CAMPAIGN_DIR";
47const SCEN: &str = "O3DB_LONE_CAMPAIGN_SCEN";
48const CUT: &str = "O3DB_LONE_CAMPAIGN_CUT";
49const CUTS: &str = "O3DB_LONE_CAMPAIGN_CUTS";
50const SEEDS: &str = "O3DB_LONE_CAMPAIGN_SEEDS";
51const NK: u64 = 12;
52
53fn cfg() -> Outcome<OzoneConfig> {
54 let mut c = res!(config(1_500));
55 c.num_zones = 2;
56 c.num_cbots_per_zone = 2;
57 c.rest_chunk_threshold = 1_000;
58 Ok(c)
59}
60
61fn key_of(i: u64) -> Dat { dat!(fmt!("campaign key {:03}", i)) }
62
63// The value carries its key, version and size, so a value read back names its write.
64fn val(i: u64, ver: u64, big: bool) -> Dat {
65 let n = if big { 2_600 } else { 90 };
66 let mut v = vec![0u8; n];
67 v[0] = i as u8;
68 v[1] = ver as u8;
69 v[2] = big as u8;
70 for j in 3..n {
71 v[j] = (i as u8) ^ (ver as u8).wrapping_mul(7) ^ (j as u8);
72 }
73 Dat::BU32(v)
74}
75
76struct Rng(u64);
77
78impl Rng {
79 fn next(&mut self) -> u64 {
80 self.0 ^= self.0 << 13;
81 self.0 ^= self.0 >> 7;
82 self.0 ^= self.0 << 17;
83 self.0
84 }
85}
86
87fn ops_path(root: &Path) -> PathBuf { PathBuf::from(fmt!("{}.ops", root.display())) }
88
89// Each line is synced before the operation it names goes on, so the log outlives the abort.
90fn log_op(f: &mut fs::File, line: &str) -> Outcome<()> {
91 res!(writeln!(f, "{}", line));
92 res!(f.sync_data());
93 Ok(())
94}
95
96fn read_value(db: &TestDb, k: &Dat) -> Outcome<Option<Dat>> {
97 match res!(db.get(k, None)) {
98 None => Ok(None),
99 Some((Dat::Tup5u64(t), _)) => Ok(Some(res!(db.api().fetch_chunks(&Dat::Tup5u64(t), None)))),
100 Some((v, _)) => Ok(Some(v)),
101 }
102}
103
104struct Ops<'a> {
105 db: &'a TestDb,
106 log: fs::File,
107}
108
109impl<'a> Ops<'a> {
110 fn put(&mut self, i: u64, ver: u64, big: bool) -> Outcome<()> {
111 res!(log_op(&mut self.log, &fmt!("try put {} {} {}", i, ver, big as u8)));
112 res!(self.db.insert(key_of(i), val(i, ver, big), Uid::default(), None));
113 log_op(&mut self.log, &fmt!("put {} {} {}", i, ver, big as u8))
114 }
115 fn del(&mut self, i: u64) -> Outcome<()> {
116 res!(log_op(&mut self.log, &fmt!("try del {}", i)));
117 let _ = res!(self.db.delete(&key_of(i), Uid::default(), None));
118 log_op(&mut self.log, &fmt!("del {}", i))
119 }
120}
121
122/// The child: writes one scenario with collection on and the old delete order, aborting at the
123/// cut. A child that reaches the end without it closes the store and says so in the log.
124fn child(root: PathBuf, scen: String, cut: u64) -> Outcome<()> {
125 log_set_level!("error");
126 let _ = fs::remove_dir_all(&root);
127 res!(fs::create_dir_all(&root));
128 hooks::set_delete_cut(true, cut);
129 let db: TestDb = res!(setup::start_db(root.clone(), Some(res!(cfg())), schemes(), None, true, true));
130 let log = res!(fs::OpenOptions::new().create(true).truncate(true).write(true).open(ops_path(&root)));
131 let mut ops = Ops { db: &db, log };
132 match scen.as_str() {
133 "over" => {
134 for i in 0..NK { res!(ops.put(i, 1, false)); }
135 for i in 0..NK { res!(ops.put(i, 2, false)); }
136 },
137 "del" => {
138 for i in 0..NK { res!(ops.put(i, 1, false)); }
139 for i in 0..NK / 2 { res!(ops.del(i)); }
140 for i in NK / 2..NK { res!(ops.put(i, 2, false)); }
141 // Pads, so that the files of tombstones seal.
142 for i in NK..NK + 12 { res!(ops.put(i, 1, false)); }
143 },
144 "chunk" => {
145 for i in 0..4 { res!(ops.put(i, 1, true)); }
146 res!(ops.put(0, 2, true));
147 res!(ops.del(1));
148 res!(ops.put(2, 3, false));
149 res!(ops.del(3));
150 res!(ops.put(3, 4, true));
151 for i in 4..NK + 4 { res!(ops.put(i, 1, false)); }
152 for i in 4..NK + 4 { res!(ops.put(i, 2, false)); }
153 },
154 seed => {
155 let mut r = Rng(res!(seed.parse::<u64>()) | 1);
156 for s in 0..150 {
157 let i = r.next() % NK;
158 match r.next() % 10 {
159 0 | 1 => res!(ops.del(i)),
160 2 | 3 => res!(ops.put(i, 10 + s, true)),
161 _ => res!(ops.put(i, 10 + s, false)),
162 }
163 }
164 },
165 }
166 let mut log = ops.log;
167 res!(log_op(&mut log, "writes done"));
168 let _ = db.api().settle_for_test(Duration::from_secs(20));
169 std::thread::sleep(Duration::from_secs(3));
170 let _ = db.api().settle_for_test(Duration::from_secs(20));
171 res!(log_op(&mut log, "closing"));
172 res!(db.close());
173 log_op(&mut log, "closed")
174}
175
176type Want = BTreeMap<u64, Option<(u64, bool)>>;
177
178// What each key must read, and the operation in flight at the cut, if any.
179fn expected(root: &Path) -> Outcome<(Want, Option<(u64, Option<(u64, bool)>)>)> {
180 let text = res!(fs::read_to_string(ops_path(root)));
181 let mut want = BTreeMap::new();
182 let mut pend = None;
183 let num = |w: Option<&&str>| -> Outcome<u64> {
184 let w = res!(w.ok_or_else(|| err!("A short line in the log of operations."; Test)));
185 Ok(res!(w.parse::<u64>()))
186 };
187 for line in text.lines() {
188 let w: Vec<&str> = line.split(' ').collect();
189 match w.as_slice() {
190 ["try", "put", ..] => pend = Some((res!(num(w.get(2))), Some((res!(num(w.get(3))), w.get(4) == Some(&"1"))))),
191 ["try", "del", ..] => pend = Some((res!(num(w.get(2))), None)),
192 ["put", ..] => {
193 pend = None;
194 want.insert(res!(num(w.get(1))), Some((res!(num(w.get(2))), w.get(3) == Some(&"1"))));
195 },
196 ["del", ..] => {
197 pend = None;
198 want.insert(res!(num(w.get(1))), None);
199 },
200 _ => (),
201 }
202 }
203 Ok((want, pend))
204}
205
206fn walk(dir: &Path, out: &mut Vec<PathBuf>) {
207 if let Ok(rd) = fs::read_dir(dir) {
208 for e in rd.flatten() {
209 let p = e.path();
210 match p.is_dir() {
211 true => walk(&p, out),
212 false => out.push(p),
213 }
214 }
215 }
216}
217
218// The indexes standing without a data file, and every index, by path.
219fn indexes(root: &Path) -> (Vec<PathBuf>, Vec<PathBuf>) {
220 let mut all = Vec::new();
221 walk(root, &mut all);
222 let inds: Vec<PathBuf> = all.into_iter()
223 .filter(|p| p.extension().map_or(false, |x| x == "ind"))
224 .collect();
225 let lone = inds.iter().filter(|p| !p.with_extension("dat").exists()).cloned().collect();
226 (lone, inds)
227}
228
229fn check(db: &TestDb, want: &Want, pend: &Option<(u64, Option<(u64, bool)>)>) -> Vec<String> {
230 let mut bad = Vec::new();
231 let mut all = want.clone();
232 if let Some((k, _)) = pend {
233 all.entry(*k).or_insert(None);
234 }
235 for (i, w) in &all {
236 let got = read_value(db, &key_of(*i));
237 let fits = |w: &Option<(u64, bool)>| match (&got, w) {
238 (Ok(Some(v)), Some((ver, big))) => *v == val(*i, *ver, *big),
239 (Ok(None), None) => true,
240 _ => false,
241 };
242 let alt = match pend {
243 Some((k, a)) if k == i => fits(a),
244 _ => false,
245 };
246 if !fits(w) && !alt {
247 bad.push(fmt!("key {} wants {:?} and reads {}", i, w, match &got {
248 Ok(Some(v)) => fmt!("version {} big {}",
249 v.bytes_ref().and_then(|b| b.get(1).copied()).unwrap_or(0),
250 v.bytes_ref().and_then(|b| b.get(2).copied()).unwrap_or(0)),
251 Ok(None) => fmt!("nothing"),
252 Err(e) => fmt!("an error: {}", e),
253 }));
254 }
255 }
256 bad
257}
258
259// Runs one image: the child to its cut, then three opens. Whether the child was cut.
260fn image(base: &Path, scen: &str, cut: u64, fails: &mut Vec<String>) -> Outcome<bool> {
261 let root = base.join(fmt!("{}_{}", scen, cut));
262 let tag = fmt!("{} cut {}", scen, cut);
263 let log = base.join(fmt!("{}_{}.log", scen, cut));
264 let out = res!(fs::File::create(&log));
265 let err = res!(out.try_clone());
266 // No core file: the abort is the point.
267 let mut kid = res!(Command::new("bash")
268 .args(["-c", "ulimit -c 0; exec \"$0\" \"$@\""])
269 .arg(res!(std::env::current_exe()))
270 .args(["main", "--exact", "--nocapture", "--test-threads=1"])
271 .env(DIR, &root)
272 .env(SCEN, scen)
273 .env(CUT, fmt!("{}", cut))
274 .stdout(Stdio::from(out))
275 .stderr(Stdio::from(err))
276 .spawn());
277 let start = Instant::now();
278 let status = loop {
279 if let Some(s) = res!(kid.try_wait()) {
280 break s;
281 }
282 if start.elapsed() > Duration::from_secs(120) {
283 let _ = kid.kill();
284 let _ = kid.wait();
285 fails.push(fmt!("{}: the child ran past two minutes; see {:?}", tag, log));
286 return Ok(false);
287 }
288 std::thread::sleep(Duration::from_millis(50));
289 };
290 let ops = res!(fs::read_to_string(ops_path(&root)).or_else(|_| Ok::<_, std::io::Error>(String::new())));
291 let cut_hit = !ops.contains("closed");
292 if !cut_hit {
293 if !status.success() {
294 fails.push(fmt!("{}: the child failed without a cut; see {:?}", tag, log));
295 }
296 let _ = fs::remove_dir_all(&root);
297 let _ = fs::remove_file(ops_path(&root));
298 let _ = fs::remove_file(&log);
299 return Ok(false);
300 }
301 let (want, pend) = res!(expected(&root));
302 let (lone, _) = indexes(&root);
303 let mut fine = true;
304 if lone.is_empty() {
305 fails.push(fmt!("{}: positive control: the cut left no index alone", tag));
306 fine = false;
307 }
308 let mut want = want;
309 for round in 0..3 {
310 let db: TestDb = match setup::start_db(root.clone(), Some(res!(cfg())), schemes(), None, false, false) {
311 Ok(db) => db,
312 Err(e) => {
313 fails.push(fmt!("{}: open {} refused: {}", tag, round, e));
314 fine = false;
315 break;
316 },
317 };
318 let bad = check(&db, &want, &pend);
319 if !bad.is_empty() {
320 fails.push(fmt!("{}: open {} reads {} keys wrong: {:?}", tag, round, bad.len(), bad));
321 fine = false;
322 }
323 let (still, before) = indexes(&root);
324 if !still.is_empty() {
325 fails.push(fmt!("{}: open {} left indexes alone: {:?}", tag, round, still));
326 fine = false;
327 }
328 if round == 0 {
329 // Enough writes to roll over to new files, none of which may take a lone number.
330 for j in 900..912 {
331 res!(db.insert(key_of(j), val(j, 1, false), Uid::default(), None));
332 want.insert(j, Some((1, false)));
333 }
334 let (_, after) = indexes(&root);
335 let reused: Vec<&PathBuf> = after.iter()
336 .filter(|p| !before.contains(p) && lone.contains(p))
337 .collect();
338 if !reused.is_empty() {
339 fails.push(fmt!("{}: new files took the numbers of lone indexes: {:?}", tag, reused));
340 fine = false;
341 }
342 }
343 res!(db.close());
344 }
345 if fine {
346 let _ = fs::remove_dir_all(&root);
347 let _ = fs::remove_file(ops_path(&root));
348 let _ = fs::remove_file(&log);
349 }
350 Ok(true)
351}
352
353fn parent() -> Outcome<()> {
354 log_set_level!("error");
355 let dir = "./test_db_lone_index_campaign";
356 let _ = fs::remove_dir_all(dir);
357 res!(fs::create_dir_all(dir));
358 let base = res!(Path::new(dir).canonicalize());
359 let cuts = std::env::var(CUTS).ok().and_then(|v| v.parse::<u64>().ok()).unwrap_or(5);
360 let seeds = std::env::var(SEEDS).unwrap_or(fmt!("7,11"));
361 let mut scens: Vec<String> = vec![fmt!("chunk"), fmt!("over"), fmt!("del")];
362 scens.extend(seeds.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()));
363 let mut fails = Vec::new();
364 let mut images = 0;
365 for scen in &scens {
366 for cut in 1..(cuts + 1) {
367 match res!(image(&base, scen, cut, &mut fails)) {
368 true => images += 1,
369 false => break, // fewer deletes than the cut: higher cuts are the same run
370 }
371 }
372 }
373 log_finish_wait!();
374 println!("{} cut images, {} failures.", images, fails.len());
375 for f in &fails {
376 println!("FAIL {}", f);
377 }
378 assert!(images >= scens.len(), "positive control: only {} images were cut", images);
379 assert!(fails.is_empty(), "{} failures over {} images: {:?}", fails.len(), images, fails);
380 let _ = fs::remove_dir_all(dir);
381 Ok(())
382}
383
384#[test]
385fn main() -> Outcome<()> {
386 match (std::env::var(DIR), std::env::var(SCEN), std::env::var(CUT)) {
387 (Ok(dir), Ok(scen), Ok(cut)) => child(PathBuf::from(dir), scen, res!(cut.parse::<u64>())),
388 _ => parent(),
389 }
390}