Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/compact_crash_pair.rs

9.3 KiB, 1 run

created by r1870400018:68519, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! A process killed in the middle of `compact_now` must leave a store that reads every key right
2//! and that `compact_now` can finish erasing (A3 R4, 2026-10-08). A collection puts two new files
3//! in place of two old ones, and no pair of renames is one step: killed between them, the data file
4//! is of one generation beside the index of the other (see `gc_crash_pair.rs`, which kills an
5//! ordinary collection). `compact_now` runs the same collection by order, on a store whose
6//! collection is switched off, and it is the call that an erasure relies on.
7//!
8//! The test runs itself twice. The child is this test binary again, started with the name of
9//! a directory in its environment: it writes a store, deletes one key in each of the first two
10//! data files, holds the next commit between the renames with `test::hooks::set_commit_delay`, and
11//! calls `compact_now`. The parent watches the files, and the moment exactly one pair has changed
12//! it kills the child with SIGKILL, which no handler in the child can answer. The collection runs
13//! one file at a time, so the other file still holds its deleted key's bytes: the control that
14//! says the last step, an erase on the reopened store, had something to erase.
15
16mod gc_pair;
17
18use gc_pair::*;
19
20use oxedyne_fe2o3_core::prelude::*;
21use oxedyne_fe2o3_iop_db::api::Database;
22use oxedyne_fe2o3_o3db_sync::{
23 base::cfg::OzoneConfig,
24 file::core::FileType,
25 test::{
26 hooks,
27 setup::{
28 self,
29 Uid,
30 },
31 },
32};
33
34use std::{
35 path::{
36 Path,
37 PathBuf,
38 },
39 process::{
40 Command,
41 Stdio,
42 },
43 thread,
44 time::Duration,
45};
46
47const ROOT: &str = "O3DB_COMPACT_CRASH_ROOT";
48const LEN: &str = "O3DB_COMPACT_CRASH_LEN";
49const READY: &str = "O3DB_COMPACT_CRASH_READY";
50
51const DELETED: [usize; 2] = [0, 5]; // the first key of file 1, and of file 2
52
53/// The child: fills the store, deletes a key from each of the first two files, then calls
54/// `compact_now` with the commit held between its renames, and waits there to be killed. It
55/// starts only when the parent has taken the sizes of the files, since a collection takes a few
56/// milliseconds.
57fn child(root: PathBuf, len: u64, ready: PathBuf) -> Outcome<()> {
58 log_set_level!("error");
59 let cfg = res!(config(5 * len + len / 2));
60 // Collection off: only the order to compact collects.
61 let db: TestDb = res!(setup::start_db(root, Some(cfg), schemes(), None, false, true));
62 res!(fill(&db));
63 for i in DELETED {
64 res!(db.delete(&key(i), Uid::default(), None));
65 }
66 res!(db.api().settle_for_test(Duration::from_secs(10)));
67 hooks::set_commit_delay(Duration::from_secs(120));
68 res!(std::fs::write(&ready, b"ready"));
69 let go = ready.with_extension("go");
70 assert!(wait_until(Duration::from_secs(120), || go.exists()), "the parent never said go");
71 let _ = db.compact_now(Duration::from_secs(150));
72 // A child that is not killed is a failure of the parent, and ends itself.
73 thread::sleep(Duration::from_secs(150));
74 std::process::exit(3);
75}
76
77// Every key reads as its version 1, and the deleted ones do not read at all.
78fn judge_after(db: &TestDb, label: &str) -> usize {
79 let mut bad = 0;
80 for i in 0..NKEYS {
81 let got = db.get(&key(i), None);
82 if DELETED.contains(&i) {
83 match got {
84 Ok(None) => (),
85 other => {
86 bad += 1;
87 let s: String = fmt!("{:?}", other).chars().take(200).collect();
88 msg!("{}: deleted key {} read back {}.", label, i, s);
89 },
90 }
91 } else {
92 match got {
93 Ok(Some((v, _))) if v == value(i, 1) => (),
94 other => {
95 bad += 1;
96 let s: String = fmt!("{:?}", other).chars().take(200).collect();
97 msg!("{}: key {} did not read back its value: {}.", label, i, s);
98 },
99 }
100 }
101 }
102 msg!("{}: {} of {} keys wrong.", label, bad, NKEYS);
103 bad
104}
105
106// How many data files hold each deleted key's value.
107fn needles(root: &Path, cfg: &OzoneConfig) -> Vec<usize> {
108 DELETED.iter().map(|i| {
109 let bytes = match value(*i, 1).bytes_ref() { Some(b) => b.to_vec(), None => Vec::new() };
110 files_holding(root, cfg, &bytes).len()
111 }).collect()
112}
113
114fn parent() -> Outcome<()> {
115 log_set_level!("error");
116 let dir = "./test_db_compact_crash_pair";
117 let _ = std::fs::remove_dir_all(dir);
118 res!(std::fs::create_dir_all(dir));
119 let root = res!(Path::new(dir).canonicalize());
120 let ready = res!(std::env::current_dir()).join("test_db_compact_crash_pair.ready");
121 let _ = std::fs::remove_file(&ready);
122 let _ = std::fs::remove_file(ready.with_extension("go"));
123 let log = res!(std::env::current_dir()).join("test_db_compact_crash_pair.log");
124
125 let probe = res!(probe("./test_db_compact_crash_pair_probe"));
126 let len = probe.len;
127 let cfg = res!(config(5 * len + len / 2));
128 let pair = |n: u32| (file(&root, &cfg, FileType::Data, n), file(&root, &cfg, FileType::Index, n));
129 let (d1, i1) = pair(1);
130 let (d2, i2) = pair(2);
131
132 let exe = res!(std::env::current_exe());
133 let out = res!(std::fs::File::create(&log));
134 let err = res!(out.try_clone());
135 let mut kid = res!(Command::new(exe)
136 .args(["main", "--exact", "--nocapture", "--test-threads=1"])
137 .env(ROOT, &root)
138 .env(LEN, fmt!("{}", len))
139 .env(READY, &ready)
140 .stdout(Stdio::from(out))
141 .stderr(Stdio::from(err))
142 .spawn());
143
144 // The child is ready when files 1 and 2 hold their five records each.
145 assert!(wait_until(Duration::from_secs(120), || ready.exists()),
146 "the child never wrote its store; see {:?}", log);
147 let (sd1, si1, sd2, si2) = (size(&d1), size(&i1), size(&d2), size(&i2));
148 assert_eq!((sd1, si1), (5 * len, probe.ind[5]), "file 1 does not hold the five records it was sized for");
149 assert_eq!((sd2, si2), (5 * len, probe.ind[5]), "file 2 does not hold the five records it was sized for");
150 res!(std::fs::write(ready.with_extension("go"), b"go"));
151
152 // Exactly one file has had one of its pair replaced: a collection is between its renames.
153 let torn = |d: &PathBuf, i: &PathBuf, sd: u64, si: u64| (size(d) != sd) != (size(i) != si);
154 let caught = wait_until(Duration::from_secs(60), ||
155 torn(&d1, &i1, sd1, si1) || torn(&d2, &i2, sd2, si2));
156 res!(kid.kill());
157 let _ = kid.wait();
158 assert!(caught, "compact_now never stopped between its renames; see {:?}", log);
159 let which = if torn(&d1, &i1, sd1, si1) { 1 } else { 2 };
160 msg!("Killed with file {} torn: file 1 data {} index {}, file 2 data {} index {}.",
161 which, size(&d1), size(&i1), size(&d2), size(&i2));
162
163 // The store the child left, reopened with collection still off.
164 let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, false, false));
165 let reopened = judge_after(&db, "After the kill");
166 let temps = temporaries(&zone_dir(&root, &cfg));
167 let size_reopened = res!(db.size_bytes());
168 let walked_reopened = zone_files_len(&root, &cfg);
169 let before = needles(&root, &cfg);
170 msg!("Reopened: size_bytes {}, walked {}, deleted keys in {:?} data files.",
171 size_reopened, walked_reopened, before);
172
173 // The order that finishes the erasure.
174 let report = res!(db.compact_now(Duration::from_secs(60)));
175 msg!("{:?}", report);
176 let after = needles(&root, &cfg);
177 let finished = judge_after(&db, "After compact_now");
178 let temps_after = temporaries(&zone_dir(&root, &cfg));
179 let size_after = res!(db.size_bytes());
180 let walked_after = zone_files_len(&root, &cfg);
181 res!(db.close());
182
183 let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, false, false));
184 let second = judge_after(&db, "After a second reopening");
185 res!(db.close());
186 log_finish_wait!();
187
188 assert_eq!(reopened, 0, "{} keys read wrong after the kill", reopened);
189 assert!(temps.is_empty(), "the start left {:?} of the killed collection behind", temps);
190 assert_eq!(size_reopened, walked_reopened, "size_bytes is not the length of the files after the kill");
191 assert!(before.iter().sum::<usize>() >= 1,
192 "positive control: the file that was not collected holds its deleted key ({:?}), file {} was torn", before, which);
193 assert_eq!(after, vec![0, 0], "a deleted key is still in a data file after compact_now: {:?}", after);
194 assert_eq!(finished, 0, "{} keys read wrong after compact_now", finished);
195 assert!(temps_after.is_empty(), "compact_now left {:?} behind", temps_after);
196 assert_eq!(size_after, walked_after, "size_bytes is not the length of the files after compact_now");
197 assert_eq!(second, 0, "{} keys read wrong after the second reopening", second);
198 let _ = std::fs::remove_dir_all(dir);
199 let _ = std::fs::remove_file(&ready);
200 let _ = std::fs::remove_file(ready.with_extension("go"));
201 let _ = std::fs::remove_file(&log);
202 Ok(())
203}
204
205#[test]
206fn main() -> Outcome<()> {
207 match (std::env::var(ROOT), std::env::var(LEN), std::env::var(READY)) {
208 (Ok(root), Ok(len), Ok(ready)) => {
209 let len = res!(len.parse::<u64>());
210 child(PathBuf::from(root), len, PathBuf::from(ready))
211 },
212 _ => parent(),
213 }
214}