Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/gc_crash_pair.rs

6.8 KiB, 1 run

created by r1870400018:62333, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! A process killed between the two renames of a collection must leave a store that reads every
2//! key right. A collection puts two new files in place of two old ones, and no pair of renames
3//! is one step: a process that dies between them leaves a data file of one generation beside the
4//! index of the other. The start used to believe such an index, and the cache then named the
5//! offsets of one file in the other (D-C, found by QA of lane o3i, 2026-09-24).
6//!
7//! The test runs itself twice. The child is this test binary again, started with the name of
8//! a directory in its environment: it writes a store, holds its next collection between the
9//! renames with `test::hooks::set_commit_delay`, and starts the collection. The parent watches
10//! the files, and the moment exactly one of the pair has changed it kills the child with SIGKILL,
11//! which no handler in the child can answer. The parent then opens the store the child left.
12
13mod gc_pair;
14
15use gc_pair::*;
16
17use oxedyne_fe2o3_core::prelude::*;
18use oxedyne_fe2o3_iop_db::api::Database;
19use oxedyne_fe2o3_o3db_sync::{
20 file::core::FileType,
21 test::{
22 hooks,
23 setup::{
24 self,
25 Uid,
26 },
27 },
28};
29
30use std::{
31 path::PathBuf,
32 process::{
33 Command,
34 Stdio,
35 },
36 thread,
37 time::Duration,
38};
39
40const ROOT: &str = "O3DB_GC_CRASH_ROOT";
41const LEN: &str = "O3DB_GC_CRASH_LEN";
42const READY: &str = "O3DB_GC_CRASH_READY";
43
44/// The child: fills the store, then starts a collection that stops between its renames, and waits
45/// there to be killed. It starts the collection only when the parent has taken the sizes of
46/// the pair, since the collection takes a few milliseconds.
47fn child(root: PathBuf, len: u64, ready: PathBuf) -> Outcome<()> {
48 log_set_level!("error");
49 let cfg = res!(config(5 * len + len / 2));
50 let db: TestDb = res!(setup::start_db(root, Some(cfg), schemes(), None, true, true));
51 res!(fill(&db));
52 hooks::set_commit_delay(Duration::from_secs(120));
53 // The parent reads the sizes of the pair when it sees this, and answers with `go`.
54 res!(std::fs::write(&ready, b"ready"));
55 let go = ready.with_extension("go");
56 assert!(wait_until(Duration::from_secs(120), || go.exists()), "the parent never said go");
57 // Superseding two of file 1's five records starts its collection.
58 res!(db.insert(key(0), value(0, 2), Uid::default(), None));
59 res!(db.insert(key(1), value(1, 2), Uid::default(), None));
60 // A child that is not killed is a failure of the parent, and ends itself.
61 thread::sleep(Duration::from_secs(150));
62 std::process::exit(3);
63}
64
65fn parent() -> Outcome<()> {
66 log_set_level!("error");
67 let dir = "./test_db_gc_crash_pair";
68 let _ = std::fs::remove_dir_all(dir);
69 res!(std::fs::create_dir_all(dir));
70 let root = res!(std::path::Path::new(dir).canonicalize());
71 let ready = res!(std::env::current_dir()).join("test_db_gc_crash_pair.ready");
72 let _ = std::fs::remove_file(&ready);
73 let _ = std::fs::remove_file(ready.with_extension("go"));
74 let log = res!(std::env::current_dir()).join("test_db_gc_crash_pair.log");
75
76 let probe = res!(probe("./test_db_gc_crash_pair_probe"));
77 let len = probe.len;
78 let cfg = res!(config(5 * len + len / 2));
79 let (d1, i1) = (file(&root, &cfg, FileType::Data, 1), file(&root, &cfg, FileType::Index, 1));
80
81 let exe = res!(std::env::current_exe());
82 let out = res!(std::fs::File::create(&log));
83 let err = res!(out.try_clone());
84 let mut kid = res!(Command::new(exe)
85 .args(["main", "--exact", "--nocapture", "--test-threads=1"])
86 .env(ROOT, &root)
87 .env(LEN, fmt!("{}", len))
88 .env(READY, &ready)
89 .stdout(Stdio::from(out))
90 .stderr(Stdio::from(err))
91 .spawn());
92
93 // The child is ready when file 1 holds its five records.
94 assert!(wait_until(Duration::from_secs(120), || ready.exists()),
95 "the child never wrote its store; see {:?}", log);
96 let (d_old, i_old) = (size(&d1), size(&i1));
97 assert_eq!((d_old, i_old), (5 * len, probe.ind[5]), "file 1 does not hold the five records it was sized for");
98 res!(std::fs::write(ready.with_extension("go"), b"go"));
99
100 // Exactly one of the pair has changed: the collection is between its renames.
101 let caught = wait_until(Duration::from_secs(60), ||
102 (size(&d1) != d_old) != (size(&i1) != i_old));
103 res!(kid.kill());
104 let _ = kid.wait();
105 assert!(caught, "the collection never stopped between its renames; see {:?}", log);
106 let (d_torn, i_torn) = (size(&d1), size(&i1));
107 msg!("Killed with file 1 at data {} (was {}) and index {} (was {}).", d_torn, d_old, i_torn, i_old);
108 assert!((d_torn != d_old) != (i_torn != i_old), "the pair was whole when the child was killed");
109
110 // The store the child left.
111 let mut want = vec![1u8; NKEYS];
112 want[0] = 2;
113 want[1] = 2;
114 let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, true, false));
115 let reopened = judge(&db, &want, "After the kill");
116 // The index has been made the data file's own, whichever file the kill left new.
117 let want_ind = probe.ind[(size(&d1) / len) as usize];
118 let rebuilt = size(&i1);
119 let temps = temporaries(&zone_dir(&root, &cfg));
120
121 // Churn so that file 1 is collected again with whatever state the start built.
122 for round in 4..7u8 {
123 for i in 2..6 {
124 res!(db.insert(key(i), value(i, round), Uid::default(), None));
125 want[i] = round;
126 }
127 }
128 thread::sleep(Duration::from_secs(3));
129 let churned = judge(&db, &want, "After more collection");
130 res!(db.close());
131
132 let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, true, false));
133 let second = judge(&db, &want, "After a second reopening");
134 res!(db.close());
135 log_finish_wait!();
136
137 assert_eq!(reopened, 0, "{} keys read wrong after the kill", reopened);
138 assert_eq!(rebuilt, want_ind,
139 "the index is {} bytes beside a data file of {}, where records of {} bytes need {}",
140 rebuilt, d_torn, len, want_ind);
141 assert!(temps.is_empty(), "the start left {:?} of the killed collection behind", temps);
142 assert_eq!(churned, 0, "{} keys read wrong after file 1 was collected again", churned);
143 assert_eq!(second, 0, "{} keys read wrong after the second reopening", second);
144 let _ = std::fs::remove_dir_all(dir);
145 let _ = std::fs::remove_file(&ready);
146 let _ = std::fs::remove_file(ready.with_extension("go"));
147 let _ = std::fs::remove_file(&log);
148 Ok(())
149}
150
151#[test]
152fn main() -> Outcome<()> {
153 match (std::env::var(ROOT), std::env::var(LEN), std::env::var(READY)) {
154 (Ok(root), Ok(len), Ok(ready)) => {
155 let len = res!(len.parse::<u64>());
156 child(PathBuf::from(root), len, PathBuf::from(ready))
157 },
158 _ => parent(),
159 }
160}