Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/fault_kill9.rs

8.1 KiB, 1 run

created by r1870400018:70852, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! A process killed with SIGKILL at each step of a barrier and of the stop after a failed one
2//! (2026-10-10, QA arm d). The child writes under the every-write policy, then stops at a
3//! `hooks::Halt`, says so through a flag file, and waits to be killed. On the reopen every
4//! confirmed value reads back, the unconfirmed one is never served once the fault record
5//! existed, a marker left before the kill refuses the open until it is cleared, and every file
6//! the record names ends at or below its mark.
7
8mod gc_pair;
9
10use gc_pair::*;
11
12use oxedyne_fe2o3_core::prelude::*;
13use oxedyne_fe2o3_iop_db::api::Database;
14use oxedyne_fe2o3_jdat::prelude::*;
15use oxedyne_fe2o3_o3db_sync::{
16 base::{
17 cfg::OzoneConfig,
18 constant,
19 },
20 file::{
21 core::FileType,
22 refusal::Refusal,
23 zdir::ZoneDir,
24 },
25 test::{
26 hooks::{
27 self,
28 Halt,
29 },
30 setup::{
31 self,
32 Uid,
33 },
34 },
35};
36
37use std::{
38 collections::BTreeMap,
39 fs,
40 path::{
41 Path,
42 PathBuf,
43 },
44 process::{
45 Command,
46 Stdio,
47 },
48 thread,
49 time::Duration,
50};
51
52const ROOT: &str = "FAULT_KILL9_ROOT";
53const HALT: &str = "FAULT_KILL9_HALT";
54const FLAG: &str = "FAULT_KILL9_FLAG";
55const NKEPT: usize = 10;
56
57const HALTS: [Halt; 5] = [
58 Halt::Synced,
59 Halt::RecordSynced,
60 Halt::Recorded,
61 Halt::Quarantined,
62 Halt::DataCut,
63];
64
65fn cfg() -> Outcome<OzoneConfig> {
66 let mut c = res!(setup::default_cfg());
67 c.num_zones = 1;
68 c.num_wbots_per_zone = 1;
69 c.zone_overrides = BTreeMap::new();
70 c.sync_on_write = true;
71 c.sync_every_n_writes = 0;
72 c.sync_interval_ms = 0;
73 Ok(c)
74}
75
76fn val(seed: u8) -> Dat {
77 Dat::BU32((0..100).map(|j| seed ^ (j as u8)).collect())
78}
79
80fn kept(i: usize) -> Dat {
81 dat!(fmt!("kill9 kept {}", i))
82}
83
84fn halt_named(name: &str) -> Outcome<Halt> {
85 match HALTS.iter().find(|h| fmt!("{:?}", h) == name) {
86 Some(h) => Ok(*h),
87 None => Err(err!("No halt is named {:?}.", name; Test, Invalid)),
88 }
89}
90
91// Writes the kept values, each confirmed durable, then one more that stops at `halt`: under
92// `Synced` its barrier succeeds, otherwise it fails.
93fn child(root: PathBuf, halt: Halt, flag: PathBuf) -> Outcome<()> {
94 log_set_level!("error");
95 let db: TestDb = res!(setup::start_db(root, Some(res!(cfg())), schemes(), None, false, true));
96 for i in 0..NKEPT {
97 res!(db.insert(kept(i), val(i as u8 + 1), Uid::default(), None));
98 }
99 hooks::set_halt(Some((halt, flag)));
100 if halt != Halt::Synced {
101 hooks::set_barrier_failure(true);
102 }
103 let _ = db.insert(dat!("kill9 last"), val(99), Uid::default(), None);
104 // A child that is not killed is a failure of the parent, and ends itself.
105 thread::sleep(Duration::from_secs(120));
106 std::process::exit(3);
107}
108
109// The cuts a fault record names, as (file number, data mark, index mark).
110fn record_cuts(root: &Path) -> Outcome<Option<Vec<(u32, u64, u64)>>> {
111 let text = match fs::read_to_string(root.join(constant::FAULT_FILENAME)) {
112 Ok(text) => text,
113 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
114 Err(e) => return Err(err!(e, "Cannot read the fault record."; Test, IO)),
115 };
116 let mut cuts = Vec::new();
117 for line in text.lines() {
118 if let Some(rest) = line.strip_prefix("cut ") {
119 let n: Vec<u64> = rest.split(' ').filter_map(|f| f.parse().ok()).collect();
120 if n.len() < 4 {
121 return Err(err!("A cut line is short: {:?}", line; Test, Invalid));
122 }
123 cuts.push((n[1] as u32, n[2], n[3]));
124 }
125 }
126 Ok(Some(cuts))
127}
128
129fn kill_at(halt: Halt) -> Outcome<()> {
130 let label = fmt!("{:?}", halt);
131 let dir = fmt!("./test_db_fault_kill9_{}", label.to_lowercase());
132 let _ = fs::remove_dir_all(&dir);
133 res!(fs::create_dir_all(&dir));
134 let root = res!(Path::new(&dir).canonicalize());
135 let flag = root.with_extension("halted");
136 let log = root.with_extension("log");
137 let _ = fs::remove_file(&flag);
138
139 let out = res!(fs::File::create(&log));
140 let err = res!(out.try_clone());
141 let mut kid = res!(Command::new(res!(std::env::current_exe()))
142 .args(["main", "--exact", "--nocapture", "--test-threads=1"])
143 .env(ROOT, &root)
144 .env(HALT, &label)
145 .env(FLAG, &flag)
146 .stdout(Stdio::from(out))
147 .stderr(Stdio::from(err))
148 .spawn());
149 let halted = wait_until(Duration::from_secs(120), || flag.exists());
150 res!(kid.kill());
151 let _ = kid.wait();
152 assert!(halted, "{}: the child never reached its halt; see {:?}", label, log);
153
154 let recorded = halt != Halt::Synced && halt != Halt::RecordSynced;
155 let marked = halt == Halt::Quarantined || halt == Halt::DataCut;
156 let cuts = res!(record_cuts(&root));
157 assert_eq!(cuts.is_some(), recorded, "{}: the fault record is {} after the kill", label,
158 if cuts.is_some() { "there" } else { "missing" });
159 assert_eq!(Refusal::path(&root).exists(), marked, "{}: the refusal marker is {} after the kill",
160 label, if marked { "missing" } else { "there" });
161
162 let c = res!(cfg());
163 let open = || -> Outcome<TestDb> {
164 log_set_level!("error");
165 setup::start_db(root.clone(), Some(c.clone()), schemes(), None, false, false)
166 };
167 if marked {
168 match open() {
169 Ok(db) => {
170 let _ = db.close();
171 return Err(err!("{}: the store opened over its refusal marker.", label; Test));
172 },
173 Err(e) => assert!(fmt!("{}", e).contains(constant::REFUSAL_FILENAME),
174 "{}: the refusal does not name its marker: {}", label, e),
175 }
176 assert!(res!(Refusal::clear(&root)));
177 }
178 let db = res!(open());
179 for i in 0..NKEPT {
180 let got = res!(db.get(&kept(i), None)).map(|(v, _)| v);
181 assert_eq!(got, Some(val(i as u8 + 1)), "{}: confirmed key {} did not read back", label, i);
182 }
183 let last = res!(db.get(&dat!("kill9 last"), None)).map(|(v, _)| v);
184 if recorded {
185 assert_eq!(last, None, "{}: the unconfirmed value was served after its fault was recorded", label);
186 } else {
187 assert!(last.is_none() || last == Some(val(99)), "{}: the last key read torn", label);
188 }
189 res!(db.close());
190 assert!(!root.join(constant::FAULT_FILENAME).exists(), "{}: the record outlived the cut", label);
191 let zdir = c.zone_root(&root).join("zone_001");
192 for (f, d, i) in cuts.unwrap_or_default() {
193 let dl = fs::metadata(zdir.join(ZoneDir::relative_file_path(&FileType::Data, f))).map(|m| m.len()).unwrap_or(0);
194 let il = fs::metadata(zdir.join(ZoneDir::relative_file_path(&FileType::Index, f))).map(|m| m.len()).unwrap_or(0);
195 assert!(dl <= d && il <= i, "{}: file {} is ({}, {}), past its mark ({}, {})", label, f, dl, il, d, i);
196 }
197 // QA r2b: the store keeps working after the cut, and a second reopen agrees with the first.
198 let db = res!(open());
199 res!(db.insert(dat!("kill9 after"), val(77), Uid::default(), None));
200 res!(db.close());
201 let db = res!(open());
202 for i in 0..NKEPT {
203 let got = res!(db.get(&kept(i), None)).map(|(v, _)| v);
204 assert_eq!(got, Some(val(i as u8 + 1)), "{}: confirmed key {} lost at the second reopen", label, i);
205 }
206 let last2 = res!(db.get(&dat!("kill9 last"), None)).map(|(v, _)| v);
207 assert_eq!(last2, last, "{}: the last key changed between reopens", label);
208 let after = res!(db.get(&dat!("kill9 after"), None)).map(|(v, _)| v);
209 assert_eq!(after, Some(val(77)), "{}: the write after the cut was lost", label);
210 res!(db.close());
211 let _ = fs::remove_dir_all(&root);
212 let _ = fs::remove_file(&flag);
213 let _ = fs::remove_file(&log);
214 Ok(())
215}
216
217#[test]
218fn main() -> Outcome<()> {
219 match (std::env::var(ROOT), std::env::var(HALT), std::env::var(FLAG)) {
220 (Ok(root), Ok(halt), Ok(flag)) =>
221 child(PathBuf::from(root), res!(halt_named(&halt)), PathBuf::from(flag)),
222 _ => {
223 for halt in HALTS {
224 res!(kill_at(halt));
225 }
226 Ok(())
227 },
228 }
229}