Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/dist_kill9.rs

7.3 KiB, 1 run

created by r1870400018:67680, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1#![cfg(feature = "dist")]
2//! A put the engine returned for survives `kill -9` of the process (A2 QA agent 2).
3//!
4//! The one-zone configuration that `DistOzone::over_o3db` is used with in production: one zone,
5//! a barrier on every write and a cache far smaller than what is stored. The test re-runs its
6//! own binary as a child that puts records in a loop and prints each id once `put_at` has
7//! returned for it. The parent reads those lines, kills the child with SIGKILL in mid-stream,
8//! reopens the root, and every acknowledged record must read back byte for byte. Three rounds
9//! reopen a root that the previous round crashed. The one write the child was inside when it
10//! died may be there or not, but if it is there it must be whole.
11//!
12//! Each run takes about half a minute at most.
13//!
14//! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\
15//! Anthropic Claude
16
17use oxedyne_fe2o3_core::prelude::*;
18use oxedyne_fe2o3_crypto::enc::EncryptionScheme;
19use oxedyne_fe2o3_hash::{
20 csum::ChecksumScheme,
21 hash::HashScheme,
22};
23use oxedyne_fe2o3_jdat::prelude::*;
24use oxedyne_fe2o3_o3db_sync::{
25 base::cfg::OzoneConfig,
26 data::core::RestSchemesInput,
27 dist::{
28 config::{
29 DistOzoneConfig,
30 TableConfig,
31 },
32 engine::DistOzone,
33 o3db_storage::O3dbStorage,
34 record::{
35 Record,
36 RecordId,
37 },
38 resolve::LastVersionWins,
39 storage::Storage,
40 },
41 kademlia::id::NodeId,
42 oam::config::OamConfig,
43 test::setup,
44};
45
46use std::{
47 collections::BTreeMap,
48 io::{
49 BufRead,
50 BufReader,
51 },
52 os::unix::process::ExitStatusExt,
53 path::PathBuf,
54 process::{
55 Command,
56 Stdio,
57 },
58 time::Duration,
59};
60
61
62type Eng = DistOzone<
63 O3dbStorage<{ setup::UID_LEN }, setup::Uid, EncryptionScheme, HashScheme, HashScheme, ChecksumScheme>,
64 LastVersionWins,
65>;
66
67const ENV_ROOT: &str = "DIST_KILL9_ROOT";
68const ENV_START: &str = "DIST_KILL9_START";
69const ENV_FRESH: &str = "DIST_KILL9_FRESH";
70const ACK: &str = "ACK ";
71
72fn rid(n: u32) -> RecordId {
73 let mut bytes = [0xABu8; 32];
74 bytes[..4].copy_from_slice(&n.to_be_bytes());
75 RecordId::from_bytes(bytes)
76}
77
78fn number(id: &RecordId) -> u32 {
79 let b = id.as_bytes();
80 u32::from_be_bytes([b[0], b[1], b[2], b[3]])
81}
82
83// 200 to 1,500 bytes, some past 256 so that a width is exercised, and different for each n.
84fn value(n: u32) -> Vec<u8> {
85 let len = 200 + (n % 97) as usize * 13;
86 (0..len).map(|i| (i as u32).wrapping_mul(n).wrapping_add(n >> 3) as u8).collect()
87}
88
89fn schemes() -> Outcome<RestSchemesInput<EncryptionScheme, HashScheme, HashScheme, ChecksumScheme>> {
90 let enckey = [0x55u8; 32];
91 let aes_gcm = res!(EncryptionScheme::new_aes_256_gcm_with_key(&enckey[..]));
92 Ok(RestSchemesInput::new(
93 Some(aes_gcm),
94 None::<HashScheme>,
95 None::<HashScheme>,
96 Some(ChecksumScheme::new_crc32()),
97 ))
98}
99
100// One zone, a barrier on every write, and a cache far smaller than what is stored. The saved
101// configuration wins on a reopen, so only a fresh root reads it.
102fn one_zone() -> Outcome<OzoneConfig> {
103 let overrides = res!(mapdat!{
104 1u16 => mapdat!{ "dir" => "", "max_size" => 104_857_600u64 },
105 }.get_map().ok_or_else(|| err!("The zone override map is not a map."; Test, Invalid)));
106 Ok(OzoneConfig {
107 num_zones: 1,
108 sync_on_write: true,
109 cache_size_limit_bytes: 4_096,
110 zone_overrides: overrides,
111 ..OzoneConfig::default()
112 })
113}
114
115fn open(root: &PathBuf, fresh: bool) -> Outcome<Eng> {
116 let oam = res!(OamConfig::new(1, 1));
117 let dist = res!(DistOzoneConfig::new(
118 NodeId::from_bytes([1u8; 32]),
119 Vec::new(),
120 oam,
121 vec![res!(TableConfig::eventual("identity"))],
122 ));
123 let cfg = if fresh { Some(res!(one_zone())) } else { None };
124 Eng::over_o3db(root.clone(), cfg, dist, res!(schemes()), setup::Uid::default(), LastVersionWins)
125}
126
127fn root_dir(name: &str) -> PathBuf {
128 PathBuf::from(fmt!("./test_db_dist_kill9_{}", name))
129}
130
131// The child. It does nothing unless the parent set its environment, so an ordinary run of this
132// binary passes it at once.
133#[test]
134fn kill9_child() -> Outcome<()> {
135 let root = match std::env::var(ENV_ROOT) {
136 Ok(r) => PathBuf::from(r),
137 Err(_) => return Ok(()),
138 };
139 let start: u32 = res!(res!(std::env::var(ENV_START)).parse::<u32>());
140 let fresh = std::env::var(ENV_FRESH).is_ok();
141 // A child the parent never kills ends itself.
142 std::thread::spawn(|| {
143 std::thread::sleep(Duration::from_secs(150));
144 std::process::exit(3);
145 });
146 let e = res!(open(&root, fresh));
147 for n in start.. {
148 let out = res!(e.put_at(Record::new(rid(n), "identity", value(n)), 1_000));
149 assert!(out.local_persisted);
150 // Only now has the put returned, so only now is the id acknowledged.
151 println!("{}{}", ACK, n);
152 }
153 Ok(())
154}
155
156#[test]
157fn kill9_acknowledged_puts_survive() -> Outcome<()> {
158 let root = root_dir("rounds");
159 if root.exists() {
160 res!(std::fs::remove_dir_all(&root));
161 }
162 res!(std::fs::create_dir_all(&root));
163 let root = res!(root.canonicalize());
164 let exe = res!(std::env::current_exe());
165
166 let mut acked: BTreeMap<u32, Vec<u8>> = BTreeMap::new();
167 let rounds: [(u32, usize); 3] = [(1, 25), (1_000_000, 50), (2_000_000, 90)];
168 for (round, (start, kill_after)) in rounds.iter().enumerate() {
169 let mut child = res!(Command::new(&exe)
170 .args(["kill9_child", "--exact", "--nocapture", "--test-threads=1"])
171 .env(ENV_ROOT, &root)
172 .env(ENV_START, start.to_string())
173 .envs(if round == 0 { Some((ENV_FRESH, "1")) } else { None })
174 .stdout(Stdio::piped())
175 .stderr(Stdio::inherit())
176 .spawn());
177 let out = res!(child.stdout.take().ok_or_else(|| err!("The child has no stdout."; Test, Missing)));
178 let mut lines = BufReader::new(out);
179 let mut this_round = 0usize;
180 let mut line = String::new();
181 let mut killed = false;
182 loop {
183 line.clear();
184 let got = res!(lines.read_line(&mut line));
185 if got == 0 {
186 break;
187 }
188 // A line cut by the kill has no newline and is not an acknowledgement.
189 if !line.ends_with('\n') {
190 continue;
191 }
192 if let Some(n) = line.trim_end().strip_prefix(ACK) {
193 let n: u32 = res!(n.parse::<u32>());
194 acked.insert(n, value(n));
195 this_round += 1;
196 if this_round == *kill_after && !killed {
197 // SIGKILL, in mid-stream: the child is already inside its next put.
198 res!(child.kill());
199 killed = true;
200 }
201 }
202 }
203 let status = res!(child.wait());
204 assert!(killed, "round {}: the child ended after {} acknowledgements, before the kill ({:?})",
205 round, this_round, status);
206 assert_eq!(status.signal(), Some(9), "round {}: the child was not ended by SIGKILL", round);
207 assert!(this_round >= *kill_after);
208
209 // Reopen what the crash left, and read everything acknowledged so far.
210 let e = res!(open(&root, false));
211 for (n, want) in &acked {
212 let got = res!(e.storage().get("identity", &rid(*n)));
213 let got = res!(got.ok_or_else(|| err!("round {}: acknowledged record {} is missing.", round, n; Test, Missing)));
214 assert_eq!(&got.value, want, "round {}: record {} came back at other bytes", round, n);
215 }
216 // A write the child was inside when it died may be present, whole, or absent.
217 let digests = res!(e.storage().digests("identity"));
218 for d in &digests {
219 let n = number(&d.id);
220 if acked.contains_key(&n) {
221 continue;
222 }
223 if let Some(r) = res!(e.storage().get("identity", &d.id)) {
224 assert_eq!(r.value, value(n), "round {}: unacknowledged record {} is torn", round, n);
225 }
226 }
227 assert!(digests.len() >= acked.len());
228 assert!(digests.len() <= acked.len() + round + 1,
229 "round {}: {} records stored, {} acknowledged", round, digests.len(), acked.len());
230 res!(e.close());
231 }
232 Ok(())
233}