oxedyne/fe2o3/fe2o3_o3db_sync/tests/compact_crash_pair.rs
9.3 KiB, 1 run
created by r1870400018:68519, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! A process killed in the middle of `compact_now` must leave a store that reads every key right |
| 2 | //! and that `compact_now` can finish erasing (A3 R4, 2026-10-08). A collection puts two new files |
| 3 | //! in place of two old ones, and no pair of renames is one step: killed between them, the data file |
| 4 | //! is of one generation beside the index of the other (see `gc_crash_pair.rs`, which kills an |
| 5 | //! ordinary collection). `compact_now` runs the same collection by order, on a store whose |
| 6 | //! collection is switched off, and it is the call that an erasure relies on. |
| 7 | //! |
| 8 | //! The test runs itself twice. The child is this test binary again, started with the name of |
| 9 | //! a directory in its environment: it writes a store, deletes one key in each of the first two |
| 10 | //! data files, holds the next commit between the renames with `test::hooks::set_commit_delay`, and |
| 11 | //! calls `compact_now`. The parent watches the files, and the moment exactly one pair has changed |
| 12 | //! it kills the child with SIGKILL, which no handler in the child can answer. The collection runs |
| 13 | //! one file at a time, so the other file still holds its deleted key's bytes: the control that |
| 14 | //! says the last step, an erase on the reopened store, had something to erase. |
| 15 | |
| 16 | mod gc_pair; |
| 17 | |
| 18 | use gc_pair::*; |
| 19 | |
| 20 | use oxedyne_fe2o3_core::prelude::*; |
| 21 | use oxedyne_fe2o3_iop_db::api::Database; |
| 22 | use oxedyne_fe2o3_o3db_sync::{ |
| 23 | base::cfg::OzoneConfig, |
| 24 | file::core::FileType, |
| 25 | test::{ |
| 26 | hooks, |
| 27 | setup::{ |
| 28 | self, |
| 29 | Uid, |
| 30 | }, |
| 31 | }, |
| 32 | }; |
| 33 | |
| 34 | use std::{ |
| 35 | path::{ |
| 36 | Path, |
| 37 | PathBuf, |
| 38 | }, |
| 39 | process::{ |
| 40 | Command, |
| 41 | Stdio, |
| 42 | }, |
| 43 | thread, |
| 44 | time::Duration, |
| 45 | }; |
| 46 | |
| 47 | const ROOT: &str = "O3DB_COMPACT_CRASH_ROOT"; |
| 48 | const LEN: &str = "O3DB_COMPACT_CRASH_LEN"; |
| 49 | const READY: &str = "O3DB_COMPACT_CRASH_READY"; |
| 50 | |
| 51 | const DELETED: [usize; 2] = [0, 5]; // the first key of file 1, and of file 2 |
| 52 | |
| 53 | /// The child: fills the store, deletes a key from each of the first two files, then calls |
| 54 | /// `compact_now` with the commit held between its renames, and waits there to be killed. It |
| 55 | /// starts only when the parent has taken the sizes of the files, since a collection takes a few |
| 56 | /// milliseconds. |
| 57 | fn child(root: PathBuf, len: u64, ready: PathBuf) -> Outcome<()> { |
| 58 | log_set_level!("error"); |
| 59 | let cfg = res!(config(5 * len + len / 2)); |
| 60 | // Collection off: only the order to compact collects. |
| 61 | let db: TestDb = res!(setup::start_db(root, Some(cfg), schemes(), None, false, true)); |
| 62 | res!(fill(&db)); |
| 63 | for i in DELETED { |
| 64 | res!(db.delete(&key(i), Uid::default(), None)); |
| 65 | } |
| 66 | res!(db.api().settle_for_test(Duration::from_secs(10))); |
| 67 | hooks::set_commit_delay(Duration::from_secs(120)); |
| 68 | res!(std::fs::write(&ready, b"ready")); |
| 69 | let go = ready.with_extension("go"); |
| 70 | assert!(wait_until(Duration::from_secs(120), || go.exists()), "the parent never said go"); |
| 71 | let _ = db.compact_now(Duration::from_secs(150)); |
| 72 | // A child that is not killed is a failure of the parent, and ends itself. |
| 73 | thread::sleep(Duration::from_secs(150)); |
| 74 | std::process::exit(3); |
| 75 | } |
| 76 | |
| 77 | // Every key reads as its version 1, and the deleted ones do not read at all. |
| 78 | fn judge_after(db: &TestDb, label: &str) -> usize { |
| 79 | let mut bad = 0; |
| 80 | for i in 0..NKEYS { |
| 81 | let got = db.get(&key(i), None); |
| 82 | if DELETED.contains(&i) { |
| 83 | match got { |
| 84 | Ok(None) => (), |
| 85 | other => { |
| 86 | bad += 1; |
| 87 | let s: String = fmt!("{:?}", other).chars().take(200).collect(); |
| 88 | msg!("{}: deleted key {} read back {}.", label, i, s); |
| 89 | }, |
| 90 | } |
| 91 | } else { |
| 92 | match got { |
| 93 | Ok(Some((v, _))) if v == value(i, 1) => (), |
| 94 | other => { |
| 95 | bad += 1; |
| 96 | let s: String = fmt!("{:?}", other).chars().take(200).collect(); |
| 97 | msg!("{}: key {} did not read back its value: {}.", label, i, s); |
| 98 | }, |
| 99 | } |
| 100 | } |
| 101 | } |
| 102 | msg!("{}: {} of {} keys wrong.", label, bad, NKEYS); |
| 103 | bad |
| 104 | } |
| 105 | |
| 106 | // How many data files hold each deleted key's value. |
| 107 | fn needles(root: &Path, cfg: &OzoneConfig) -> Vec<usize> { |
| 108 | DELETED.iter().map(|i| { |
| 109 | let bytes = match value(*i, 1).bytes_ref() { Some(b) => b.to_vec(), None => Vec::new() }; |
| 110 | files_holding(root, cfg, &bytes).len() |
| 111 | }).collect() |
| 112 | } |
| 113 | |
| 114 | fn parent() -> Outcome<()> { |
| 115 | log_set_level!("error"); |
| 116 | let dir = "./test_db_compact_crash_pair"; |
| 117 | let _ = std::fs::remove_dir_all(dir); |
| 118 | res!(std::fs::create_dir_all(dir)); |
| 119 | let root = res!(Path::new(dir).canonicalize()); |
| 120 | let ready = res!(std::env::current_dir()).join("test_db_compact_crash_pair.ready"); |
| 121 | let _ = std::fs::remove_file(&ready); |
| 122 | let _ = std::fs::remove_file(ready.with_extension("go")); |
| 123 | let log = res!(std::env::current_dir()).join("test_db_compact_crash_pair.log"); |
| 124 | |
| 125 | let probe = res!(probe("./test_db_compact_crash_pair_probe")); |
| 126 | let len = probe.len; |
| 127 | let cfg = res!(config(5 * len + len / 2)); |
| 128 | let pair = |n: u32| (file(&root, &cfg, FileType::Data, n), file(&root, &cfg, FileType::Index, n)); |
| 129 | let (d1, i1) = pair(1); |
| 130 | let (d2, i2) = pair(2); |
| 131 | |
| 132 | let exe = res!(std::env::current_exe()); |
| 133 | let out = res!(std::fs::File::create(&log)); |
| 134 | let err = res!(out.try_clone()); |
| 135 | let mut kid = res!(Command::new(exe) |
| 136 | .args(["main", "--exact", "--nocapture", "--test-threads=1"]) |
| 137 | .env(ROOT, &root) |
| 138 | .env(LEN, fmt!("{}", len)) |
| 139 | .env(READY, &ready) |
| 140 | .stdout(Stdio::from(out)) |
| 141 | .stderr(Stdio::from(err)) |
| 142 | .spawn()); |
| 143 | |
| 144 | // The child is ready when files 1 and 2 hold their five records each. |
| 145 | assert!(wait_until(Duration::from_secs(120), || ready.exists()), |
| 146 | "the child never wrote its store; see {:?}", log); |
| 147 | let (sd1, si1, sd2, si2) = (size(&d1), size(&i1), size(&d2), size(&i2)); |
| 148 | assert_eq!((sd1, si1), (5 * len, probe.ind[5]), "file 1 does not hold the five records it was sized for"); |
| 149 | assert_eq!((sd2, si2), (5 * len, probe.ind[5]), "file 2 does not hold the five records it was sized for"); |
| 150 | res!(std::fs::write(ready.with_extension("go"), b"go")); |
| 151 | |
| 152 | // Exactly one file has had one of its pair replaced: a collection is between its renames. |
| 153 | let torn = |d: &PathBuf, i: &PathBuf, sd: u64, si: u64| (size(d) != sd) != (size(i) != si); |
| 154 | let caught = wait_until(Duration::from_secs(60), || |
| 155 | torn(&d1, &i1, sd1, si1) || torn(&d2, &i2, sd2, si2)); |
| 156 | res!(kid.kill()); |
| 157 | let _ = kid.wait(); |
| 158 | assert!(caught, "compact_now never stopped between its renames; see {:?}", log); |
| 159 | let which = if torn(&d1, &i1, sd1, si1) { 1 } else { 2 }; |
| 160 | msg!("Killed with file {} torn: file 1 data {} index {}, file 2 data {} index {}.", |
| 161 | which, size(&d1), size(&i1), size(&d2), size(&i2)); |
| 162 | |
| 163 | // The store the child left, reopened with collection still off. |
| 164 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, false, false)); |
| 165 | let reopened = judge_after(&db, "After the kill"); |
| 166 | let temps = temporaries(&zone_dir(&root, &cfg)); |
| 167 | let size_reopened = res!(db.size_bytes()); |
| 168 | let walked_reopened = zone_files_len(&root, &cfg); |
| 169 | let before = needles(&root, &cfg); |
| 170 | msg!("Reopened: size_bytes {}, walked {}, deleted keys in {:?} data files.", |
| 171 | size_reopened, walked_reopened, before); |
| 172 | |
| 173 | // The order that finishes the erasure. |
| 174 | let report = res!(db.compact_now(Duration::from_secs(60))); |
| 175 | msg!("{:?}", report); |
| 176 | let after = needles(&root, &cfg); |
| 177 | let finished = judge_after(&db, "After compact_now"); |
| 178 | let temps_after = temporaries(&zone_dir(&root, &cfg)); |
| 179 | let size_after = res!(db.size_bytes()); |
| 180 | let walked_after = zone_files_len(&root, &cfg); |
| 181 | res!(db.close()); |
| 182 | |
| 183 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, false, false)); |
| 184 | let second = judge_after(&db, "After a second reopening"); |
| 185 | res!(db.close()); |
| 186 | log_finish_wait!(); |
| 187 | |
| 188 | assert_eq!(reopened, 0, "{} keys read wrong after the kill", reopened); |
| 189 | assert!(temps.is_empty(), "the start left {:?} of the killed collection behind", temps); |
| 190 | assert_eq!(size_reopened, walked_reopened, "size_bytes is not the length of the files after the kill"); |
| 191 | assert!(before.iter().sum::<usize>() >= 1, |
| 192 | "positive control: the file that was not collected holds its deleted key ({:?}), file {} was torn", before, which); |
| 193 | assert_eq!(after, vec![0, 0], "a deleted key is still in a data file after compact_now: {:?}", after); |
| 194 | assert_eq!(finished, 0, "{} keys read wrong after compact_now", finished); |
| 195 | assert!(temps_after.is_empty(), "compact_now left {:?} behind", temps_after); |
| 196 | assert_eq!(size_after, walked_after, "size_bytes is not the length of the files after compact_now"); |
| 197 | assert_eq!(second, 0, "{} keys read wrong after the second reopening", second); |
| 198 | let _ = std::fs::remove_dir_all(dir); |
| 199 | let _ = std::fs::remove_file(&ready); |
| 200 | let _ = std::fs::remove_file(ready.with_extension("go")); |
| 201 | let _ = std::fs::remove_file(&log); |
| 202 | Ok(()) |
| 203 | } |
| 204 | |
| 205 | #[test] |
| 206 | fn main() -> Outcome<()> { |
| 207 | match (std::env::var(ROOT), std::env::var(LEN), std::env::var(READY)) { |
| 208 | (Ok(root), Ok(len), Ok(ready)) => { |
| 209 | let len = res!(len.parse::<u64>()); |
| 210 | child(PathBuf::from(root), len, PathBuf::from(ready)) |
| 211 | }, |
| 212 | _ => parent(), |
| 213 | } |
| 214 | } |