oxedyne/fe2o3/fe2o3_o3db_sync/tests/gc_crash_pair.rs
6.8 KiB, 1 run
created by r1870400018:62333, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! A process killed between the two renames of a collection must leave a store that reads every |
| 2 | //! key right. A collection puts two new files in place of two old ones, and no pair of renames |
| 3 | //! is one step: a process that dies between them leaves a data file of one generation beside the |
| 4 | //! index of the other. The start used to believe such an index, and the cache then named the |
| 5 | //! offsets of one file in the other (D-C, found by QA of lane o3i, 2026-09-24). |
| 6 | //! |
| 7 | //! The test runs itself twice. The child is this test binary again, started with the name of |
| 8 | //! a directory in its environment: it writes a store, holds its next collection between the |
| 9 | //! renames with `test::hooks::set_commit_delay`, and starts the collection. The parent watches |
| 10 | //! the files, and the moment exactly one of the pair has changed it kills the child with SIGKILL, |
| 11 | //! which no handler in the child can answer. The parent then opens the store the child left. |
| 12 | |
| 13 | mod gc_pair; |
| 14 | |
| 15 | use gc_pair::*; |
| 16 | |
| 17 | use oxedyne_fe2o3_core::prelude::*; |
| 18 | use oxedyne_fe2o3_iop_db::api::Database; |
| 19 | use oxedyne_fe2o3_o3db_sync::{ |
| 20 | file::core::FileType, |
| 21 | test::{ |
| 22 | hooks, |
| 23 | setup::{ |
| 24 | self, |
| 25 | Uid, |
| 26 | }, |
| 27 | }, |
| 28 | }; |
| 29 | |
| 30 | use std::{ |
| 31 | path::PathBuf, |
| 32 | process::{ |
| 33 | Command, |
| 34 | Stdio, |
| 35 | }, |
| 36 | thread, |
| 37 | time::Duration, |
| 38 | }; |
| 39 | |
| 40 | const ROOT: &str = "O3DB_GC_CRASH_ROOT"; |
| 41 | const LEN: &str = "O3DB_GC_CRASH_LEN"; |
| 42 | const READY: &str = "O3DB_GC_CRASH_READY"; |
| 43 | |
| 44 | /// The child: fills the store, then starts a collection that stops between its renames, and waits |
| 45 | /// there to be killed. It starts the collection only when the parent has taken the sizes of |
| 46 | /// the pair, since the collection takes a few milliseconds. |
| 47 | fn child(root: PathBuf, len: u64, ready: PathBuf) -> Outcome<()> { |
| 48 | log_set_level!("error"); |
| 49 | let cfg = res!(config(5 * len + len / 2)); |
| 50 | let db: TestDb = res!(setup::start_db(root, Some(cfg), schemes(), None, true, true)); |
| 51 | res!(fill(&db)); |
| 52 | hooks::set_commit_delay(Duration::from_secs(120)); |
| 53 | // The parent reads the sizes of the pair when it sees this, and answers with `go`. |
| 54 | res!(std::fs::write(&ready, b"ready")); |
| 55 | let go = ready.with_extension("go"); |
| 56 | assert!(wait_until(Duration::from_secs(120), || go.exists()), "the parent never said go"); |
| 57 | // Superseding two of file 1's five records starts its collection. |
| 58 | res!(db.insert(key(0), value(0, 2), Uid::default(), None)); |
| 59 | res!(db.insert(key(1), value(1, 2), Uid::default(), None)); |
| 60 | // A child that is not killed is a failure of the parent, and ends itself. |
| 61 | thread::sleep(Duration::from_secs(150)); |
| 62 | std::process::exit(3); |
| 63 | } |
| 64 | |
| 65 | fn parent() -> Outcome<()> { |
| 66 | log_set_level!("error"); |
| 67 | let dir = "./test_db_gc_crash_pair"; |
| 68 | let _ = std::fs::remove_dir_all(dir); |
| 69 | res!(std::fs::create_dir_all(dir)); |
| 70 | let root = res!(std::path::Path::new(dir).canonicalize()); |
| 71 | let ready = res!(std::env::current_dir()).join("test_db_gc_crash_pair.ready"); |
| 72 | let _ = std::fs::remove_file(&ready); |
| 73 | let _ = std::fs::remove_file(ready.with_extension("go")); |
| 74 | let log = res!(std::env::current_dir()).join("test_db_gc_crash_pair.log"); |
| 75 | |
| 76 | let probe = res!(probe("./test_db_gc_crash_pair_probe")); |
| 77 | let len = probe.len; |
| 78 | let cfg = res!(config(5 * len + len / 2)); |
| 79 | let (d1, i1) = (file(&root, &cfg, FileType::Data, 1), file(&root, &cfg, FileType::Index, 1)); |
| 80 | |
| 81 | let exe = res!(std::env::current_exe()); |
| 82 | let out = res!(std::fs::File::create(&log)); |
| 83 | let err = res!(out.try_clone()); |
| 84 | let mut kid = res!(Command::new(exe) |
| 85 | .args(["main", "--exact", "--nocapture", "--test-threads=1"]) |
| 86 | .env(ROOT, &root) |
| 87 | .env(LEN, fmt!("{}", len)) |
| 88 | .env(READY, &ready) |
| 89 | .stdout(Stdio::from(out)) |
| 90 | .stderr(Stdio::from(err)) |
| 91 | .spawn()); |
| 92 | |
| 93 | // The child is ready when file 1 holds its five records. |
| 94 | assert!(wait_until(Duration::from_secs(120), || ready.exists()), |
| 95 | "the child never wrote its store; see {:?}", log); |
| 96 | let (d_old, i_old) = (size(&d1), size(&i1)); |
| 97 | assert_eq!((d_old, i_old), (5 * len, probe.ind[5]), "file 1 does not hold the five records it was sized for"); |
| 98 | res!(std::fs::write(ready.with_extension("go"), b"go")); |
| 99 | |
| 100 | // Exactly one of the pair has changed: the collection is between its renames. |
| 101 | let caught = wait_until(Duration::from_secs(60), || |
| 102 | (size(&d1) != d_old) != (size(&i1) != i_old)); |
| 103 | res!(kid.kill()); |
| 104 | let _ = kid.wait(); |
| 105 | assert!(caught, "the collection never stopped between its renames; see {:?}", log); |
| 106 | let (d_torn, i_torn) = (size(&d1), size(&i1)); |
| 107 | msg!("Killed with file 1 at data {} (was {}) and index {} (was {}).", d_torn, d_old, i_torn, i_old); |
| 108 | assert!((d_torn != d_old) != (i_torn != i_old), "the pair was whole when the child was killed"); |
| 109 | |
| 110 | // The store the child left. |
| 111 | let mut want = vec![1u8; NKEYS]; |
| 112 | want[0] = 2; |
| 113 | want[1] = 2; |
| 114 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, true, false)); |
| 115 | let reopened = judge(&db, &want, "After the kill"); |
| 116 | // The index has been made the data file's own, whichever file the kill left new. |
| 117 | let want_ind = probe.ind[(size(&d1) / len) as usize]; |
| 118 | let rebuilt = size(&i1); |
| 119 | let temps = temporaries(&zone_dir(&root, &cfg)); |
| 120 | |
| 121 | // Churn so that file 1 is collected again with whatever state the start built. |
| 122 | for round in 4..7u8 { |
| 123 | for i in 2..6 { |
| 124 | res!(db.insert(key(i), value(i, round), Uid::default(), None)); |
| 125 | want[i] = round; |
| 126 | } |
| 127 | } |
| 128 | thread::sleep(Duration::from_secs(3)); |
| 129 | let churned = judge(&db, &want, "After more collection"); |
| 130 | res!(db.close()); |
| 131 | |
| 132 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, true, false)); |
| 133 | let second = judge(&db, &want, "After a second reopening"); |
| 134 | res!(db.close()); |
| 135 | log_finish_wait!(); |
| 136 | |
| 137 | assert_eq!(reopened, 0, "{} keys read wrong after the kill", reopened); |
| 138 | assert_eq!(rebuilt, want_ind, |
| 139 | "the index is {} bytes beside a data file of {}, where records of {} bytes need {}", |
| 140 | rebuilt, d_torn, len, want_ind); |
| 141 | assert!(temps.is_empty(), "the start left {:?} of the killed collection behind", temps); |
| 142 | assert_eq!(churned, 0, "{} keys read wrong after file 1 was collected again", churned); |
| 143 | assert_eq!(second, 0, "{} keys read wrong after the second reopening", second); |
| 144 | let _ = std::fs::remove_dir_all(dir); |
| 145 | let _ = std::fs::remove_file(&ready); |
| 146 | let _ = std::fs::remove_file(ready.with_extension("go")); |
| 147 | let _ = std::fs::remove_file(&log); |
| 148 | Ok(()) |
| 149 | } |
| 150 | |
| 151 | #[test] |
| 152 | fn main() -> Outcome<()> { |
| 153 | match (std::env::var(ROOT), std::env::var(LEN), std::env::var(READY)) { |
| 154 | (Ok(root), Ok(len), Ok(ready)) => { |
| 155 | let len = res!(len.parse::<u64>()); |
| 156 | child(PathBuf::from(root), len, PathBuf::from(ready)) |
| 157 | }, |
| 158 | _ => parent(), |
| 159 | } |
| 160 | } |