oxedyne/fe2o3/fe2o3_o3db_sync/tests/store_lock.rs
5.9 KiB, 1 run
created by r1870400018:70427, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! One process per store (Ozone R2-2, 2026-10-10). A start takes an exclusive lock on a file in |
| 2 | //! the store's root, so a second open, in another process or this one, is refused by name rather |
| 3 | //! than surveying and appending to the same live files. The lock is the kernel's: a holder killed |
| 4 | //! with SIGKILL leaves no stale lock behind. |
| 5 | //! |
| 6 | //! The test runs itself as a child, the pattern of `compact_crash_pair.rs`: with the variables |
| 7 | //! below in its environment it opens the store, says so, and waits to be killed. |
| 8 | |
| 9 | mod gc_pair; |
| 10 | |
| 11 | use gc_pair::*; |
| 12 | |
| 13 | use oxedyne_fe2o3_core::prelude::*; |
| 14 | use oxedyne_fe2o3_iop_db::api::Database; |
| 15 | use oxedyne_fe2o3_jdat::prelude::*; |
| 16 | use oxedyne_fe2o3_o3db_sync::{ |
| 17 | gateway, |
| 18 | sweep, |
| 19 | test::setup::{ |
| 20 | self, |
| 21 | Uid, |
| 22 | }, |
| 23 | }; |
| 24 | |
| 25 | use std::{ |
| 26 | path::{ |
| 27 | Path, |
| 28 | PathBuf, |
| 29 | }, |
| 30 | process::{ |
| 31 | Command, |
| 32 | Stdio, |
| 33 | }, |
| 34 | thread, |
| 35 | time::Duration, |
| 36 | }; |
| 37 | |
| 38 | const ROOT: &str = "O3DB_STORE_LOCK_ROOT"; |
| 39 | const READY: &str = "O3DB_STORE_LOCK_READY"; |
| 40 | const GATEWAY: &str = "O3DB_STORE_LOCK_GATEWAY"; |
| 41 | |
| 42 | const KEY: [u8; gateway::DB_KEY_LEN] = [7u8; gateway::DB_KEY_LEN]; // a test store's, allowlist secret |
| 43 | |
| 44 | // The child: opens the store, writes a key, says it is ready, and waits to be killed. |
| 45 | fn child(root: PathBuf, ready: PathBuf, as_gateway: bool) -> Outcome<()> { |
| 46 | log_set_level!("error"); |
| 47 | if as_gateway { |
| 48 | let (db, _) = res!(gateway::open_store(&root, Some(res!(config(16_000))), &KEY, true, "child")); |
| 49 | res!(db.insert(dat!("held"), dat!("by the child"), Uid::default(), None)); |
| 50 | res!(std::fs::write(&ready, b"ready")); |
| 51 | thread::sleep(Duration::from_secs(150)); |
| 52 | } else { |
| 53 | let db: TestDb = res!(setup::start_db(root, Some(res!(config(16_000))), schemes(), None, false, true)); |
| 54 | res!(db.insert(dat!("held"), dat!("by the child"), Uid::default(), None)); |
| 55 | res!(std::fs::write(&ready, b"ready")); |
| 56 | thread::sleep(Duration::from_secs(150)); |
| 57 | } |
| 58 | // A child that is not killed is a failure of the parent, and ends itself. |
| 59 | std::process::exit(3); |
| 60 | } |
| 61 | |
| 62 | fn spawn_holder(name: &str, as_gateway: bool) -> Outcome<(std::process::Child, PathBuf, PathBuf)> { |
| 63 | let dir = PathBuf::from(fmt!("./test_db_store_lock_{}", name)); |
| 64 | let _ = std::fs::remove_dir_all(&dir); |
| 65 | res!(std::fs::create_dir_all(&dir)); |
| 66 | let root = res!(dir.canonicalize()); |
| 67 | let ready = res!(std::env::current_dir()).join(fmt!("test_db_store_lock_{}.ready", name)); |
| 68 | let _ = std::fs::remove_file(&ready); |
| 69 | let log = res!(std::env::current_dir()).join(fmt!("test_db_store_lock_{}.log", name)); |
| 70 | let out = res!(std::fs::File::create(&log)); |
| 71 | let err = res!(out.try_clone()); |
| 72 | let kid = res!(Command::new(res!(std::env::current_exe())) |
| 73 | .args(["main", "--exact", "--nocapture", "--test-threads=1"]) |
| 74 | .env(ROOT, &root) |
| 75 | .env(READY, &ready) |
| 76 | .env(GATEWAY, if as_gateway { "1" } else { "0" }) |
| 77 | .stdout(Stdio::from(out)) |
| 78 | .stderr(Stdio::from(err)) |
| 79 | .spawn()); |
| 80 | assert!(wait_until(Duration::from_secs(120), || ready.exists()), |
| 81 | "the child never opened its store; see {:?}", log); |
| 82 | Ok((kid, root, ready)) |
| 83 | } |
| 84 | |
| 85 | fn refusal<T>(r: Outcome<T>, root: &Path, what: &str) -> Error<ErrTag> { |
| 86 | match r { |
| 87 | Ok(_) => panic!("{}: a second open of a held store at {:?} was allowed", what, root), |
| 88 | Err(e) => { |
| 89 | let tags = e.tags(); |
| 90 | assert!(tags.contains(&ErrTag::Init) && tags.contains(&ErrTag::Lock) |
| 91 | && tags.contains(&ErrTag::Conflict), "{}: refused with tags {:?}: {}", what, tags, e); |
| 92 | let text = fmt!("{}", e); |
| 93 | assert!(text.contains(&fmt!("{}", root.display())), "{}: the refusal does not name the store: {}", what, text); |
| 94 | e |
| 95 | }, |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | fn parent() -> Outcome<()> { |
| 100 | log_set_level!("error"); |
| 101 | |
| 102 | // Held by another process, then that process killed with SIGKILL. |
| 103 | let (mut kid, root, ready) = res!(spawn_holder("kill", false)); |
| 104 | let cfg = res!(config(16_000)); |
| 105 | refusal(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, false, false), |
| 106 | &root, "held by a child"); |
| 107 | res!(kid.kill()); |
| 108 | let _ = kid.wait(); |
| 109 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, false, false)); |
| 110 | let held = res!(db.get(&dat!("held"), None)); |
| 111 | assert!(matches!(&held, Some((v, _)) if *v == dat!("by the child")), "the killed child's write reads {:?}", held); |
| 112 | |
| 113 | // Held in this process: a second open is refused here too. |
| 114 | refusal(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, false, false), |
| 115 | &root, "held by this process"); |
| 116 | |
| 117 | // Closed, then opened again. |
| 118 | res!(db.close()); |
| 119 | drop(db); |
| 120 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, false, false)); |
| 121 | res!(db.close()); |
| 122 | drop(db); |
| 123 | let _ = std::fs::remove_dir_all(&root); |
| 124 | let _ = std::fs::remove_file(&ready); |
| 125 | |
| 126 | // The sweep tool against a store the gateway holds: refused, saying what to do instead. |
| 127 | let (mut kid, root, ready) = res!(spawn_holder("sweep", true)); |
| 128 | let e = refusal(sweep::open_store_alone(&root, &KEY, "sweep"), &root, "the sweep tool"); |
| 129 | let text = fmt!("{}", e); |
| 130 | res!(kid.kill()); |
| 131 | let _ = kid.wait(); |
| 132 | assert!(text.contains("Stop the gateway") && text.contains("sweep::sweep_orphans"), |
| 133 | "the sweep tool's refusal does not say what to do: {}", text); |
| 134 | // With the gateway gone the tool opens it. |
| 135 | let (db, n) = res!(sweep::open_store_alone(&root, &KEY, "sweep")); |
| 136 | res!(gateway::require_answer(n, "sweep", &root)); |
| 137 | res!(db.shutdown()); |
| 138 | let _ = std::fs::remove_dir_all(&root); |
| 139 | let _ = std::fs::remove_file(&ready); |
| 140 | log_finish_wait!(); |
| 141 | Ok(()) |
| 142 | } |
| 143 | |
| 144 | #[test] |
| 145 | fn main() -> Outcome<()> { |
| 146 | match (std::env::var(ROOT), std::env::var(READY), std::env::var(GATEWAY)) { |
| 147 | (Ok(root), Ok(ready), Ok(g)) => child(PathBuf::from(root), PathBuf::from(ready), g == "1"), |
| 148 | _ => parent(), |
| 149 | } |
| 150 | } |