oxedyne/fe2o3/fe2o3_o3db_sync/tests/scan_fd_bound.rs
5.5 KiB, 7 runs
created by r1870400018:76352, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! A scan of a zone holds a bounded number of open files, whatever the number of files in it. |
| 2 | //! |
| 3 | //! The scan bot held every data file of a pass open until the pass ended, so that the tombstone |
| 4 | //! check after the walk could read each record from the generation its index named. A zone with |
| 5 | //! more files than the process may hold open came up short: each index open past the limit |
| 6 | //! failed, the failure was taken for a collection that had removed the file, and the scan |
| 7 | //! refused with "Scan of this zone is short" on a store that was whole (Daimond restore |
| 8 | //! rehearsal under systemd's default soft limit of 1024 open files, 7,939 file pairs, |
| 9 | //! 2026-11-01). The test runs itself again in a child whose soft limit is below the number of |
| 10 | //! files in the zone, since the limit is per process. |
| 11 | use oxedyne_fe2o3_core::prelude::*; |
| 12 | use oxedyne_fe2o3_crypto::enc::EncryptionScheme; |
| 13 | use oxedyne_fe2o3_hash::{ |
| 14 | csum::ChecksumScheme, |
| 15 | hash::HashScheme, |
| 16 | }; |
| 17 | use oxedyne_fe2o3_iop_db::api::{ |
| 18 | Database, |
| 19 | ScanOpts, |
| 20 | }; |
| 21 | use oxedyne_fe2o3_jdat::prelude::*; |
| 22 | use oxedyne_fe2o3_o3db_sync::{ |
| 23 | base::cfg::OzoneConfig, |
| 24 | data::core::RestSchemesInput, |
| 25 | test::setup, |
| 26 | }; |
| 27 | |
| 28 | use std::{ |
| 29 | collections::BTreeSet, |
| 30 | env, |
| 31 | path::{ |
| 32 | Path, |
| 33 | PathBuf, |
| 34 | }, |
| 35 | process::Command, |
| 36 | time::Duration, |
| 37 | }; |
| 38 | |
| 39 | const N: usize = 400; // Records, one to a data file. |
| 40 | const FD_LIMIT: u32 = 160; // The child's soft limit on open files, well below N. |
| 41 | const CHILD_VAR: &str = "O3DB_SCAN_FD_BOUND_CHILD"; |
| 42 | const TEST_NAME: &str = "scan_of_a_zone_with_more_files_than_the_fd_limit_is_whole"; |
| 43 | |
| 44 | type TestDb = oxedyne_fe2o3_o3db_sync::O3db< |
| 45 | { setup::UID_LEN }, |
| 46 | setup::Uid, |
| 47 | EncryptionScheme, |
| 48 | HashScheme, |
| 49 | HashScheme, |
| 50 | ChecksumScheme, |
| 51 | >; |
| 52 | |
| 53 | fn schemes() -> RestSchemesInput<EncryptionScheme, HashScheme, HashScheme, ChecksumScheme> { |
| 54 | RestSchemesInput::new( |
| 55 | None::<EncryptionScheme>, |
| 56 | None::<HashScheme>, |
| 57 | None::<HashScheme>, |
| 58 | Some(ChecksumScheme::new_crc32()), |
| 59 | ) |
| 60 | } |
| 61 | |
| 62 | fn root() -> Outcome<PathBuf> { |
| 63 | let dir = "./test_db_scan_fd_bound"; |
| 64 | let _ = std::fs::remove_dir_all(dir); |
| 65 | res!(std::fs::create_dir_all(dir)); |
| 66 | Ok(res!(Path::new(dir).canonicalize())) |
| 67 | } |
| 68 | |
| 69 | fn config() -> Outcome<OzoneConfig> { |
| 70 | let mut cfg = res!(setup::default_cfg()); |
| 71 | cfg.num_zones = 1; |
| 72 | cfg.sync_on_write = true; |
| 73 | // Smaller than two records, so that every record starts a data file of its own, with the |
| 74 | // chunking threshold and chunk size inside the ratios the configuration allows. |
| 75 | cfg.data_file_max_bytes = 2_000; |
| 76 | cfg.rest_chunk_threshold = 1_500; |
| 77 | cfg.rest_chunk_bytes = 500; |
| 78 | cfg.zone_overrides = DaticleMap::new(); |
| 79 | Ok(cfg) |
| 80 | } |
| 81 | |
| 82 | fn key(i: usize) -> String { fmt!("k:{:04}", i) } |
| 83 | |
| 84 | fn scanned(db: &TestDb) -> Outcome<BTreeSet<String>> { |
| 85 | let mut out = BTreeSet::new(); |
| 86 | for (k, _, _) in res!(db.scan(&ScanOpts::all(), None)) { |
| 87 | if let Dat::Str(s) = k { |
| 88 | out.insert(s); |
| 89 | } |
| 90 | } |
| 91 | Ok(out) |
| 92 | } |
| 93 | |
| 94 | fn zone_data_files(db: &TestDb) -> Outcome<usize> { |
| 95 | let mut n = 0; |
| 96 | for (_, zdir) in res!(db.api().get_zone_dirs()) { |
| 97 | for entry in res!(std::fs::read_dir(&zdir.dir)) { |
| 98 | let path = res!(entry).path(); |
| 99 | if path.extension().and_then(|e| e.to_str()) == Some("dat") { |
| 100 | n += 1; |
| 101 | } |
| 102 | } |
| 103 | } |
| 104 | Ok(n) |
| 105 | } |
| 106 | |
| 107 | // The body, run in the child under the low limit. |
| 108 | fn scan_under_the_limit() -> Outcome<()> { |
| 109 | log_set_level!("error"); |
| 110 | let root = res!(root()); |
| 111 | let cfg = res!(config()); |
| 112 | let user = setup::Uid::default(); |
| 113 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, true, true)); |
| 114 | let mut want = BTreeSet::new(); |
| 115 | for i in 0..N { |
| 116 | res!(db.insert(dat!(key(i)), dat!("v".repeat(1_100)), user, None)); |
| 117 | want.insert(key(i)); |
| 118 | } |
| 119 | res!(db.api().settle_for_test(Duration::from_secs(30))); |
| 120 | let files = res!(zone_data_files(&db)); |
| 121 | assert!(files > FD_LIMIT as usize, |
| 122 | "the zone must hold more data files ({}) than the limit on open files ({})", files, FD_LIMIT); |
| 123 | let got = res!(scanned(&db)); |
| 124 | assert_eq!(got, want, "the live store's scan of {} files must list all {} keys", files, N); |
| 125 | res!(db.shutdown()); |
| 126 | // Restarted on the index files, as a restored store is. |
| 127 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg), schemes(), None, true, false)); |
| 128 | let got = res!(scanned(&db)); |
| 129 | assert_eq!(got, want, "the restarted store's scan of {} files must list all {} keys", files, N); |
| 130 | res!(db.shutdown()); |
| 131 | let _ = std::fs::remove_dir_all(&root); |
| 132 | Ok(()) |
| 133 | } |
| 134 | |
| 135 | #[test] |
| 136 | fn scan_of_a_zone_with_more_files_than_the_fd_limit_is_whole() -> Outcome<()> { |
| 137 | if env::var_os(CHILD_VAR).is_some() { |
| 138 | return scan_under_the_limit(); |
| 139 | } |
| 140 | let exe = res!(env::current_exe()); |
| 141 | // The shell lowers the soft limit and becomes this test binary, running this test alone. |
| 142 | let out = res!(Command::new("sh") |
| 143 | .arg("-c") |
| 144 | .arg("ulimit -Sn \"$1\" && exec \"$2\" --exact \"$3\" --nocapture --test-threads=1") |
| 145 | .arg("sh") |
| 146 | .arg(FD_LIMIT.to_string()) |
| 147 | .arg(&exe) |
| 148 | .arg(TEST_NAME) |
| 149 | .env(CHILD_VAR, "1") |
| 150 | .output()); |
| 151 | let stdout = String::from_utf8_lossy(&out.stdout); |
| 152 | let stderr = String::from_utf8_lossy(&out.stderr); |
| 153 | assert!(out.status.success(), |
| 154 | "the scan failed with at most {} open files: {}\n{}\n{}", FD_LIMIT, out.status, stdout, stderr); |
| 155 | assert!(stdout.contains("1 passed"), "the child ran no test: {}", stdout); |
| 156 | Ok(()) |
| 157 | } |