oxedyne/fe2o3/fe2o3_o3db_sync/tests/open_flush.rs
13.3 KiB, 1 run
created by r1870400018:70856, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! The flush before replay (R2, 2026-10-10). An open makes every data and index file durable |
| 2 | //! before replay calls their records durable. Where the kernel reports write-back errors through |
| 3 | //! `syncfs` that is one flush per filesystem, not one per file, which on a store of thousands of |
| 4 | //! files took seconds. Either way a failed flush refuses the open. The hooks are process-wide, |
| 5 | //! so the tests hold one lock. |
| 6 | |
| 7 | mod gc_pair; |
| 8 | |
| 9 | use gc_pair::*; |
| 10 | |
| 11 | use oxedyne_fe2o3_core::prelude::*; |
| 12 | use oxedyne_fe2o3_iop_db::api::Database; |
| 13 | use oxedyne_fe2o3_o3db_sync::{ |
| 14 | base::{ |
| 15 | cfg::OzoneConfig, |
| 16 | constant, |
| 17 | }, |
| 18 | db::O3db, |
| 19 | file::refusal::Refusal, |
| 20 | test::{ |
| 21 | hooks::{ |
| 22 | self, |
| 23 | ReplayFlush, |
| 24 | }, |
| 25 | setup::Uid, |
| 26 | }, |
| 27 | }; |
| 28 | |
| 29 | use std::{ |
| 30 | fs, |
| 31 | io::Write, |
| 32 | path::{ |
| 33 | Path, |
| 34 | PathBuf, |
| 35 | }, |
| 36 | sync::{ |
| 37 | Arc, |
| 38 | Mutex, |
| 39 | MutexGuard, |
| 40 | atomic::{ |
| 41 | AtomicBool, |
| 42 | Ordering, |
| 43 | }, |
| 44 | }, |
| 45 | thread, |
| 46 | time::{ |
| 47 | Duration, |
| 48 | Instant, |
| 49 | }, |
| 50 | }; |
| 51 | |
| 52 | const MAX: u64 = 1_000; // data file length, so the store holds many files |
| 53 | const NVAL: usize = 120; // 100-byte values |
| 54 | |
| 55 | static HOOKS: Mutex<()> = Mutex::new(()); |
| 56 | |
| 57 | fn lock() -> MutexGuard<'static, ()> { |
| 58 | match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() } |
| 59 | } |
| 60 | |
| 61 | // Two zones on one filesystem, so a flush per directory would show. |
| 62 | fn cfg() -> Outcome<OzoneConfig> { |
| 63 | let mut c = res!(config(MAX)); |
| 64 | c.num_zones = 2; |
| 65 | Ok(c) |
| 66 | } |
| 67 | |
| 68 | fn open(root: &Path) -> Outcome<TestDb> { |
| 69 | let mut db = res!(O3db::new(root.to_path_buf(), Some(res!(cfg())), schemes(), Uid::default())); |
| 70 | res!(db.start("open_flush")); |
| 71 | Ok(db) |
| 72 | } |
| 73 | |
| 74 | // A store of many small files, closed. |
| 75 | fn written(dir: &str) -> Outcome<PathBuf> { |
| 76 | written_n(dir, NVAL) |
| 77 | } |
| 78 | |
| 79 | fn written_n(dir: &str, nval: usize) -> Outcome<PathBuf> { |
| 80 | log_set_level!("error"); |
| 81 | let _ = std::fs::remove_dir_all(dir); |
| 82 | res!(std::fs::create_dir_all(dir)); |
| 83 | let root = res!(Path::new(dir).canonicalize()); |
| 84 | let _ = std::fs::remove_file(OzoneConfig::config_path(&root)); |
| 85 | let db = res!(open(&root)); |
| 86 | for i in 0..nval { |
| 87 | res!(db.insert(key(i), value(i, 1), Uid::default(), None)); |
| 88 | } |
| 89 | res!(db.close()); |
| 90 | Ok(root) |
| 91 | } |
| 92 | |
| 93 | // The data and index files on disk, in both zones. |
| 94 | fn nfiles(root: &Path) -> Outcome<usize> { |
| 95 | let zroot = res!(cfg()).zone_root(root); |
| 96 | let mut n = 0; |
| 97 | for z in ["zone_001", "zone_002"] { |
| 98 | for item in res!(std::fs::read_dir(zroot.join(z))) { |
| 99 | let path = res!(item).path(); |
| 100 | match path.extension().and_then(|e| e.to_str()) { |
| 101 | Some("dat") | Some("ind") => n += 1, |
| 102 | _ => (), |
| 103 | } |
| 104 | } |
| 105 | } |
| 106 | Ok(n) |
| 107 | } |
| 108 | |
| 109 | fn all_read(db: &TestDb) -> bool { |
| 110 | all_read_n(db, NVAL) |
| 111 | } |
| 112 | |
| 113 | fn all_read_n(db: &TestDb, nval: usize) -> bool { |
| 114 | (0..nval).all(|i| matches!(db.get(&key(i), None), Ok(Some((v, _))) if v == value(i, 1))) |
| 115 | } |
| 116 | |
| 117 | // Opens and closes the store with the flush made one way, returning how many flushes it made. |
| 118 | fn flushes_on_open(root: &Path, way: ReplayFlush) -> Outcome<u64> { |
| 119 | hooks::set_replay_flush(way); |
| 120 | let n0 = hooks::replay_flushes(); |
| 121 | let db = res!(open(root)); |
| 122 | let n = hooks::replay_flushes() - n0; |
| 123 | let ok = all_read(&db); |
| 124 | res!(db.close()); |
| 125 | hooks::set_replay_flush(ReplayFlush::Kernel); |
| 126 | if !ok { |
| 127 | return Err(err!("A key did not read back after an open flushed by {:?}.", way; Test)); |
| 128 | } |
| 129 | Ok(n) |
| 130 | } |
| 131 | |
| 132 | /// One flush per filesystem with `syncfs`, one per file without, and the kernel's choice is one |
| 133 | /// of the two. |
| 134 | #[test] |
| 135 | fn an_open_flushes_each_filesystem_once() -> Outcome<()> { |
| 136 | let _g = lock(); |
| 137 | let root = res!(written("./test_db_open_flush_count")); |
| 138 | let files = res!(nfiles(&root)); |
| 139 | assert!(files >= 20, "the store holds only {} files, too few to tell the ways apart", files); |
| 140 | |
| 141 | let by_fs = res!(flushes_on_open(&root, ReplayFlush::Syncfs)); |
| 142 | if cfg!(target_os = "linux") { |
| 143 | assert_eq!(by_fs, 1, "two zones on one filesystem were flushed {} times by syncfs", by_fs); |
| 144 | } |
| 145 | let by_file = res!(flushes_on_open(&root, ReplayFlush::PerFile)); |
| 146 | assert!(by_file as usize >= files, |
| 147 | "the per-file flush made {} syncs for {} files", by_file, files); |
| 148 | let chosen = res!(flushes_on_open(&root, ReplayFlush::Kernel)); |
| 149 | assert!(chosen == by_fs || chosen as usize >= files, |
| 150 | "the kernel's choice made {} flushes, neither way's count", chosen); |
| 151 | |
| 152 | let _ = std::fs::remove_dir_all(&root); |
| 153 | Ok(()) |
| 154 | } |
| 155 | |
| 156 | /// A failed flush refuses the open, made either way, and the refusal stands: the next open is |
| 157 | /// refused too, naming the marker, though the disk reports nothing now, as a kernel that has |
| 158 | /// reported a write-back error once would. Once the marker is cleared the store opens whole. |
| 159 | /// The hook replaces the answer of the real `syncfs` or `sync_data`, so the error path is the |
| 160 | /// real one. |
| 161 | #[test] |
| 162 | fn a_failed_flush_refuses_the_open() -> Outcome<()> { |
| 163 | let _g = lock(); |
| 164 | let root = res!(written("./test_db_open_flush_fail")); |
| 165 | for (way, says) in [ |
| 166 | (ReplayFlush::Syncfs, if cfg!(target_os = "linux") { "write-back error" } else { "durable" }), |
| 167 | (ReplayFlush::PerFile, "durable before replaying"), |
| 168 | ] { |
| 169 | hooks::set_replay_flush(way); |
| 170 | hooks::set_replay_sync_failure(true); |
| 171 | let opened = open(&root); |
| 172 | hooks::set_replay_sync_failure(false); |
| 173 | match opened { |
| 174 | Ok(db) => { |
| 175 | let _ = db.close(); |
| 176 | hooks::set_replay_flush(ReplayFlush::Kernel); |
| 177 | return Err(err!("The store opened though its flush by {:?} failed.", way; Test)); |
| 178 | }, |
| 179 | Err(e) => { |
| 180 | let s = fmt!("{}", e); |
| 181 | assert!(s.contains(says), "the refusal by {:?} does not say '{}': {}", way, says, s); |
| 182 | assert!(s.contains(constant::REFUSAL_FILENAME), |
| 183 | "the refusal by {:?} does not name the marker it kept: {}", way, s); |
| 184 | }, |
| 185 | } |
| 186 | assert!(Refusal::path(&root).exists(), "the refused open by {:?} left no marker", way); |
| 187 | match open(&root) { |
| 188 | Ok(db) => { |
| 189 | let _ = db.close(); |
| 190 | hooks::set_replay_flush(ReplayFlush::Kernel); |
| 191 | return Err(err!("The store opened over the refusal by {:?}, as a retry under \ |
| 192 | systemd would.", way; Test)); |
| 193 | }, |
| 194 | Err(e) => { |
| 195 | let s = fmt!("{}", e); |
| 196 | assert!(s.contains(constant::REFUSAL_FILENAME) && s.contains("Refusal::clear"), |
| 197 | "the second open after {:?} was refused without naming the marker and its \ |
| 198 | verb: {}", way, s); |
| 199 | }, |
| 200 | } |
| 201 | hooks::set_replay_flush(ReplayFlush::Kernel); |
| 202 | assert!(res!(Refusal::clear(&root)), "there was no marker to clear after {:?}", way); |
| 203 | assert!(!res!(Refusal::clear(&root)), "a second clear after {:?} found a marker", way); |
| 204 | let db = res!(open(&root)); |
| 205 | let ok = all_read(&db); |
| 206 | res!(db.close()); |
| 207 | assert!(ok, "a key did not read back after the refused open by {:?}", way); |
| 208 | } |
| 209 | let _ = std::fs::remove_dir_all(&root); |
| 210 | Ok(()) |
| 211 | } |
| 212 | |
| 213 | /// Before Linux 5.8 `syncfs` reports no write-back error, so the open syncs each file; a release |
| 214 | /// that cannot be read is taken as one of those. From 5.8 one `syncfs` serves the filesystem. |
| 215 | #[test] |
| 216 | fn the_kernel_release_picks_the_flush() -> Outcome<()> { |
| 217 | let _g = lock(); |
| 218 | let root = res!(written("./test_db_open_flush_release")); |
| 219 | let files = res!(nfiles(&root)); |
| 220 | let mut counts = Vec::new(); |
| 221 | for rel in ["5.4.0-150-generic", "", "6.1.0"] { |
| 222 | hooks::set_kernel_release(Some(rel)); |
| 223 | let n = flushes_on_open(&root, ReplayFlush::Kernel); |
| 224 | hooks::set_kernel_release(None); |
| 225 | counts.push((rel, res!(n))); |
| 226 | } |
| 227 | for (rel, n) in &counts[..2] { |
| 228 | assert!(*n as usize >= files, |
| 229 | "kernel release {:?} made {} flushes for {} files, not one per file", rel, n, files); |
| 230 | } |
| 231 | if cfg!(target_os = "linux") { |
| 232 | assert_eq!(counts[2].1, 1, "kernel release 6.1.0 made {} flushes, not one syncfs", counts[2].1); |
| 233 | } |
| 234 | let _ = std::fs::remove_dir_all(&root); |
| 235 | Ok(()) |
| 236 | } |
| 237 | |
| 238 | // Writes without syncing beside the store until stopped, as `dd` would, so the filesystem holds |
| 239 | // dirty pages that a `syncfs` must write back. Returns the bytes written. |
| 240 | fn dirtier(dir: &Path, stop: Arc<AtomicBool>) -> thread::JoinHandle<u64> { |
| 241 | let path = dir.with_extension("dirt"); |
| 242 | thread::spawn(move || { |
| 243 | let block = vec![0x5au8; 1 << 20]; |
| 244 | let mut n = 0u64; |
| 245 | let mut f = match fs::File::create(&path) { |
| 246 | Ok(f) => f, |
| 247 | Err(_) => return 0, |
| 248 | }; |
| 249 | while !stop.load(Ordering::SeqCst) { |
| 250 | if n >= 256 << 20 { |
| 251 | // Start again, so the dirt stays bounded but keeps coming. |
| 252 | if f.set_len(0).is_err() { |
| 253 | break; |
| 254 | } |
| 255 | n = 0; |
| 256 | } |
| 257 | if f.write_all(&block).is_err() { |
| 258 | break; |
| 259 | } |
| 260 | n += block.len() as u64; |
| 261 | } |
| 262 | drop(f); |
| 263 | let _ = fs::remove_file(&path); |
| 264 | n |
| 265 | }) |
| 266 | } |
| 267 | |
| 268 | // Opens the store at `root` while a writer dirties its filesystem, and reads every key. |
| 269 | fn loaded_open(root: &Path, nval: usize) -> Outcome<Duration> { |
| 270 | let stop = Arc::new(AtomicBool::new(false)); |
| 271 | let h = dirtier(root, stop.clone()); |
| 272 | thread::sleep(Duration::from_millis(500)); |
| 273 | let t0 = Instant::now(); |
| 274 | let opened = open(root); |
| 275 | let took = t0.elapsed(); |
| 276 | stop.store(true, Ordering::SeqCst); |
| 277 | let dirt = h.join().unwrap_or(0); |
| 278 | let db = res!(opened); |
| 279 | let ok = all_read_n(&db, nval); |
| 280 | res!(db.close()); |
| 281 | println!("loaded open of {} files: {:?}, with {} MiB written beside it", |
| 282 | res!(nfiles(root)), took, dirt >> 20); |
| 283 | if !ok { |
| 284 | return Err(err!("A key did not read back after a loaded open."; Test)); |
| 285 | } |
| 286 | Ok(took) |
| 287 | } |
| 288 | |
| 289 | /// An open while another writer dirties the same filesystem: `syncfs` writes back the other |
| 290 | /// writer's pages too, so the open costs with the dirt, not the file count. Always run, small. |
| 291 | #[test] |
| 292 | fn a_loaded_open_is_bounded() -> Outcome<()> { |
| 293 | let _g = lock(); |
| 294 | let root = res!(written("./test_db_open_flush_loaded")); |
| 295 | let took = res!(loaded_open(&root, NVAL)); |
| 296 | assert!(took < Duration::from_secs(30), "a loaded open of a small store took {:?}", took); |
| 297 | let _ = std::fs::remove_dir_all(&root); |
| 298 | Ok(()) |
| 299 | } |
| 300 | |
| 301 | /// The same at 9.6k files, the size whose per-file flush once cost 10-25 s; QA measured 466 ms |
| 302 | /// warm and 880 ms cold unloaded. Run alone, under the slot's I/O cap: |
| 303 | /// `cargo test -p oxedyne_fe2o3_o3db_sync --test open_flush -- --ignored --nocapture |
| 304 | /// a_loaded_open_of_9600_files`. |
| 305 | #[test] |
| 306 | #[ignore] |
| 307 | fn a_loaded_open_of_9600_files() -> Outcome<()> { |
| 308 | let _g = lock(); |
| 309 | let nval = 48_000; // ten values a pair, so about 9.6k data and index files |
| 310 | let root = res!(written_n("./test_db_open_flush_loaded_9600", nval)); |
| 311 | let files = res!(nfiles(&root)); |
| 312 | assert!(files >= 9_000, "the store holds only {} files", files); |
| 313 | let took = res!(loaded_open(&root, nval)); |
| 314 | assert!(took < Duration::from_secs(60), "a loaded open of {} files took {:?}", files, took); |
| 315 | let _ = std::fs::remove_dir_all(&root); |
| 316 | Ok(()) |
| 317 | } |
| 318 | |
| 319 | /// M11: the open really calls `syncfs`, once per filesystem. `hooks::replay_flushes` counts the |
| 320 | /// flush before it is made, so only the kernel can tell whether it was: this test runs itself |
| 321 | /// again under `strace -f -e trace=syncfs` and counts the calls the child's open makes. It |
| 322 | /// passes with a note when strace is absent. Runner: |
| 323 | /// `cargo test -p oxedyne_fe2o3_o3db_sync --test open_flush -- --ignored --nocapture |
| 324 | /// an_open_calls_syncfs_once_per_filesystem`. |
| 325 | #[test] |
| 326 | #[ignore] |
| 327 | fn an_open_calls_syncfs_once_per_filesystem() -> Outcome<()> { |
| 328 | const CHILD: &str = "OPEN_FLUSH_STRACE_ROOT"; |
| 329 | const NAME: &str = "an_open_calls_syncfs_once_per_filesystem"; |
| 330 | let _g = lock(); |
| 331 | if let Ok(root) = std::env::var(CHILD) { |
| 332 | // The child: one open and close of the store its parent wrote, flushed by syncfs. |
| 333 | hooks::set_replay_flush(ReplayFlush::Syncfs); |
| 334 | let db = res!(open(Path::new(&root))); |
| 335 | res!(db.close()); |
| 336 | return Ok(()); |
| 337 | } |
| 338 | if !cfg!(target_os = "linux") { |
| 339 | return Ok(()); |
| 340 | } |
| 341 | let have = std::process::Command::new("strace").arg("-V").output(); |
| 342 | if !matches!(have, Ok(ref o) if o.status.success()) { |
| 343 | println!("{}: strace is not installed, so the kernel's count is not checked.", NAME); |
| 344 | return Ok(()); |
| 345 | } |
| 346 | let root = res!(written("./test_db_open_flush_strace")); |
| 347 | let trace = root.with_extension("strace"); |
| 348 | let exe = res!(std::env::current_exe()); |
| 349 | let out = res!(std::process::Command::new("strace") |
| 350 | .args(["-f", "-qq", "-e", "trace=syncfs", "-o"]) |
| 351 | .arg(&trace) |
| 352 | .arg(&exe) |
| 353 | .args(["--ignored", "--exact", NAME, "--test-threads=1"]) |
| 354 | .env(CHILD, &root) |
| 355 | .output()); |
| 356 | let text = fs::read_to_string(&trace).unwrap_or_default(); |
| 357 | let _ = fs::remove_file(&trace); |
| 358 | let _ = fs::remove_dir_all(&root); |
| 359 | assert!(out.status.success(), "the traced open failed: {}", |
| 360 | String::from_utf8_lossy(&out.stdout)); |
| 361 | let calls = text.lines().filter(|l| l.contains("syncfs(")).count(); |
| 362 | assert_eq!(calls, 1, "two zones on one filesystem made {} syncfs calls:\n{}", calls, text); |
| 363 | Ok(()) |
| 364 | } |