Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/open_flush.rs

13.3 KiB, 1 run

created by r1870400018:70856, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! The flush before replay (R2, 2026-10-10). An open makes every data and index file durable
2//! before replay calls their records durable. Where the kernel reports write-back errors through
3//! `syncfs` that is one flush per filesystem, not one per file, which on a store of thousands of
4//! files took seconds. Either way a failed flush refuses the open. The hooks are process-wide,
5//! so the tests hold one lock.
6
7mod gc_pair;
8
9use gc_pair::*;
10
11use oxedyne_fe2o3_core::prelude::*;
12use oxedyne_fe2o3_iop_db::api::Database;
13use oxedyne_fe2o3_o3db_sync::{
14 base::{
15 cfg::OzoneConfig,
16 constant,
17 },
18 db::O3db,
19 file::refusal::Refusal,
20 test::{
21 hooks::{
22 self,
23 ReplayFlush,
24 },
25 setup::Uid,
26 },
27};
28
29use std::{
30 fs,
31 io::Write,
32 path::{
33 Path,
34 PathBuf,
35 },
36 sync::{
37 Arc,
38 Mutex,
39 MutexGuard,
40 atomic::{
41 AtomicBool,
42 Ordering,
43 },
44 },
45 thread,
46 time::{
47 Duration,
48 Instant,
49 },
50};
51
52const MAX: u64 = 1_000; // data file length, so the store holds many files
53const NVAL: usize = 120; // 100-byte values
54
55static HOOKS: Mutex<()> = Mutex::new(());
56
57fn lock() -> MutexGuard<'static, ()> {
58 match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() }
59}
60
61// Two zones on one filesystem, so a flush per directory would show.
62fn cfg() -> Outcome<OzoneConfig> {
63 let mut c = res!(config(MAX));
64 c.num_zones = 2;
65 Ok(c)
66}
67
68fn open(root: &Path) -> Outcome<TestDb> {
69 let mut db = res!(O3db::new(root.to_path_buf(), Some(res!(cfg())), schemes(), Uid::default()));
70 res!(db.start("open_flush"));
71 Ok(db)
72}
73
74// A store of many small files, closed.
75fn written(dir: &str) -> Outcome<PathBuf> {
76 written_n(dir, NVAL)
77}
78
79fn written_n(dir: &str, nval: usize) -> Outcome<PathBuf> {
80 log_set_level!("error");
81 let _ = std::fs::remove_dir_all(dir);
82 res!(std::fs::create_dir_all(dir));
83 let root = res!(Path::new(dir).canonicalize());
84 let _ = std::fs::remove_file(OzoneConfig::config_path(&root));
85 let db = res!(open(&root));
86 for i in 0..nval {
87 res!(db.insert(key(i), value(i, 1), Uid::default(), None));
88 }
89 res!(db.close());
90 Ok(root)
91}
92
93// The data and index files on disk, in both zones.
94fn nfiles(root: &Path) -> Outcome<usize> {
95 let zroot = res!(cfg()).zone_root(root);
96 let mut n = 0;
97 for z in ["zone_001", "zone_002"] {
98 for item in res!(std::fs::read_dir(zroot.join(z))) {
99 let path = res!(item).path();
100 match path.extension().and_then(|e| e.to_str()) {
101 Some("dat") | Some("ind") => n += 1,
102 _ => (),
103 }
104 }
105 }
106 Ok(n)
107}
108
109fn all_read(db: &TestDb) -> bool {
110 all_read_n(db, NVAL)
111}
112
113fn all_read_n(db: &TestDb, nval: usize) -> bool {
114 (0..nval).all(|i| matches!(db.get(&key(i), None), Ok(Some((v, _))) if v == value(i, 1)))
115}
116
117// Opens and closes the store with the flush made one way, returning how many flushes it made.
118fn flushes_on_open(root: &Path, way: ReplayFlush) -> Outcome<u64> {
119 hooks::set_replay_flush(way);
120 let n0 = hooks::replay_flushes();
121 let db = res!(open(root));
122 let n = hooks::replay_flushes() - n0;
123 let ok = all_read(&db);
124 res!(db.close());
125 hooks::set_replay_flush(ReplayFlush::Kernel);
126 if !ok {
127 return Err(err!("A key did not read back after an open flushed by {:?}.", way; Test));
128 }
129 Ok(n)
130}
131
132/// One flush per filesystem with `syncfs`, one per file without, and the kernel's choice is one
133/// of the two.
134#[test]
135fn an_open_flushes_each_filesystem_once() -> Outcome<()> {
136 let _g = lock();
137 let root = res!(written("./test_db_open_flush_count"));
138 let files = res!(nfiles(&root));
139 assert!(files >= 20, "the store holds only {} files, too few to tell the ways apart", files);
140
141 let by_fs = res!(flushes_on_open(&root, ReplayFlush::Syncfs));
142 if cfg!(target_os = "linux") {
143 assert_eq!(by_fs, 1, "two zones on one filesystem were flushed {} times by syncfs", by_fs);
144 }
145 let by_file = res!(flushes_on_open(&root, ReplayFlush::PerFile));
146 assert!(by_file as usize >= files,
147 "the per-file flush made {} syncs for {} files", by_file, files);
148 let chosen = res!(flushes_on_open(&root, ReplayFlush::Kernel));
149 assert!(chosen == by_fs || chosen as usize >= files,
150 "the kernel's choice made {} flushes, neither way's count", chosen);
151
152 let _ = std::fs::remove_dir_all(&root);
153 Ok(())
154}
155
156/// A failed flush refuses the open, made either way, and the refusal stands: the next open is
157/// refused too, naming the marker, though the disk reports nothing now, as a kernel that has
158/// reported a write-back error once would. Once the marker is cleared the store opens whole.
159/// The hook replaces the answer of the real `syncfs` or `sync_data`, so the error path is the
160/// real one.
161#[test]
162fn a_failed_flush_refuses_the_open() -> Outcome<()> {
163 let _g = lock();
164 let root = res!(written("./test_db_open_flush_fail"));
165 for (way, says) in [
166 (ReplayFlush::Syncfs, if cfg!(target_os = "linux") { "write-back error" } else { "durable" }),
167 (ReplayFlush::PerFile, "durable before replaying"),
168 ] {
169 hooks::set_replay_flush(way);
170 hooks::set_replay_sync_failure(true);
171 let opened = open(&root);
172 hooks::set_replay_sync_failure(false);
173 match opened {
174 Ok(db) => {
175 let _ = db.close();
176 hooks::set_replay_flush(ReplayFlush::Kernel);
177 return Err(err!("The store opened though its flush by {:?} failed.", way; Test));
178 },
179 Err(e) => {
180 let s = fmt!("{}", e);
181 assert!(s.contains(says), "the refusal by {:?} does not say '{}': {}", way, says, s);
182 assert!(s.contains(constant::REFUSAL_FILENAME),
183 "the refusal by {:?} does not name the marker it kept: {}", way, s);
184 },
185 }
186 assert!(Refusal::path(&root).exists(), "the refused open by {:?} left no marker", way);
187 match open(&root) {
188 Ok(db) => {
189 let _ = db.close();
190 hooks::set_replay_flush(ReplayFlush::Kernel);
191 return Err(err!("The store opened over the refusal by {:?}, as a retry under \
192 systemd would.", way; Test));
193 },
194 Err(e) => {
195 let s = fmt!("{}", e);
196 assert!(s.contains(constant::REFUSAL_FILENAME) && s.contains("Refusal::clear"),
197 "the second open after {:?} was refused without naming the marker and its \
198 verb: {}", way, s);
199 },
200 }
201 hooks::set_replay_flush(ReplayFlush::Kernel);
202 assert!(res!(Refusal::clear(&root)), "there was no marker to clear after {:?}", way);
203 assert!(!res!(Refusal::clear(&root)), "a second clear after {:?} found a marker", way);
204 let db = res!(open(&root));
205 let ok = all_read(&db);
206 res!(db.close());
207 assert!(ok, "a key did not read back after the refused open by {:?}", way);
208 }
209 let _ = std::fs::remove_dir_all(&root);
210 Ok(())
211}
212
213/// Before Linux 5.8 `syncfs` reports no write-back error, so the open syncs each file; a release
214/// that cannot be read is taken as one of those. From 5.8 one `syncfs` serves the filesystem.
215#[test]
216fn the_kernel_release_picks_the_flush() -> Outcome<()> {
217 let _g = lock();
218 let root = res!(written("./test_db_open_flush_release"));
219 let files = res!(nfiles(&root));
220 let mut counts = Vec::new();
221 for rel in ["5.4.0-150-generic", "", "6.1.0"] {
222 hooks::set_kernel_release(Some(rel));
223 let n = flushes_on_open(&root, ReplayFlush::Kernel);
224 hooks::set_kernel_release(None);
225 counts.push((rel, res!(n)));
226 }
227 for (rel, n) in &counts[..2] {
228 assert!(*n as usize >= files,
229 "kernel release {:?} made {} flushes for {} files, not one per file", rel, n, files);
230 }
231 if cfg!(target_os = "linux") {
232 assert_eq!(counts[2].1, 1, "kernel release 6.1.0 made {} flushes, not one syncfs", counts[2].1);
233 }
234 let _ = std::fs::remove_dir_all(&root);
235 Ok(())
236}
237
238// Writes without syncing beside the store until stopped, as `dd` would, so the filesystem holds
239// dirty pages that a `syncfs` must write back. Returns the bytes written.
240fn dirtier(dir: &Path, stop: Arc<AtomicBool>) -> thread::JoinHandle<u64> {
241 let path = dir.with_extension("dirt");
242 thread::spawn(move || {
243 let block = vec![0x5au8; 1 << 20];
244 let mut n = 0u64;
245 let mut f = match fs::File::create(&path) {
246 Ok(f) => f,
247 Err(_) => return 0,
248 };
249 while !stop.load(Ordering::SeqCst) {
250 if n >= 256 << 20 {
251 // Start again, so the dirt stays bounded but keeps coming.
252 if f.set_len(0).is_err() {
253 break;
254 }
255 n = 0;
256 }
257 if f.write_all(&block).is_err() {
258 break;
259 }
260 n += block.len() as u64;
261 }
262 drop(f);
263 let _ = fs::remove_file(&path);
264 n
265 })
266}
267
268// Opens the store at `root` while a writer dirties its filesystem, and reads every key.
269fn loaded_open(root: &Path, nval: usize) -> Outcome<Duration> {
270 let stop = Arc::new(AtomicBool::new(false));
271 let h = dirtier(root, stop.clone());
272 thread::sleep(Duration::from_millis(500));
273 let t0 = Instant::now();
274 let opened = open(root);
275 let took = t0.elapsed();
276 stop.store(true, Ordering::SeqCst);
277 let dirt = h.join().unwrap_or(0);
278 let db = res!(opened);
279 let ok = all_read_n(&db, nval);
280 res!(db.close());
281 println!("loaded open of {} files: {:?}, with {} MiB written beside it",
282 res!(nfiles(root)), took, dirt >> 20);
283 if !ok {
284 return Err(err!("A key did not read back after a loaded open."; Test));
285 }
286 Ok(took)
287}
288
289/// An open while another writer dirties the same filesystem: `syncfs` writes back the other
290/// writer's pages too, so the open costs with the dirt, not the file count. Always run, small.
291#[test]
292fn a_loaded_open_is_bounded() -> Outcome<()> {
293 let _g = lock();
294 let root = res!(written("./test_db_open_flush_loaded"));
295 let took = res!(loaded_open(&root, NVAL));
296 assert!(took < Duration::from_secs(30), "a loaded open of a small store took {:?}", took);
297 let _ = std::fs::remove_dir_all(&root);
298 Ok(())
299}
300
301/// The same at 9.6k files, the size whose per-file flush once cost 10-25 s; QA measured 466 ms
302/// warm and 880 ms cold unloaded. Run alone, under the slot's I/O cap:
303/// `cargo test -p oxedyne_fe2o3_o3db_sync --test open_flush -- --ignored --nocapture
304/// a_loaded_open_of_9600_files`.
305#[test]
306#[ignore]
307fn a_loaded_open_of_9600_files() -> Outcome<()> {
308 let _g = lock();
309 let nval = 48_000; // ten values a pair, so about 9.6k data and index files
310 let root = res!(written_n("./test_db_open_flush_loaded_9600", nval));
311 let files = res!(nfiles(&root));
312 assert!(files >= 9_000, "the store holds only {} files", files);
313 let took = res!(loaded_open(&root, nval));
314 assert!(took < Duration::from_secs(60), "a loaded open of {} files took {:?}", files, took);
315 let _ = std::fs::remove_dir_all(&root);
316 Ok(())
317}
318
319/// M11: the open really calls `syncfs`, once per filesystem. `hooks::replay_flushes` counts the
320/// flush before it is made, so only the kernel can tell whether it was: this test runs itself
321/// again under `strace -f -e trace=syncfs` and counts the calls the child's open makes. It
322/// passes with a note when strace is absent. Runner:
323/// `cargo test -p oxedyne_fe2o3_o3db_sync --test open_flush -- --ignored --nocapture
324/// an_open_calls_syncfs_once_per_filesystem`.
325#[test]
326#[ignore]
327fn an_open_calls_syncfs_once_per_filesystem() -> Outcome<()> {
328 const CHILD: &str = "OPEN_FLUSH_STRACE_ROOT";
329 const NAME: &str = "an_open_calls_syncfs_once_per_filesystem";
330 let _g = lock();
331 if let Ok(root) = std::env::var(CHILD) {
332 // The child: one open and close of the store its parent wrote, flushed by syncfs.
333 hooks::set_replay_flush(ReplayFlush::Syncfs);
334 let db = res!(open(Path::new(&root)));
335 res!(db.close());
336 return Ok(());
337 }
338 if !cfg!(target_os = "linux") {
339 return Ok(());
340 }
341 let have = std::process::Command::new("strace").arg("-V").output();
342 if !matches!(have, Ok(ref o) if o.status.success()) {
343 println!("{}: strace is not installed, so the kernel's count is not checked.", NAME);
344 return Ok(());
345 }
346 let root = res!(written("./test_db_open_flush_strace"));
347 let trace = root.with_extension("strace");
348 let exe = res!(std::env::current_exe());
349 let out = res!(std::process::Command::new("strace")
350 .args(["-f", "-qq", "-e", "trace=syncfs", "-o"])
351 .arg(&trace)
352 .arg(&exe)
353 .args(["--ignored", "--exact", NAME, "--test-threads=1"])
354 .env(CHILD, &root)
355 .output());
356 let text = fs::read_to_string(&trace).unwrap_or_default();
357 let _ = fs::remove_file(&trace);
358 let _ = fs::remove_dir_all(&root);
359 assert!(out.status.success(), "the traced open failed: {}",
360 String::from_utf8_lossy(&out.stdout));
361 let calls = text.lines().filter(|l| l.contains("syncfs(")).count();
362 assert_eq!(calls, 1, "two zones on one filesystem made {} syncfs calls:\n{}", calls, text);
363 Ok(())
364}