Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_test/src/scratch.rs

13.9 KiB, 1 run

created by r1870400018:20966, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! Scratch directories for fixtures that need a real filesystem.
2//!
3//! A test that wants a directory of its own must not take one from
4//! `std::env::temp_dir()`. On the machines this library is developed on `/tmp` is a
5//! tmpfs, so every byte a fixture writes is resident memory charged to the process
6//! that wrote it, and nothing reclaims it when the test binary exits. A suite that
7//! leaves a directory behind per test leaves memory behind per test.
8//!
9//! [`scratch_dir`] answers with a directory under the user's cache instead: on disk,
10//! never under the system temporary directory, unique to the call, and swept clean of
11//! the fixtures left by processes that have since exited.
12//!
13//! ```no_run
14//! use oxedyne_fe2o3_core::prelude::*;
15//! use oxedyne_fe2o3_test::scratch::scratch_dir;
16//!
17//! # fn main() -> Outcome<()> {
18//! let dir = res!(scratch_dir("my_crate_widget"));
19//! res!(std::fs::write(dir.join("in.txt"), b"hello"));
20//! # Ok(())
21//! # }
22//! ```
23
24use oxedyne_fe2o3_core::prelude::*;
25
26use std::{
27 fs,
28 path::{
29 Path,
30 PathBuf,
31 },
32 sync::{
33 atomic::{
34 AtomicU64,
35 Ordering,
36 },
37 Once,
38 },
39 time::{
40 Duration,
41 SystemTime,
42 UNIX_EPOCH,
43 },
44};
45
46/// Where scratch directories are gathered, relative to the user's cache directory.
47pub const SCRATCH_REL: &str = "oxedyne/scratch";
48
49/// Names the scratch root outright, overriding the cache directory.
50pub const SCRATCH_ENV: &str = "OXEDYNE_SCRATCH_DIR";
51
52/// The last-resort root, relative to the current directory, when no cache
53/// directory can be written to.
54pub const SCRATCH_FALLBACK: &str = ".oxedyne-scratch";
55
56/// How old a fixture must be before the sweep removes it when it cannot tell
57/// whether the process that made it is still running.
58const UNKNOWN_LIFE_AGE: Duration = Duration::from_secs(60 * 60);
59
60/// How old a fixture must be before the sweep removes it regardless of who
61/// appears to own it. Process identifiers are reused, so a live-looking owner
62/// is not proof for ever.
63const MAX_AGE: Duration = Duration::from_secs(24 * 60 * 60);
64
65/// How many names one call tries before giving up.
66const NAME_ATTEMPTS: u32 = 64;
67
68/// Distinguishes two calls that land in the same nanosecond.
69static SEQ: AtomicU64 = AtomicU64::new(0);
70
71/// Guards the sweep, which is worth doing once per process and no more.
72static SWEPT: Once = Once::new();
73
74/// The directory scratch fixtures are gathered under, created if it is absent.
75///
76/// Resolved from the first of these that can be created:
77///
78/// 1. `$OXEDYNE_SCRATCH_DIR`, used as given.
79/// 2. `$XDG_CACHE_HOME/oxedyne/scratch`.
80/// 3. `$HOME/.cache/oxedyne/scratch`.
81/// 4. `./.oxedyne-scratch`.
82///
83/// # Errors
84/// Fails if the resolved root lies under the system temporary directory, which is
85/// the thing this module exists to avoid, or if no candidate can be created. There
86/// is deliberately no fall back to `/tmp`: a fixture that quietly lands in a tmpfs
87/// is the failure being prevented, so it is reported instead.
88pub fn scratch_root() -> Outcome<PathBuf> {
89 // An explicit request is honoured or refused, never quietly replaced.
90 if let Some(dir) = env_dir(SCRATCH_ENV) {
91 if under_temp(&dir) {
92 return Err(err!(
93 "{} names {:?}, which is under the system temporary directory {:?}. \
94 Scratch fixtures must not be written to a tmpfs.",
95 SCRATCH_ENV, dir, std::env::temp_dir();
96 Invalid, Input));
97 }
98 res!(fs::create_dir_all(&dir), IO, File);
99 return Ok(dir);
100 }
101
102 let mut tried: Vec<String> = Vec::new();
103 let mut cands: Vec<PathBuf> = Vec::new();
104 if let Some(cache) = env_dir("XDG_CACHE_HOME") {
105 cands.push(cache.join(SCRATCH_REL));
106 }
107 if let Some(home) = env_dir("HOME") {
108 cands.push(home.join(".cache").join(SCRATCH_REL));
109 }
110 cands.push(PathBuf::from(SCRATCH_FALLBACK));
111
112 for cand in cands {
113 if under_temp(&cand) {
114 tried.push(fmt!("{:?} (under the system temporary directory)", cand));
115 continue;
116 }
117 match fs::create_dir_all(&cand) {
118 Ok(()) => return Ok(cand),
119 Err(e) => tried.push(fmt!("{:?} ({})", cand, e)),
120 }
121 }
122 Err(err!(
123 "No scratch root could be created; tried {}. Set {} to a writable path \
124 outside {:?}.",
125 tried.join(", "), SCRATCH_ENV, std::env::temp_dir();
126 IO, File, Create))
127}
128
129/// Creates a fresh, empty scratch directory for `label` and returns its path.
130///
131/// The name carries the process identifier, so a second process never shares a
132/// directory with this one, and the directory is created rather than merely named:
133/// uniqueness is settled by the filesystem, which is the only party that can settle
134/// it. A clash is retried under a new name.
135///
136/// The first call in a process also sweeps the scratch root of fixtures whose owning
137/// process has exited, which is what keeps the root from growing run over run. There
138/// is no reliable teardown hook in a Rust test binary, so the cleaning is done on the
139/// way in rather than on the way out.
140///
141/// # Errors
142/// Fails if the scratch root cannot be resolved (see [`scratch_root`]) or if
143/// [`NAME_ATTEMPTS`] names in a row are all taken.
144pub fn scratch_dir(label: &str) -> Outcome<PathBuf> {
145 let root = res!(scratch_root());
146 SWEPT.call_once(|| sweep(&root));
147
148 let label = clean_label(label);
149 let pid = std::process::id();
150 for _ in 0..NAME_ATTEMPTS {
151 let nanos = match SystemTime::now().duration_since(UNIX_EPOCH) {
152 Ok(d) => d.as_nanos(),
153 Err(_) => 0,
154 };
155 let seq = SEQ.fetch_add(1, Ordering::Relaxed);
156 let dir = root.join(fmt!("{}.{}.{}.{}", label, pid, nanos, seq));
157 match fs::create_dir(&dir) {
158 Ok(()) => return Ok(dir),
159 // Taken: some other call got this name first, so take another.
160 Err(ref e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue,
161 Err(e) => return Err(err!(e,
162 "Could not create the scratch directory {:?}.", dir;
163 IO, File, Create)),
164 }
165 }
166 Err(err!(
167 "Could not find an unused scratch name for {:?} under {:?} in {} attempts.",
168 label, root, NAME_ATTEMPTS;
169 IO, File, Conflict))
170}
171
172/// Reads an environment variable as a directory, ignoring it when unset or empty.
173fn env_dir(name: &str) -> Option<PathBuf> {
174 match std::env::var(name) {
175 Ok(s) if !s.trim().is_empty() => Some(PathBuf::from(s)),
176 _ => None,
177 }
178}
179
180/// Whether `p` lies under the system temporary directory.
181///
182/// `/tmp` is checked as well as `std::env::temp_dir()`, because `TMPDIR` may point
183/// somewhere else while `/tmp` is still the tmpfs that must be kept clear.
184fn under_temp(p: &Path) -> bool {
185 let tmp = std::env::temp_dir();
186 p.starts_with(&tmp) || p.starts_with("/tmp")
187}
188
189/// Reduces `label` to characters that are safe in a directory name.
190///
191/// A dot is not one of them: the sweep reads the trailing fields of a name back,
192/// and a dotted label would make that ambiguous.
193fn clean_label(label: &str) -> String {
194 let mut out = String::with_capacity(label.len());
195 for c in label.chars() {
196 if c.is_ascii_alphanumeric() || c == '_' || c == '-' {
197 out.push(c);
198 } else {
199 out.push('_');
200 }
201 }
202 if out.is_empty() {
203 out.push_str("scratch");
204 }
205 out
206}
207
208/// Whether a process with this identifier is still running.
209///
210/// Answered from `/proc`, which is Linux's; where there is no `/proc` there is no
211/// answer without a system call, so `None` comes back and the caller falls back to
212/// the age of the directory.
213fn pid_alive(pid: u32) -> Option<bool> {
214 let proc = Path::new("/proc");
215 if !proc.is_dir() {
216 return None;
217 }
218 Some(proc.join(pid.to_string()).is_dir())
219}
220
221/// The owning process identifier encoded in a scratch directory name.
222///
223/// `None` for anything not written by [`scratch_dir`], which is then left alone --
224/// the sweep only removes what it made.
225fn owner_pid(name: &str) -> Option<u32> {
226 // Read back to front: sequence, nanoseconds, process identifier, label. A
227 // fourth field must be there, or the label is missing and the name is not ours.
228 let parts: Vec<&str> = name.rsplitn(4, '.').collect();
229 if parts.len() < 4 {
230 return None;
231 }
232 if parts[0].parse::<u64>().is_err() || parts[1].parse::<u128>().is_err() {
233 return None;
234 }
235 match parts[2].parse::<u32>() {
236 Ok(pid) => Some(pid),
237 Err(_) => None,
238 }
239}
240
241/// Removes the scratch directories left behind by processes that have exited.
242///
243/// Directories owned by a running process are left where they are, including this
244/// process's own, so two suites running side by side never tread on each other. A
245/// directory older than [`MAX_AGE`] goes regardless, because process identifiers are
246/// reused and a stale one can look alive for ever.
247///
248/// Errors are swallowed: a sweep that cannot remove something has not stopped the
249/// caller from getting the directory it asked for.
250pub fn sweep(root: &Path) {
251 let self_pid = std::process::id();
252 let entries = match fs::read_dir(root) {
253 Ok(e) => e,
254 Err(_) => return,
255 };
256 for entry in entries.flatten() {
257 let name = entry.file_name().to_string_lossy().to_string();
258 let pid = match owner_pid(&name) {
259 Some(p) => p,
260 None => continue, // Not ours to remove.
261 };
262 // Our own, and never stale. On Linux the liveness test below reaches the
263 // same answer, so this only saves a `/proc` lookup; where there is no
264 // `/proc` it is the whole protection, since a suite that runs longer than
265 // [`UNKNOWN_LIFE_AGE`] would otherwise sweep its own working directories
266 // away underneath itself.
267 if pid == self_pid {
268 continue;
269 }
270 let age = entry
271 .metadata()
272 .ok()
273 .and_then(|m| m.modified().ok())
274 .and_then(|t| SystemTime::now().duration_since(t).ok());
275 let stale = match pid_alive(pid) {
276 Some(true) => age.map(|a| a > MAX_AGE).unwrap_or(false),
277 Some(false) => true,
278 None => age.map(|a| a > UNKNOWN_LIFE_AGE).unwrap_or(false),
279 };
280 if stale {
281 let _ = fs::remove_dir_all(entry.path());
282 }
283 }
284}
285
286
287// ┌───────────────────────────────────────────────────────────────┐
288// │ Tests │
289// └───────────────────────────────────────────────────────────────┘
290
291#[cfg(test)]
292mod tests {
293 use super::*;
294
295 /// A directory to plant fixtures in and sweep.
296 ///
297 /// Taken from [`scratch_dir`] like everything else, so that a run cut short
298 /// leaves behind something the next run knows how to remove.
299 fn test_root(name: &str) -> PathBuf {
300 match scratch_dir(&fmt!("selftest_{}", name)) {
301 Ok(d) => d,
302 Err(e) => panic!("a self-test root: {}", e),
303 }
304 }
305
306 #[test]
307 fn test_scratch_dir_is_never_under_tmp() {
308 // The whole point: a fixture in a tmpfs is resident memory nothing frees.
309 let dir = match scratch_dir("fe2o3_test_selfcheck") {
310 Ok(d) => d,
311 Err(e) => panic!("a scratch directory: {}", e),
312 };
313 assert!(!dir.starts_with("/tmp"),
314 "scratch directory landed in the tmpfs: {:?}", dir);
315 assert!(!dir.starts_with(std::env::temp_dir()),
316 "scratch directory landed in the system temporary directory: {:?}", dir);
317 assert!(dir.is_dir(), "scratch directory was not created: {:?}", dir);
318 let _ = fs::remove_dir_all(&dir);
319 }
320
321 #[test]
322 fn test_scratch_dir_is_unique_per_call() {
323 // Two calls in the same tick used to be able to agree on a name, which is
324 // how two tests come to share one directory.
325 let mut seen = std::collections::HashSet::new();
326 let mut made = Vec::new();
327 for _ in 0..50 {
328 let dir = match scratch_dir("fe2o3_test_unique") {
329 Ok(d) => d,
330 Err(e) => panic!("a scratch directory: {}", e),
331 };
332 assert!(seen.insert(dir.clone()), "two calls shared {:?}", dir);
333 made.push(dir);
334 }
335 for dir in made {
336 let _ = fs::remove_dir_all(&dir);
337 }
338 }
339
340 #[test]
341 fn test_sweep_removes_dead_owners_and_spares_live_ones() {
342 // Bounded accumulation rests entirely on this: a test binary has no
343 // teardown hook, so what the last run left is removed by the next one.
344 let root = test_root("sweep");
345 // Process 0 is not a process, so nothing owns this one.
346 let dead = root.join("fixture.0.123456789.0");
347 // Process 1 always is one, and it is not this process -- so sparing it can
348 // only be the liveness test doing the work, not the shortcut for our own.
349 let other = root.join("fixture.1.123456789.1");
350 let mine = root.join(fmt!("fixture.{}.123456789.2", std::process::id()));
351 let alien = root.join("someone-elses-directory");
352 for d in [&dead, &other, &mine, &alien] {
353 match fs::create_dir_all(d) {
354 Ok(()) => {}
355 Err(e) => panic!("a fixture: {}", e),
356 }
357 }
358 sweep(&root);
359 assert!(!dead.exists(), "a dead owner's fixture survived the sweep");
360 assert!(other.exists(), "a live owner's fixture was swept away");
361 assert!(mine.exists(), "this process's own fixture was swept away");
362 assert!(alien.exists(), "the sweep removed a directory it did not create");
363 let _ = fs::remove_dir_all(&root);
364 }
365
366 #[test]
367 fn test_a_tmp_scratch_root_is_refused() {
368 // Refused rather than obeyed: an override is the one way a caller could
369 // still put fixtures in the tmpfs by hand.
370 let dir = std::env::temp_dir().join("fe2o3_test_should_never_exist");
371 assert!(under_temp(&dir), "the temporary directory should read as temporary");
372 assert!(under_temp(Path::new("/tmp/anything")),
373 "/tmp should read as temporary whatever TMPDIR says");
374 assert!(!under_temp(Path::new("/home/someone/.cache/oxedyne/scratch")),
375 "a cache path should not read as temporary");
376 // And nothing above created it.
377 assert!(!dir.exists(), "the refusal should not have made the directory");
378 }
379
380 #[test]
381 fn test_only_our_own_names_are_understood() {
382 assert_eq!(owner_pid("label.42.123.7"), Some(42));
383 assert_eq!(owner_pid("has-dashes_and_underscores.42.123.7"), Some(42));
384 assert_eq!(owner_pid("nodots"), None);
385 assert_eq!(owner_pid("42.123.7"), None); // No label field.
386 assert_eq!(owner_pid("label.notapid.123.7"), None);
387 }
388
389 #[test]
390 fn test_a_label_cannot_smuggle_a_path_or_a_dot() {
391 assert_eq!(clean_label("../../etc"), "______etc");
392 assert_eq!(clean_label("a.b"), "a_b");
393 assert_eq!(clean_label(""), "scratch");
394 assert_eq!(clean_label("daimond-gw_test"), "daimond-gw_test");
395 }
396}