Oregami
Repositories/oxedyne/ore

oxedyne/ore/store/src/sweep.rs

14.5 KiB, 9 runs

created by r2848102244:1177, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! Reading the whole store the slow way, on a schedule, because the fast way
2//! stopped looking.
3//!
4//! [`oxedyne_fe2o3_ore::segment::Integrity::Vouched`] took the per-record digest
5//! out of every command that reads a vouched sealed segment. That digest was the
6//! only thing left catching a body byte that flipped on the disk: a signature is
7//! skipped on the same warrant, the file's length and modification time do not
8//! move, and [`crate::verdict`]'s fold is over the digests written beside the
9//! bodies rather than over the bodies themselves. So a store read only that way
10//! is a store nothing checks, and this is where the checking went.
11//!
12//! # What it is, exactly
13//!
14//! One checked read of every segment, signatures and all, ignoring every verdict
15//! and leaving fresh ones behind. It is the read a command does when it has no
16//! verdict to go on -- [`Store::checked_fold`] enters the same code path with the
17//! vouching turned off -- so what a sweep checks is what a command would have
18//! checked, and not a second opinion that could come to differ.
19//!
20//! # Two triggers, and the second is why the first may fail
21//!
22//! A verdict goes off after [`crate::verdict::VERDICT_LIFE`], so a store nobody
23//! sweeps is not a store nobody checks: the next command that reads it pays for a
24//! full checked replay and files fresh notes. This runs on a timer so that the
25//! cost lands at four in the morning instead of in front of somebody, and so that
26//! a repository nobody has run a verb in for a year is still read -- which
27//! matters, because rot is a function of time on the disk and not of use.
28//!
29//! **That is the answer to a sweep that quietly stopped.** It cannot take the
30//! checking with it. What a dead timer costs is speed, and the symptom is
31//! commands going slow again, which somebody notices and asks about; it is not a
32//! guarantee that lapsed while a green tick stayed green.
33//!
34//! # What it does about damage
35//!
36//! Nothing, and that is deliberate. It cannot repair a segment -- the bytes are
37//! gone and no other copy of them is here -- so what it does is stop the store
38//! lying about them: the damaged segment gets no verdict, and the next verb that
39//! touches it reads it checked, fails its integrity check and refuses the history
40//! naming the operation. The sweep withdraws the licence to skip and lets the
41//! ordinary refusal do the rest, rather than inventing a second way to fail.
42//!
43//! Beside that it leaves [`SWEEP_FILE`], which says when the last sweep finished,
44//! how much it read, and every segment it would not vouch for, with the sentence
45//! the read refused with. That file is what `ore flags` reads to say how long it
46//! has been.
47//!
48//! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\
49//! Anthropic Claude
50
51use crate::store::{
52 Lock,
53 SEGMENT_LIMIT,
54 Store,
55 Verify,
56};
57use crate::verdict::{
58 self,
59 Verdicts,
60};
61
62use oxedyne_fe2o3_core::prelude::*;
63
64use std::fs;
65use std::path::{
66 Path,
67 PathBuf,
68};
69
70
71// The file, and what it says about itself
72pub const SWEEP_FILE: &str = "checked";
73pub const SWEEP_FORMAT: &str = "ORESWEEP 1"; // a later format is left alone, not misread
74
75
76/// One segment the sweep would not vouch for.
77#[derive(Clone, Debug, Eq, PartialEq)]
78pub struct Rot {
79 pub name: String, // the segment file, without its directory
80 pub said: String, // the sentence the checked read refused with
81}
82
83
84/// What one sweep found.
85#[derive(Clone, Debug, Default, Eq, PartialEq)]
86pub struct Swept {
87 pub at: u128, // nanoseconds since the epoch, when it finished
88 pub segments: usize, // segments read
89 pub bytes: u64, // bytes read
90 pub rotten: Vec<Rot>, // what it would not vouch for, in log order
91}
92
93impl Swept {
94
95 pub fn path_of(dir: &Path) -> PathBuf {
96 dir.join(SWEEP_FILE)
97 }
98
99 /// What the last sweep at `dir` left, where it left anything this build
100 /// reads.
101 ///
102 /// Every way of not being able to read it is the same answer, for the reason
103 /// [`Verdicts::read`] gives: this file is a note about work, not the work,
104 /// and the cost of not having it is a sweep that runs when it need not have.
105 pub fn read(dir: &Path) -> Option<Self> {
106 let text = match fs::read_to_string(Self::path_of(dir)) {
107 Ok(t) => t,
108 Err(_) => return None,
109 };
110 let mut lines = text.lines();
111 match lines.next() {
112 Some(first) if first.trim() == SWEEP_FORMAT => (),
113 _ => return None,
114 }
115 let mut out = Self::default();
116 let mut finished = false;
117 for line in lines {
118 if line.trim().is_empty() {
119 continue;
120 }
121 // Anything not understood refuses the file, rather than being stepped
122 // over. A line this cannot place is a file something is wrong with, and
123 // what it would cost to be wrong about that is a sweep believed to have
124 // found nothing when it found something.
125 let (word, rest) = match line.split_once(' ') {
126 Some(pair) => pair,
127 None => return None,
128 };
129 match word {
130 "finished" => match rest.trim().parse::<u128>() {
131 Ok(n) => { out.at = n; finished = true; },
132 Err(_) => return None,
133 },
134 "read" => {
135 let field: Vec<&str> = rest.split_whitespace().collect();
136 if field.len() != 2 {
137 return None;
138 }
139 match (field[0].parse::<usize>(), field[1].parse::<u64>()) {
140 (Ok(n), Ok(b)) => { out.segments = n; out.bytes = b; },
141 _ => return None,
142 }
143 },
144 "rot" => match rest.split_once(' ') {
145 Some((name, said)) => out.rotten.push(Rot {
146 name: fmt!("{}", name),
147 said: fmt!("{}", said),
148 }),
149 None => return None,
150 },
151 _ => return None,
152 }
153 }
154 match finished {
155 true => Some(out),
156 false => None,
157 }
158 }
159
160 /// How long ago this sweep finished, in nanoseconds, or `None` where the
161 /// clock says it finished in the future.
162 pub fn age(&self, at: u128) -> Option<u128> {
163 at.checked_sub(self.at)
164 }
165
166 /// Writes the record to `dir`, replacing whatever was there.
167 ///
168 /// Written aside and renamed over, for the reason [`Verdicts::write`] is.
169 pub fn write(&self, dir: &Path)
170 -> Outcome<()>
171 {
172 let path = Self::path_of(dir);
173 let mut text = fmt!("{}\n", SWEEP_FORMAT);
174 text.push_str(&fmt!("finished {}\n", self.at));
175 text.push_str(&fmt!("read {} {}\n", self.segments, self.bytes));
176 for rot in &self.rotten {
177 // One line each, and the sentence is the last field, so a name with no
178 // space in it -- which every segment file has -- reads back whole.
179 text.push_str(&fmt!("rot {} {}\n", rot.name, rot.said.replace('\n', " ")));
180 }
181 let aside = path.with_extension("new");
182 match fs::write(&aside, text.as_bytes()) {
183 Ok(()) => (),
184 Err(e) => return Err(err!(e,
185 "The sweep record could not be written aside to {:?}.", aside;
186 IO, File, Write)),
187 }
188 match fs::rename(&aside, &path) {
189 Ok(()) => Ok(()),
190 Err(e) => {
191 let _ = fs::remove_file(&aside);
192 Err(err!(e,
193 "The sweep record written to {:?} could not be moved over {:?}.",
194 aside, path;
195 IO, File, Write))
196 },
197 }
198 }
199}
200
201
202/// Reads every segment at `dir` the slow way and files fresh verdicts for the
203/// ones that hold.
204///
205/// The lock is taken for the whole of it, because an append part way through
206/// would seal a segment this has already measured and leave a verdict about a
207/// length the file no longer has. Nothing is written to the log.
208///
209/// **A damaged segment does not stop the sweep.** The point of running it is to
210/// learn how much is wrong, and stopping at the first bad file answers a
211/// different question. Every failure is collected, named, and left without a
212/// verdict; what comes back says what happened and the caller decides what that
213/// is worth.
214pub fn verify(dir: &Path, how: Verify)
215 -> Outcome<Swept>
216{
217 let _lock = res!(Lock::take(dir));
218 let store = Store::at(dir);
219 let segments = res!(store.segments());
220 let newest = segments.last().map(|(n, _)| *n);
221 let mut out = Swept::default();
222 let mut said = Verdicts::new();
223 // One reading of the clock, so that every note this leaves goes off together
224 // and the next sweep has one date to beat rather than forty-four.
225 let at = verdict::now();
226 for (n, path) in &segments {
227 let name = match path.file_name() {
228 Some(n) => n.to_string_lossy().into_owned(),
229 None => continue,
230 };
231 let (len, modified) = res!(Store::stamp(path));
232 out.segments += 1;
233 out.bytes += len;
234 let whence = fmt!("the segment {}", path.display());
235 let fold = match Store::checked_fold(path, how, &whence) {
236 Ok(f) => f,
237 Err(e) => {
238 out.rotten.push(Rot { name, said: fmt!("{}", e.plain()) });
239 continue;
240 },
241 };
242 // The same rule the replay files notes by: the tail is the one the next
243 // append continues, and a fold over it names a state it may already have
244 // left. It is still read, because rot does not wait for a segment to fill.
245 let sealed = Some(*n) != newest || len >= SEGMENT_LIMIT;
246 if let (true, Verify::Signatures(_)) = (sealed, how) {
247 said.record(&name, len, modified, fold, at);
248 }
249 }
250 // A relay checks nothing and therefore knows nothing, and writing down what
251 // it did not do is exactly the mistake this must not make.
252 if let Verify::Signatures(_) = how {
253 res!(said.write(dir));
254 }
255 out.at = verdict::now();
256 res!(out.write(dir));
257 Ok(out)
258}
259
260
261#[cfg(test)]
262mod tests {
263 use super::*;
264
265 /// A directory that removes itself however the test ends.
266 struct Scratch {
267 path: PathBuf,
268 }
269
270 impl Scratch {
271 fn new(what: &str)
272 -> Outcome<Self>
273 {
274 let stamp = res!(std::time::SystemTime::now()
275 .duration_since(std::time::UNIX_EPOCH));
276 let path = std::env::temp_dir().join(fmt!(
277 "ore_sweep_{}_{}_{}", what, std::process::id(), stamp.as_nanos()));
278 res!(fs::create_dir_all(&path));
279 Ok(Self { path })
280 }
281 }
282
283 impl Drop for Scratch {
284 fn drop(&mut self) {
285 let _ = fs::remove_dir_all(&self.path);
286 }
287 }
288
289 /// A record written and read back is the record that was written.
290 ///
291 /// The sentence a damaged segment is described by holds spaces, quotes,
292 /// brackets and hex, and it is the last field of its line, so it is the one
293 /// thing here that a parser splitting on whitespace would quietly cut in half.
294 /// The fixture carries one of each rather than a placeholder, because a round
295 /// trip over a word proves nothing about the field it is really for.
296 #[test]
297 fn a_sweep_record_round_trips() -> Outcome<()> {
298 let scratch = res!(Scratch::new("round"));
299 let was = Swept {
300 at: 1_787_395_459_262_654_054,
301 segments: 44,
302 bytes: 88_976_008,
303 rotten: vec![
304 Rot {
305 name: fmt!("000006.seg"),
306 said: fmt!("The segment \"/a/b/.ore/log/000006.seg\" could not be \
307 decoded. Record 178, carrying r1870400018:1324, fails its \
308 integrity check: 22153922 bytes hash to [7d, f2], and [f8, b7] \
309 was recorded."),
310 },
311 Rot {
312 name: fmt!("000031.seg"),
313 said: fmt!("something else"),
314 },
315 ],
316 };
317 res!(was.write(&scratch.path));
318 let now = res!(Swept::read(&scratch.path).ok_or_else(|| err!(
319 "The record just written did not read back."; Test, Missing)));
320 assert_eq!(now, was, "what was written is what comes back");
321 Ok(())
322 }
323
324 /// A newline inside a sentence does not become a second entry.
325 ///
326 /// One line per damaged segment is the whole of the format, so a sentence that
327 /// carried a newline through would read back as a line beginning with a word
328 /// this does not know -- silently dropping the rest of the sentence, and, were
329 /// the word ever `rot`, inventing a damaged segment that does not exist.
330 #[test]
331 fn a_newline_in_a_sentence_does_not_become_a_line() -> Outcome<()> {
332 let scratch = res!(Scratch::new("newline"));
333 let was = Swept {
334 at: 1,
335 segments: 1,
336 bytes: 2,
337 rotten: vec![Rot {
338 name: fmt!("000000.seg"),
339 said: fmt!("first part\nrot 000001.seg invented"),
340 }],
341 };
342 res!(was.write(&scratch.path));
343 let now = res!(Swept::read(&scratch.path).ok_or_else(|| err!(
344 "The record just written did not read back."; Test, Missing)));
345 assert_eq!(now.rotten.len(), 1, "one damaged segment, not two: {:?}", now.rotten);
346 assert_eq!(now.rotten[0].name, "000000.seg", "and it is the one that was written");
347 assert!(now.rotten[0].said.contains("first part"),
348 "the sentence survives: {}", now.rotten[0].said);
349 assert!(now.rotten[0].said.contains("invented"),
350 "all of it, on one line: {}", now.rotten[0].said);
351 Ok(())
352 }
353
354 /// A record this build cannot read is nothing known, and never an error.
355 ///
356 /// Every way of failing gives the same answer for the reason [`Verdicts::read`]
357 /// gives: the only consequence of knowing nothing is a sweep that runs when it
358 /// need not have. A half-read record is refused too -- one that stopped before
359 /// it said it had finished is a sweep that was killed part way, and its counts
360 /// describe a reading that never ended.
361 #[test]
362 fn a_record_this_build_cannot_read_is_nothing_known() -> Outcome<()> {
363 let scratch = res!(Scratch::new("unreadable"));
364 let path = Swept::path_of(&scratch.path);
365 assert!(Swept::read(&scratch.path).is_none(), "absent");
366 for (what, text) in [
367 ("a format from later", fmt!("ORESWEEP 2\nfinished 1\nread 1 2\n")),
368 ("no format line at all", fmt!("finished 1\nread 1 2\n")),
369 ("empty", fmt!("")),
370 ("a date that is not one", fmt!("ORESWEEP 1\nfinished soon\nread 1 2\n")),
371 ("counts that are not", fmt!("ORESWEEP 1\nfinished 1\nread one two\n")),
372 ("one count, not two", fmt!("ORESWEEP 1\nfinished 1\nread 1\n")),
373 ("a sweep that never finished", fmt!("ORESWEEP 1\nread 1 2\n")),
374 ("a rot line with no sentence", fmt!("ORESWEEP 1\nfinished 1\nread 1 2\nrot\n")),
375 ("a rot line naming nothing", fmt!("ORESWEEP 1\nfinished 1\nread 1 2\nrot 000000.seg\n")),
376 ("a word this build does not know",
377 fmt!("ORESWEEP 1\nfinished 1\nread 1 2\nswept everything\n")),
378 ] {
379 res!(fs::write(&path, text.as_bytes()));
380 assert!(Swept::read(&scratch.path).is_none(), "{} reads as nothing known", what);
381 }
382 // And the sound one still reads, so the refusals above are not a parser
383 // that refuses everything.
384 res!(fs::write(&path, b"ORESWEEP 1\n\nfinished 7\nread 3 400\n\n"));
385 let got = res!(Swept::read(&scratch.path).ok_or_else(|| err!(
386 "A sound record was refused."; Test, Invalid)));
387 assert_eq!((got.at, got.segments, got.bytes), (7, 3, 400));
388 assert!(got.rotten.is_empty(), "and nothing damaged");
389 Ok(())
390 }
391
392 /// An age is only ever reported where the clock agrees it is in the past.
393 #[test]
394 fn a_record_from_the_future_has_no_age() -> Outcome<()> {
395 let swept = Swept { at: 100, segments: 1, bytes: 1, rotten: Vec::new() };
396 assert_eq!(swept.age(250), Some(150), "an ordinary age");
397 assert_eq!(swept.age(100), Some(0), "and the moment it was written");
398 assert_eq!(swept.age(99), None,
399 "a clock that has gone back says nothing rather than a large number");
400 Ok(())
401 }
402}