Oregami
Repositories/oxedyne/ore

oxedyne/ore/cli/src/listing.rs

17.3 KiB, 1 run

created by r2848102244:1081, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! The history as a listing reads it, kept beside the log so that asking for it
2//! twice costs one reading and not two.
3//!
4//! `ore log` prints an account of the history: how many operations there are,
5//! what the frontier is, how many of them are signed, and every mark somebody
6//! named with the work counted between them. None of that needs a single byte of
7//! what any operation *says*. Reading the store does: 85 MB of segments on the
8//! history this was measured against, 465 ms of it, and 80% of that is recomputing
9//! the SHA-256 each record is framed with. The command took 0.52 s and 161 MB
10//! against git's 0.01 s and 8.6 MB.
11//!
12//! So this is the reading, written down. It is 60 KB where the log is 85 MB,
13//! because it holds the shape of the history and none of its content: the counts,
14//! the frontier, and one line per mark.
15//!
16//! # It cannot be stale, and the argument is not a timestamp
17//!
18//! A listing carries the [`Consumed`] cursor the read that produced it stopped
19//! at, and [`Store::settled_at`] puts that cursor to the store before a word of
20//! the listing is believed. Every segment must still be there under the same
21//! name; every sealed one must have the length and modification time it was read
22//! at; the one segment an append may extend must still begin with the bytes it
23//! began with, which is read and hashed rather than guessed; and nothing may have
24//! been appended past any of it. That is the same evidence
25//! [`Store::replay_since`] takes up on, and what it cannot see is what that
26//! cannot see -- a sealed segment rewritten with its length and modification time
27//! both put back.
28//!
29//! **Another process appending is exactly the case it catches**, and it is the
30//! case that matters: several sessions share one working copy here. An append
31//! lands in the newest segment, which moves its length; a rollover puts a segment
32//! in the directory that the cursor does not name. Both are refused, by name, and
33//! the whole log is read.
34//!
35//! # It is only written where the cursor still describes the store
36//!
37//! Which is to say by a command that appended nothing. A command that appends has
38//! moved the store past the cursor its own read stopped at, and it cannot say
39//! where the new bytes end without reading them again -- so it removes the
40//! listing instead, and the next command pays one whole read and writes a fresh
41//! one. That is the rule [`crate::stat`] already follows for the working copy
42//! index and it is followed here for the same reason: derived state is written
43//! only at a moment it is known to be true.
44//!
45//! What that costs is one slow command after every command that wrote something.
46//! What it buys is that there is no way for this file to describe a store it was
47//! not taken from.
48//!
49//! # A read that appends nothing still writes nothing
50//!
51//! `ore log` on an unchanged working copy writes no operation and says so. It
52//! must not start writing a file every time somebody asks a question either, so
53//! the listing is written only where what is on the disk is not already what
54//! would be written -- which, after the first read, it is.
55//!
56//! # It is replica-local, derived, and safe to delete
57//!
58//! Like the verdict cache and the working copy index, it is never put in a
59//! segment or a sync message, nothing in the history depends on it, and losing it
60//! costs one whole read. Anything about it that does not parse is treated as an
61//! absent listing, which is the same failure the absence itself is.
62
63use crate::repo::{
64 self,
65 Repo,
66};
67
68use ore_store::keys::{
69 self,
70 Prov,
71};
72use ore_store::store::{
73 Consumed,
74 Settled,
75 Store,
76};
77
78use oxedyne_fe2o3_core::prelude::*;
79use oxedyne_fe2o3_ore::id::OpId;
80use oxedyne_fe2o3_ore::op::Op;
81
82use std::fs;
83use std::path::{
84 Path,
85 PathBuf,
86};
87use std::str::FromStr;
88
89
90/// Name of the listing, within the repository directory.
91pub const LISTING_FILE: &str = "listing";
92/// What the listing says about itself. A later format is left alone, not misread.
93pub const LISTING_FORMAT: &str = "ORELIST 1";
94
95// The words each kind of line begins with. A cursor's own lines are its business
96// and are asked about through [`Consumed::owns`].
97const OPS_LINE: &str = "ops";
98const AT_LINE: &str = "at";
99const PROV_LINE: &str = "prov";
100const MARK_LINE: &str = "mark";
101
102
103/// One mark, and where it stands among the operations.
104pub struct Point {
105 pub at: usize, // its position in the log, counting from the start
106 pub id: OpId,
107 pub prov: Prov,
108 pub name: String,
109 pub parents: Vec<OpId>,
110 pub auto: bool, // written by this tool at the end of a command
111}
112
113
114/// Everything a history listing is made of, and nothing else.
115pub struct Listing {
116 pub cursor: Consumed, // where the read that produced this stopped
117 pub ops: usize, // how many operations the log holds
118 pub frontier: Vec<OpId>,
119 // Provenance, counted rather than listed
120 pub signed: usize,
121 pub unknown: usize,
122 pub bare: usize,
123 pub marks: Vec<Point>, // every mark, in log order
124}
125
126
127/// Whether the listing on disk still describes the store.
128pub enum Held {
129 /// It does, and here it is.
130 Current(Listing),
131 /// It does not, and this says why, in a sentence meant to be printed.
132 Stale(String),
133}
134
135
136impl Listing {
137
138 /// Reads the history out of an open repository.
139 ///
140 /// One pass over the log. This is the only place a listing is derived from
141 /// operations, so what is read from the disk and what is read from the log
142 /// cannot come to disagree about what a listing is.
143 pub fn of(repo: &Repo) -> Self {
144 let mut out = Self {
145 cursor: repo.cursor.clone(),
146 ops: 0,
147 frontier: repo.log.frontier(),
148 signed: 0,
149 unknown: 0,
150 bare: 0,
151 marks: Vec::new(),
152 };
153 for (at, rec) in repo.log.iter().enumerate() {
154 let id = rec.head.id();
155 match repo.prov.get(&id) {
156 Some(Prov::Verified) => out.signed += 1,
157 Some(Prov::Unknown) => out.unknown += 1,
158 _ => out.bare += 1,
159 }
160 if let Op::Mark { name, .. } = &rec.op {
161 out.marks.push(Point {
162 at,
163 id,
164 prov: repo.prov.get(&id).copied().unwrap_or(Prov::Bare),
165 name: name.clone(),
166 parents: rec.parents().to_vec(),
167 auto: repo::is_auto_mark(name),
168 });
169 }
170 out.ops += 1;
171 }
172 out
173 }
174
175 pub fn path_of(dir: &Path) -> PathBuf {
176 dir.join(LISTING_FILE)
177 }
178
179 /// Reads the listing at `dir`, and never fails.
180 ///
181 /// Absent, unreadable, a format this build does not know, a line that is not
182 /// the shape it should be, a mark name that is not the text encoding: all of
183 /// them answer that nothing is known, because the only consequence of knowing
184 /// nothing is reading the store, which is what a caller holding no listing
185 /// does anyway. A partial read is refused too -- half a cache is not worth
186 /// the question of which half.
187 pub fn read(dir: &Path)
188 -> Option<Self>
189 {
190 match fs::read_to_string(Self::path_of(dir)) {
191 Ok(t) => Self::from_text(&t),
192 Err(_) => None,
193 }
194 }
195
196 /// Reads back what [`Listing::to_text`] wrote, or nothing where the text is
197 /// not exactly that. See [`Listing::read`], which is this over a file.
198 pub fn from_text(text: &str)
199 -> Option<Self>
200 {
201 let mut lines = text.lines();
202 match lines.next() {
203 Some(first) if first.trim() == LISTING_FORMAT => (),
204 _ => return None,
205 }
206 let mut cursor: Vec<&str> = Vec::new();
207 let mut ops: Option<usize> = None;
208 let mut frontier: Option<Vec<OpId>> = None;
209 let mut tally: Option<(usize, usize, usize)> = None;
210 let mut marks: Vec<Point> = Vec::new();
211 for line in lines {
212 if line.trim().is_empty() {
213 continue;
214 }
215 let field: Vec<&str> = line.split_whitespace().collect();
216 let word = match field.first() {
217 Some(w) => *w,
218 None => continue,
219 };
220 if Consumed::owns(word) {
221 cursor.push(line);
222 continue;
223 }
224 match word {
225 OPS_LINE => {
226 if field.len() != 2 || ops.is_some() {
227 return None;
228 }
229 ops = match field[1].parse::<usize>() {
230 Ok(n) => Some(n),
231 Err(_) => return None,
232 };
233 },
234 AT_LINE => {
235 if frontier.is_some() {
236 return None;
237 }
238 let mut heads = Vec::new();
239 for said in &field[1..] {
240 match OpId::from_str(said) {
241 Ok(id) => heads.push(id),
242 Err(_) => return None,
243 }
244 }
245 frontier = Some(heads);
246 },
247 PROV_LINE => {
248 if field.len() != 4 || tally.is_some() {
249 return None;
250 }
251 let mut n = [0usize; 3];
252 for (i, said) in field[1..].iter().enumerate() {
253 n[i] = match said.parse::<usize>() {
254 Ok(v) => v,
255 Err(_) => return None,
256 };
257 }
258 tally = Some((n[0], n[1], n[2]));
259 },
260 MARK_LINE => {
261 // at, id, provenance, auto, name, then however many parents.
262 if field.len() < 6 {
263 return None;
264 }
265 let at = match field[1].parse::<usize>() {
266 Ok(n) => n,
267 Err(_) => return None,
268 };
269 let id = match OpId::from_str(field[2]) {
270 Ok(v) => v,
271 Err(_) => return None,
272 };
273 let prov = match prov_of(field[3]) {
274 Some(p) => p,
275 None => return None,
276 };
277 let auto = match field[4] {
278 "1" => true,
279 "0" => false,
280 _ => return None,
281 };
282 let name = match keys::bytes_of(field[5]) {
283 Ok(b) => match String::from_utf8(b) {
284 Ok(s) => s,
285 Err(_) => return None,
286 },
287 Err(_) => return None,
288 };
289 let mut parents = Vec::new();
290 for said in &field[6..] {
291 match OpId::from_str(said) {
292 Ok(p) => parents.push(p),
293 Err(_) => return None,
294 }
295 }
296 marks.push(Point { at, id, prov, name, parents, auto });
297 },
298 _ => return None,
299 }
300 }
301 // A listing whose marks do not stand in ascending order inside the log it
302 // says it read is not a listing of that log, and the counts taken between
303 // two of them would be nonsense rather than wrong by a little. Refused
304 // here, where refusing costs one whole read.
305 let mut last: Option<usize> = None;
306 for point in &marks {
307 match last {
308 Some(was) if point.at <= was => return None,
309 _ => last = Some(point.at),
310 }
311 }
312 if let (Some(n), Some(last)) = (ops, last) {
313 if last >= n {
314 return None;
315 }
316 }
317 let cursor = match Consumed::from_lines(cursor.into_iter()) {
318 Some(c) => c,
319 None => return None,
320 };
321 match (ops, frontier, tally) {
322 (Some(ops), Some(frontier), Some((signed, unknown, bare))) => Some(Self {
323 cursor, ops, frontier, signed, unknown, bare, marks,
324 }),
325 _ => None,
326 }
327 }
328
329 /// Writes the listing to `dir`, replacing whatever was there.
330 ///
331 /// Aside and renamed over, because two commands reading one store may both
332 /// reach here and a half-written file is one the next reader would discard
333 /// anyway. Both writers are right, so the last one wins and nothing is lost
334 /// but a little work.
335 pub fn write(&self, dir: &Path)
336 -> Outcome<()>
337 {
338 let path = Self::path_of(dir);
339 let text = self.to_text();
340 let aside = path.with_extension("new");
341 match fs::write(&aside, text.as_bytes()) {
342 Ok(()) => (),
343 Err(e) => return Err(err!(e,
344 "The history listing could not be written aside to {:?}.", aside;
345 IO, File, Write)),
346 }
347 match fs::rename(&aside, &path) {
348 Ok(()) => Ok(()),
349 Err(e) => {
350 let _ = fs::remove_file(&aside);
351 Err(err!(e,
352 "The history listing at {:?} could not be moved over {:?}.",
353 aside, path;
354 IO, File, Write))
355 },
356 }
357 }
358
359 /// Writes the listing only where what is on the disk is not already this.
360 ///
361 /// A question must not become a write. `ore log` on an unchanged working copy
362 /// appends nothing and says so, and it would be a poor trade if it wrote a
363 /// file every time somebody asked; after the first read it does not, because
364 /// what it would write is what is there.
365 pub fn keep(&self, dir: &Path)
366 -> Outcome<()>
367 {
368 let path = Self::path_of(dir);
369 if let Ok(had) = fs::read_to_string(&path) {
370 if had == self.to_text() {
371 return Ok(());
372 }
373 }
374 self.write(dir)
375 }
376
377 /// The listing as the bytes that stand in the file.
378 pub fn to_text(&self) -> String {
379 let mut text = fmt!("{}\n", LISTING_FORMAT);
380 for line in self.cursor.to_lines() {
381 text.push_str(&fmt!("{}\n", line));
382 }
383 text.push_str(&fmt!("{} {}\n", OPS_LINE, self.ops));
384 let heads: Vec<String> = self.frontier.iter().map(|id| fmt!("{}", id)).collect();
385 text.push_str(&fmt!("{} {}\n", AT_LINE, heads.join(" ")));
386 text.push_str(&fmt!("{} {} {} {}\n",
387 PROV_LINE, self.signed, self.unknown, self.bare));
388 for point in &self.marks {
389 let parents: Vec<String> = point.parents.iter()
390 .map(|id| fmt!("{}", id))
391 .collect();
392 text.push_str(&fmt!("{} {} {} {} {} {}{}{}\n",
393 MARK_LINE, point.at, point.id, point.prov.mark(),
394 if point.auto { 1 } else { 0 },
395 keys::text_of(point.name.as_bytes()),
396 if parents.is_empty() { "" } else { " " },
397 parents.join(" ")));
398 }
399 text
400 }
401
402 /// Removes the listing, because what it described is no longer the store.
403 pub fn forget(dir: &Path) {
404 let _ = fs::remove_file(Self::path_of(dir));
405 }
406}
407
408
409/// The listing at `dir`, where it still describes `store`.
410///
411/// Every way of not being able to use one comes back as [`Held::Stale`] carrying
412/// the sentence that says which, so that a caller falling back to the whole read
413/// can say why rather than fall back in silence.
414pub fn current(dir: &Path, store: &Store)
415 -> Outcome<Held>
416{
417 let listing = match Listing::read(dir) {
418 Some(l) => l,
419 None => return Ok(Held::Stale(fmt!(
420 "there is no history listing at {:?}, or it is not one this build reads",
421 Listing::path_of(dir)))),
422 };
423 match res!(store.settled_at(&listing.cursor)) {
424 Settled::Same => Ok(Held::Current(listing)),
425 Settled::Moved(why) => Ok(Held::Stale(why)),
426 }
427}
428
429
430/// The provenance one character stands for, or nothing where it stands for none.
431fn prov_of(said: &str)
432 -> Option<Prov>
433{
434 for one in [Prov::Verified, Prov::Unknown, Prov::Bare] {
435 if said.len() == one.mark().len_utf8() && said.starts_with(one.mark()) {
436 return Some(one);
437 }
438 }
439 None
440}
441
442
443#[cfg(test)]
444mod tests {
445 use super::*;
446
447 use ore_store::store::Consumed;
448
449 use oxedyne_fe2o3_ore::id::ReplicaId;
450
451 /// A listing with both kinds of mark in it and a cursor over one segment.
452 fn a_listing() -> Listing {
453 let cursor = match Consumed::from_lines(
454 ["seg 000000.seg 4096 17 3 1 8 sealed", "deferred 0"].iter().copied())
455 {
456 Some(c) => c,
457 None => panic_no_cursor(),
458 };
459 Listing {
460 cursor,
461 ops: 9,
462 frontier: vec![OpId::new(ReplicaId::new(8), 9)],
463 signed: 7,
464 unknown: 1,
465 bare: 1,
466 marks: vec![
467 Point {
468 at: 2,
469 id: OpId::new(ReplicaId::new(8), 3),
470 prov: Prov::Verified,
471 // Spaces, a quotation mark and a character outside ASCII,
472 // because a name is what somebody typed and the encoding is
473 // what carries it.
474 name: fmt!("release \"1.0\" -- caf\u{e9}"),
475 parents: vec![OpId::new(ReplicaId::new(8), 2)],
476 auto: false,
477 },
478 Point {
479 at: 8,
480 id: OpId::new(ReplicaId::new(8), 9),
481 prov: Prov::Bare,
482 name: fmt!("@2026-08-22T04:12:09.482913Z"),
483 parents: vec![
484 OpId::new(ReplicaId::new(8), 8),
485 OpId::new(ReplicaId::new(3), 4),
486 ],
487 auto: true,
488 },
489 ],
490 }
491 }
492
493 fn panic_no_cursor() -> ! {
494 unreachable!("the fixture's own cursor must read")
495 }
496
497 /// A listing written down and read back is the listing that was written.
498 #[test]
499 fn a_listing_survives_being_written_down() -> Outcome<()> {
500 let was = a_listing();
501 let back = res!(Listing::from_text(&was.to_text())
502 .ok_or_else(|| err!("What a listing wrote did not read back."; Test)));
503 assert_eq!(back.to_text(), was.to_text(), "the text is the text");
504 assert_eq!(back.ops, was.ops);
505 assert_eq!(back.frontier, was.frontier);
506 assert_eq!((back.signed, back.unknown, back.bare), (7, 1, 1));
507 assert_eq!(back.marks.len(), 2);
508 assert_eq!(back.marks[0].name, was.marks[0].name,
509 "a name is carried as the bytes somebody typed");
510 assert_eq!(back.marks[0].prov, Prov::Verified);
511 assert_eq!(back.marks[1].parents, was.marks[1].parents,
512 "and a mark with two parents keeps both");
513 assert!(back.marks[1].auto && !back.marks[0].auto,
514 "and which of them this tool wrote");
515 Ok(())
516 }
517
518 /// Nothing a listing cannot make sense of is an error, and none of it is half
519 /// a listing either.
520 #[test]
521 fn a_listing_that_does_not_read_is_no_listing() -> Outcome<()> {
522 let whole = a_listing().to_text();
523 let lines: Vec<&str> = whole.lines().collect();
524 let joined = |v: Vec<String>| v.join("\n") + "\n";
525 let cases: Vec<(&str, String)> = vec![
526 ("a format line this build does not know", joined({
527 let mut v: Vec<String> = lines.iter().map(|l| fmt!("{}", l)).collect();
528 v[0] = fmt!("{} and then some", LISTING_FORMAT);
529 v
530 })),
531 ("no count of operations", joined(lines.iter()
532 .filter(|l| !l.starts_with(OPS_LINE))
533 .map(|l| fmt!("{}", l)).collect())),
534 ("no frontier", joined(lines.iter()
535 .filter(|l| !l.starts_with(AT_LINE))
536 .map(|l| fmt!("{}", l)).collect())),
537 ("no provenance", joined(lines.iter()
538 .filter(|l| !l.starts_with(PROV_LINE))
539 .map(|l| fmt!("{}", l)).collect())),
540 ("no cursor", joined(lines.iter()
541 .filter(|l| !Consumed::owns(l.split_whitespace().next().unwrap_or("")))
542 .map(|l| fmt!("{}", l)).collect())),
543 ("a line nobody wrote", fmt!("{}stanza\n", whole)),
544 ("a mark standing past the end of the log",
545 whole.replace("ops 9", "ops 3")),
546 ];
547 for (what, text) in cases {
548 assert!(Listing::from_text(&text).is_none(),
549 "a listing with {} is no listing:\n{}", what, text);
550 }
551 assert!(Listing::from_text(&whole).is_some(),
552 "and the listing they vary from must itself read");
553 Ok(())
554 }
555}