Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/src/lone_index.rs

19.6 KiB, 7 runs

created by r1870400018:72466, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! An index file standing without its data file, judged at the open (2026-10-11).
2//!
3//! A collection deletes a file only once every record in it is old, and removes the pair with two
4//! unlinks. Builds before 2026-10-11 removed the data file first, so a crash between the two left
5//! the index alone, and a filesystem that does not order two unlinks before a directory sync can do
6//! the same under the present order. The same state also arises when a data file is really lost.
7//! The difference is in the records the index names: after an interrupted delete each one is
8//! superseded by a newer record elsewhere, or is a chunk no current value names, and a refusal to
9//! open would be the wrong answer to a crash. So the zones replay without such an index, and once
10//! every zone is up `settle` reads each one and either completes the delete or refuses the open,
11//! naming the record that would be lost.
12//!
13//! A user key's record cannot lose its only shadow: a key's tombstone is forgotten by no cache and
14//! collected only when a newer record supersedes it. A chunk tombstone is forgotten once the older
15//! records of its key are gone, which a replay that skipped this index believes, so a chunk record
16//! with nothing newer at its key is judged by whether a current value names it.
17
18use crate::{
19 prelude::*,
20 api::legacy_chunk_set_id,
21 base::{
22 alias,
23 constant,
24 index::{
25 BotPoolInd,
26 WorkerInd,
27 ZoneInd,
28 },
29 },
30 bots::worker::worker_deps::WorkerType,
31 comm::msg::OzoneMsg,
32 data::{
33 cache::supersedes,
34 choose::ChooseCache,
35 core::Key,
36 },
37 file::{
38 floc::{
39 FileLocation,
40 FileNum,
41 },
42 stored::{
43 RecordDigest,
44 StoredIndex,
45 StoredKey,
46 },
47 zdir::{
48 ZoneDir,
49 sync_dir,
50 },
51 },
52};
53
54use oxedyne_fe2o3_iop_db::api::Meta;
55use oxedyne_fe2o3_jdat::{
56 bdat::DecodeLimits,
57 prelude::*,
58 id::NumIdDat,
59};
60
61use std::{
62 collections::BTreeMap,
63 fs,
64 io::BufReader,
65 path::PathBuf,
66};
67
68
69// One record an index names.
70struct Named<
71 const UIDL: usize,
72 UID: NumIdDat<UIDL>,
73> {
74 kbyts: Vec<u8>,
75 meta: Meta<UIDL, UID>,
76 floc: FileLocation,
77 chash: alias::ChooseHashUint,
78}
79
80// An index standing alone, and what it names.
81struct Lone<
82 const UIDL: usize,
83 UID: NumIdDat<UIDL>,
84> {
85 zdir: PathBuf,
86 fnum: FileNum,
87 ind: PathBuf,
88 dat: PathBuf,
89 recs: Vec<Named<UIDL, UID>>,
90}
91
92/// Completes each delete a crash interrupted, leaving an index without its data file, and refuses
93/// when such an index names a record nothing else holds. Run once every zone has replayed and
94/// before the store is handed out. Gives the number of indexes removed.
95pub fn settle<
96 const UIDL: usize,
97 UID: NumIdDat<UIDL> + 'static,
98 ENC: Encrypter + 'static,
99 KH: Hasher + 'static,
100 PR: Hasher + 'static,
101 CS: Checksummer + 'static,
102>(
103 api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>,
104)
105 -> Outcome<usize>
106{
107 let mut lones = Vec::new();
108 for (_, zdir) in res!(api.get_zone_dirs()) {
109 res!(find(api, &zdir, &mut lones));
110 }
111 if lones.is_empty() {
112 return Ok(0);
113 }
114
115 // 1. A record is superseded when its cache holds a newer record of the key, the order every
116 // reader decides by. Asked of each cache bot once, behind its replay.
117 let mut asks: BTreeMap<(usize, usize), Vec<(usize, usize)>> = BTreeMap::new();
118 for (l, lone) in lones.iter().enumerate() {
119 for (r, rec) in lone.recs.iter().enumerate() {
120 let w = ChooseCache::<PR>::choose_cbot_select(
121 rec.chash,
122 api.cfg().num_zones,
123 api.cfg().num_cbots_per_zone,
124 );
125 asks.entry((**w.zind(), **w.bpind())).or_default().push((l, r));
126 }
127 }
128 let mut chunks = Vec::new(); // (lone, record, chunk key) with nothing newer at its key
129 for ((z, b), which) in asks {
130 let keys: Vec<Vec<u8>> = which.iter().map(|(l, r)| lones[*l].recs[*r].kbyts.clone()).collect();
131 let held = res!(records_of(api, z, b, keys));
132 if held.len() != which.len() {
133 return Err(err!(
134 "{}: Cache bot {} of zone {} answered {} records for {} keys while judging an \
135 index without its data file.", api.ozid(), b, z, held.len(), which.len();
136 Bug, Mismatch));
137 }
138 for ((l, r), now) in which.into_iter().zip(held) {
139 let rec = &lones[l].recs[r];
140 let newer = match &now {
141 Some((m, f)) => f.fnum != rec.floc.fnum
142 && supersedes(&m.time, f, &rec.meta.time, &rec.floc),
143 None => false,
144 };
145 if newer {
146 continue;
147 }
148 // A key of the chunk form is judged as a chunk. The write has refused a user key of
149 // that form only since cb05687c (2026-10-09), so an older store could hold one, but on
150 // 2026-10-11 no caller writes such a key: a read-only grep for Tup5u64, t5u64 and
151 // [u64; 5] over the Daimond gateway source, the Oxegen tree and fe2o3's callers of
152 // Ozone (steel, ore, sbj, net) found none, and jarrah's r545 store copy checked on
153 // 2026-10-10 held no chunked value at all (review F2).
154 match Dat::from_bytes_limited(&rec.kbyts, &DecodeLimits::STORED) {
155 Ok((Dat::Tup5u64(tup), n)) if n == rec.kbyts.len() => chunks.push((l, r, tup)),
156 _ => return Err(refusal(api, &lones[l], rec, "no other file holds a newer record \
157 of its key")),
158 }
159 }
160 }
161
162 // 2. A chunk with nothing newer at its key is lost only if a current value names it. One
163 // write stamps its head and every chunk with one metadata, so the head of the chunk's own
164 // write carries its stamp. The exception is key reuse: the builds of 2026-09 took the set
165 // identifier from the key alone, so a torn rewrite of one key at one length leaves the
166 // first write's head naming the second write's chunks, and that head is found by its key
167 // (2026-10-11). So the caches are asked for those heads alone and only they are read,
168 // never every value in the store (review F3, 2026-10-11). A head that cannot be read is a
169 // head that may name the chunk, so it refuses; any other unreadable value is never looked
170 // at.
171 if !chunks.is_empty() {
172 let mut metas: Vec<Meta<UIDL, UID>> = Vec::new();
173 let mut set_ids: Vec<u64> = Vec::new();
174 for (l, r, tup) in &chunks {
175 let m = &lones[*l].recs[*r].meta;
176 if !metas.contains(m) {
177 metas.push(m.clone());
178 }
179 if !set_ids.contains(&tup[0]) {
180 set_ids.push(tup[0]);
181 }
182 }
183 for z in 0..api.cfg().num_zones as usize {
184 for b in 0..api.cfg().num_cbots_per_zone as usize {
185 for (kbyts, hmeta) in res!(heads_of(api, z, b, metas.clone(), set_ids.clone())) {
186 let named = part_key_of(api, z, b, &kbyts);
187 let legacy = legacy_chunk_set_id(&kbyts);
188 for (l, r, tup) in &chunks {
189 let rec = &lones[*l].recs[*r];
190 if rec.meta != hmeta && legacy != tup[0] {
191 continue;
192 }
193 match &named {
194 Ok(Some(pk)) if names(pk, tup) => return Err(refusal(api,
195 &lones[*l], rec, "it is a chunk of a current value")),
196 Ok(Some(_)) => (), // another write's set, under the same stamp
197 Ok(None) => return Err(refusal(api, &lones[*l], rec,
198 "it is a chunk written with the head of a current value, and \
199 that head does not read as a part key, so whether it names this \
200 chunk cannot be told")),
201 Err(e) => return Err(err!(e.clone(), "{}", refusal(api, &lones[*l],
202 rec, "it is a chunk written with the head of a current value, and \
203 that head cannot be read to tell whether it names this chunk");
204 Init, File, Missing)),
205 }
206 }
207 }
208 }
209 }
210 }
211
212 // 3. Every record is superseded, so the file was deleted whole: finish the delete.
213 let mut synced = Vec::new();
214 for lone in &lones {
215 if let Err(e) = fs::remove_file(&lone.ind) {
216 return Err(err!(e,
217 "{}: Completing the interrupted delete of index {:?}, whose data file {:?} is \
218 gone and every record of which is superseded, failed at the unlink, so the \
219 store is not opened. No record was changed: fix the cause and reopen.", api.ozid(), lone.ind, lone.dat;
220 Init, IO, File, Write));
221 }
222 info!(sync_log::stream(), "{}: Removed index {:?}, left by an interrupted collection: its \
223 data file {:?} was deleted and each of its {} records is superseded.",
224 api.ozid(), lone.ind, lone.dat, lone.recs.len());
225 if !synced.contains(&lone.zdir) {
226 synced.push(lone.zdir.clone());
227 }
228 }
229 for dir in &synced {
230 if let Err(e) = sync_dir(dir) {
231 let done = lones.iter()
232 .filter(|l| &l.zdir == dir)
233 .map(|l| fmt!("index {:?} (data file {:?})", l.ind, l.dat))
234 .collect::<Vec<_>>()
235 .join(", ");
236 return Err(err!(e,
237 "{}: Completing the interrupted delete of {}, whose data files are gone and \
238 every record superseded, failed at the sync of directory {:?} after the unlink, so \
239 the store is not opened. No record was changed: fix the cause and reopen.", api.ozid(), done, dir;
240 Init, IO, File, Write));
241 }
242 }
243 Ok(lones.len())
244}
245
246// Each index of the zone with records in it and no data file.
247fn find<
248 const UIDL: usize,
249 UID: NumIdDat<UIDL> + 'static,
250 ENC: Encrypter + 'static,
251 KH: Hasher + 'static,
252 PR: Hasher + 'static,
253 CS: Checksummer + 'static,
254>(
255 api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>,
256 zdir: &ZoneDir,
257 out: &mut Vec<Lone<UIDL, UID>>,
258)
259 -> Outcome<()>
260{
261 for entry in res!(fs::read_dir(&zdir.dir)) {
262 let path = res!(entry).path();
263 if !path.is_file() || ZoneDir::is_gc_temp_file(&path) {
264 continue;
265 }
266 let (fnum, typ) = match ZoneDir::ozone_file_number_and_type(&path) {
267 Ok(pair) => pair,
268 Err(_) => continue,
269 };
270 if typ != FileType::Index || res!(fs::metadata(&path)).len() == 0 {
271 continue;
272 }
273 let dat = zdir.dir.join(ZoneDir::relative_file_path(&FileType::Data, fnum));
274 if dat.exists() {
275 continue;
276 }
277 let recs = res!(read(api, &path, fnum));
278 out.push(Lone { zdir: zdir.dir.clone(), fnum, ind: path, dat, recs });
279 }
280 Ok(())
281}
282
283// The records an index names, read whole. One that cannot be read cannot be judged.
284fn read<
285 const UIDL: usize,
286 UID: NumIdDat<UIDL> + 'static,
287 ENC: Encrypter + 'static,
288 KH: Hasher + 'static,
289 PR: Hasher + 'static,
290 CS: Checksummer + 'static,
291>(
292 api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>,
293 ind: &PathBuf,
294 fnum: FileNum,
295)
296 -> Outcome<Vec<Named<UIDL, UID>>>
297{
298 let csum = api.schemes().checksummer().clone();
299 let mut rdr = BufReader::new(res!(fs::File::open(ind)));
300 let mut recs = Vec::new();
301 let mut pos = 0;
302 loop {
303 let (key, meta, chash) = match StoredKey::load(&mut rdr, csum.clone()) {
304 Ok(None) => break,
305 Ok(Some((skey, _, n))) => {
306 pos += n;
307 let meta = skey.meta().clone();
308 let chash = skey.ref_chash().clone();
309 (skey.into_key(), meta, chash)
310 },
311 Err(e) => return Err(err!(e,
312 "{}: The index {:?} stands without its data file and cannot be read at byte \
313 {}, so whether it names a record nothing else holds cannot be told. Restore \
314 its data file from a backup or a replica and reopen; only if losing the records \
315 of that file is accepted, remove {:?} and reopen. The open is refused.",
316 api.ozid(), ind, pos, ind;
317 Init, File, Missing)),
318 };
319 let floc = match StoredIndex::read(&mut rdr, fnum, csum.clone()) {
320 Ok((Some(sindex), n)) => {
321 pos += n;
322 sindex.ref_file_location().clone()
323 },
324 Ok((None, _)) => return Err(err!(
325 "{}: The index {:?} stands without its data file and ends inside an entry at byte \
326 {}. Restore its data file from a backup or a replica and reopen; only if losing \
327 the records of that file is accepted, remove {:?} and reopen. The open is \
328 refused.", api.ozid(), ind, pos, ind;
329 Init, File, Missing)),
330 Err(e) => return Err(err!(e,
331 "{}: The index {:?} stands without its data file and cannot be read at byte \
332 {}. Restore its data file from a backup or a replica and reopen; only if losing \
333 the records of that file is accepted, remove {:?} and reopen. The open is \
334 refused.", api.ozid(), ind, pos, ind;
335 Init, File, Missing)),
336 };
337 let chash = res!(<alias::ChooseHash>::try_from(&chash[..constant::CACHE_HASH_BYTES]));
338 recs.push(Named {
339 kbyts: key.into_bytes(),
340 meta,
341 floc,
342 chash: alias::ChooseHashUint::from_be_bytes(chash),
343 });
344 }
345 Ok(recs)
346}
347
348// The record cache bot `b` of zone `z` holds for each stored key.
349fn records_of<
350 const UIDL: usize,
351 UID: NumIdDat<UIDL> + 'static,
352 ENC: Encrypter + 'static,
353 KH: Hasher + 'static,
354 PR: Hasher + 'static,
355 CS: Checksummer + 'static,
356>(
357 api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>,
358 z: usize,
359 b: usize,
360 keys: Vec<Vec<u8>>,
361)
362 -> Outcome<Vec<Option<(Meta<UIDL, UID>, FileLocation)>>>
363{
364 let resp = api.responder();
365 let cbots = res!(api.chans().get_workers_of_type_in_zone(&WorkerType::Cache, &ZoneInd::new(z)));
366 let bot = res!(cbots.get_bot(b));
367 if let Err(e) = bot.send(OzoneMsg::RecordsOf(keys, resp.clone())) {
368 return Err(err!(e,
369 "{}: Asking cache bot {} of zone {} for the records of an index without its data \
370 file.", api.ozid(), b, z;
371 Channel, Write));
372 }
373 match res!(resp.recv_timeout(constant::CONTROL_REQUEST_TIMEOUT)) {
374 OzoneMsg::RecordsAt(held) => Ok(held),
375 OzoneMsg::Error(e) => Err(err!(e,
376 "{}: Cache bot {} of zone {} could not give the records of an index without its \
377 data file.", api.ozid(), b, z;
378 Channel)),
379 msg => Err(err!(
380 "{}: Unexpected answer from cache bot {} of zone {} for the records of an index \
381 without its data file: {:?}", api.ozid(), b, z, msg;
382 Channel, Unexpected)),
383 }
384}
385
386// The current heads cache bot `b` of zone `z` holds written with any of `metas`.
387fn heads_of<
388 const UIDL: usize,
389 UID: NumIdDat<UIDL> + 'static,
390 ENC: Encrypter + 'static,
391 KH: Hasher + 'static,
392 PR: Hasher + 'static,
393 CS: Checksummer + 'static,
394>(
395 api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>,
396 z: usize,
397 b: usize,
398 metas: Vec<Meta<UIDL, UID>>,
399 set_ids: Vec<u64>,
400)
401 -> Outcome<Vec<(Vec<u8>, Meta<UIDL, UID>)>>
402{
403 let resp = api.responder();
404 let cbots = res!(api.chans().get_workers_of_type_in_zone(&WorkerType::Cache, &ZoneInd::new(z)));
405 let bot = res!(cbots.get_bot(b));
406 if let Err(e) = bot.send(OzoneMsg::HeadsStamped(metas, set_ids, resp.clone())) {
407 return Err(err!(e,
408 "{}: Asking cache bot {} of zone {} for the heads written with the chunks of an index \
409 without its data file.", api.ozid(), b, z;
410 Channel, Write));
411 }
412 match res!(resp.recv_timeout(constant::CONTROL_REQUEST_TIMEOUT)) {
413 OzoneMsg::HeadsAt(heads) => Ok(heads),
414 OzoneMsg::Error(e) => Err(err!(e,
415 "{}: Cache bot {} of zone {} could not give the heads written with the chunks of an \
416 index without its data file.", api.ozid(), b, z;
417 Channel)),
418 msg => Err(err!(
419 "{}: Unexpected answer from cache bot {} of zone {} for the heads written with the \
420 chunks of an index without its data file: {:?}", api.ozid(), b, z, msg;
421 Channel, Unexpected)),
422 }
423}
424
425// The part key the head at stored key `kbyts`, held by cache bot `b` of zone `z`, names, read with
426// the store's own schemes.
427fn part_key_of<
428 const UIDL: usize,
429 UID: NumIdDat<UIDL> + 'static,
430 ENC: Encrypter + 'static,
431 KH: Hasher + 'static,
432 PR: Hasher + 'static,
433 CS: Checksummer + 'static,
434>(
435 api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>,
436 z: usize,
437 b: usize,
438 kbyts: &[u8],
439)
440 -> Outcome<Option<[u64; 5]>>
441{
442 let cbwind = WorkerInd::new(ZoneInd::new(z), BotPoolInd::new(b));
443 let resp = res!(api.fetch_using_key(Key::Complete(kbyts.to_vec()), cbwind));
444 match res!(resp.recv_daticle(api.schemes().encrypter(), None)) {
445 (Some((Dat::Tup5u64(tup), _)), _) => Ok(Some(tup)),
446 _ => Ok(None),
447 }
448}
449
450// Does the part key `pk` of a head name the chunk key `tup`? The chunk keys of a set are the part
451// key with its index running from 1 to the number of parts.
452fn names(pk: &[u64; 5], tup: &[u64; 5]) -> bool {
453 pk[0] == tup[0]
454 && pk[2] == tup[2]
455 && pk[3] == tup[3]
456 && pk[4] == tup[4]
457 && tup[1] >= 1
458 && tup[1] <= pk[3]
459}
460
461// The refusal for a record nothing else holds: the index, the record and the data file it is in,
462// and the way out.
463fn refusal<
464 const UIDL: usize,
465 UID: NumIdDat<UIDL> + 'static,
466 ENC: Encrypter + 'static,
467 KH: Hasher + 'static,
468 PR: Hasher + 'static,
469 CS: Checksummer + 'static,
470>(
471 api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>,
472 lone: &Lone<UIDL, UID>,
473 rec: &Named<UIDL, UID>,
474 why: &str,
475)
476 -> Error<ErrTag>
477{
478 let key = match Dat::from_bytes_limited(&rec.kbyts, &DecodeLimits::STORED) {
479 Ok((dat, n)) if n == rec.kbyts.len() => {
480 let s = fmt!("{:?}", dat);
481 match s.char_indices().nth(120) {
482 Some((i, _)) => fmt!("{}...", &s[..i]),
483 None => s,
484 }
485 },
486 _ => fmt!("0x{}", rec.kbyts.iter().take(32).map(|b| fmt!("{:02x}", b)).collect::<String>()),
487 };
488 let rid = match RecordDigest::new(&rec.kbyts, &rec.meta) {
489 Ok(rid) => fmt!("{:?}", rid),
490 Err(_) => fmt!("?"),
491 };
492 err!(
493 "{}: The data file {:?} is missing while its index {:?} stands, naming {} records. The \
494 record of key {} (record {}, at byte {} of the missing data file) would be lost: {}. \
495 Restore the data file from a backup or a replica and reopen. Only if losing the records \
496 of that file is accepted, remove the index file {:?} and reopen. The open is refused.",
497 api.ozid(), lone.dat, lone.ind, lone.recs.len(), key, rid, rec.floc.start, why, lone.ind;
498 Init, File, Missing)
499}