oxedyne/fe2o3/fe2o3_o3db_sync/src/verify.rs
7.4 KiB, 25 runs
created by r1870400018:69891, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! A read-only check that every live value of an Ozone store reads back whole. |
| 2 | //! |
| 3 | //! Every live user key is read through the ordinary read path, so a chunked value is gathered, |
| 4 | //! rejoined, decrypted and decoded exactly as a caller would have it. In an encrypted store the |
| 5 | //! GCM tag covers the whole rejoined ciphertext, so a value whose chunks belong to two different |
| 6 | //! writes (a torn value, A3 QA B1 and B2, 2026-10-09) fails to decrypt and is reported here. In |
| 7 | //! a store without encryption nothing covers a value across its chunks, and a torn value is found |
| 8 | //! only when its bytes no longer decode. |
| 9 | //! |
| 10 | //! [`verify_key`] is the same check for one key, and issues one ordinary read, so a running |
| 11 | //! process may call it on its own open store to find a record its index holds and its bytes no |
| 12 | //! longer back. |
| 13 | //! |
| 14 | //! Opening a store starts its bots, which survey and may settle its files. Check a `cp -a` copy |
| 15 | //! taken while the store's own process is stopped, never a live store, and never with a second |
| 16 | //! process on the same store. |
| 17 | |
| 18 | use crate::{ |
| 19 | prelude::*, |
| 20 | comm::response::Wait, |
| 21 | }; |
| 22 | |
| 23 | use oxedyne_fe2o3_iop_db::api::{ |
| 24 | RestSchemesOverride, |
| 25 | ScanOpts, |
| 26 | }; |
| 27 | use oxedyne_fe2o3_jdat::{ |
| 28 | prelude::*, |
| 29 | id::NumIdDat, |
| 30 | }; |
| 31 | |
| 32 | |
| 33 | /// A live key that did not read back. |
| 34 | #[derive(Clone, Debug)] |
| 35 | pub struct ReadFailure { |
| 36 | pub key: Dat, |
| 37 | pub chunked: bool, // stored under a bunch key |
| 38 | pub damaged: bool, // the bytes failed, not the store's answer |
| 39 | pub error: String, |
| 40 | } |
| 41 | |
| 42 | /// What [`verify_live_set`] found. |
| 43 | #[derive(Clone, Debug, Default)] |
| 44 | pub struct VerifyReport { |
| 45 | pub keys: usize, // live keys the scan emitted |
| 46 | pub present: usize, // read back whole |
| 47 | pub absent: usize, // deleted after the scan listed it |
| 48 | pub chunked: usize, // stored under a bunch key, read back or not |
| 49 | pub failures: Vec<ReadFailure>, |
| 50 | } |
| 51 | |
| 52 | impl VerifyReport { |
| 53 | |
| 54 | /// Did every live key read back? |
| 55 | pub fn clean(&self) -> bool { self.failures.is_empty() } |
| 56 | |
| 57 | /// A multi-line summary for a console, naming each key that failed. `max_key` caps the |
| 58 | /// characters of each key shown. |
| 59 | pub fn summary(&self, max_key: usize) -> String { |
| 60 | let mut s = fmt!("Read check {}:\n", if self.clean() { "CLEAN" } else { "FAILED" }); |
| 61 | s.push_str(&fmt!(" live keys : {}\n", self.keys)); |
| 62 | s.push_str(&fmt!(" read whole : {}\n", self.present)); |
| 63 | s.push_str(&fmt!(" deleted : {}\n", self.absent)); |
| 64 | s.push_str(&fmt!(" chunked : {}\n", self.chunked)); |
| 65 | s.push_str(&fmt!(" failed to read : {}\n", self.failures.len())); |
| 66 | for f in &self.failures { |
| 67 | let k: String = fmt!("{:?}", f.key).chars().take(max_key).collect(); |
| 68 | s.push_str(&fmt!(" FAILED{}{} {}\n {}\n", |
| 69 | if f.chunked { " (chunked)" } else { "" }, |
| 70 | if f.damaged { "" } else { " (unanswered)" }, k, f.error)); |
| 71 | } |
| 72 | s |
| 73 | } |
| 74 | } |
| 75 | |
| 76 | /// What reading one key back through the ordinary read path found. |
| 77 | /// |
| 78 | /// `Damaged` and `Unavailable` are kept apart because only the first says anything about the |
| 79 | /// bytes. A store too busy to answer within its deadline fails a read just as a damaged record |
| 80 | /// does, and a caller that removes damaged records must never remove one on a timeout |
| 81 | /// (CHUNK-VERIFY D1, 2026-10-17). |
| 82 | #[derive(Clone, Debug, Eq, PartialEq)] |
| 83 | pub enum KeyCheck { |
| 84 | Whole { chunked: bool }, // read, rejoined, decrypted and decoded |
| 85 | Absent, // no record, or the newest is a deletion |
| 86 | Damaged { chunked: bool, error: String }, // the bytes fail checksum, decryption or decoding |
| 87 | Unavailable { chunked: bool, error: String }, // no answer: a timeout, a channel, anything else |
| 88 | } |
| 89 | |
| 90 | impl KeyCheck { |
| 91 | |
| 92 | /// Classifies a failed read by the tags anywhere in its chain. Only `Mismatch` (a checksum, |
| 93 | /// or a record other than the one named, on every attempt), `Decrypt` and `Decode` say the |
| 94 | /// bytes are bad; every other failure, a responder timeout included, is `Unavailable`, so |
| 95 | /// an error nobody foresaw never condemns a record. |
| 96 | pub fn of_error(chunked: bool, e: &Error<ErrTag>) -> Self { |
| 97 | let error = fmt!("{}", e); |
| 98 | let damaged = e.tags().iter() |
| 99 | .any(|t| matches!(t, ErrTag::Mismatch | ErrTag::Decrypt | ErrTag::Decode)); |
| 100 | if damaged { |
| 101 | Self::Damaged { chunked, error } |
| 102 | } else { |
| 103 | Self::Unavailable { chunked, error } |
| 104 | } |
| 105 | } |
| 106 | } |
| 107 | |
| 108 | /// Reads `key` back whole, gathering and rejoining a chunked value, and says what it found. It |
| 109 | /// issues no write, so unlike [`verify_live_set`] it is safe against a live store: it is one |
| 110 | /// ordinary read. A record the key index holds but whose bytes fail their checksum, decryption |
| 111 | /// or decoding is [`KeyCheck::Damaged`], which a presence lookup (`contains_wait`) cannot see. |
| 112 | pub fn verify_key< |
| 113 | const UIDL: usize, |
| 114 | UID: NumIdDat<UIDL> + 'static, |
| 115 | ENC: Encrypter + 'static, |
| 116 | KH: Hasher + 'static, |
| 117 | PR: Hasher + 'static, |
| 118 | CS: Checksummer + 'static, |
| 119 | >( |
| 120 | api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>, |
| 121 | key: &Dat, |
| 122 | schms2: Option<&RestSchemesOverride<ENC, KH>>, |
| 123 | ) |
| 124 | -> KeyCheck |
| 125 | { |
| 126 | // The head first, so that a chunked value is known as one whether or not its chunks read. |
| 127 | let head = match api.get_head_wait(key, schms2) { |
| 128 | Err(e) => return KeyCheck::of_error(false, &e), |
| 129 | Ok(None) => return KeyCheck::Absent, |
| 130 | Ok(Some((head, _))) => head, |
| 131 | }; |
| 132 | if !matches!(head, Dat::Tup5u64(_)) { |
| 133 | return KeyCheck::Whole { chunked: false }; |
| 134 | } |
| 135 | match api.fetch_chunks(&head, schms2) { |
| 136 | Ok(_) => KeyCheck::Whole { chunked: true }, |
| 137 | Err(e) => KeyCheck::of_error(true, &e), |
| 138 | } |
| 139 | } |
| 140 | |
| 141 | /// Reads every live key of the store behind `api` and reports each that fails to read, decrypt |
| 142 | /// or decode, with counts of the keys and of the chunked ones among them. It issues no write. |
| 143 | /// An error is returned only when the store cannot be scanned; a key that cannot be read is a |
| 144 | /// finding, not an error. |
| 145 | /// |
| 146 | /// # Arguments |
| 147 | /// * `scan_wait` - how long each zone's scan may take; a large store needs a generous wait. |
| 148 | pub fn verify_live_set< |
| 149 | const UIDL: usize, |
| 150 | UID: NumIdDat<UIDL> + 'static, |
| 151 | ENC: Encrypter + 'static, |
| 152 | KH: Hasher + 'static, |
| 153 | PR: Hasher + 'static, |
| 154 | CS: Checksummer + 'static, |
| 155 | >( |
| 156 | api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>, |
| 157 | schms2: Option<&RestSchemesOverride<ENC, KH>>, |
| 158 | scan_wait: Wait, |
| 159 | ) |
| 160 | -> Outcome<VerifyReport> |
| 161 | { |
| 162 | let entries = res!(api.scan_with_wait(&ScanOpts::all(), schms2, scan_wait)); |
| 163 | let mut rep = VerifyReport { keys: entries.len(), ..Default::default() }; |
| 164 | for (k, _, _) in entries { |
| 165 | match verify_key(api, &k, schms2) { |
| 166 | KeyCheck::Whole { chunked } => { |
| 167 | rep.present += 1; |
| 168 | if chunked { rep.chunked += 1; } |
| 169 | }, |
| 170 | KeyCheck::Absent => rep.absent += 1, |
| 171 | KeyCheck::Damaged { chunked, error } => { |
| 172 | if chunked { rep.chunked += 1; } |
| 173 | rep.failures.push(ReadFailure { key: k, chunked, damaged: true, error }); |
| 174 | }, |
| 175 | KeyCheck::Unavailable { chunked, error } => { |
| 176 | if chunked { rep.chunked += 1; } |
| 177 | rep.failures.push(ReadFailure { key: k, chunked, damaged: false, error }); |
| 178 | }, |
| 179 | } |
| 180 | } |
| 181 | Ok(rep) |
| 182 | } |