oxedyne/fe2o3/fe2o3_jdat/src/string/scan.rs
7.3 KiB, 1 run
created by r1870400018:67307, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! A read-only check of JDAT text a store holds, against the decoder as it now refuses. |
| 2 | //! |
| 3 | //! The decoder refuses what it once read loosely (a value glued to another with no ',' between |
| 4 | //! them, a ',' that follows nothing, a fraction in a whole-number kind). Before a build that |
| 5 | //! refuses goes live, the text a store holds is read through it and the refusals are counted by |
| 6 | //! class. This module reads **framed text files** (`.log` and `.tbl`): each record is a 4-byte |
| 7 | //! big-endian length and the JDAT text of one entry, as Oxegen's peer store keeps its tables. A |
| 8 | //! truncated tail is reported as such. A store that holds text inside binary values (an Ozone |
| 9 | //! store) hands each text to [`refusal`] and records what it finds with [`TextScan::refuse`]. |
| 10 | //! |
| 11 | //! **Nothing a store holds is ever printed.** A refusal is reported by the file, the record |
| 12 | //! number, its byte offset in the file, a 16-hex-digit tag to find it by, the line and column of |
| 13 | //! the refusal and its class. The scan is read-only, and is run on a `cp -a` COPY of a store, |
| 14 | //! never on a live one. |
| 15 | |
| 16 | use crate::{ |
| 17 | prelude::*, |
| 18 | string::dec::{ |
| 19 | DecoderConfig, |
| 20 | Located, |
| 21 | }, |
| 22 | usr::{ |
| 23 | UsrKind, |
| 24 | UsrKindCode, |
| 25 | UsrKindId, |
| 26 | }, |
| 27 | }; |
| 28 | |
| 29 | use oxedyne_fe2o3_core::prelude::*; |
| 30 | |
| 31 | use std::{ |
| 32 | collections::BTreeMap, |
| 33 | fs, |
| 34 | path::Path, |
| 35 | }; |
| 36 | |
| 37 | |
| 38 | type Cfg = DecoderConfig<BTreeMap<UsrKindCode, UsrKind>, BTreeMap<String, UsrKindId>>; |
| 39 | |
| 40 | // The scan's own class, for a framed record that is not text at all. |
| 41 | pub const NOT_UTF8: &str = "not UTF-8"; |
| 42 | |
| 43 | // The classes that are always printed, so that a count of nought is seen as one. |
| 44 | const NAMED: [&str; 3] = [ |
| 45 | "no separator after a comment", |
| 46 | "whole-number kind with a fraction", |
| 47 | "no separator", |
| 48 | ]; |
| 49 | |
| 50 | #[derive(Clone, Debug)] |
| 51 | pub struct Refused { |
| 52 | pub store: String, // The file name, or "o3db". |
| 53 | pub rec: usize, // The record, or the text within the value, counting from 1. |
| 54 | pub at: u64, // The record's byte offset in its file, 0 for an Ozone value. |
| 55 | pub hash: String, // The tag of the record, or of the key's text. |
| 56 | pub line: usize, |
| 57 | pub col: usize, |
| 58 | pub class: &'static str, |
| 59 | } |
| 60 | |
| 61 | #[derive(Clone, Debug, Default)] |
| 62 | pub struct TextScan { |
| 63 | pub files: usize, // Framed files read. |
| 64 | pub records: usize, // Complete records, or live keys, read. |
| 65 | pub texts: usize, // Texts given to the decoder. |
| 66 | pub skipped: usize, // Strings that do not start like JDAT. |
| 67 | pub truncated: Vec<(String, u64)>, // A file and the offset of its broken tail. |
| 68 | pub refused: Vec<Refused>, |
| 69 | pub classes: BTreeMap<&'static str, usize>, |
| 70 | } |
| 71 | |
| 72 | impl TextScan { |
| 73 | pub fn refuse(&mut self, r: Refused) { |
| 74 | *self.classes.entry(r.class).or_insert(0) += 1; |
| 75 | self.refused.push(r); |
| 76 | } |
| 77 | |
| 78 | /// Was every text read without a refusal, and every file read to its end? |
| 79 | pub fn is_clean(&self) -> bool { |
| 80 | self.refused.is_empty() && self.truncated.is_empty() |
| 81 | } |
| 82 | |
| 83 | /// The report: a line for each refusal and broken tail, the counts, and the counts by class. |
| 84 | pub fn report(&self) -> String { |
| 85 | let mut s = String::new(); |
| 86 | for r in &self.refused { |
| 87 | s.push_str(&fmt!( |
| 88 | "REFUSED store={} rec={} at={} hash={} line={} col={} class={}\n", |
| 89 | r.store, r.rec, r.at, r.hash, r.line, r.col, r.class)); |
| 90 | } |
| 91 | for (store, at) in &self.truncated { |
| 92 | s.push_str(&fmt!("TRUNCATED store={} at={}\n", store, at)); |
| 93 | } |
| 94 | s.push_str(&fmt!( |
| 95 | "files={} records={} texts={} skipped={} refused={} truncated={}\n", |
| 96 | self.files, self.records, self.texts, self.skipped, |
| 97 | self.refused.len(), self.truncated.len())); |
| 98 | for label in Located::CLASSES.iter().chain([NOT_UTF8].iter()) { |
| 99 | let n = self.classes.get(label).copied().unwrap_or(0); |
| 100 | if n > 0 || NAMED.contains(label) { |
| 101 | s.push_str(&fmt!("class {}: {}\n", label, n)); |
| 102 | } |
| 103 | } |
| 104 | s |
| 105 | } |
| 106 | } |
| 107 | |
| 108 | /// A 16-hex-digit tag of some bytes (FNV-1a, 64 bits), to find a record by without printing it. |
| 109 | pub fn tag(bytes: &[u8]) -> String { |
| 110 | let mut h = 0xcbf2_9ce4_8422_2325u64; |
| 111 | for b in bytes { |
| 112 | h ^= *b as u64; |
| 113 | h = h.wrapping_mul(0x0000_0100_0000_01b3); |
| 114 | } |
| 115 | fmt!("{:016x}", h) |
| 116 | } |
| 117 | |
| 118 | /// Where and why the decoder refuses a text, as line, column and class, or None when it reads. |
| 119 | pub fn refusal(text: &str) -> Option<(usize, usize, &'static str)> { |
| 120 | match Dat::decode_string_located(text, &Cfg::default()) { |
| 121 | Ok(_) => None, |
| 122 | Err(at) => Some((at.line, at.col, at.class(text))), |
| 123 | } |
| 124 | } |
| 125 | |
| 126 | /// Is a file's text jdat, and where does it stop? The line `jdat_check` prints for a file, which |
| 127 | /// is `path: OK` or `path: REFUSED line L col C: tags`: the place and the error's tags, never the |
| 128 | /// refusal's words and never a key or a value. |
| 129 | pub fn check_line(path: &str, text: &str) -> (bool, String) { |
| 130 | match Dat::decode_string_located(text, &Cfg::default()) { |
| 131 | Ok(_) => (true, fmt!("{}: OK", path)), |
| 132 | Err(at) => { |
| 133 | let kind = Error::<ErrTag>::tags_display(at.error.tags()); |
| 134 | (false, fmt!("{}: REFUSED line {} col {}: {}", path, at.line, at.col, kind)) |
| 135 | }, |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | /// Reads one framed file's bytes: records of a 4-byte big-endian length and JDAT text. |
| 140 | pub fn scan_framed( |
| 141 | store: &str, |
| 142 | bytes: &[u8], |
| 143 | out: &mut TextScan, |
| 144 | ) { |
| 145 | out.files += 1; |
| 146 | let mut i = 0usize; |
| 147 | let mut rec = 0usize; |
| 148 | while i < bytes.len() { |
| 149 | if i + 4 > bytes.len() { |
| 150 | out.truncated.push((store.to_string(), i as u64)); |
| 151 | break; |
| 152 | } |
| 153 | let len = u32::from_be_bytes([bytes[i], bytes[i + 1], bytes[i + 2], bytes[i + 3]]) as usize; |
| 154 | let start = i + 4; |
| 155 | if start + len > bytes.len() { |
| 156 | out.truncated.push((store.to_string(), i as u64)); |
| 157 | break; |
| 158 | } |
| 159 | let body = &bytes[start..start + len]; |
| 160 | rec += 1; |
| 161 | out.records += 1; |
| 162 | let spot = match std::str::from_utf8(body) { |
| 163 | Ok(text) => { |
| 164 | out.texts += 1; |
| 165 | refusal(text) |
| 166 | }, |
| 167 | Err(_) => Some((0, 0, NOT_UTF8)), |
| 168 | }; |
| 169 | if let Some((line, col, class)) = spot { |
| 170 | out.refuse(Refused { |
| 171 | store: store.to_string(), |
| 172 | rec, |
| 173 | at: i as u64, |
| 174 | hash: tag(body), |
| 175 | line, |
| 176 | col, |
| 177 | class, |
| 178 | }); |
| 179 | } |
| 180 | i = start + len; |
| 181 | } |
| 182 | } |
| 183 | |
| 184 | /// Reads every `.log` and `.tbl` file in a directory as framed text, in name order. |
| 185 | pub fn scan_dir( |
| 186 | dir: &Path, |
| 187 | out: &mut TextScan, |
| 188 | ) |
| 189 | -> Outcome<()> |
| 190 | { |
| 191 | let mut paths = Vec::new(); |
| 192 | for entry in res!(fs::read_dir(dir)) { |
| 193 | let path = res!(entry).path(); |
| 194 | let table = match path.extension().and_then(|e| e.to_str()) { |
| 195 | Some("log") | Some("tbl") => true, |
| 196 | _ => false, |
| 197 | }; |
| 198 | if table && path.is_file() { |
| 199 | paths.push(path); |
| 200 | } |
| 201 | } |
| 202 | paths.sort(); |
| 203 | for path in paths { |
| 204 | let bytes = res!(fs::read(&path)); |
| 205 | let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("?").to_string(); |
| 206 | scan_framed(&name, &bytes, out); |
| 207 | } |
| 208 | Ok(()) |
| 209 | } |