Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_jdat/src/string/scan.rs

7.3 KiB, 1 run

created by r1870400018:67307, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! A read-only check of JDAT text a store holds, against the decoder as it now refuses.
2//!
3//! The decoder refuses what it once read loosely (a value glued to another with no ',' between
4//! them, a ',' that follows nothing, a fraction in a whole-number kind). Before a build that
5//! refuses goes live, the text a store holds is read through it and the refusals are counted by
6//! class. This module reads **framed text files** (`.log` and `.tbl`): each record is a 4-byte
7//! big-endian length and the JDAT text of one entry, as Oxegen's peer store keeps its tables. A
8//! truncated tail is reported as such. A store that holds text inside binary values (an Ozone
9//! store) hands each text to [`refusal`] and records what it finds with [`TextScan::refuse`].
10//!
11//! **Nothing a store holds is ever printed.** A refusal is reported by the file, the record
12//! number, its byte offset in the file, a 16-hex-digit tag to find it by, the line and column of
13//! the refusal and its class. The scan is read-only, and is run on a `cp -a` COPY of a store,
14//! never on a live one.
15
16use crate::{
17 prelude::*,
18 string::dec::{
19 DecoderConfig,
20 Located,
21 },
22 usr::{
23 UsrKind,
24 UsrKindCode,
25 UsrKindId,
26 },
27};
28
29use oxedyne_fe2o3_core::prelude::*;
30
31use std::{
32 collections::BTreeMap,
33 fs,
34 path::Path,
35};
36
37
38type Cfg = DecoderConfig<BTreeMap<UsrKindCode, UsrKind>, BTreeMap<String, UsrKindId>>;
39
40// The scan's own class, for a framed record that is not text at all.
41pub const NOT_UTF8: &str = "not UTF-8";
42
43// The classes that are always printed, so that a count of nought is seen as one.
44const NAMED: [&str; 3] = [
45 "no separator after a comment",
46 "whole-number kind with a fraction",
47 "no separator",
48];
49
50#[derive(Clone, Debug)]
51pub struct Refused {
52 pub store: String, // The file name, or "o3db".
53 pub rec: usize, // The record, or the text within the value, counting from 1.
54 pub at: u64, // The record's byte offset in its file, 0 for an Ozone value.
55 pub hash: String, // The tag of the record, or of the key's text.
56 pub line: usize,
57 pub col: usize,
58 pub class: &'static str,
59}
60
61#[derive(Clone, Debug, Default)]
62pub struct TextScan {
63 pub files: usize, // Framed files read.
64 pub records: usize, // Complete records, or live keys, read.
65 pub texts: usize, // Texts given to the decoder.
66 pub skipped: usize, // Strings that do not start like JDAT.
67 pub truncated: Vec<(String, u64)>, // A file and the offset of its broken tail.
68 pub refused: Vec<Refused>,
69 pub classes: BTreeMap<&'static str, usize>,
70}
71
72impl TextScan {
73 pub fn refuse(&mut self, r: Refused) {
74 *self.classes.entry(r.class).or_insert(0) += 1;
75 self.refused.push(r);
76 }
77
78 /// Was every text read without a refusal, and every file read to its end?
79 pub fn is_clean(&self) -> bool {
80 self.refused.is_empty() && self.truncated.is_empty()
81 }
82
83 /// The report: a line for each refusal and broken tail, the counts, and the counts by class.
84 pub fn report(&self) -> String {
85 let mut s = String::new();
86 for r in &self.refused {
87 s.push_str(&fmt!(
88 "REFUSED store={} rec={} at={} hash={} line={} col={} class={}\n",
89 r.store, r.rec, r.at, r.hash, r.line, r.col, r.class));
90 }
91 for (store, at) in &self.truncated {
92 s.push_str(&fmt!("TRUNCATED store={} at={}\n", store, at));
93 }
94 s.push_str(&fmt!(
95 "files={} records={} texts={} skipped={} refused={} truncated={}\n",
96 self.files, self.records, self.texts, self.skipped,
97 self.refused.len(), self.truncated.len()));
98 for label in Located::CLASSES.iter().chain([NOT_UTF8].iter()) {
99 let n = self.classes.get(label).copied().unwrap_or(0);
100 if n > 0 || NAMED.contains(label) {
101 s.push_str(&fmt!("class {}: {}\n", label, n));
102 }
103 }
104 s
105 }
106}
107
108/// A 16-hex-digit tag of some bytes (FNV-1a, 64 bits), to find a record by without printing it.
109pub fn tag(bytes: &[u8]) -> String {
110 let mut h = 0xcbf2_9ce4_8422_2325u64;
111 for b in bytes {
112 h ^= *b as u64;
113 h = h.wrapping_mul(0x0000_0100_0000_01b3);
114 }
115 fmt!("{:016x}", h)
116}
117
118/// Where and why the decoder refuses a text, as line, column and class, or None when it reads.
119pub fn refusal(text: &str) -> Option<(usize, usize, &'static str)> {
120 match Dat::decode_string_located(text, &Cfg::default()) {
121 Ok(_) => None,
122 Err(at) => Some((at.line, at.col, at.class(text))),
123 }
124}
125
126/// Is a file's text jdat, and where does it stop? The line `jdat_check` prints for a file, which
127/// is `path: OK` or `path: REFUSED line L col C: tags`: the place and the error's tags, never the
128/// refusal's words and never a key or a value.
129pub fn check_line(path: &str, text: &str) -> (bool, String) {
130 match Dat::decode_string_located(text, &Cfg::default()) {
131 Ok(_) => (true, fmt!("{}: OK", path)),
132 Err(at) => {
133 let kind = Error::<ErrTag>::tags_display(at.error.tags());
134 (false, fmt!("{}: REFUSED line {} col {}: {}", path, at.line, at.col, kind))
135 },
136 }
137}
138
139/// Reads one framed file's bytes: records of a 4-byte big-endian length and JDAT text.
140pub fn scan_framed(
141 store: &str,
142 bytes: &[u8],
143 out: &mut TextScan,
144) {
145 out.files += 1;
146 let mut i = 0usize;
147 let mut rec = 0usize;
148 while i < bytes.len() {
149 if i + 4 > bytes.len() {
150 out.truncated.push((store.to_string(), i as u64));
151 break;
152 }
153 let len = u32::from_be_bytes([bytes[i], bytes[i + 1], bytes[i + 2], bytes[i + 3]]) as usize;
154 let start = i + 4;
155 if start + len > bytes.len() {
156 out.truncated.push((store.to_string(), i as u64));
157 break;
158 }
159 let body = &bytes[start..start + len];
160 rec += 1;
161 out.records += 1;
162 let spot = match std::str::from_utf8(body) {
163 Ok(text) => {
164 out.texts += 1;
165 refusal(text)
166 },
167 Err(_) => Some((0, 0, NOT_UTF8)),
168 };
169 if let Some((line, col, class)) = spot {
170 out.refuse(Refused {
171 store: store.to_string(),
172 rec,
173 at: i as u64,
174 hash: tag(body),
175 line,
176 col,
177 class,
178 });
179 }
180 i = start + len;
181 }
182}
183
184/// Reads every `.log` and `.tbl` file in a directory as framed text, in name order.
185pub fn scan_dir(
186 dir: &Path,
187 out: &mut TextScan,
188)
189 -> Outcome<()>
190{
191 let mut paths = Vec::new();
192 for entry in res!(fs::read_dir(dir)) {
193 let path = res!(entry).path();
194 let table = match path.extension().and_then(|e| e.to_str()) {
195 Some("log") | Some("tbl") => true,
196 _ => false,
197 };
198 if table && path.is_file() {
199 paths.push(path);
200 }
201 }
202 paths.sort();
203 for path in paths {
204 let bytes = res!(fs::read(&path));
205 let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("?").to_string();
206 scan_framed(&name, &bytes, out);
207 }
208 Ok(())
209}