Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/textscan.rs

8.7 KiB, 1 run

created by r1870400018:67329, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! The Ozone store text scan (`textscan`) on synthetic, ENCRYPTED Ozone stores, one with planted
2//! bad text and one clean, and the gateway's key and store opening (`gateway`).
3//!
4//! The scan walks the value tree (a list, a map, a box, a byte string) and finds the planted text
5//! where it lies; a store without any reports no refusal. The framed-file scan has its own tests
6//! in `oxedyne_fe2o3_jdat` (`tests/textscan.rs`).
7//!
8//! `TEXTSCAN_KEEP=<dir>` keeps the two stores as `<dir>/planted` and `<dir>/clean`, with their
9//! key (32 bytes of 0x5a) as `<dir>/key`, to run the `jdat_store_check` binary on COPIES of them.
10//!
11//! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\
12//! Anthropic Claude
13
14use oxedyne_fe2o3_core::prelude::*;
15use oxedyne_fe2o3_crypto::enc::EncryptionScheme;
16use oxedyne_fe2o3_hash::{
17 csum::ChecksumScheme,
18 hash::HashScheme,
19};
20use oxedyne_fe2o3_jdat::{
21 prelude::*,
22 string::scan::TextScan,
23};
24use oxedyne_fe2o3_o3db_sync::{
25 base::{
26 cfg::OzoneConfig,
27 constant,
28 },
29 comm::response::Wait,
30 data::core::RestSchemesInput,
31 gateway,
32 test::setup,
33 textscan,
34};
35
36use std::{
37 collections::BTreeMap,
38 fs,
39 path::{
40 Path,
41 PathBuf,
42 },
43 thread,
44 time::Duration,
45};
46
47
48type Enc = EncryptionScheme;
49type Kh = HashScheme;
50type Cs = ChecksumScheme;
51
52const SCAN_WAIT: Wait = Wait {
53 max_wait: Duration::from_secs(120),
54 check_interval: constant::CHECK_INTERVAL,
55};
56
57// Words planted in the bad text, which no report may ever repeat.
58const SECRET: &str = "s3cret";
59
60#[test]
61fn test_an_ozone_store_scan_finds_planted_text_00() -> Outcome<()> {
62 log_set_level!("warn");
63 let outcome = run_stores();
64 log_finish_wait!();
65 outcome
66}
67
68fn run_stores() -> Outcome<()> {
69 let user = setup::Uid::default();
70
71 // The planted store: clean text, text glued to a text, a comma-less comment in a byte string
72 // inside a list inside a map, a fraction in a box, a plain string and a number; the map's own key is a string that is not text.
73 let planted_root = res!(store_root("planted"));
74 let mut inner = BTreeMap::new();
75 inner.insert(Dat::Str(fmt!("k")), Dat::BU8(b"[1 # a note\n 2]".to_vec()));
76 let rows = vec![
77 ("a:1", Dat::Str(fmt!("{{\"ok\": 1}}"))),
78 ("a:2", Dat::Str(fmt!("{{\"k\": 1 \"{}\": 2}}", SECRET))),
79 ("a:3", Dat::List(vec![Dat::U8(1), Dat::Map(inner)])),
80 ("a:4", Dat::Str(fmt!("hello"))),
81 ("a:5", Dat::U64(7)),
82 ("a:6", Dat::Box(Box::new(Dat::Str(fmt!("(u8|1.5)"))))),
83 ];
84 res!(write_store(&planted_root, &rows, user));
85 let planted = res!(scan_store(&planted_root));
86 let got = |class: &str| planted.classes.get(class).copied().unwrap_or(0);
87 req!(planted.records, 6);
88 req!(planted.texts, 4);
89 req!(planted.skipped, 2);
90 req!(planted.refused.len(), 3);
91 // Every refusal says where, a line and a column, and the report carries both.
92 let report = planted.report();
93 for r in &planted.refused {
94 req!((r.line >= 1 && r.col >= 1), true);
95 req!(report.contains(&fmt!("line={} col={}", r.line, r.col)), true);
96 }
97 req!(got("no separator"), 1);
98 req!(got("no separator after a comment"), 1);
99 req!(got("whole-number kind with a fraction"), 1);
100 req!(planted.is_clean(), false);
101 for r in &planted.refused {
102 req!((r.store.as_str(), r.at, r.hash.len()), ("o3db", 0, 16));
103 }
104 for word in [SECRET, "hello", "a:2", "ok", "a note"] {
105 if report.contains(word) {
106 return Err(err!("The report repeats {:?}: {}", word, report; Test, Data));
107 }
108 }
109 drop_store(&planted_root);
110
111 // The clean store reports no refusal.
112 let clean_root = res!(store_root("clean"));
113 let rows = vec![
114 ("a:1", Dat::Str(fmt!("{{\"ok\": 1}}"))),
115 ("a:2", Dat::List(vec![Dat::Str(fmt!("[1, 2]"))])),
116 ("a:3", Dat::U64(7)),
117 ];
118 res!(write_store(&clean_root, &rows, user));
119 let clean = res!(scan_store(&clean_root));
120 req!((clean.records, clean.texts, clean.refused.len()), (3, 2, 0));
121 req!(clean.is_clean(), true);
122 drop_store(&clean_root);
123 Ok(())
124}
125
126// A 32-byte at-rest key, fixed so the test is deterministic; never a real one.
127fn test_key() -> [u8; 32] { [0x5au8; 32] }
128
129fn schemes_input() -> Outcome<RestSchemesInput<Enc, Kh, Kh, Cs>> {
130 let aes_gcm = res!(EncryptionScheme::new_aes_256_gcm_with_key(&test_key()[..]));
131 let crc32 = ChecksumScheme::new_crc32();
132 Ok(RestSchemesInput::new(
133 Some(aes_gcm),
134 None::<HashScheme>,
135 None::<HashScheme>,
136 Some(crc32),
137 ))
138}
139
140fn fresh_root(name: &str) -> Outcome<PathBuf> {
141 let _ = fs::remove_dir_all(name);
142 res!(fs::create_dir_all(name));
143 Ok(res!(Path::new(name).canonicalize()))
144}
145
146fn base_cfg() -> Outcome<OzoneConfig> {
147 let mut cfg = res!(setup::default_cfg());
148 cfg.num_zones = 3;
149 cfg.num_cbots_per_zone = 1;
150 cfg.num_fbots_per_zone = 1;
151 cfg.num_wbots_per_zone = 1;
152 cfg.num_igbots_per_zone = 1;
153 cfg.data_file_max_bytes = 4_000;
154 cfg.rest_chunk_threshold = 500;
155 cfg.rest_chunk_bytes = 128;
156 cfg.cache_size_limit_bytes = 40_000_000;
157 cfg.zone_overrides = BTreeMap::new();
158 Ok(cfg)
159}
160
161fn write_store(
162 root: &Path,
163 rows: &[(&str, Dat)],
164 user: setup::Uid,
165)
166 -> Outcome<()>
167{
168 let db = res!(setup::start_db(
169 root.to_path_buf(),
170 Some(res!(base_cfg())),
171 res!(schemes_input()),
172 None,
173 false,
174 true,
175 ));
176 thread::sleep(Duration::from_secs(1));
177 for (k, v) in rows {
178 let resp = res!(db.api().store(Dat::Str(k.to_string()), v.clone(), user));
179 res!(resp.recv_store_ack());
180 }
181 thread::sleep(Duration::from_secs(1));
182 res!(db.shutdown());
183 thread::sleep(Duration::from_secs(1));
184 Ok(())
185}
186
187// Opens the store as the examples do, through the gateway: its own configuration, garbage
188// collection off.
189fn scan_store(root: &Path) -> Outcome<TextScan> {
190 let (db, answered) = res!(gateway::open_store(root, None, &test_key(), false, "textscan-test"));
191 // A live store answers, and the requirement the scanning tool makes of it is met.
192 req!((answered > 0), true);
193 req!(res!(gateway::require_answer(answered, "textscan-test", root)), answered);
194 thread::sleep(Duration::from_secs(1));
195 let mut out = TextScan::default();
196 res!(textscan::scan_o3db(db.api(), SCAN_WAIT, &mut out));
197 res!(db.shutdown());
198 thread::sleep(Duration::from_secs(1));
199 Ok(out)
200}
201
202// A store's root: under TEXTSCAN_KEEP, with the key file beside it, or a scratch directory.
203fn store_root(name: &str) -> Outcome<PathBuf> {
204 match std::env::var("TEXTSCAN_KEEP") {
205 Ok(keep) => {
206 let dir = Path::new(&keep);
207 res!(fs::create_dir_all(dir));
208 res!(fs::write(dir.join("key"), test_key()));
209 fresh_root(&dir.join(name).to_string_lossy())
210 },
211 Err(_) => fresh_root(&fmt!("./test_db_textscan_{}", name)),
212 }
213}
214
215// Removes a scratch store, unless the run was asked to keep them.
216fn drop_store(root: &Path) {
217 if std::env::var("TEXTSCAN_KEEP").is_err() {
218 let _ = fs::remove_dir_all(root);
219 }
220}
221
222#[test]
223fn test_the_gateway_key_is_read_whole_or_refused_00() -> Outcome<()> {
224 let dir = res!(fresh_root("./test_gateway_key"));
225 let good = dir.join("good");
226 let short = dir.join("short");
227 res!(fs::write(&good, test_key()));
228 res!(fs::write(&short, &test_key()[..31]));
229 req!(res!(gateway::load_key(&good)), test_key());
230 req!(gateway::load_key(&short).is_err(), true);
231 req!(gateway::load_key(&dir.join("absent")).is_err(), true);
232 let _ = fs::remove_dir_all(&dir);
233 Ok(())
234}
235
236#[test]
237fn test_a_store_that_gives_no_answer_is_refused_by_the_requirement_alone_00() -> Outcome<()> {
238 // `open_store` hands the store back whatever the count, as it did before the helper, so that
239 // `o3db_migrate` and `o3db_sweep` carry on. Only the tool that reports on the store asks for
240 // an answer. No store gives a zero count to open (with every bot dead the ping itself fails,
241 // and a configuration with no bot divides by zero at start), so the zero is driven here
242 // through the requirement.
243 let root = Path::new("/nowhere");
244 let e = match gateway::require_answer(0, "scan", root) {
245 Ok(n) => return Err(err!("A count of 0 should be refused, and gave {}", n; Test, Data)),
246 Err(e) => e,
247 };
248 if !e.plain().contains("no bot of the store") {
249 return Err(err!("The refusal should say that no bot answered: {}", e.plain(); Test, Data));
250 }
251 req!(res!(gateway::require_answer(28, "scan", root)), 28);
252 req!(res!(gateway::require_answer(1, "scan", root)), 1);
253 Ok(())
254}