oxedyne/fe2o3/fe2o3_o3db_sync/tests/textscan.rs
8.7 KiB, 1 run
created by r1870400018:67329, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! The Ozone store text scan (`textscan`) on synthetic, ENCRYPTED Ozone stores, one with planted |
| 2 | //! bad text and one clean, and the gateway's key and store opening (`gateway`). |
| 3 | //! |
| 4 | //! The scan walks the value tree (a list, a map, a box, a byte string) and finds the planted text |
| 5 | //! where it lies; a store without any reports no refusal. The framed-file scan has its own tests |
| 6 | //! in `oxedyne_fe2o3_jdat` (`tests/textscan.rs`). |
| 7 | //! |
| 8 | //! `TEXTSCAN_KEEP=<dir>` keeps the two stores as `<dir>/planted` and `<dir>/clean`, with their |
| 9 | //! key (32 bytes of 0x5a) as `<dir>/key`, to run the `jdat_store_check` binary on COPIES of them. |
| 10 | //! |
| 11 | //! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\ |
| 12 | //! Anthropic Claude |
| 13 | |
| 14 | use oxedyne_fe2o3_core::prelude::*; |
| 15 | use oxedyne_fe2o3_crypto::enc::EncryptionScheme; |
| 16 | use oxedyne_fe2o3_hash::{ |
| 17 | csum::ChecksumScheme, |
| 18 | hash::HashScheme, |
| 19 | }; |
| 20 | use oxedyne_fe2o3_jdat::{ |
| 21 | prelude::*, |
| 22 | string::scan::TextScan, |
| 23 | }; |
| 24 | use oxedyne_fe2o3_o3db_sync::{ |
| 25 | base::{ |
| 26 | cfg::OzoneConfig, |
| 27 | constant, |
| 28 | }, |
| 29 | comm::response::Wait, |
| 30 | data::core::RestSchemesInput, |
| 31 | gateway, |
| 32 | test::setup, |
| 33 | textscan, |
| 34 | }; |
| 35 | |
| 36 | use std::{ |
| 37 | collections::BTreeMap, |
| 38 | fs, |
| 39 | path::{ |
| 40 | Path, |
| 41 | PathBuf, |
| 42 | }, |
| 43 | thread, |
| 44 | time::Duration, |
| 45 | }; |
| 46 | |
| 47 | |
| 48 | type Enc = EncryptionScheme; |
| 49 | type Kh = HashScheme; |
| 50 | type Cs = ChecksumScheme; |
| 51 | |
| 52 | const SCAN_WAIT: Wait = Wait { |
| 53 | max_wait: Duration::from_secs(120), |
| 54 | check_interval: constant::CHECK_INTERVAL, |
| 55 | }; |
| 56 | |
| 57 | // Words planted in the bad text, which no report may ever repeat. |
| 58 | const SECRET: &str = "s3cret"; |
| 59 | |
| 60 | #[test] |
| 61 | fn test_an_ozone_store_scan_finds_planted_text_00() -> Outcome<()> { |
| 62 | log_set_level!("warn"); |
| 63 | let outcome = run_stores(); |
| 64 | log_finish_wait!(); |
| 65 | outcome |
| 66 | } |
| 67 | |
| 68 | fn run_stores() -> Outcome<()> { |
| 69 | let user = setup::Uid::default(); |
| 70 | |
| 71 | // The planted store: clean text, text glued to a text, a comma-less comment in a byte string |
| 72 | // inside a list inside a map, a fraction in a box, a plain string and a number; the map's own key is a string that is not text. |
| 73 | let planted_root = res!(store_root("planted")); |
| 74 | let mut inner = BTreeMap::new(); |
| 75 | inner.insert(Dat::Str(fmt!("k")), Dat::BU8(b"[1 # a note\n 2]".to_vec())); |
| 76 | let rows = vec![ |
| 77 | ("a:1", Dat::Str(fmt!("{{\"ok\": 1}}"))), |
| 78 | ("a:2", Dat::Str(fmt!("{{\"k\": 1 \"{}\": 2}}", SECRET))), |
| 79 | ("a:3", Dat::List(vec![Dat::U8(1), Dat::Map(inner)])), |
| 80 | ("a:4", Dat::Str(fmt!("hello"))), |
| 81 | ("a:5", Dat::U64(7)), |
| 82 | ("a:6", Dat::Box(Box::new(Dat::Str(fmt!("(u8|1.5)"))))), |
| 83 | ]; |
| 84 | res!(write_store(&planted_root, &rows, user)); |
| 85 | let planted = res!(scan_store(&planted_root)); |
| 86 | let got = |class: &str| planted.classes.get(class).copied().unwrap_or(0); |
| 87 | req!(planted.records, 6); |
| 88 | req!(planted.texts, 4); |
| 89 | req!(planted.skipped, 2); |
| 90 | req!(planted.refused.len(), 3); |
| 91 | // Every refusal says where, a line and a column, and the report carries both. |
| 92 | let report = planted.report(); |
| 93 | for r in &planted.refused { |
| 94 | req!((r.line >= 1 && r.col >= 1), true); |
| 95 | req!(report.contains(&fmt!("line={} col={}", r.line, r.col)), true); |
| 96 | } |
| 97 | req!(got("no separator"), 1); |
| 98 | req!(got("no separator after a comment"), 1); |
| 99 | req!(got("whole-number kind with a fraction"), 1); |
| 100 | req!(planted.is_clean(), false); |
| 101 | for r in &planted.refused { |
| 102 | req!((r.store.as_str(), r.at, r.hash.len()), ("o3db", 0, 16)); |
| 103 | } |
| 104 | for word in [SECRET, "hello", "a:2", "ok", "a note"] { |
| 105 | if report.contains(word) { |
| 106 | return Err(err!("The report repeats {:?}: {}", word, report; Test, Data)); |
| 107 | } |
| 108 | } |
| 109 | drop_store(&planted_root); |
| 110 | |
| 111 | // The clean store reports no refusal. |
| 112 | let clean_root = res!(store_root("clean")); |
| 113 | let rows = vec![ |
| 114 | ("a:1", Dat::Str(fmt!("{{\"ok\": 1}}"))), |
| 115 | ("a:2", Dat::List(vec![Dat::Str(fmt!("[1, 2]"))])), |
| 116 | ("a:3", Dat::U64(7)), |
| 117 | ]; |
| 118 | res!(write_store(&clean_root, &rows, user)); |
| 119 | let clean = res!(scan_store(&clean_root)); |
| 120 | req!((clean.records, clean.texts, clean.refused.len()), (3, 2, 0)); |
| 121 | req!(clean.is_clean(), true); |
| 122 | drop_store(&clean_root); |
| 123 | Ok(()) |
| 124 | } |
| 125 | |
| 126 | // A 32-byte at-rest key, fixed so the test is deterministic; never a real one. |
| 127 | fn test_key() -> [u8; 32] { [0x5au8; 32] } |
| 128 | |
| 129 | fn schemes_input() -> Outcome<RestSchemesInput<Enc, Kh, Kh, Cs>> { |
| 130 | let aes_gcm = res!(EncryptionScheme::new_aes_256_gcm_with_key(&test_key()[..])); |
| 131 | let crc32 = ChecksumScheme::new_crc32(); |
| 132 | Ok(RestSchemesInput::new( |
| 133 | Some(aes_gcm), |
| 134 | None::<HashScheme>, |
| 135 | None::<HashScheme>, |
| 136 | Some(crc32), |
| 137 | )) |
| 138 | } |
| 139 | |
| 140 | fn fresh_root(name: &str) -> Outcome<PathBuf> { |
| 141 | let _ = fs::remove_dir_all(name); |
| 142 | res!(fs::create_dir_all(name)); |
| 143 | Ok(res!(Path::new(name).canonicalize())) |
| 144 | } |
| 145 | |
| 146 | fn base_cfg() -> Outcome<OzoneConfig> { |
| 147 | let mut cfg = res!(setup::default_cfg()); |
| 148 | cfg.num_zones = 3; |
| 149 | cfg.num_cbots_per_zone = 1; |
| 150 | cfg.num_fbots_per_zone = 1; |
| 151 | cfg.num_wbots_per_zone = 1; |
| 152 | cfg.num_igbots_per_zone = 1; |
| 153 | cfg.data_file_max_bytes = 4_000; |
| 154 | cfg.rest_chunk_threshold = 500; |
| 155 | cfg.rest_chunk_bytes = 128; |
| 156 | cfg.cache_size_limit_bytes = 40_000_000; |
| 157 | cfg.zone_overrides = BTreeMap::new(); |
| 158 | Ok(cfg) |
| 159 | } |
| 160 | |
| 161 | fn write_store( |
| 162 | root: &Path, |
| 163 | rows: &[(&str, Dat)], |
| 164 | user: setup::Uid, |
| 165 | ) |
| 166 | -> Outcome<()> |
| 167 | { |
| 168 | let db = res!(setup::start_db( |
| 169 | root.to_path_buf(), |
| 170 | Some(res!(base_cfg())), |
| 171 | res!(schemes_input()), |
| 172 | None, |
| 173 | false, |
| 174 | true, |
| 175 | )); |
| 176 | thread::sleep(Duration::from_secs(1)); |
| 177 | for (k, v) in rows { |
| 178 | let resp = res!(db.api().store(Dat::Str(k.to_string()), v.clone(), user)); |
| 179 | res!(resp.recv_store_ack()); |
| 180 | } |
| 181 | thread::sleep(Duration::from_secs(1)); |
| 182 | res!(db.shutdown()); |
| 183 | thread::sleep(Duration::from_secs(1)); |
| 184 | Ok(()) |
| 185 | } |
| 186 | |
| 187 | // Opens the store as the examples do, through the gateway: its own configuration, garbage |
| 188 | // collection off. |
| 189 | fn scan_store(root: &Path) -> Outcome<TextScan> { |
| 190 | let (db, answered) = res!(gateway::open_store(root, None, &test_key(), false, "textscan-test")); |
| 191 | // A live store answers, and the requirement the scanning tool makes of it is met. |
| 192 | req!((answered > 0), true); |
| 193 | req!(res!(gateway::require_answer(answered, "textscan-test", root)), answered); |
| 194 | thread::sleep(Duration::from_secs(1)); |
| 195 | let mut out = TextScan::default(); |
| 196 | res!(textscan::scan_o3db(db.api(), SCAN_WAIT, &mut out)); |
| 197 | res!(db.shutdown()); |
| 198 | thread::sleep(Duration::from_secs(1)); |
| 199 | Ok(out) |
| 200 | } |
| 201 | |
| 202 | // A store's root: under TEXTSCAN_KEEP, with the key file beside it, or a scratch directory. |
| 203 | fn store_root(name: &str) -> Outcome<PathBuf> { |
| 204 | match std::env::var("TEXTSCAN_KEEP") { |
| 205 | Ok(keep) => { |
| 206 | let dir = Path::new(&keep); |
| 207 | res!(fs::create_dir_all(dir)); |
| 208 | res!(fs::write(dir.join("key"), test_key())); |
| 209 | fresh_root(&dir.join(name).to_string_lossy()) |
| 210 | }, |
| 211 | Err(_) => fresh_root(&fmt!("./test_db_textscan_{}", name)), |
| 212 | } |
| 213 | } |
| 214 | |
| 215 | // Removes a scratch store, unless the run was asked to keep them. |
| 216 | fn drop_store(root: &Path) { |
| 217 | if std::env::var("TEXTSCAN_KEEP").is_err() { |
| 218 | let _ = fs::remove_dir_all(root); |
| 219 | } |
| 220 | } |
| 221 | |
| 222 | #[test] |
| 223 | fn test_the_gateway_key_is_read_whole_or_refused_00() -> Outcome<()> { |
| 224 | let dir = res!(fresh_root("./test_gateway_key")); |
| 225 | let good = dir.join("good"); |
| 226 | let short = dir.join("short"); |
| 227 | res!(fs::write(&good, test_key())); |
| 228 | res!(fs::write(&short, &test_key()[..31])); |
| 229 | req!(res!(gateway::load_key(&good)), test_key()); |
| 230 | req!(gateway::load_key(&short).is_err(), true); |
| 231 | req!(gateway::load_key(&dir.join("absent")).is_err(), true); |
| 232 | let _ = fs::remove_dir_all(&dir); |
| 233 | Ok(()) |
| 234 | } |
| 235 | |
| 236 | #[test] |
| 237 | fn test_a_store_that_gives_no_answer_is_refused_by_the_requirement_alone_00() -> Outcome<()> { |
| 238 | // `open_store` hands the store back whatever the count, as it did before the helper, so that |
| 239 | // `o3db_migrate` and `o3db_sweep` carry on. Only the tool that reports on the store asks for |
| 240 | // an answer. No store gives a zero count to open (with every bot dead the ping itself fails, |
| 241 | // and a configuration with no bot divides by zero at start), so the zero is driven here |
| 242 | // through the requirement. |
| 243 | let root = Path::new("/nowhere"); |
| 244 | let e = match gateway::require_answer(0, "scan", root) { |
| 245 | Ok(n) => return Err(err!("A count of 0 should be refused, and gave {}", n; Test, Data)), |
| 246 | Err(e) => e, |
| 247 | }; |
| 248 | if !e.plain().contains("no bot of the store") { |
| 249 | return Err(err!("The refusal should say that no bot answered: {}", e.plain(); Test, Data)); |
| 250 | } |
| 251 | req!(res!(gateway::require_answer(28, "scan", root)), 28); |
| 252 | req!(res!(gateway::require_answer(1, "scan", root)), 1); |
| 253 | Ok(()) |
| 254 | } |