oxedyne/fe2o3/fe2o3_jdat/tests/textscan.rs
6.5 KiB, 1 run
created by r1870400018:67321, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! The framed text scan (`string::scan`), on a framed log built in memory and on a directory. |
| 2 | //! |
| 3 | //! The scan counts a clean record, a value glued to another, a comma-less comment, a fraction in |
| 4 | //! a whole-number kind, a record that is not UTF-8 and a truncated tail, each in its class, and |
| 5 | //! the report holds the place (line and column) and a tag but never a word of what was planted, |
| 6 | //! and `jdat_check`'s line for a file does the same. |
| 7 | |
| 8 | use oxedyne_fe2o3_core::prelude::*; |
| 9 | use oxedyne_fe2o3_jdat::string::scan; |
| 10 | |
| 11 | use std::fs; |
| 12 | |
| 13 | // Words planted in the bad text, which no report may ever repeat. |
| 14 | const SECRET: &str = "s3cret"; |
| 15 | |
| 16 | fn frame(text: &[u8]) -> Vec<u8> { |
| 17 | let mut f = (text.len() as u32).to_be_bytes().to_vec(); |
| 18 | f.extend_from_slice(text); |
| 19 | f |
| 20 | } |
| 21 | |
| 22 | #[test] |
| 23 | fn test_a_framed_scan_counts_by_class_and_prints_no_text_00() -> Outcome<()> { |
| 24 | let glued = fmt!("{{\"seq\": 3, \"{}\": \"x\" \"y\": 1}}", SECRET); |
| 25 | let note = "{\"seq\": 4, \"a\": 1 # note\n \"b\": 2}"; |
| 26 | let frac = "{\"seq\": 5, \"n\": (u8|2.5)}"; |
| 27 | let mut file = Vec::new(); |
| 28 | let mut offs = Vec::new(); |
| 29 | for rec in [ |
| 30 | frame(b"{\"seq\": 1, \"k\": \"v\"}"), |
| 31 | frame(b"{\"seq\": 2, \"k\": [1, 2]}"), |
| 32 | frame(glued.as_bytes()), |
| 33 | frame(note.as_bytes()), |
| 34 | frame(frac.as_bytes()), |
| 35 | frame(&[0xff, 0xfe, 0x41]), |
| 36 | ] { |
| 37 | offs.push(file.len() as u64); |
| 38 | file.extend_from_slice(&rec); |
| 39 | } |
| 40 | // A length that promises more than the file holds. |
| 41 | offs.push(file.len() as u64); |
| 42 | file.extend_from_slice(&100u32.to_be_bytes()); |
| 43 | file.extend_from_slice(b"abc"); |
| 44 | |
| 45 | let mut out = scan::TextScan::default(); |
| 46 | scan::scan_framed("things.log", &file, &mut out); |
| 47 | let got = |class: &str| out.classes.get(class).copied().unwrap_or(0); |
| 48 | req!(out.records, 6); |
| 49 | req!(out.texts, 5); |
| 50 | req!(out.refused.len(), 4); |
| 51 | req!(out.truncated, vec![("things.log".to_string(), offs[6])]); |
| 52 | req!(got("no separator"), 1); |
| 53 | req!(got("no separator after a comment"), 1); |
| 54 | req!(got("whole-number kind with a fraction"), 1); |
| 55 | req!(got(scan::NOT_UTF8), 1); |
| 56 | // The first refusal is the third record, at its own offset, with its own tag. |
| 57 | let first = &out.refused[0]; |
| 58 | req!((first.rec, first.at, first.line), (3, offs[2], 1)); |
| 59 | req!(first.hash.len(), 16); |
| 60 | req!(first.hash.clone(), scan::tag(glued.as_bytes())); |
| 61 | req!(first.class, "no separator"); |
| 62 | req!(out.is_clean(), false); |
| 63 | // Every refusal of text says where, as a line and a column, and a report line carries both. |
| 64 | for r in &out.refused { |
| 65 | if r.class != scan::NOT_UTF8 { |
| 66 | req!((r.line >= 1 && r.col >= 1), true); |
| 67 | } |
| 68 | } |
| 69 | let report = out.report(); |
| 70 | for line in report.lines().filter(|l| l.starts_with("REFUSED ") && !l.ends_with(scan::NOT_UTF8)) { |
| 71 | let num = |label: &str| line.split(' ').filter_map(|w| w.strip_prefix(label)) |
| 72 | .filter_map(|n| n.parse::<usize>().ok()).next().unwrap_or(0); |
| 73 | if num("line=") < 1 || num("col=") < 1 { |
| 74 | return Err(err!("A REFUSED line does not say where: {}", line; Test, Data)); |
| 75 | } |
| 76 | } |
| 77 | req!(report.contains(&fmt!("line={} col={}", first.line, first.col)), true); |
| 78 | for word in [SECRET, "seq", "note", "2.5"] { |
| 79 | if report.contains(word) { |
| 80 | return Err(err!("The report repeats {:?}: {}", word, report; Test, Data)); |
| 81 | } |
| 82 | } |
| 83 | for line in ["REFUSED store=things.log rec=3", "TRUNCATED store=things.log at=", |
| 84 | "class no separator after a comment: 1", "class whole-number kind with a fraction: 1"] |
| 85 | { |
| 86 | if !report.contains(line) { |
| 87 | return Err(err!("The report lacks {:?}: {}", line, report; Test, Data)); |
| 88 | } |
| 89 | } |
| 90 | |
| 91 | // A clean file reports nothing, and says so. |
| 92 | let mut clean = scan::TextScan::default(); |
| 93 | scan::scan_framed("ok.log", &file[..offs[2] as usize], &mut clean); |
| 94 | req!((clean.records, clean.texts, clean.refused.len()), (2, 2, 0)); |
| 95 | req!(clean.is_clean(), true); |
| 96 | Ok(()) |
| 97 | } |
| 98 | |
| 99 | #[test] |
| 100 | fn test_a_directory_scan_reads_logs_and_tables_only_00() -> Outcome<()> { |
| 101 | let name = "./test_textscan_dir"; |
| 102 | let _ = fs::remove_dir_all(name); |
| 103 | res!(fs::create_dir_all(name)); |
| 104 | let dir = res!(std::path::Path::new(name).canonicalize()); |
| 105 | res!(fs::write(dir.join("b.log"), frame(b"{\"a\": 1 \"b\": 2}"))); |
| 106 | res!(fs::write(dir.join("a.tbl"), frame(b"{\"a\": 1}"))); |
| 107 | res!(fs::write(dir.join("notes.txt"), b"{\"a\": 1 \"b\": 2}")); |
| 108 | let mut out = scan::TextScan::default(); |
| 109 | res!(scan::scan_dir(&dir, &mut out)); |
| 110 | req!((out.files, out.records, out.refused.len()), (2, 2, 1)); |
| 111 | req!(out.refused[0].store.as_str(), "b.log"); |
| 112 | // A directory that does not exist is an error, not an empty scan. |
| 113 | let mut none = scan::TextScan::default(); |
| 114 | req!(scan::scan_dir(&dir.join("absent"), &mut none).is_err(), true); |
| 115 | let _ = fs::remove_dir_all(&dir); |
| 116 | Ok(()) |
| 117 | } |
| 118 | |
| 119 | #[test] |
| 120 | fn test_a_tag_is_sixteen_hex_digits_and_fixed_00() -> Outcome<()> { |
| 121 | // The FNV-1a 64 test vectors: the empty input is its offset basis, "a" is af63dc4c8601ec8c. |
| 122 | req!(scan::tag(b""), "cbf29ce484222325".to_string()); |
| 123 | req!(scan::tag(b"a"), "af63dc4c8601ec8c".to_string()); |
| 124 | req!(scan::tag(b"foobar"), "85944171f73967e8".to_string()); |
| 125 | Ok(()) |
| 126 | } |
| 127 | |
| 128 | #[test] |
| 129 | fn test_jdat_check_says_where_and_prints_no_text_00() -> Outcome<()> { |
| 130 | // A refused file is one line: its path, the line and the column, and the error's tags. No |
| 131 | // key, no value, and none of the refusal's words, which a caller's log would hold. |
| 132 | let key = "Qv7kXp"; |
| 133 | let val = "Zq9xW"; |
| 134 | for (text, line) in [ |
| 135 | (fmt!("{{\"{}\": {} \"{}2\": 1}}", key, val, key), 1), |
| 136 | (fmt!("{{\n\"a\": 1,\n\"{}\": (u8|{}.5)}}", key, val), 3), |
| 137 | (fmt!("{{\"{}\": 1,\n \"{}\": 2}}", key, key), 2), |
| 138 | (fmt!("{{\"{}\": [1,, {}]}}", key, val), 1), |
| 139 | ] { |
| 140 | let (read, said) = scan::check_line("f.jdat", &text); |
| 141 | req!(read, false); |
| 142 | let want = fmt!("f.jdat: REFUSED line {} col ", line); |
| 143 | if !said.starts_with(&want) { |
| 144 | return Err(err!("The line should start {:?}, and reads {:?}", want, said; Test, Data)); |
| 145 | } |
| 146 | let col = said[want.len()..].split(':').next().and_then(|n| n.parse::<usize>().ok()).unwrap_or(0); |
| 147 | req!((col >= 1), true); |
| 148 | for word in [key, val, "7351846"] { |
| 149 | if said.contains(word) { |
| 150 | return Err(err!("The line repeats {:?}: {}", word, said; Test, Data)); |
| 151 | } |
| 152 | } |
| 153 | } |
| 154 | req!(scan::check_line("g.jdat", "{\"a\": [1, 2]}"), (true, "g.jdat: OK".to_string())); |
| 155 | Ok(()) |
| 156 | } |