oxedyne/fe2o3/fe2o3_jdat/tests/decode_roundtrip.rs
12.6 KiB, 1 run
created by r1870400018:67313, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! The encoder's own output must read back as the daticle it was written from. |
| 2 | |
| 3 | use oxedyne_fe2o3_core::prelude::*; |
| 4 | use oxedyne_fe2o3_jdat::{prelude::*, map::MapKey, note::NoteConfig, string::dec::DecoderConfig, usr::{UsrKind, UsrKindCode, UsrKindId, UsrKinds}}; |
| 5 | use oxedyne_fe2o3_num::float::{Float32, Float64}; |
| 6 | use num_bigint::BigInt; |
| 7 | use std::collections::BTreeMap; |
| 8 | |
| 9 | struct Rng(u64); |
| 10 | impl Rng { |
| 11 | fn next(&mut self) -> u64 { let mut x = self.0; x ^= x << 13; x ^= x >> 7; x ^= x << 17; self.0 = x; x } |
| 12 | fn below(&mut self, n: u64) -> u64 { self.next() % n } |
| 13 | } |
| 14 | |
| 15 | const ALPHA: &[char] = &['a','b','Z','0','9',' ','"','\'','\\','{','}','[',']','(',')','|',':',',','!','#','\n','\t','\r','é','😀','\u{1}','\u{7f}','\u{feff}','u','/','-','.','e','x']; |
| 16 | const CALPHA: &[char] = &['a','b',' ','"','\'','{','}','[',']','(',')','|',':',',','\\','x','.']; |
| 17 | |
| 18 | fn rstr(r: &mut Rng, alpha: &[char], max: u64) -> String { |
| 19 | let n = r.below(max); |
| 20 | (0..n).map(|_| alpha[r.below(alpha.len() as u64) as usize]).collect() |
| 21 | } |
| 22 | |
| 23 | fn gen(r: &mut Rng, depth: u32, notes: bool) -> Dat { |
| 24 | let leaf = depth == 0 || r.below(3) == 0; |
| 25 | if leaf { |
| 26 | match r.below(20) { |
| 27 | 0 => Dat::Empty, |
| 28 | 1 => Dat::Bool(r.below(2) == 0), |
| 29 | 2 => Dat::U8(r.next() as u8), |
| 30 | 3 => Dat::U16(r.next() as u16), |
| 31 | 4 => Dat::U32(r.next() as u32), |
| 32 | 5 => Dat::U64(r.next()), |
| 33 | 6 => Dat::U128(((r.next() as u128) << 64) | r.next() as u128), |
| 34 | 7 => Dat::I8(r.next() as i8), |
| 35 | 8 => Dat::I16(r.next() as i16), |
| 36 | 9 => Dat::I32(r.next() as i32), |
| 37 | 10 => Dat::I64(r.next() as i64), |
| 38 | 11 => Dat::I128((((r.next() as u128) << 64) | r.next() as u128) as i128), |
| 39 | 12 => { let f = f32::from_bits(r.next() as u32); Dat::F32(Float32(if f.is_finite() { f } else { 1.5 })) } |
| 40 | 13 => { let f = f64::from_bits(r.next()); Dat::F64(Float64(if f.is_finite() { f } else { -2.5 })) } |
| 41 | 14 => Dat::Aint(BigInt::from(r.next() as i64) * BigInt::from(r.next()) * BigInt::from(r.next())), |
| 42 | 15 => Dat::C64(r.next() >> r.below(64)), |
| 43 | 16 => Dat::BU8((0..r.below(6)).map(|_| r.next() as u8).collect()), |
| 44 | 17 => Dat::B2([r.next() as u8, r.next() as u8]), |
| 45 | 18 => Dat::Opt(Box::new(None)), |
| 46 | _ => Dat::Str(rstr(r, ALPHA, 12)), |
| 47 | } |
| 48 | } else { |
| 49 | let d = depth - 1; |
| 50 | match r.below(8) { |
| 51 | 0 => Dat::List((0..r.below(5)).map(|_| child(r, d, notes)).collect()), |
| 52 | 1 => Dat::Box(Box::new(gen(r, d, notes))), |
| 53 | 2 => Dat::Opt(Box::new(Some(gen(r, d, notes)))), |
| 54 | 3 => Dat::Tup2(Box::new([gen(r, d, notes), gen(r, d, notes)])), |
| 55 | 4 | 5 => { |
| 56 | let mut m = BTreeMap::new(); |
| 57 | for _ in 0..r.below(5) { |
| 58 | let k = if r.below(4) == 0 { Dat::U32(r.next() as u32) } else { Dat::Str(rstr(r, ALPHA, 8)) }; |
| 59 | m.insert(k, child(r, d, notes)); |
| 60 | } |
| 61 | Dat::Map(m) |
| 62 | } |
| 63 | 6 => { |
| 64 | let mut m = BTreeMap::new(); |
| 65 | let mut used = std::collections::BTreeSet::new(); |
| 66 | let mut ord = Dat::OMAP_ORDER_START_DEFAULT; // The text holds no order, the reader numbers the keys. |
| 67 | for _ in 0..r.below(5) { |
| 68 | let k = Dat::Str(rstr(r, ALPHA, 8)); |
| 69 | if !used.insert(k.clone()) { continue; } |
| 70 | m.insert(MapKey::new(ord, k), child(r, d, notes)); |
| 71 | ord += Dat::OMAP_ORDER_DELTA_DEFAULT; |
| 72 | } |
| 73 | Dat::OrdMap(m) |
| 74 | } |
| 75 | _ => Dat::List(vec![child(r, d, notes)]), |
| 76 | } |
| 77 | } |
| 78 | } |
| 79 | |
| 80 | // A list item or a map value, which may carry a note; the reader keeps a note nowhere else. |
| 81 | fn child(r: &mut Rng, depth: u32, notes: bool) -> Dat { |
| 82 | let v = gen(r, depth, notes); |
| 83 | if notes && r.below(4) == 0 { |
| 84 | // A note is kept trimmed, and an empty one is not kept at all. |
| 85 | let note = rstr(r, CALPHA, 10).trim().to_string(); |
| 86 | if note.is_empty() { v } else { Dat::ABox(NoteConfig::default(), Box::new(v), note) } |
| 87 | } else { v } |
| 88 | } |
| 89 | |
| 90 | |
| 91 | // Where two debug forms part, with a little of each side. |
| 92 | fn first_diff(a: &str, b: &str) -> String { |
| 93 | let n = a.chars().zip(b.chars()).take_while(|(x, y)| x == y).count(); |
| 94 | let from = n.saturating_sub(40); |
| 95 | let cut = |s: &str| s.chars().skip(from).take(100).collect::<String>(); |
| 96 | fmt!("at {}: want {:?} got {:?}", n, cut(a), cut(b)) |
| 97 | } |
| 98 | |
| 99 | // Round trip: every daticle the encoder writes must read back equal, in both encodings. |
| 100 | fn run(n: u64, notes: bool) -> (u64, Vec<String>) { |
| 101 | let mut r = Rng(0x9E3779B97F4A7C15); |
| 102 | let mut ok = 0; |
| 103 | let mut bad = Vec::new(); |
| 104 | for i in 0..n { |
| 105 | let dat = gen(&mut r, 4, notes); |
| 106 | let encs = vec![("jdat", dat.jdat()), ("lines", dat.jdat_to_lines(" "))]; |
| 107 | for (mode, enc) in encs { |
| 108 | let s = match enc { Ok(s) => s, Err(e) => { bad.push(fmt!("#{} {} encoder: {}", i, mode, e.plain())); continue; } }; |
| 109 | match Dat::decode_string(&s) { |
| 110 | Ok(d2) if d2 == dat => ok += 1, |
| 111 | Ok(d2) => bad.push(fmt!("#{} {} DIFF {}", i, mode, first_diff(&fmt!("{:?}", dat), &fmt!("{:?}", d2)))), |
| 112 | Err(e) => bad.push(fmt!("#{} {} ERR text={:?} {}", i, mode, s, e.plain().replace('\n', " "))), |
| 113 | } |
| 114 | } |
| 115 | } |
| 116 | (ok, bad) |
| 117 | } |
| 118 | |
| 119 | // A note keeps its value in the lines encoding, in a list and in a map; only a note on nothing is a bare comment line. |
| 120 | #[test] |
| 121 | fn test_jdat_to_lines_keeps_a_notes_value_00() { |
| 122 | let note = |v: Dat| Dat::ABox(NoteConfig::default(), Box::new(v), fmt!("c")); |
| 123 | let mut m = BTreeMap::new(); |
| 124 | m.insert(Dat::Str(fmt!("k")), note(Dat::U8(1))); |
| 125 | let mut km = BTreeMap::new(); |
| 126 | km.insert(note(Dat::Str(fmt!("k"))), Dat::U8(2)); |
| 127 | let cases = vec![ |
| 128 | Dat::Map(m), |
| 129 | Dat::Map(km), |
| 130 | Dat::List(vec![Dat::U8(7), note(Dat::U8(1)), note(Dat::Empty), Dat::U8(8)]), |
| 131 | ]; |
| 132 | for dat in cases { |
| 133 | let s = match dat.jdat_to_lines(" ") { Ok(s) => s, Err(e) => panic!("encoder: {}", e.plain()) }; |
| 134 | assert_eq!(Dat::decode_string(&s).ok(), Some(dat.clone()), "lines text {:?} for {:?}", s, dat); |
| 135 | assert_eq!(s.contains('1') || s.contains('2'), true, "a value was dropped: {:?}", s); |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | #[test] |
| 140 | fn test_roundtrip_random_00() { |
| 141 | random(false); |
| 142 | } |
| 143 | |
| 144 | #[test] |
| 145 | fn test_roundtrip_random_notes_00() { |
| 146 | random(true); |
| 147 | } |
| 148 | |
| 149 | fn random(notes: bool) { |
| 150 | let (ok, bad) = run(3000, notes); |
| 151 | for b in bad.iter().take(25) { |
| 152 | println!("{}", b.chars().take(400).collect::<String>()); |
| 153 | } |
| 154 | assert!(bad.is_empty(), "{} of {} decodes failed, the first being: {}", bad.len(), ok + bad.len() as u64, |
| 155 | bad.first().map(|b| b.chars().take(600).collect::<String>()).unwrap_or_default()); |
| 156 | } |
| 157 | |
| 158 | |
| 159 | fn leaves() -> Vec<Dat> { |
| 160 | vec![ |
| 161 | Dat::Empty, |
| 162 | Dat::Bool(true), |
| 163 | Dat::Opt(Box::new(None)), |
| 164 | Dat::U8(1), |
| 165 | Dat::Str(fmt!("a")), |
| 166 | Dat::List(Vec::new()), |
| 167 | Dat::Map(BTreeMap::new()), |
| 168 | Dat::OrdMap(BTreeMap::new()), |
| 169 | Dat::Tup2(Box::new([Dat::U8(1), Dat::U8(2)])), |
| 170 | ] |
| 171 | } |
| 172 | |
| 173 | fn wrap(x: &Dat) -> Vec<Dat> { |
| 174 | let mut m = BTreeMap::new(); |
| 175 | m.insert(Dat::Str(fmt!("k")), x.clone()); |
| 176 | let mut o = BTreeMap::new(); |
| 177 | o.insert(MapKey::new(Dat::OMAP_ORDER_START_DEFAULT, Dat::Str(fmt!("k"))), x.clone()); |
| 178 | vec![ |
| 179 | Dat::Box(Box::new(x.clone())), |
| 180 | Dat::Opt(Box::new(Some(x.clone()))), |
| 181 | Dat::List(vec![x.clone()]), |
| 182 | Dat::List(vec![x.clone(), Dat::U8(3)]), |
| 183 | Dat::Map(m), |
| 184 | Dat::OrdMap(o), |
| 185 | Dat::Tup2(Box::new([x.clone(), Dat::U8(1)])), |
| 186 | Dat::Tup2(Box::new([Dat::U8(1), x.clone()])), |
| 187 | ] |
| 188 | } |
| 189 | |
| 190 | #[test] |
| 191 | fn test_roundtrip_systematic_00() { |
| 192 | let mut all = leaves(); |
| 193 | let mut level = all.clone(); |
| 194 | for _ in 0..2 { |
| 195 | let mut next = Vec::new(); |
| 196 | for x in &level { next.extend(wrap(x)); } |
| 197 | all.extend(next.iter().cloned()); |
| 198 | level = next; |
| 199 | } |
| 200 | let mut bad = Vec::new(); |
| 201 | for dat in &all { |
| 202 | for (mode, enc) in [("jdat", dat.jdat()), ("lines", dat.jdat_to_lines(" "))] { |
| 203 | let s = match enc { Ok(s) => s, Err(e) => { bad.push(fmt!("{} encoder: {}", mode, e.plain())); continue; } }; |
| 204 | match Dat::decode_string(&s) { |
| 205 | Ok(d2) if d2 == *dat => (), |
| 206 | Ok(d2) => bad.push(fmt!("{} DIFF text={:?} got={:?}", mode, s, d2)), |
| 207 | Err(e) => bad.push(fmt!("{} ERR text={:?} {}", mode, s, e.plain().replace('\n', " "))), |
| 208 | } |
| 209 | } |
| 210 | } |
| 211 | bad.sort_by_key(|b| b.len()); |
| 212 | for b in bad.iter().take(25) { |
| 213 | println!("{}", b.chars().take(260).collect::<String>()); |
| 214 | } |
| 215 | assert!(bad.is_empty(), "{} of {} decodes failed, the shortest being: {}", bad.len(), 2 * all.len(), |
| 216 | bad.first().map(|b| b.chars().take(400).collect::<String>()).unwrap_or_default()); |
| 217 | } |
| 218 | |
| 219 | |
| 220 | fn node_kinds() -> (UsrKindId, UsrKinds<BTreeMap<UsrKindCode, UsrKind>, BTreeMap<String, UsrKindId>>) { |
| 221 | let mut uks = UsrKinds::new(BTreeMap::new(), BTreeMap::new()); |
| 222 | let ukid = UsrKindId::new(1, Some("node"), None); |
| 223 | uks.add(ukid.clone()).unwrap_or_default(); |
| 224 | (ukid, uks) |
| 225 | } |
| 226 | |
| 227 | #[test] |
| 228 | fn test_a_user_kind_payload_round_trips_of_every_shape_00() { |
| 229 | // `(node|"abc")` read as `(node)` and the payload was dropped without a word (U0-13 H1). |
| 230 | // Every payload shape the encoder can write for a user kind is read back exactly, bare, |
| 231 | // inside a list and inside a map, through both encodings. |
| 232 | let (ukid, uks) = node_kinds(); |
| 233 | let cfg = DecoderConfig::<_, _>::jdat(Some(uks.clone())); |
| 234 | let usr = |d: Option<Dat>| Dat::Usr(ukid.clone(), d.map(Box::new)); |
| 235 | let mut m = BTreeMap::new(); |
| 236 | m.insert(Dat::Str(fmt!("k")), Dat::U8(1)); |
| 237 | let payloads = vec![ |
| 238 | usr(Some(Dat::Str(fmt!("abc")))), |
| 239 | usr(Some(Dat::Str(fmt!("")))), |
| 240 | usr(Some(Dat::Str(fmt!("a word with ) and | in it")))), |
| 241 | usr(Some(Dat::U8(1))), |
| 242 | usr(Some(Dat::I32(-7))), |
| 243 | usr(Some(Dat::Bool(true))), |
| 244 | usr(Some(Dat::Map(m))), |
| 245 | usr(Some(Dat::List(vec![Dat::U8(1), Dat::Str(fmt!("x"))]))), |
| 246 | usr(Some(Dat::List(Vec::new()))), |
| 247 | usr(Some(usr(None))), |
| 248 | usr(Some(usr(Some(Dat::Str(fmt!("deep")))))), |
| 249 | usr(Some(Dat::Box(Box::new(Dat::Str(fmt!("b")))))), |
| 250 | usr(None), |
| 251 | ]; |
| 252 | let mut bad = Vec::new(); |
| 253 | for p in &payloads { |
| 254 | let mut mk = BTreeMap::new(); |
| 255 | mk.insert(Dat::Str(fmt!("n")), p.clone()); |
| 256 | for dat in [ |
| 257 | p.clone(), |
| 258 | Dat::List(vec![p.clone(), Dat::U8(3)]), |
| 259 | Dat::Map(mk), |
| 260 | ] { |
| 261 | let encs = [ |
| 262 | ("jdat", dat.jdat_with_usr_kinds(Some(uks.clone()))), |
| 263 | ("lines", dat.jdat_with_usr_kinds_to_lines(Some(uks.clone()), " ")), |
| 264 | ]; |
| 265 | for (mode, enc) in encs { |
| 266 | let s = match enc { |
| 267 | Ok(s) => s, |
| 268 | Err(e) => { bad.push(fmt!("{} encoder: {}", mode, e.plain())); continue; }, |
| 269 | }; |
| 270 | match Dat::decode_string_with_config(&s, &cfg) { |
| 271 | Ok(d2) if d2 == dat => (), |
| 272 | Ok(d2) => bad.push(fmt!("{} DIFF text={:?} got={:?}", mode, s, d2)), |
| 273 | Err(e) => bad.push(fmt!("{} ERR text={:?} {}", mode, s, e.plain().replace('\n', " "))), |
| 274 | } |
| 275 | } |
| 276 | } |
| 277 | } |
| 278 | for b in bad.iter().take(12) { println!("{}", b.chars().take(300).collect::<String>()); } |
| 279 | assert!(bad.is_empty(), "{} decodes failed, the first being: {}", bad.len(), |
| 280 | bad.first().map(|b| b.chars().take(400).collect::<String>()).unwrap_or_default()); |
| 281 | } |
| 282 | |
| 283 | #[test] |
| 284 | fn test_a_user_kind_atom_payload_written_by_hand_is_kept_00() { |
| 285 | let (ukid, uks) = node_kinds(); |
| 286 | let cfg = DecoderConfig::<_, _>::jdat(Some(uks)); |
| 287 | let usr = |d: Dat| Dat::Usr(ukid.clone(), Some(Box::new(d))); |
| 288 | let read = |s: &str| match Dat::decode_string_with_config(s, &cfg) { |
| 289 | Ok(d) => d, |
| 290 | Err(e) => panic!("{:?} should read, and was refused: {}", s, e.plain()), |
| 291 | }; |
| 292 | assert_eq!(read("(node|\"abc\")"), usr(Dat::Str(fmt!("abc")))); |
| 293 | assert_eq!(read("(node|'abc')"), usr(Dat::Str(fmt!("abc")))); |
| 294 | assert_eq!(read("(node|\"\")"), usr(Dat::Str(fmt!("")))); |
| 295 | assert_eq!(read("(node|1)"), usr(Dat::U8(1))); |
| 296 | assert_eq!(read("(node|word)"), usr(Dat::Str(fmt!("word")))); |
| 297 | assert_eq!(read("(node| \"abc\" )"), usr(Dat::Str(fmt!("abc")))); |
| 298 | assert_eq!(read("(node)"), Dat::Usr(ukid.clone(), None)); |
| 299 | assert_eq!(read("[(node|\"a\"), (node|2), (node|w)]"), Dat::List(vec![ |
| 300 | usr(Dat::Str(fmt!("a"))), usr(Dat::U8(2)), usr(Dat::Str(fmt!("w"))), |
| 301 | ])); |
| 302 | let mut m = BTreeMap::new(); |
| 303 | m.insert(Dat::Str(fmt!("k")), usr(Dat::Str(fmt!("v")))); |
| 304 | assert_eq!(read("{\"k\": (node|\"v\")}"), Dat::Map(m)); |
| 305 | } |