Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_net/examples/pmtiles_check.rs

7.7 KiB, 1 run

created by r1870400018:60840, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! Reads tiles from a remote PMTiles archive by HTTP range and checks them against the
2//! reference JavaScript readers' output.
3//!
4//! ```text
5//! cargo run --release -p oxedyne_fe2o3_net --example pmtiles_check -- \
6//! --expect fe2o3_geom/tests/data/protomaps/expect.json [--dumps DIR] [--url URL]
7//! ```
8//!
9//! `--expect` is the file the capture wrote: the archive's URL, and for each tile its length
10//! and FNV-1a-64 once decompressed, as `pmtiles` 4.5.0 read it. `--dumps` is a directory of
11//! `protomaps_<z>_<x>_<y>.json.gz`, `@mapbox/vector-tile` 3.0.0's reading of each tile; with it
12//! every tile is also decoded here and compared feature by feature. A few megabytes are read.
13
14use oxedyne_fe2o3_geom::{
15 mvt::{
16 self,
17 Value,
18 },
19 tile::pmtiles::{
20 self,
21 Archive,
22 Compression,
23 },
24};
25use oxedyne_fe2o3_net::{
26 http::range_source::HttpRangeSource,
27 tls::default_client_config,
28};
29
30use oxedyne_fe2o3_core::prelude::*;
31use oxedyne_fe2o3_jdat::{
32 prelude::*,
33 string::dec::DecoderConfig,
34 usr::{
35 UsrKind,
36 UsrKindCode,
37 UsrKindId,
38 },
39};
40
41use std::{
42 collections::BTreeMap,
43 sync::Arc,
44};
45
46fn json(text: &str) -> Outcome<Dat> {
47 let cfg = DecoderConfig::<BTreeMap<UsrKindCode, UsrKind>, BTreeMap<String, UsrKindId>>::json(None);
48 Dat::decode_string_with_config(text, &cfg)
49}
50
51fn field<'a>(d: &'a Dat, key: &str) -> Outcome<&'a Dat> {
52 res!(d.map_get(&dat!(key))).ok_or_else(|| err!("No {} in {:?}.", key, d.kind(); Input, Missing))
53}
54
55fn int(d: &Dat) -> Outcome<i128> {
56 match d {
57 Dat::U8(v) => Ok(*v as i128),
58 Dat::U16(v) => Ok(*v as i128),
59 Dat::U32(v) => Ok(*v as i128),
60 Dat::U64(v) => Ok(*v as i128),
61 Dat::I8(v) => Ok(*v as i128),
62 Dat::I16(v) => Ok(*v as i128),
63 Dat::I32(v) => Ok(*v as i128),
64 Dat::I64(v) => Ok(*v as i128),
65 other => Err(err!("Expected an integer, found {:?}.", other; Input, Mismatch)),
66 }
67}
68
69fn list(d: &Dat) -> Outcome<&Vec<Dat>> {
70 match d {
71 Dat::List(v) => Ok(v),
72 other => Err(err!("Expected a list, found {:?}.", other.kind(); Input, Mismatch)),
73 }
74}
75
76fn fnv1a64(b: &[u8]) -> String {
77 let mut h: u64 = 0xcbf29ce484222325;
78 for x in b {
79 h ^= *x as u64;
80 h = h.wrapping_mul(0x100000001b3);
81 }
82 fmt!("{:016x}", h)
83}
84
85/// Compares a decoded tile with the reference decoder's reading, returning the feature count.
86fn compare(t: &mvt::Tile, want: &Dat, name: &str) -> Outcome<usize> {
87 let layers = res!(list(want));
88 if t.layers.len() != layers.len() {
89 return Err(err!("{}: {} layers, the reference {}.", name, t.layers.len(), layers.len(); Mismatch));
90 }
91 let mut n = 0;
92 for (l, w) in t.layers.iter().zip(layers.iter()) {
93 let wf = res!(list(res!(field(w, "features"))));
94 if l.features.len() != wf.len() {
95 return Err(err!("{} {}: {} features, the reference {}.", name, l.name, l.features.len(),
96 wf.len(); Mismatch));
97 }
98 for (f, w) in l.features.iter().zip(wf.iter()) {
99 n += 1;
100 if f.kind.code() as i128 != res!(int(res!(field(w, "type")))) {
101 return Err(err!("{} {}: a type differs.", name, l.name; Mismatch));
102 }
103 let wp = res!(field(w, "props"));
104 for (k, v) in res!(f.properties(l)) {
105 let d = res!(field(wp, k));
106 let ok = match (v, d) {
107 (Value::Str(s), Dat::Str(r)) => s == r,
108 (Value::Bool(b), Dat::Bool(r)) => b == r,
109 (v, d) => match (v.as_f64(), d.get_float64()) {
110 (Some(a), Some(b)) => a == b.0,
111 _ => false,
112 },
113 };
114 if !ok {
115 return Err(err!("{} {}: {} is {:?}, the reference {:?}.", name, l.name, k, v, d; Mismatch));
116 }
117 }
118 let g = res!(f.geometry());
119 let wg = res!(list(res!(field(w, "geom"))));
120 if g.len() != wg.len() {
121 return Err(err!("{} {}: ring counts differ.", name, l.name; Mismatch));
122 }
123 for (r, wr) in g.iter().zip(wg.iter()) {
124 let wr = res!(list(wr));
125 if r.len() != wr.len() {
126 return Err(err!("{} {}: point counts differ.", name, l.name; Mismatch));
127 }
128 for (p, q) in r.iter().zip(wr.iter()) {
129 let q = res!(list(q));
130 if (p.0 as i128, p.1 as i128) != (res!(int(&q[0])), res!(int(&q[1]))) {
131 return Err(err!("{} {}: a point differs.", name, l.name; Mismatch));
132 }
133 }
134 }
135 }
136 }
137 Ok(n)
138}
139
140fn main() -> Outcome<()> {
141 log_set_level!("warn");
142 let mut url: Option<String> = None;
143 let mut expect_path: Option<String> = None;
144 let mut dumps: Option<String> = None;
145 let mut it = std::env::args().skip(1);
146 while let Some(a) = it.next() {
147 let v = res!(it.next().ok_or_else(|| err!("{} wants a value.", a; Input, Missing)));
148 match a.as_str() {
149 "--url" => url = Some(v),
150 "--expect" => expect_path = Some(v),
151 "--dumps" => dumps = Some(v),
152 other => return Err(err!("Unknown argument {:?}.", other; Input, Invalid)),
153 }
154 }
155 let expect_path = res!(expect_path.ok_or_else(|| err!("--expect is needed."; Input, Missing)));
156 let expect = res!(json(&res!(std::fs::read_to_string(&expect_path), File, Read)));
157 let url = match url {
158 Some(u) => u,
159 None => match res!(field(&expect, "url")) {
160 Dat::Str(s) => s.clone(),
161 _ => return Err(err!("The expectations name no URL."; Input, Missing)),
162 },
163 };
164 let tls = Arc::new(res!(default_client_config()));
165 let archive = res!(Archive::open(res!(HttpRangeSource::new(&url, Some(tls)))));
166 println!("{}: zooms {} to {}, {} tile contents", url, archive.header().min_zoom,
167 archive.header().max_zoom, archive.header().tile_contents);
168 let tiles = res!(list(res!(field(&expect, "tiles"))));
169 let (mut same, mut decoded, mut features) = (0, 0, 0);
170 for t in tiles {
171 let (z, x, y) = (res!(int(res!(field(t, "z")))) as u8, res!(int(res!(field(t, "x")))) as u32,
172 res!(int(res!(field(t, "y")))) as u32);
173 let name = fmt!("{}/{}/{}", z, x, y);
174 let tile = res!(res!(archive.tile_decoded(z, x, y)).ok_or_else(|| err!("{} is missing.", name; Mismatch)));
175 let len = res!(int(res!(field(t, "len"))));
176 let fnv = match res!(field(t, "fnv")) { Dat::Str(s) => s.clone(), _ => String::new() };
177 if tile.len() as i128 != len || fnv1a64(&tile) != fnv {
178 return Err(err!("{}: {} bytes {}, the reference {} bytes {}.", name, tile.len(),
179 fnv1a64(&tile), len, fnv; Mismatch));
180 }
181 same += 1;
182 if let Some(dir) = &dumps {
183 let path = fmt!("{}/protomaps_{}_{}_{}.json.gz", dir, z, x, y);
184 if let Ok(gz) = std::fs::read(&path) {
185 let text = match String::from_utf8(res!(pmtiles::decompress(&gz, Compression::Gzip))) {
186 Ok(s) => s,
187 Err(_) => return Err(err!("{} is not text.", path; Input, UTF8)),
188 };
189 let want = res!(json(&text));
190 let t = res!(mvt::decode(&tile));
191 features += res!(compare(&t, &want, &name));
192 decoded += 1;
193 }
194 }
195 }
196 println!("{} of {} tiles byte-identical to the pmtiles reader's", same, tiles.len());
197 if dumps.is_some() {
198 println!("{} tiles, {} features decoded identically to @mapbox/vector-tile", decoded, features);
199 }
200 Ok(())
201}