Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/presence.rs

8.7 KiB, 1 run

created by r1870400018:74305, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! A presence query reads no value.
2//!
3//! Asking whether a store holds a key was answered by reading the value, decrypting and decoding
4//! it, and throwing it away. The Daimond gateway's chunk presence query asked that of 2,000
5//! values of 256 KB at a time, half a gigabyte read and decrypted to answer "which of these do you
6//! hold?", and on a one-CPU host the phone gave up after 63 s (2026-10-16). The zone caches hold
7//! every key's record in memory, and a tombstone has one length, so the index answers.
8//!
9//! This store is the gateway's: 2,000 encrypted values of 256 KB, under a 100 MB cache, so most
10//! values are not cached and a read goes to disk.
11//!
12//! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\
13//! Anthropic Claude
14
15use oxedyne_fe2o3_core::{
16 prelude::*,
17 rand::Rand,
18};
19use oxedyne_fe2o3_crypto::enc::EncryptionScheme;
20use oxedyne_fe2o3_hash::{
21 csum::ChecksumScheme,
22 hash::HashScheme,
23};
24use oxedyne_fe2o3_iop_db::api::Database;
25use oxedyne_fe2o3_jdat::prelude::*;
26use oxedyne_fe2o3_o3db_sync::{
27 O3db,
28 base::constant,
29 comm::response::Wait,
30 data::core::RestSchemesInput,
31 test::{
32 hooks,
33 setup,
34 },
35};
36
37use std::{
38 path::PathBuf,
39 sync::Arc,
40 thread,
41 time::{
42 Duration,
43 Instant,
44 },
45};
46
47
48type Db = O3db<{ setup::UID_LEN }, setup::Uid, EncryptionScheme, HashScheme, HashScheme, ChecksumScheme>;
49
50const N: usize = 2_000; // the gateway's HAVE_QUERY_BATCH
51const VAL_BYTES: usize = 256 * 1024; // one Daimond chunk
52const DELETED: usize = 50; // of the N, deleted after writing
53const ABSENT: usize = 200; // never written
54const QUERIES: usize = 8; // concurrent queries, as the phone's retries stacked
55// Measured on argonaut 2026-10-16: 8 concurrent 2,000-key queries of the index took 8.9 ms in
56// all, 33 ms pinned to one CPU, and one query of the old path, reading 524 MB of values, took
57// 218-286 s. Fifteen times the index's one-CPU figure.
58const BOUND: Duration = Duration::from_millis(500);
59
60fn key(i: usize) -> Dat {
61 dat!(fmt!("chunk:{:05}", i))
62}
63
64fn wait() -> Wait {
65 Wait {
66 max_wait: Duration::from_secs(30),
67 check_interval: constant::CHECK_INTERVAL,
68 }
69}
70
71fn open(root: &PathBuf, enc: &EncryptionScheme, wipe: bool) -> Outcome<Db> {
72 let schms_input = RestSchemesInput::new(
73 Some(enc.clone()),
74 None::<HashScheme>,
75 None::<HashScheme>,
76 Some(ChecksumScheme::new_crc32()),
77 );
78 let mut cfg = res!(setup::default_cfg());
79 cfg.zone_overrides = DaticleMap::new();
80 cfg.cache_size_limit_bytes = 100_000_000; // as the live gateway's
81 cfg.data_file_max_bytes = 64_000_000;
82 cfg.rest_chunk_threshold = 716_800; // so a chunk is one record, as live
83 cfg.rest_chunk_bytes = 102_400;
84 setup::start_db(root.clone(), Some(cfg), schms_input, None, false, wipe)
85}
86
87#[test]
88fn presence_reads_no_value() -> Outcome<()> {
89 log_set_level!("warn");
90 let outcome = run();
91 log_finish_wait!();
92 outcome
93}
94
95fn run() -> Outcome<()> {
96 let root = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join("test_db_presence");
97 let _ = std::fs::remove_dir_all(&root);
98 res!(std::fs::create_dir_all(&root));
99 let root = res!(root.canonicalize());
100
101 let mut enckey = [0u8; 32];
102 Rand::fill_u8(&mut enckey);
103 let aes = res!(EncryptionScheme::new_aes_256_gcm_with_key(&enckey[..]));
104
105 let user = setup::Uid::default();
106 let mut val = vec![0u8; VAL_BYTES];
107 {
108 let db = res!(open(&root, &aes, true));
109 let t = Instant::now();
110 for i in 0..N {
111 val[..8].copy_from_slice(&(i as u64).to_le_bytes());
112 res!(db.insert(key(i), Dat::BU32(val.clone()), user, None));
113 }
114 println!("wrote {} values of {} bytes in {:?}", N, VAL_BYTES, t.elapsed());
115 res!(db.close());
116 }
117
118 // Reopened, so the cache holds what a restarted gateway's does: the index, and no values.
119 let db = Arc::new(res!(open(&root, &aes, false)));
120 let all: Vec<Dat> = (0..N).map(key).collect();
121
122 // (a) No value is read to answer for present keys.
123 let before = hooks::record_bytes_read();
124 let t = Instant::now();
125 let held = res!(db.contains_many(&all, None));
126 let took = t.elapsed();
127 let read = hooks::record_bytes_read() - before;
128 println!("(a) presence of {} present keys: {} bytes read from data files, {:?}", N, read, took);
129 if held.iter().any(|h| !h) {
130 return Err(err!("{} of {} present keys were reported absent.",
131 held.iter().filter(|h| !**h).count(), N; Test, Missing));
132 }
133 if read != 0 {
134 return Err(err!(
135 "Answering whether {} present keys are held read {} bytes of records from the data \
136 files, about {} values of {} bytes; the index alone holds the answer.",
137 N, read, read / VAL_BYTES as u64, VAL_BYTES;
138 Test, Excessive));
139 }
140
141 // (b) Eight concurrent queries of all 2,000 finish inside the bound.
142 let t = Instant::now();
143 let mut handles = Vec::with_capacity(QUERIES);
144 for _ in 0..QUERIES {
145 let db = db.clone();
146 let all = all.clone();
147 handles.push(thread::spawn(move || -> Outcome<(usize, Duration)> {
148 let t = Instant::now();
149 let held = res!(db.contains_many(&all, None));
150 Ok((held.iter().filter(|h| **h).count(), t.elapsed()))
151 }));
152 }
153 let mut slowest = Duration::ZERO;
154 for h in handles {
155 match h.join() {
156 Ok(Ok((n, took))) => {
157 if n != N {
158 return Err(err!("A concurrent query found {} of {} keys.", n, N; Test, Missing));
159 }
160 slowest = std::cmp::max(slowest, took);
161 },
162 Ok(Err(e)) => return Err(e),
163 Err(_) => return Err(err!("A query thread panicked."; Test, Thread)),
164 }
165 }
166 let all_took = t.elapsed();
167 println!("(b) {} concurrent queries of {} keys: slowest {:?}, all {:?}", QUERIES, N, slowest, all_took);
168 if all_took > BOUND {
169 return Err(err!(
170 "{} concurrent presence queries of {} keys took {:?}, over the {:?} bound.",
171 QUERIES, N, all_took, BOUND;
172 Test, Excessive));
173 }
174
175 // The old path, for the record: `get` on every key, which is what `has` did.
176 let before = hooks::record_bytes_read();
177 let t = Instant::now();
178 let mut found = 0;
179 for k in &all {
180 if res!(db.get(k, None)).is_some() {
181 found += 1;
182 }
183 }
184 println!("old path, get().is_some() on {} keys: found {}, {} bytes read, {:?}",
185 N, found, hooks::record_bytes_read() - before, t.elapsed());
186
187 // (c) A deleted key is absent, its tombstone cached and, after the cache is cleared, not.
188 for i in 0..DELETED {
189 res!(db.delete(&key(i), user, None));
190 }
191 let gone: Vec<Dat> = (0..DELETED).map(key).collect();
192 let held = res!(db.contains_many(&gone, None));
193 if let Some(i) = held.iter().position(|h| *h) {
194 return Err(err!("The deleted key {:?} is reported held, its tombstone cached.", key(i);
195 Test, Mismatch));
196 }
197 res!(db.api().clear_cache_values(wait()));
198 let before = hooks::record_bytes_read();
199 let held = res!(db.contains_many(&gone, None));
200 let read = hooks::record_bytes_read() - before;
201 if let Some(i) = held.iter().position(|h| *h) {
202 return Err(err!("The deleted key {:?} is reported held, its tombstone uncached.", key(i);
203 Test, Mismatch));
204 }
205 println!("(c) {} deleted keys absent; uncached, their tombstones cost {} bytes read", DELETED, read);
206 if read > (DELETED * 64) as u64 {
207 return Err(err!("Reading {} uncached tombstones took {} bytes.", DELETED, read;
208 Test, Excessive));
209 }
210 // Only the deleted ones: their neighbours stay held.
211 let held = res!(db.contains_many(&all, None));
212 let n = held.iter().filter(|h| **h).count();
213 if n != N - DELETED || held[..DELETED].iter().any(|h| *h) {
214 return Err(err!("After {} deletes, {} of {} keys are held.", DELETED, n, N; Test, Mismatch));
215 }
216
217 // (d) A key never written is absent.
218 let never: Vec<Dat> = (N..N + ABSENT).map(key).collect();
219 let held = res!(db.contains_many(&never, None));
220 if held.iter().any(|h| *h) {
221 return Err(err!("A key never written is reported held."; Test, Mismatch));
222 }
223 if res!(db.contains(&key(N + ABSENT), None)) {
224 return Err(err!("A key never written is reported held by contains."; Test, Mismatch));
225 }
226 println!("(d) {} keys never written absent", ABSENT);
227
228 res!(db.close());
229 let _ = std::fs::remove_dir_all(&root);
230 Ok(())
231}