Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/dist_over_o3db.rs

15.8 KiB, 3 runs

created by r1870400018:67488, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1#![cfg(feature = "dist")]
2//! Integration tests for [`DistOzone::over_o3db`] and [`DistOzone::close`].
3//!
4//! An application opens a distributed engine over an Ozone database in one call and closes it in
5//! another, and what a put returned for must survive the pair and a reopen of the same root.
6//! Each test works in its own `./test_db_dist_over_o3db_<name>` root, wiped when the test begins.
7//!
8//! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\
9//! Anthropic Claude
10
11use oxedyne_fe2o3_core::prelude::*;
12use oxedyne_fe2o3_crypto::enc::EncryptionScheme;
13use oxedyne_fe2o3_hash::{
14 csum::ChecksumScheme,
15 hash::HashScheme,
16};
17use oxedyne_fe2o3_jdat::prelude::*;
18use oxedyne_fe2o3_o3db_sync::{
19 base::cfg::OzoneConfig,
20 data::core::RestSchemesInput,
21 dist::{
22 config::{
23 DistOzoneConfig,
24 TableConfig,
25 },
26 engine::DistOzone,
27 o3db_storage::O3dbStorage,
28 record::{
29 Record,
30 RecordDigest,
31 RecordId,
32 content_hash,
33 },
34 resolve::{
35 LastVersionWins,
36 ReadView,
37 ResolveCtx,
38 Resolver,
39 Verdict,
40 },
41 storage::{
42 MemoryStorage,
43 Storage,
44 },
45 transport::{
46 Envelope,
47 MsgKind,
48 },
49 },
50 kademlia::id::NodeId,
51 oam::config::OamConfig,
52 test::setup,
53};
54
55use std::{
56 path::PathBuf,
57 sync::{
58 Arc,
59 Mutex,
60 },
61};
62
63
64// One test at a time, so that the process's thread count belongs to the test that reads it.
65static ONE_AT_A_TIME: Mutex<()> = Mutex::new(());
66
67
68type Eng<R> = DistOzone<
69 O3dbStorage<{ setup::UID_LEN }, setup::Uid, EncryptionScheme, HashScheme, HashScheme, ChecksumScheme>,
70 R,
71>;
72
73fn node(b: u8) -> NodeId {
74 let mut bytes = [0u8; 32];
75 bytes[31] = b;
76 NodeId::from_bytes(bytes)
77}
78
79fn rid(b: u8) -> RecordId {
80 RecordId::from_bytes([b; 32])
81}
82
83fn rec(table: &str, id: RecordId, value: &[u8]) -> Record {
84 Record::new(id, table, value.to_vec())
85}
86
87/// The threads this process runs, which a database that is open adds its bots to.
88fn threads() -> Outcome<usize> {
89 Ok(res!(std::fs::read_dir("/proc/self/task")).count())
90}
91
92/// A root of its own for the test `name`, emptied before use and left behind after it.
93fn fresh_root(name: &str) -> Outcome<PathBuf> {
94 let dir = PathBuf::from(fmt!("./test_db_dist_over_o3db_{}", name));
95 if dir.exists() {
96 res!(std::fs::remove_dir_all(&dir));
97 }
98 res!(std::fs::create_dir_all(&dir));
99 Ok(res!(dir.canonicalize()))
100}
101
102fn schemes() -> Outcome<RestSchemesInput<EncryptionScheme, HashScheme, HashScheme, ChecksumScheme>> {
103 let enckey = [0x55u8; 32];
104 let aes_gcm = res!(EncryptionScheme::new_aes_256_gcm_with_key(&enckey[..]));
105 Ok(RestSchemesInput::new(
106 Some(aes_gcm),
107 None::<HashScheme>,
108 None::<HashScheme>,
109 Some(ChecksumScheme::new_crc32()),
110 ))
111}
112
113/// The stock store: two zones, the stock cache, the default durability barrier.
114fn ozone_cfg() -> OzoneConfig {
115 OzoneConfig::default()
116}
117
118/// One peer holding everything, with these eventual tables.
119fn single(tables: &[&str]) -> Outcome<DistOzoneConfig> {
120 let oam = res!(OamConfig::new(1, 1));
121 let mut tcs = Vec::with_capacity(tables.len());
122 for t in tables {
123 tcs.push(res!(TableConfig::eventual(*t)));
124 }
125 DistOzoneConfig::new(node(1), Vec::new(), oam, tcs)
126}
127
128/// Two peers that both hold everything, seen from peer `local`.
129fn pair(local: u8, remote: u8, table: &str) -> Outcome<DistOzoneConfig> {
130 let oam = res!(OamConfig::new(2, 2));
131 let tcs = vec![res!(TableConfig::eventual(table))];
132 DistOzoneConfig::new(node(local), vec![node(remote)], oam, tcs)
133}
134
135fn open<R: Resolver>(
136 root: &PathBuf,
137 ozone_cfg: Option<OzoneConfig>,
138 dist_cfg: DistOzoneConfig,
139 resolver: R,
140)
141 -> Outcome<Eng<R>>
142{
143 Eng::<R>::over_o3db(
144 root.clone(),
145 ozone_cfg,
146 dist_cfg,
147 res!(schemes()),
148 setup::Uid::default(),
149 resolver,
150 )
151}
152
153fn held<S: Storage, R: Resolver>(
154 e: &DistOzone<S, R>,
155 table: &str,
156 id: RecordId,
157)
158 -> Outcome<Option<Vec<u8>>>
159{
160 Ok(res!(e.storage().get(table, &id)).map(|r| r.value))
161}
162
163fn digests<S: Storage, R: Resolver>(e: &DistOzone<S, R>, table: &str) -> Outcome<Vec<RecordDigest>> {
164 e.storage().digests(table)
165}
166
167/// Refuses a value that begins with "BAD", and takes any other.
168struct RefuseBad;
169
170impl Resolver for RefuseBad {
171 fn resolve<V: ReadView>(
172 &self,
173 _ctx: &ResolveCtx,
174 _view: &V,
175 _table: &str,
176 _id: &RecordId,
177 _held: Option<&[u8]>,
178 incoming: &[u8],
179 )
180 -> Outcome<Verdict>
181 {
182 if incoming.starts_with(b"BAD") {
183 return Ok(Verdict::Refuse("marked bad".to_string()));
184 }
185 Ok(Verdict::Take(incoming.to_vec()))
186 }
187}
188
189/// What the engine's answer to a peer's anti-entropy digest says: the records it would send, the
190/// ids it asks for, and whether it fell back to a bulk reply.
191fn answer<S1: Storage, R1: Resolver, S2: Storage, R2: Resolver>(
192 asker: &DistOzone<S1, R1>,
193 answerer: &DistOzone<S2, R2>,
194 answerer_id: u8,
195 table: &str,
196)
197 -> Outcome<(Vec<Record>, Vec<RecordId>, bool)>
198{
199 let req = res!(asker.build_anti_entropy_request(table, node(answerer_id)));
200 let mut got = res!(answerer.handle_envelope_at(req, 1_000));
201 assert_eq!(got.outbound.len(), 1);
202 let env: Envelope = got.outbound.remove(0);
203 match env.body {
204 MsgKind::AntiEntropyReply { records, requested_ids, bulk, .. } => Ok((records, requested_ids, bulk)),
205 other => Err(err!("Expected an AntiEntropyReply, got {:?}.", other; Test, Mismatch)),
206 }
207}
208
209
210#[test]
211fn opens_puts_closes_reopens_reads_back() -> Outcome<()> {
212 let _one = lock_mutex!(ONE_AT_A_TIME);
213 let root = res!(fresh_root("reopen"));
214 let dist = || single(&["identity", "escrow"]);
215 let idle = res!(threads());
216 let e = res!(open(&root, Some(ozone_cfg()), res!(dist()), LastVersionWins));
217 let running = res!(threads());
218 assert!(running > idle + 10, "an open database should run its bots: {} threads before, {} after", idle, running);
219
220 // Two values that differ only by a trailing zero, in one table, and a longer one in another.
221 let puts = [
222 ("identity", rid(1), vec![0xABu8]),
223 ("identity", rid(2), vec![0xABu8, 0x00]),
224 ("escrow", rid(1), b"gamma, longer than one chunk of eight bytes".to_vec()),
225 ];
226 for (table, id, value) in &puts {
227 let out = res!(e.put_at(rec(table, *id, value), 1_000));
228 assert!(out.local_persisted, "the put of {:?} in {} was not stored", id, table);
229 }
230 let before_identity = res!(digests(&e, "identity"));
231 let before_escrow = res!(digests(&e, "escrow"));
232 assert_eq!(before_identity.len(), 2);
233 assert_eq!(before_escrow.len(), 1);
234 assert_ne!(before_identity[0].content, before_identity[1].content);
235 res!(e.close());
236 // A close returns once the bots have stopped, so their threads are gone, or going.
237 let closed = res!(threads());
238 assert!(closed < idle + (running - idle) / 2,
239 "close left the bots running: {} threads before the open, {} open, {} after the close", idle, running, closed);
240
241 // The same root, no Ozone configuration given: the one saved there is used.
242 let e = res!(open(&root, None, res!(dist()), LastVersionWins));
243 for (table, id, value) in &puts {
244 assert_eq!(res!(held(&e, table, *id)).as_ref(), Some(value),
245 "{} {:?} did not come back after the reopen", table, id);
246 }
247 assert_eq!(res!(digests(&e, "identity")), before_identity);
248 assert_eq!(res!(digests(&e, "escrow")), before_escrow);
249 for d in &before_identity {
250 let value = if d.id == rid(1) { vec![0xABu8] } else { vec![0xABu8, 0x00] };
251 assert_eq!(d.content, content_hash(&value));
252 }
253 res!(e.close());
254 Ok(())
255}
256
257#[test]
258fn a_dropped_engine_stops_its_bots_and_keeps_what_it_stored() -> Outcome<()> {
259 let _one = lock_mutex!(ONE_AT_A_TIME);
260 let root = res!(fresh_root("dropped"));
261 let idle = res!(threads());
262 let e = res!(open(&root, Some(ozone_cfg()), res!(single(&["identity"])), LastVersionWins));
263 let running = res!(threads());
264 assert!(running > idle + 10, "an open database should run its bots: {} threads before, {} after", idle, running);
265 let out = res!(e.put_at(rec("identity", rid(1), b"kept without a close"), 1_000));
266 assert!(out.local_persisted, "the put was not stored");
267
268 // Dropped as an error path or a panic unwinds it, with no close.
269 drop(e);
270 let dropped = res!(threads());
271 assert!(dropped < idle + (running - idle) / 2,
272 "dropping the engine left the bots running: {} threads before the open, {} open, {} after the drop",
273 idle, running, dropped);
274
275 // The bots were stopped in order, so the same root opens again with the record in it.
276 let e = res!(open(&root, None, res!(single(&["identity"])), LastVersionWins));
277 assert_eq!(res!(held(&e, "identity", rid(1))), Some(b"kept without a close".to_vec()));
278 res!(e.close());
279 Ok(())
280}
281
282#[test]
283fn dropping_a_shared_storage_leaves_the_database_to_its_other_holder() -> Outcome<()> {
284 let _one = lock_mutex!(ONE_AT_A_TIME);
285 let root = res!(fresh_root("dropshared"));
286 let e = res!(open(&root, Some(ozone_cfg()), res!(single(&["identity"])), LastVersionWins));
287 res!(e.put_at(rec("identity", rid(1), b"still open"), 1_000));
288 let extra = Arc::clone(e.storage().db());
289
290 // The engine goes, the database does not: someone else still holds it.
291 drop(e);
292 let storage = O3dbStorage::new(extra, setup::Uid::default());
293 let back = res!(storage.get("identity", &rid(1)));
294 assert_eq!(back.map(|r| r.value), Some(b"still open".to_vec()));
295 res!(storage.close());
296 Ok(())
297}
298
299#[test]
300fn over_o3db_uses_the_given_resolver() -> Outcome<()> {
301 let _one = lock_mutex!(ONE_AT_A_TIME);
302 let root = res!(fresh_root("resolver"));
303 let e = res!(open(&root, Some(ozone_cfg()), res!(single(&["identity"])), RefuseBad));
304
305 let refused = res!(e.put_at(rec("identity", rid(1), b"BAD value"), 1_000));
306 assert!(!refused.local_persisted);
307 assert!(matches!(refused.verdict, Some(Verdict::Refuse(_))), "verdict was {:?}", refused.verdict);
308 let taken = res!(e.put_at(rec("identity", rid(2), b"good value"), 1_000));
309 assert!(taken.local_persisted);
310 assert_eq!(res!(held(&e, "identity", rid(1))), None);
311 res!(e.close());
312
313 // Reopened under the same resolver, the refused value is still absent and the other present.
314 let e = res!(open(&root, None, res!(single(&["identity"])), RefuseBad));
315 assert_eq!(res!(held(&e, "identity", rid(1))), None);
316 assert_eq!(res!(held(&e, "identity", rid(2))), Some(b"good value".to_vec()));
317 assert_eq!(res!(digests(&e, "identity")).len(), 1);
318 res!(e.close());
319 Ok(())
320}
321
322/// What a peer answers to another's anti-entropy digest: the records it would send, the ids it
323/// asks for, and whether it fell back to a bulk reply.
324type Answer = (Vec<Record>, Vec<RecordId>, bool);
325
326#[test]
327fn memory_and_o3db_peers_decode_empty_difference() -> Outcome<()> {
328 let _one = lock_mutex!(ONE_AT_A_TIME);
329 let root = res!(fresh_root("mixed"));
330 // Peer 1 holds in memory. Peer 2 is held by O3db, and a second peer 2 in memory is the
331 // reference: whatever the reference pair does, the mixed pair must do alike.
332 let mem: DistOzone<MemoryStorage> = res!(DistOzone::new(res!(pair(1, 2, "identity")), MemoryStorage::new()));
333 let o3 = res!(open(&root, Some(ozone_cfg()), res!(pair(2, 1, "identity")), LastVersionWins));
334 let refr: DistOzone<MemoryStorage> = res!(DistOzone::new(res!(pair(2, 1, "identity")), MemoryStorage::new()));
335
336 let long = b"a longer value than one chunk of eight bytes".to_vec();
337 let records = [
338 (rid(1), vec![0xABu8]),
339 (rid(2), vec![0xABu8, 0x00]),
340 (rid(3), long.clone()),
341 ];
342 for (id, value) in &records {
343 for e in [&mem, &refr] {
344 res!(e.put_at(rec("identity", *id, value), 1_000));
345 }
346 res!(o3.put_at(rec("identity", *id, value), 1_000));
347 }
348 assert_eq!(res!(digests(&mem, "identity")), res!(digests(&o3, "identity")));
349
350 // The two answers each way, from the mixed pair and from the reference pair.
351 let both = |_: ()| -> Outcome<(Answer, Answer, Answer, Answer)> {
352 Ok((
353 res!(answer(&mem, &o3, 2, "identity")),
354 res!(answer(&o3, &mem, 1, "identity")),
355 res!(answer(&mem, &refr, 2, "identity")),
356 res!(answer(&refr, &mem, 1, "identity")),
357 ))
358 };
359
360 // The same three records on both sides: nothing to send, nothing to ask for, no bulk fallback.
361 let (m_asks, o_asks, m_ref, o_ref) = res!(both(()));
362 for (who, a) in [("memory asked, O3db answered", &m_asks), ("O3db asked, memory answered", &o_asks)] {
363 assert!(a.0.is_empty() && a.1.is_empty() && !a.2, "{}: {:?}", who, a);
364 }
365 assert_eq!((&m_asks, &o_asks), (&m_ref, &o_ref));
366
367 // One value revised at O3db and at the reference. The mixed pair answers as the reference
368 // pair does, and each way the answer is exactly one record, the answerer's own copy, with the
369 // id asked for in return and no bulk fallback.
370 let newer = b"a longer value than one chunk of eight bytes, revised".to_vec();
371 for e in [&refr] {
372 assert!(res!(e.put_at(rec("identity", rid(3), &newer), 2_000)).local_persisted);
373 }
374 assert!(res!(o3.put_at(rec("identity", rid(3), &newer), 2_000)).local_persisted);
375 let (m_asks, o_asks, m_ref, o_ref) = res!(both(()));
376 assert_eq!((&m_asks, &o_asks), (&m_ref, &o_ref));
377 assert_eq!(m_asks, (vec![rec("identity", rid(3), &newer)], vec![rid(3)], false));
378 assert_eq!(o_asks, (vec![rec("identity", rid(3), &long)], vec![rid(3)], false));
379
380 // The memory peer takes the revision, and the pair agrees again.
381 assert!(res!(mem.put_at(rec("identity", rid(3), &newer), 2_000)).local_persisted);
382 assert_eq!(res!(digests(&mem, "identity")), res!(digests(&o3, "identity")));
383 let (m_asks, o_asks, _, _) = res!(both(()));
384 for a in [&m_asks, &o_asks] {
385 assert!(a.0.is_empty() && a.1.is_empty() && !a.2, "after the revision: {:?}", a);
386 }
387
388 // One record only the O3db peer holds: exactly one record one way and one request the other.
389 let extra = vec![0x04u8; 40];
390 for e in [&refr] {
391 res!(e.put_at(rec("identity", rid(4), &extra), 3_000));
392 }
393 res!(o3.put_at(rec("identity", rid(4), &extra), 3_000));
394 let (m_asks, o_asks, m_ref, o_ref) = res!(both(()));
395 assert_eq!((&m_asks, &o_asks), (&m_ref, &o_ref));
396 assert_eq!(m_asks, (vec![rec("identity", rid(4), &extra)], Vec::new(), false));
397 assert_eq!(o_asks, (Vec::new(), vec![rid(4)], false));
398
399 res!(o3.close());
400 Ok(())
401}
402
403#[test]
404fn close_refuses_while_shared() -> Outcome<()> {
405 let _one = lock_mutex!(ONE_AT_A_TIME);
406 let root = res!(fresh_root("shared"));
407 let e = res!(open(&root, Some(ozone_cfg()), res!(single(&["identity"])), LastVersionWins));
408 res!(e.put_at(rec("identity", rid(1), b"kept"), 1_000));
409
410 // A second holder of the database.
411 let extra = Arc::clone(e.storage().db());
412 let refused = match e.close() {
413 Ok(()) => return Err(err!("close succeeded with another reference to the database held."; Test, Unexpected)),
414 Err(refused) => refused,
415 };
416 let said = fmt!("{}", refused);
417 assert!(said.contains("1 other reference"), "the refusal does not name the holder: {}", said);
418
419 // The refusal left the database open and the record in it.
420 let storage = O3dbStorage::new(extra, setup::Uid::default());
421 let back = res!(storage.get("identity", &rid(1)));
422 assert_eq!(back.map(|r| r.value), Some(b"kept".to_vec()));
423 res!(storage.close());
424 Ok(())
425}
426
427#[test]
428fn a_put_returned_before_close_reads_back_after_reopen_on_one_zone() -> Outcome<()> {
429 let _one = lock_mutex!(ONE_AT_A_TIME);
430 let root = res!(fresh_root("one_zone"));
431 // One zone, a barrier on every write, and a cache far smaller than what is stored.
432 let overrides = res!(mapdat!{
433 1u16 => mapdat!{ "dir" => "", "max_size" => 104_857_600u64 },
434 }.get_map().ok_or_else(|| err!("The zone override map is not a map."; Test, Invalid)));
435 let cfg = OzoneConfig {
436 num_zones: 1,
437 sync_on_write: true,
438 cache_size_limit_bytes: 4_096,
439 zone_overrides: overrides,
440 ..OzoneConfig::default()
441 };
442 let e = res!(open(&root, Some(cfg), res!(single(&["identity"])), LastVersionWins));
443 let mut puts = Vec::new();
444 for n in 1..=24u8 {
445 let value: Vec<u8> = (0..300u32).map(|i| (i as u8).wrapping_mul(n).wrapping_add(n)).collect();
446 let out = res!(e.put_at(rec("identity", rid(n), &value), 1_000));
447 assert!(out.local_persisted);
448 puts.push((rid(n), value));
449 }
450 res!(e.close());
451
452 let e = res!(open(&root, None, res!(single(&["identity"])), LastVersionWins));
453 for (id, value) in &puts {
454 assert_eq!(res!(held(&e, "identity", *id)).as_ref(), Some(value),
455 "{:?} did not come back after the reopen", id);
456 }
457 assert_eq!(res!(digests(&e, "identity")).len(), puts.len());
458 res!(e.close());
459 Ok(())
460}