Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/qa_a2_agent1.rs

7.9 KiB, 1 run

created by r1870400018:67684, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1#![cfg(feature = "dist")]
2//! Post-landing QA of distributed Ozone A2, agent 1 (identity, anti-entropy, convergence).
3//! Every test here fails on 96d0c015 and names the defect it shows.
4//!
5//! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\
6//! Anthropic Claude
7
8use oxedyne_fe2o3_core::prelude::*;
9use oxedyne_fe2o3_o3db_sync::{
10 dist::{
11 config::{
12 Consistency,
13 DistOzoneConfig,
14 TableConfig,
15 },
16 engine::DistOzone,
17 record::{
18 Record,
19 RecordId,
20 content_hash,
21 },
22 resolve::{
23 Convergence,
24 LastVersionWins,
25 ReadView,
26 ResolveCtx,
27 Resolver,
28 Verdict,
29 check_convergence,
30 },
31 storage::{
32 MemoryStorage,
33 Storage,
34 },
35 transport::{
36 Envelope,
37 MsgKind,
38 },
39 },
40 kademlia::id::NodeId,
41 oam::config::OamConfig,
42};
43
44use std::time::Duration;
45
46
47fn node(b: u8) -> NodeId {
48 let mut bytes = [0u8; 32];
49 bytes[31] = b;
50 NodeId::from_bytes(bytes)
51}
52
53fn rid2(i: u16) -> RecordId {
54 let mut bytes = [0u8; 32];
55 bytes[0] = (i >> 8) as u8;
56 bytes[1] = i as u8;
57 RecordId::from_bytes(bytes)
58}
59
60fn pair() -> Outcome<(DistOzone<MemoryStorage>, DistOzone<MemoryStorage>)> {
61 let mk = |local: u8, remote: u8| -> Outcome<DistOzoneConfig> {
62 let oam = res!(OamConfig::new(2, 2));
63 let table = res!(TableConfig::new("identity", Consistency::Eventual, Duration::from_secs(30), 256));
64 DistOzoneConfig::new(node(local), vec![node(remote)], oam, vec![table])
65 };
66 Ok((
67 res!(DistOzone::new(res!(mk(1, 2)), MemoryStorage::new())),
68 res!(DistOzone::new(res!(mk(2, 1)), MemoryStorage::new())),
69 ))
70}
71
72// One exchange as a real transport runs it: every envelope is encoded and decoded, and one the
73// codec refuses to write is lost, as it would be on the wire.
74fn wire_exchange(d: &DistOzone<MemoryStorage>, l: &DistOzone<MemoryStorage>, l_id: u8) -> Outcome<usize> {
75 let mut lost = 0;
76 let req = res!(d.build_anti_entropy_request("identity", node(l_id)));
77 let req = res!(Envelope::decode(&res!(req.encode())));
78 let out = res!(l.handle_envelope_at(req, 1_000));
79 for reply in out.outbound {
80 let bytes = match reply.encode() {
81 Ok(b) => b,
82 Err(_) => { lost += 1; continue; },
83 };
84 let back = res!(d.handle_envelope_at(res!(Envelope::decode(&bytes)), 1_000));
85 for push in back.outbound {
86 match push.encode() {
87 Ok(b) => { res!(l.handle_envelope_at(res!(Envelope::decode(&b)), 1_000)); },
88 Err(_) => lost += 1,
89 }
90 }
91 }
92 Ok(lost)
93}
94
95// F1 (S1). A peer that rejoins a table holding more than MAX_ENVELOPE_BYTES never converges: its
96// empty sketch overloads, the bulk reply is the listener's whole table, and the codec refuses to
97// write it, on every exchange, for ever. 300 records of 64 KiB is 19.7 MB.
98#[test]
99fn qa1_rejoin_over_16mib_table_converges() -> Outcome<()> {
100 let (d, l) = res!(pair());
101 for i in 0..300u16 {
102 res!(l.storage().put(&Record::new(rid2(i), "identity", vec![(i & 0xff) as u8; 64 * 1024])));
103 }
104 let mut lost = 0;
105 for _ in 0..3 {
106 lost += res!(wire_exchange(&d, &l, 2));
107 }
108 let held = res!(d.storage().digests("identity")).len();
109 assert_eq!(held, 300, "the rejoining peer holds {} of 300 after 3 exchanges ({} envelopes the codec refused)", held, lost);
110 Ok(())
111}
112
113// F1 (S1), without overload. A difference the sketch decodes (100 entries, under the ~170 that
114// 256 cells hold) of 200 KiB records is a 20 MB reply, refused alike, so it never repairs.
115#[test]
116fn qa1_decoded_difference_over_16mib_converges() -> Outcome<()> {
117 let (d, l) = res!(pair());
118 for i in 0..100u16 {
119 res!(l.storage().put(&Record::new(rid2(i), "identity", vec![(i & 0xff) as u8; 200 * 1024])));
120 }
121 let mut lost = 0;
122 for _ in 0..3 {
123 lost += res!(wire_exchange(&d, &l, 2));
124 }
125 let held = res!(d.storage().digests("identity")).len();
126 assert_eq!(held, 100, "the dialler holds {} of 100 after 3 exchanges ({} envelopes refused)", held, lost);
127 Ok(())
128}
129
130// F2 (S2). A sketch whose header claims a key length near usize::MAX panics the listener inside
131// Iblt::from_bytes (an unchecked `key_len + value_len`, then `vec![0; num_cells * key_len]`)
132// before the engine's config check can refuse it. The codec passes the sketch as opaque bytes.
133#[test]
134fn qa1_hostile_sketch_header_is_an_err_not_a_panic() -> Outcome<()> {
135 let (_, l) = res!(pair());
136 let mut sketch = Vec::with_capacity(40 + 76);
137 sketch.extend_from_slice(&1u64.to_le_bytes()); // num_cells
138 sketch.extend_from_slice(&1u64.to_le_bytes()); // num_hashes
139 sketch.extend_from_slice(&(1u64 << 63).to_le_bytes()); // key_len
140 sketch.extend_from_slice(&((1u64 << 63) + 64).to_le_bytes()); // value_len; the sum wraps to 64
141 sketch.extend_from_slice(&0u64.to_le_bytes()); // seed
142 sketch.extend_from_slice(&[0u8; 76]); // one cell of the wrapped width
143 let env = Envelope::new(node(1), node(2), MsgKind::AntiEntropyDigest {
144 table: "identity".to_string(),
145 sketch,
146 after: None,
147 });
148 // The envelope survives the codec, so a peer can send it.
149 let env = res!(Envelope::decode(&res!(env.encode())));
150 let got = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| l.handle_envelope_at(env, 1_000)));
151 match got {
152 Err(_) => panic!("a hostile sketch header panicked the listener"),
153 Ok(Ok(_)) => panic!("a hostile sketch header was accepted"),
154 Ok(Err(_)) => {},
155 }
156 Ok(())
157}
158
159/// Defers a value stamped after the clock; once the clock passes, the first value an id
160/// receives wins, which diverges by arrival order.
161struct FirstWinsAfter;
162
163impl Resolver for FirstWinsAfter {
164 fn resolve<V: ReadView>(
165 &self,
166 ctx: &ResolveCtx,
167 _view: &V,
168 _table: &str,
169 _id: &RecordId,
170 held: Option<&[u8]>,
171 incoming: &[u8],
172 )
173 -> Outcome<Verdict>
174 {
175 if let Some(stamp) = LastVersionWins::version(incoming) {
176 if stamp > ctx.now_ms {
177 return Ok(Verdict::Defer);
178 }
179 }
180 Ok(match held {
181 Some(_) => Verdict::Keep,
182 None => Verdict::Take(incoming.to_vec()),
183 })
184 }
185}
186
187// F3 (S2). check_convergence calls a run quiet at the first full round that stores nothing, even
188// while local puts it deferred are still pending and the skewed clocks are still moving. So a
189// time-deferred record is never offered past its time, and a rule that diverges once the time
190// comes passes, while the same rule with a nearer time fails.
191#[test]
192fn qa1_check_reports_divergence_after_a_deferred_time() -> Outcome<()> {
193 const NOW: u64 = 1_000;
194 let run = |stamp: u64| -> Outcome<()> {
195 let x = RecordId::from_bytes([9u8; 32]);
196 let records = vec![
197 Record::new(x, "identity", LastVersionWins::value(stamp, &[1])),
198 Record::new(x, "identity", LastVersionWins::value(stamp, &[2])),
199 ];
200 let cfg = Convergence::new(3, 0..16, NOW, 16).with_skew_ms(400);
201 check_convergence(&cfg, &["identity"], &records, |_| FirstWinsAfter, |_, _, _| Ok(Vec::new()))
202 };
203 // Control: inside the clocks' first step, the divergence is reported.
204 assert!(run(NOW + 200).is_err(), "control: the near stamp did not fail");
205 // Five steps out, within max_rounds = 16: the same rule passes.
206 assert!(run(NOW + 2_000).is_err(), "a rule that diverges after its deferred time passed the check");
207 Ok(())
208}
209
210// F4 (S3). When the dialler takes the listener's value, it pushes that value straight back,
211// although the listener holds those very bytes, so each such repair crosses the wire twice. The
212// bulk path already skips this case.
213#[test]
214fn qa1_push_does_not_echo_the_listeners_own_bytes() -> Outcome<()> {
215 let (d, l) = res!(pair());
216 let x = RecordId::from_bytes([7u8; 32]);
217 let old = LastVersionWins::value(1, b"old");
218 let new = LastVersionWins::value(2, b"new");
219 res!(d.storage().put(&Record::new(x, "identity", old)));
220 res!(l.storage().put(&Record::new(x, "identity", new.clone())));
221 let req = res!(d.build_anti_entropy_request("identity", node(2)));
222 let mut out = res!(l.handle_envelope_at(req, 1_000));
223 let back = res!(d.handle_envelope_at(out.outbound.remove(0), 1_000));
224 for env in &back.outbound {
225 if let MsgKind::AntiEntropyPush { records, .. } = &env.body {
226 for r in records {
227 assert!(content_hash(&r.value) != content_hash(&new),
228 "the push sends the listener the bytes it already holds at {:?}", r.id);
229 }
230 }
231 }
232 Ok(())
233}