oxedyne/fe2o3/fe2o3_o3db_sync/tests/qa_a2_agent1.rs
7.9 KiB, 1 run
created by r1870400018:67684, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | #![cfg(feature = "dist")] |
| 2 | //! Post-landing QA of distributed Ozone A2, agent 1 (identity, anti-entropy, convergence). |
| 3 | //! Every test here fails on 96d0c015 and names the defect it shows. |
| 4 | //! |
| 5 | //! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\ |
| 6 | //! Anthropic Claude |
| 7 | |
| 8 | use oxedyne_fe2o3_core::prelude::*; |
| 9 | use oxedyne_fe2o3_o3db_sync::{ |
| 10 | dist::{ |
| 11 | config::{ |
| 12 | Consistency, |
| 13 | DistOzoneConfig, |
| 14 | TableConfig, |
| 15 | }, |
| 16 | engine::DistOzone, |
| 17 | record::{ |
| 18 | Record, |
| 19 | RecordId, |
| 20 | content_hash, |
| 21 | }, |
| 22 | resolve::{ |
| 23 | Convergence, |
| 24 | LastVersionWins, |
| 25 | ReadView, |
| 26 | ResolveCtx, |
| 27 | Resolver, |
| 28 | Verdict, |
| 29 | check_convergence, |
| 30 | }, |
| 31 | storage::{ |
| 32 | MemoryStorage, |
| 33 | Storage, |
| 34 | }, |
| 35 | transport::{ |
| 36 | Envelope, |
| 37 | MsgKind, |
| 38 | }, |
| 39 | }, |
| 40 | kademlia::id::NodeId, |
| 41 | oam::config::OamConfig, |
| 42 | }; |
| 43 | |
| 44 | use std::time::Duration; |
| 45 | |
| 46 | |
| 47 | fn node(b: u8) -> NodeId { |
| 48 | let mut bytes = [0u8; 32]; |
| 49 | bytes[31] = b; |
| 50 | NodeId::from_bytes(bytes) |
| 51 | } |
| 52 | |
| 53 | fn rid2(i: u16) -> RecordId { |
| 54 | let mut bytes = [0u8; 32]; |
| 55 | bytes[0] = (i >> 8) as u8; |
| 56 | bytes[1] = i as u8; |
| 57 | RecordId::from_bytes(bytes) |
| 58 | } |
| 59 | |
| 60 | fn pair() -> Outcome<(DistOzone<MemoryStorage>, DistOzone<MemoryStorage>)> { |
| 61 | let mk = |local: u8, remote: u8| -> Outcome<DistOzoneConfig> { |
| 62 | let oam = res!(OamConfig::new(2, 2)); |
| 63 | let table = res!(TableConfig::new("identity", Consistency::Eventual, Duration::from_secs(30), 256)); |
| 64 | DistOzoneConfig::new(node(local), vec![node(remote)], oam, vec![table]) |
| 65 | }; |
| 66 | Ok(( |
| 67 | res!(DistOzone::new(res!(mk(1, 2)), MemoryStorage::new())), |
| 68 | res!(DistOzone::new(res!(mk(2, 1)), MemoryStorage::new())), |
| 69 | )) |
| 70 | } |
| 71 | |
| 72 | // One exchange as a real transport runs it: every envelope is encoded and decoded, and one the |
| 73 | // codec refuses to write is lost, as it would be on the wire. |
| 74 | fn wire_exchange(d: &DistOzone<MemoryStorage>, l: &DistOzone<MemoryStorage>, l_id: u8) -> Outcome<usize> { |
| 75 | let mut lost = 0; |
| 76 | let req = res!(d.build_anti_entropy_request("identity", node(l_id))); |
| 77 | let req = res!(Envelope::decode(&res!(req.encode()))); |
| 78 | let out = res!(l.handle_envelope_at(req, 1_000)); |
| 79 | for reply in out.outbound { |
| 80 | let bytes = match reply.encode() { |
| 81 | Ok(b) => b, |
| 82 | Err(_) => { lost += 1; continue; }, |
| 83 | }; |
| 84 | let back = res!(d.handle_envelope_at(res!(Envelope::decode(&bytes)), 1_000)); |
| 85 | for push in back.outbound { |
| 86 | match push.encode() { |
| 87 | Ok(b) => { res!(l.handle_envelope_at(res!(Envelope::decode(&b)), 1_000)); }, |
| 88 | Err(_) => lost += 1, |
| 89 | } |
| 90 | } |
| 91 | } |
| 92 | Ok(lost) |
| 93 | } |
| 94 | |
| 95 | // F1 (S1). A peer that rejoins a table holding more than MAX_ENVELOPE_BYTES never converges: its |
| 96 | // empty sketch overloads, the bulk reply is the listener's whole table, and the codec refuses to |
| 97 | // write it, on every exchange, for ever. 300 records of 64 KiB is 19.7 MB. |
| 98 | #[test] |
| 99 | fn qa1_rejoin_over_16mib_table_converges() -> Outcome<()> { |
| 100 | let (d, l) = res!(pair()); |
| 101 | for i in 0..300u16 { |
| 102 | res!(l.storage().put(&Record::new(rid2(i), "identity", vec![(i & 0xff) as u8; 64 * 1024]))); |
| 103 | } |
| 104 | let mut lost = 0; |
| 105 | for _ in 0..3 { |
| 106 | lost += res!(wire_exchange(&d, &l, 2)); |
| 107 | } |
| 108 | let held = res!(d.storage().digests("identity")).len(); |
| 109 | assert_eq!(held, 300, "the rejoining peer holds {} of 300 after 3 exchanges ({} envelopes the codec refused)", held, lost); |
| 110 | Ok(()) |
| 111 | } |
| 112 | |
| 113 | // F1 (S1), without overload. A difference the sketch decodes (100 entries, under the ~170 that |
| 114 | // 256 cells hold) of 200 KiB records is a 20 MB reply, refused alike, so it never repairs. |
| 115 | #[test] |
| 116 | fn qa1_decoded_difference_over_16mib_converges() -> Outcome<()> { |
| 117 | let (d, l) = res!(pair()); |
| 118 | for i in 0..100u16 { |
| 119 | res!(l.storage().put(&Record::new(rid2(i), "identity", vec![(i & 0xff) as u8; 200 * 1024]))); |
| 120 | } |
| 121 | let mut lost = 0; |
| 122 | for _ in 0..3 { |
| 123 | lost += res!(wire_exchange(&d, &l, 2)); |
| 124 | } |
| 125 | let held = res!(d.storage().digests("identity")).len(); |
| 126 | assert_eq!(held, 100, "the dialler holds {} of 100 after 3 exchanges ({} envelopes refused)", held, lost); |
| 127 | Ok(()) |
| 128 | } |
| 129 | |
| 130 | // F2 (S2). A sketch whose header claims a key length near usize::MAX panics the listener inside |
| 131 | // Iblt::from_bytes (an unchecked `key_len + value_len`, then `vec![0; num_cells * key_len]`) |
| 132 | // before the engine's config check can refuse it. The codec passes the sketch as opaque bytes. |
| 133 | #[test] |
| 134 | fn qa1_hostile_sketch_header_is_an_err_not_a_panic() -> Outcome<()> { |
| 135 | let (_, l) = res!(pair()); |
| 136 | let mut sketch = Vec::with_capacity(40 + 76); |
| 137 | sketch.extend_from_slice(&1u64.to_le_bytes()); // num_cells |
| 138 | sketch.extend_from_slice(&1u64.to_le_bytes()); // num_hashes |
| 139 | sketch.extend_from_slice(&(1u64 << 63).to_le_bytes()); // key_len |
| 140 | sketch.extend_from_slice(&((1u64 << 63) + 64).to_le_bytes()); // value_len; the sum wraps to 64 |
| 141 | sketch.extend_from_slice(&0u64.to_le_bytes()); // seed |
| 142 | sketch.extend_from_slice(&[0u8; 76]); // one cell of the wrapped width |
| 143 | let env = Envelope::new(node(1), node(2), MsgKind::AntiEntropyDigest { |
| 144 | table: "identity".to_string(), |
| 145 | sketch, |
| 146 | after: None, |
| 147 | }); |
| 148 | // The envelope survives the codec, so a peer can send it. |
| 149 | let env = res!(Envelope::decode(&res!(env.encode()))); |
| 150 | let got = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| l.handle_envelope_at(env, 1_000))); |
| 151 | match got { |
| 152 | Err(_) => panic!("a hostile sketch header panicked the listener"), |
| 153 | Ok(Ok(_)) => panic!("a hostile sketch header was accepted"), |
| 154 | Ok(Err(_)) => {}, |
| 155 | } |
| 156 | Ok(()) |
| 157 | } |
| 158 | |
| 159 | /// Defers a value stamped after the clock; once the clock passes, the first value an id |
| 160 | /// receives wins, which diverges by arrival order. |
| 161 | struct FirstWinsAfter; |
| 162 | |
| 163 | impl Resolver for FirstWinsAfter { |
| 164 | fn resolve<V: ReadView>( |
| 165 | &self, |
| 166 | ctx: &ResolveCtx, |
| 167 | _view: &V, |
| 168 | _table: &str, |
| 169 | _id: &RecordId, |
| 170 | held: Option<&[u8]>, |
| 171 | incoming: &[u8], |
| 172 | ) |
| 173 | -> Outcome<Verdict> |
| 174 | { |
| 175 | if let Some(stamp) = LastVersionWins::version(incoming) { |
| 176 | if stamp > ctx.now_ms { |
| 177 | return Ok(Verdict::Defer); |
| 178 | } |
| 179 | } |
| 180 | Ok(match held { |
| 181 | Some(_) => Verdict::Keep, |
| 182 | None => Verdict::Take(incoming.to_vec()), |
| 183 | }) |
| 184 | } |
| 185 | } |
| 186 | |
| 187 | // F3 (S2). check_convergence calls a run quiet at the first full round that stores nothing, even |
| 188 | // while local puts it deferred are still pending and the skewed clocks are still moving. So a |
| 189 | // time-deferred record is never offered past its time, and a rule that diverges once the time |
| 190 | // comes passes, while the same rule with a nearer time fails. |
| 191 | #[test] |
| 192 | fn qa1_check_reports_divergence_after_a_deferred_time() -> Outcome<()> { |
| 193 | const NOW: u64 = 1_000; |
| 194 | let run = |stamp: u64| -> Outcome<()> { |
| 195 | let x = RecordId::from_bytes([9u8; 32]); |
| 196 | let records = vec![ |
| 197 | Record::new(x, "identity", LastVersionWins::value(stamp, &[1])), |
| 198 | Record::new(x, "identity", LastVersionWins::value(stamp, &[2])), |
| 199 | ]; |
| 200 | let cfg = Convergence::new(3, 0..16, NOW, 16).with_skew_ms(400); |
| 201 | check_convergence(&cfg, &["identity"], &records, |_| FirstWinsAfter, |_, _, _| Ok(Vec::new())) |
| 202 | }; |
| 203 | // Control: inside the clocks' first step, the divergence is reported. |
| 204 | assert!(run(NOW + 200).is_err(), "control: the near stamp did not fail"); |
| 205 | // Five steps out, within max_rounds = 16: the same rule passes. |
| 206 | assert!(run(NOW + 2_000).is_err(), "a rule that diverges after its deferred time passed the check"); |
| 207 | Ok(()) |
| 208 | } |
| 209 | |
| 210 | // F4 (S3). When the dialler takes the listener's value, it pushes that value straight back, |
| 211 | // although the listener holds those very bytes, so each such repair crosses the wire twice. The |
| 212 | // bulk path already skips this case. |
| 213 | #[test] |
| 214 | fn qa1_push_does_not_echo_the_listeners_own_bytes() -> Outcome<()> { |
| 215 | let (d, l) = res!(pair()); |
| 216 | let x = RecordId::from_bytes([7u8; 32]); |
| 217 | let old = LastVersionWins::value(1, b"old"); |
| 218 | let new = LastVersionWins::value(2, b"new"); |
| 219 | res!(d.storage().put(&Record::new(x, "identity", old))); |
| 220 | res!(l.storage().put(&Record::new(x, "identity", new.clone()))); |
| 221 | let req = res!(d.build_anti_entropy_request("identity", node(2))); |
| 222 | let mut out = res!(l.handle_envelope_at(req, 1_000)); |
| 223 | let back = res!(d.handle_envelope_at(out.outbound.remove(0), 1_000)); |
| 224 | for env in &back.outbound { |
| 225 | if let MsgKind::AntiEntropyPush { records, .. } = &env.body { |
| 226 | for r in records { |
| 227 | assert!(content_hash(&r.value) != content_hash(&new), |
| 228 | "the push sends the listener the bytes it already holds at {:?}", r.id); |
| 229 | } |
| 230 | } |
| 231 | } |
| 232 | Ok(()) |
| 233 | } |