oxedyne/fe2o3/fe2o3_o3db_sync/tests/dist_over_o3db.rs
15.8 KiB, 3 runs
created by r1870400018:67488, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | #![cfg(feature = "dist")] |
| 2 | //! Integration tests for [`DistOzone::over_o3db`] and [`DistOzone::close`]. |
| 3 | //! |
| 4 | //! An application opens a distributed engine over an Ozone database in one call and closes it in |
| 5 | //! another, and what a put returned for must survive the pair and a reopen of the same root. |
| 6 | //! Each test works in its own `./test_db_dist_over_o3db_<name>` root, wiped when the test begins. |
| 7 | //! |
| 8 | //! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\ |
| 9 | //! Anthropic Claude |
| 10 | |
| 11 | use oxedyne_fe2o3_core::prelude::*; |
| 12 | use oxedyne_fe2o3_crypto::enc::EncryptionScheme; |
| 13 | use oxedyne_fe2o3_hash::{ |
| 14 | csum::ChecksumScheme, |
| 15 | hash::HashScheme, |
| 16 | }; |
| 17 | use oxedyne_fe2o3_jdat::prelude::*; |
| 18 | use oxedyne_fe2o3_o3db_sync::{ |
| 19 | base::cfg::OzoneConfig, |
| 20 | data::core::RestSchemesInput, |
| 21 | dist::{ |
| 22 | config::{ |
| 23 | DistOzoneConfig, |
| 24 | TableConfig, |
| 25 | }, |
| 26 | engine::DistOzone, |
| 27 | o3db_storage::O3dbStorage, |
| 28 | record::{ |
| 29 | Record, |
| 30 | RecordDigest, |
| 31 | RecordId, |
| 32 | content_hash, |
| 33 | }, |
| 34 | resolve::{ |
| 35 | LastVersionWins, |
| 36 | ReadView, |
| 37 | ResolveCtx, |
| 38 | Resolver, |
| 39 | Verdict, |
| 40 | }, |
| 41 | storage::{ |
| 42 | MemoryStorage, |
| 43 | Storage, |
| 44 | }, |
| 45 | transport::{ |
| 46 | Envelope, |
| 47 | MsgKind, |
| 48 | }, |
| 49 | }, |
| 50 | kademlia::id::NodeId, |
| 51 | oam::config::OamConfig, |
| 52 | test::setup, |
| 53 | }; |
| 54 | |
| 55 | use std::{ |
| 56 | path::PathBuf, |
| 57 | sync::{ |
| 58 | Arc, |
| 59 | Mutex, |
| 60 | }, |
| 61 | }; |
| 62 | |
| 63 | |
| 64 | // One test at a time, so that the process's thread count belongs to the test that reads it. |
| 65 | static ONE_AT_A_TIME: Mutex<()> = Mutex::new(()); |
| 66 | |
| 67 | |
| 68 | type Eng<R> = DistOzone< |
| 69 | O3dbStorage<{ setup::UID_LEN }, setup::Uid, EncryptionScheme, HashScheme, HashScheme, ChecksumScheme>, |
| 70 | R, |
| 71 | >; |
| 72 | |
| 73 | fn node(b: u8) -> NodeId { |
| 74 | let mut bytes = [0u8; 32]; |
| 75 | bytes[31] = b; |
| 76 | NodeId::from_bytes(bytes) |
| 77 | } |
| 78 | |
| 79 | fn rid(b: u8) -> RecordId { |
| 80 | RecordId::from_bytes([b; 32]) |
| 81 | } |
| 82 | |
| 83 | fn rec(table: &str, id: RecordId, value: &[u8]) -> Record { |
| 84 | Record::new(id, table, value.to_vec()) |
| 85 | } |
| 86 | |
| 87 | /// The threads this process runs, which a database that is open adds its bots to. |
| 88 | fn threads() -> Outcome<usize> { |
| 89 | Ok(res!(std::fs::read_dir("/proc/self/task")).count()) |
| 90 | } |
| 91 | |
| 92 | /// A root of its own for the test `name`, emptied before use and left behind after it. |
| 93 | fn fresh_root(name: &str) -> Outcome<PathBuf> { |
| 94 | let dir = PathBuf::from(fmt!("./test_db_dist_over_o3db_{}", name)); |
| 95 | if dir.exists() { |
| 96 | res!(std::fs::remove_dir_all(&dir)); |
| 97 | } |
| 98 | res!(std::fs::create_dir_all(&dir)); |
| 99 | Ok(res!(dir.canonicalize())) |
| 100 | } |
| 101 | |
| 102 | fn schemes() -> Outcome<RestSchemesInput<EncryptionScheme, HashScheme, HashScheme, ChecksumScheme>> { |
| 103 | let enckey = [0x55u8; 32]; |
| 104 | let aes_gcm = res!(EncryptionScheme::new_aes_256_gcm_with_key(&enckey[..])); |
| 105 | Ok(RestSchemesInput::new( |
| 106 | Some(aes_gcm), |
| 107 | None::<HashScheme>, |
| 108 | None::<HashScheme>, |
| 109 | Some(ChecksumScheme::new_crc32()), |
| 110 | )) |
| 111 | } |
| 112 | |
| 113 | /// The stock store: two zones, the stock cache, the default durability barrier. |
| 114 | fn ozone_cfg() -> OzoneConfig { |
| 115 | OzoneConfig::default() |
| 116 | } |
| 117 | |
| 118 | /// One peer holding everything, with these eventual tables. |
| 119 | fn single(tables: &[&str]) -> Outcome<DistOzoneConfig> { |
| 120 | let oam = res!(OamConfig::new(1, 1)); |
| 121 | let mut tcs = Vec::with_capacity(tables.len()); |
| 122 | for t in tables { |
| 123 | tcs.push(res!(TableConfig::eventual(*t))); |
| 124 | } |
| 125 | DistOzoneConfig::new(node(1), Vec::new(), oam, tcs) |
| 126 | } |
| 127 | |
| 128 | /// Two peers that both hold everything, seen from peer `local`. |
| 129 | fn pair(local: u8, remote: u8, table: &str) -> Outcome<DistOzoneConfig> { |
| 130 | let oam = res!(OamConfig::new(2, 2)); |
| 131 | let tcs = vec![res!(TableConfig::eventual(table))]; |
| 132 | DistOzoneConfig::new(node(local), vec![node(remote)], oam, tcs) |
| 133 | } |
| 134 | |
| 135 | fn open<R: Resolver>( |
| 136 | root: &PathBuf, |
| 137 | ozone_cfg: Option<OzoneConfig>, |
| 138 | dist_cfg: DistOzoneConfig, |
| 139 | resolver: R, |
| 140 | ) |
| 141 | -> Outcome<Eng<R>> |
| 142 | { |
| 143 | Eng::<R>::over_o3db( |
| 144 | root.clone(), |
| 145 | ozone_cfg, |
| 146 | dist_cfg, |
| 147 | res!(schemes()), |
| 148 | setup::Uid::default(), |
| 149 | resolver, |
| 150 | ) |
| 151 | } |
| 152 | |
| 153 | fn held<S: Storage, R: Resolver>( |
| 154 | e: &DistOzone<S, R>, |
| 155 | table: &str, |
| 156 | id: RecordId, |
| 157 | ) |
| 158 | -> Outcome<Option<Vec<u8>>> |
| 159 | { |
| 160 | Ok(res!(e.storage().get(table, &id)).map(|r| r.value)) |
| 161 | } |
| 162 | |
| 163 | fn digests<S: Storage, R: Resolver>(e: &DistOzone<S, R>, table: &str) -> Outcome<Vec<RecordDigest>> { |
| 164 | e.storage().digests(table) |
| 165 | } |
| 166 | |
| 167 | /// Refuses a value that begins with "BAD", and takes any other. |
| 168 | struct RefuseBad; |
| 169 | |
| 170 | impl Resolver for RefuseBad { |
| 171 | fn resolve<V: ReadView>( |
| 172 | &self, |
| 173 | _ctx: &ResolveCtx, |
| 174 | _view: &V, |
| 175 | _table: &str, |
| 176 | _id: &RecordId, |
| 177 | _held: Option<&[u8]>, |
| 178 | incoming: &[u8], |
| 179 | ) |
| 180 | -> Outcome<Verdict> |
| 181 | { |
| 182 | if incoming.starts_with(b"BAD") { |
| 183 | return Ok(Verdict::Refuse("marked bad".to_string())); |
| 184 | } |
| 185 | Ok(Verdict::Take(incoming.to_vec())) |
| 186 | } |
| 187 | } |
| 188 | |
| 189 | /// What the engine's answer to a peer's anti-entropy digest says: the records it would send, the |
| 190 | /// ids it asks for, and whether it fell back to a bulk reply. |
| 191 | fn answer<S1: Storage, R1: Resolver, S2: Storage, R2: Resolver>( |
| 192 | asker: &DistOzone<S1, R1>, |
| 193 | answerer: &DistOzone<S2, R2>, |
| 194 | answerer_id: u8, |
| 195 | table: &str, |
| 196 | ) |
| 197 | -> Outcome<(Vec<Record>, Vec<RecordId>, bool)> |
| 198 | { |
| 199 | let req = res!(asker.build_anti_entropy_request(table, node(answerer_id))); |
| 200 | let mut got = res!(answerer.handle_envelope_at(req, 1_000)); |
| 201 | assert_eq!(got.outbound.len(), 1); |
| 202 | let env: Envelope = got.outbound.remove(0); |
| 203 | match env.body { |
| 204 | MsgKind::AntiEntropyReply { records, requested_ids, bulk, .. } => Ok((records, requested_ids, bulk)), |
| 205 | other => Err(err!("Expected an AntiEntropyReply, got {:?}.", other; Test, Mismatch)), |
| 206 | } |
| 207 | } |
| 208 | |
| 209 | |
| 210 | #[test] |
| 211 | fn opens_puts_closes_reopens_reads_back() -> Outcome<()> { |
| 212 | let _one = lock_mutex!(ONE_AT_A_TIME); |
| 213 | let root = res!(fresh_root("reopen")); |
| 214 | let dist = || single(&["identity", "escrow"]); |
| 215 | let idle = res!(threads()); |
| 216 | let e = res!(open(&root, Some(ozone_cfg()), res!(dist()), LastVersionWins)); |
| 217 | let running = res!(threads()); |
| 218 | assert!(running > idle + 10, "an open database should run its bots: {} threads before, {} after", idle, running); |
| 219 | |
| 220 | // Two values that differ only by a trailing zero, in one table, and a longer one in another. |
| 221 | let puts = [ |
| 222 | ("identity", rid(1), vec![0xABu8]), |
| 223 | ("identity", rid(2), vec![0xABu8, 0x00]), |
| 224 | ("escrow", rid(1), b"gamma, longer than one chunk of eight bytes".to_vec()), |
| 225 | ]; |
| 226 | for (table, id, value) in &puts { |
| 227 | let out = res!(e.put_at(rec(table, *id, value), 1_000)); |
| 228 | assert!(out.local_persisted, "the put of {:?} in {} was not stored", id, table); |
| 229 | } |
| 230 | let before_identity = res!(digests(&e, "identity")); |
| 231 | let before_escrow = res!(digests(&e, "escrow")); |
| 232 | assert_eq!(before_identity.len(), 2); |
| 233 | assert_eq!(before_escrow.len(), 1); |
| 234 | assert_ne!(before_identity[0].content, before_identity[1].content); |
| 235 | res!(e.close()); |
| 236 | // A close returns once the bots have stopped, so their threads are gone, or going. |
| 237 | let closed = res!(threads()); |
| 238 | assert!(closed < idle + (running - idle) / 2, |
| 239 | "close left the bots running: {} threads before the open, {} open, {} after the close", idle, running, closed); |
| 240 | |
| 241 | // The same root, no Ozone configuration given: the one saved there is used. |
| 242 | let e = res!(open(&root, None, res!(dist()), LastVersionWins)); |
| 243 | for (table, id, value) in &puts { |
| 244 | assert_eq!(res!(held(&e, table, *id)).as_ref(), Some(value), |
| 245 | "{} {:?} did not come back after the reopen", table, id); |
| 246 | } |
| 247 | assert_eq!(res!(digests(&e, "identity")), before_identity); |
| 248 | assert_eq!(res!(digests(&e, "escrow")), before_escrow); |
| 249 | for d in &before_identity { |
| 250 | let value = if d.id == rid(1) { vec![0xABu8] } else { vec![0xABu8, 0x00] }; |
| 251 | assert_eq!(d.content, content_hash(&value)); |
| 252 | } |
| 253 | res!(e.close()); |
| 254 | Ok(()) |
| 255 | } |
| 256 | |
| 257 | #[test] |
| 258 | fn a_dropped_engine_stops_its_bots_and_keeps_what_it_stored() -> Outcome<()> { |
| 259 | let _one = lock_mutex!(ONE_AT_A_TIME); |
| 260 | let root = res!(fresh_root("dropped")); |
| 261 | let idle = res!(threads()); |
| 262 | let e = res!(open(&root, Some(ozone_cfg()), res!(single(&["identity"])), LastVersionWins)); |
| 263 | let running = res!(threads()); |
| 264 | assert!(running > idle + 10, "an open database should run its bots: {} threads before, {} after", idle, running); |
| 265 | let out = res!(e.put_at(rec("identity", rid(1), b"kept without a close"), 1_000)); |
| 266 | assert!(out.local_persisted, "the put was not stored"); |
| 267 | |
| 268 | // Dropped as an error path or a panic unwinds it, with no close. |
| 269 | drop(e); |
| 270 | let dropped = res!(threads()); |
| 271 | assert!(dropped < idle + (running - idle) / 2, |
| 272 | "dropping the engine left the bots running: {} threads before the open, {} open, {} after the drop", |
| 273 | idle, running, dropped); |
| 274 | |
| 275 | // The bots were stopped in order, so the same root opens again with the record in it. |
| 276 | let e = res!(open(&root, None, res!(single(&["identity"])), LastVersionWins)); |
| 277 | assert_eq!(res!(held(&e, "identity", rid(1))), Some(b"kept without a close".to_vec())); |
| 278 | res!(e.close()); |
| 279 | Ok(()) |
| 280 | } |
| 281 | |
| 282 | #[test] |
| 283 | fn dropping_a_shared_storage_leaves_the_database_to_its_other_holder() -> Outcome<()> { |
| 284 | let _one = lock_mutex!(ONE_AT_A_TIME); |
| 285 | let root = res!(fresh_root("dropshared")); |
| 286 | let e = res!(open(&root, Some(ozone_cfg()), res!(single(&["identity"])), LastVersionWins)); |
| 287 | res!(e.put_at(rec("identity", rid(1), b"still open"), 1_000)); |
| 288 | let extra = Arc::clone(e.storage().db()); |
| 289 | |
| 290 | // The engine goes, the database does not: someone else still holds it. |
| 291 | drop(e); |
| 292 | let storage = O3dbStorage::new(extra, setup::Uid::default()); |
| 293 | let back = res!(storage.get("identity", &rid(1))); |
| 294 | assert_eq!(back.map(|r| r.value), Some(b"still open".to_vec())); |
| 295 | res!(storage.close()); |
| 296 | Ok(()) |
| 297 | } |
| 298 | |
| 299 | #[test] |
| 300 | fn over_o3db_uses_the_given_resolver() -> Outcome<()> { |
| 301 | let _one = lock_mutex!(ONE_AT_A_TIME); |
| 302 | let root = res!(fresh_root("resolver")); |
| 303 | let e = res!(open(&root, Some(ozone_cfg()), res!(single(&["identity"])), RefuseBad)); |
| 304 | |
| 305 | let refused = res!(e.put_at(rec("identity", rid(1), b"BAD value"), 1_000)); |
| 306 | assert!(!refused.local_persisted); |
| 307 | assert!(matches!(refused.verdict, Some(Verdict::Refuse(_))), "verdict was {:?}", refused.verdict); |
| 308 | let taken = res!(e.put_at(rec("identity", rid(2), b"good value"), 1_000)); |
| 309 | assert!(taken.local_persisted); |
| 310 | assert_eq!(res!(held(&e, "identity", rid(1))), None); |
| 311 | res!(e.close()); |
| 312 | |
| 313 | // Reopened under the same resolver, the refused value is still absent and the other present. |
| 314 | let e = res!(open(&root, None, res!(single(&["identity"])), RefuseBad)); |
| 315 | assert_eq!(res!(held(&e, "identity", rid(1))), None); |
| 316 | assert_eq!(res!(held(&e, "identity", rid(2))), Some(b"good value".to_vec())); |
| 317 | assert_eq!(res!(digests(&e, "identity")).len(), 1); |
| 318 | res!(e.close()); |
| 319 | Ok(()) |
| 320 | } |
| 321 | |
| 322 | /// What a peer answers to another's anti-entropy digest: the records it would send, the ids it |
| 323 | /// asks for, and whether it fell back to a bulk reply. |
| 324 | type Answer = (Vec<Record>, Vec<RecordId>, bool); |
| 325 | |
| 326 | #[test] |
| 327 | fn memory_and_o3db_peers_decode_empty_difference() -> Outcome<()> { |
| 328 | let _one = lock_mutex!(ONE_AT_A_TIME); |
| 329 | let root = res!(fresh_root("mixed")); |
| 330 | // Peer 1 holds in memory. Peer 2 is held by O3db, and a second peer 2 in memory is the |
| 331 | // reference: whatever the reference pair does, the mixed pair must do alike. |
| 332 | let mem: DistOzone<MemoryStorage> = res!(DistOzone::new(res!(pair(1, 2, "identity")), MemoryStorage::new())); |
| 333 | let o3 = res!(open(&root, Some(ozone_cfg()), res!(pair(2, 1, "identity")), LastVersionWins)); |
| 334 | let refr: DistOzone<MemoryStorage> = res!(DistOzone::new(res!(pair(2, 1, "identity")), MemoryStorage::new())); |
| 335 | |
| 336 | let long = b"a longer value than one chunk of eight bytes".to_vec(); |
| 337 | let records = [ |
| 338 | (rid(1), vec![0xABu8]), |
| 339 | (rid(2), vec![0xABu8, 0x00]), |
| 340 | (rid(3), long.clone()), |
| 341 | ]; |
| 342 | for (id, value) in &records { |
| 343 | for e in [&mem, &refr] { |
| 344 | res!(e.put_at(rec("identity", *id, value), 1_000)); |
| 345 | } |
| 346 | res!(o3.put_at(rec("identity", *id, value), 1_000)); |
| 347 | } |
| 348 | assert_eq!(res!(digests(&mem, "identity")), res!(digests(&o3, "identity"))); |
| 349 | |
| 350 | // The two answers each way, from the mixed pair and from the reference pair. |
| 351 | let both = |_: ()| -> Outcome<(Answer, Answer, Answer, Answer)> { |
| 352 | Ok(( |
| 353 | res!(answer(&mem, &o3, 2, "identity")), |
| 354 | res!(answer(&o3, &mem, 1, "identity")), |
| 355 | res!(answer(&mem, &refr, 2, "identity")), |
| 356 | res!(answer(&refr, &mem, 1, "identity")), |
| 357 | )) |
| 358 | }; |
| 359 | |
| 360 | // The same three records on both sides: nothing to send, nothing to ask for, no bulk fallback. |
| 361 | let (m_asks, o_asks, m_ref, o_ref) = res!(both(())); |
| 362 | for (who, a) in [("memory asked, O3db answered", &m_asks), ("O3db asked, memory answered", &o_asks)] { |
| 363 | assert!(a.0.is_empty() && a.1.is_empty() && !a.2, "{}: {:?}", who, a); |
| 364 | } |
| 365 | assert_eq!((&m_asks, &o_asks), (&m_ref, &o_ref)); |
| 366 | |
| 367 | // One value revised at O3db and at the reference. The mixed pair answers as the reference |
| 368 | // pair does, and each way the answer is exactly one record, the answerer's own copy, with the |
| 369 | // id asked for in return and no bulk fallback. |
| 370 | let newer = b"a longer value than one chunk of eight bytes, revised".to_vec(); |
| 371 | for e in [&refr] { |
| 372 | assert!(res!(e.put_at(rec("identity", rid(3), &newer), 2_000)).local_persisted); |
| 373 | } |
| 374 | assert!(res!(o3.put_at(rec("identity", rid(3), &newer), 2_000)).local_persisted); |
| 375 | let (m_asks, o_asks, m_ref, o_ref) = res!(both(())); |
| 376 | assert_eq!((&m_asks, &o_asks), (&m_ref, &o_ref)); |
| 377 | assert_eq!(m_asks, (vec![rec("identity", rid(3), &newer)], vec![rid(3)], false)); |
| 378 | assert_eq!(o_asks, (vec![rec("identity", rid(3), &long)], vec![rid(3)], false)); |
| 379 | |
| 380 | // The memory peer takes the revision, and the pair agrees again. |
| 381 | assert!(res!(mem.put_at(rec("identity", rid(3), &newer), 2_000)).local_persisted); |
| 382 | assert_eq!(res!(digests(&mem, "identity")), res!(digests(&o3, "identity"))); |
| 383 | let (m_asks, o_asks, _, _) = res!(both(())); |
| 384 | for a in [&m_asks, &o_asks] { |
| 385 | assert!(a.0.is_empty() && a.1.is_empty() && !a.2, "after the revision: {:?}", a); |
| 386 | } |
| 387 | |
| 388 | // One record only the O3db peer holds: exactly one record one way and one request the other. |
| 389 | let extra = vec![0x04u8; 40]; |
| 390 | for e in [&refr] { |
| 391 | res!(e.put_at(rec("identity", rid(4), &extra), 3_000)); |
| 392 | } |
| 393 | res!(o3.put_at(rec("identity", rid(4), &extra), 3_000)); |
| 394 | let (m_asks, o_asks, m_ref, o_ref) = res!(both(())); |
| 395 | assert_eq!((&m_asks, &o_asks), (&m_ref, &o_ref)); |
| 396 | assert_eq!(m_asks, (vec![rec("identity", rid(4), &extra)], Vec::new(), false)); |
| 397 | assert_eq!(o_asks, (Vec::new(), vec![rid(4)], false)); |
| 398 | |
| 399 | res!(o3.close()); |
| 400 | Ok(()) |
| 401 | } |
| 402 | |
| 403 | #[test] |
| 404 | fn close_refuses_while_shared() -> Outcome<()> { |
| 405 | let _one = lock_mutex!(ONE_AT_A_TIME); |
| 406 | let root = res!(fresh_root("shared")); |
| 407 | let e = res!(open(&root, Some(ozone_cfg()), res!(single(&["identity"])), LastVersionWins)); |
| 408 | res!(e.put_at(rec("identity", rid(1), b"kept"), 1_000)); |
| 409 | |
| 410 | // A second holder of the database. |
| 411 | let extra = Arc::clone(e.storage().db()); |
| 412 | let refused = match e.close() { |
| 413 | Ok(()) => return Err(err!("close succeeded with another reference to the database held."; Test, Unexpected)), |
| 414 | Err(refused) => refused, |
| 415 | }; |
| 416 | let said = fmt!("{}", refused); |
| 417 | assert!(said.contains("1 other reference"), "the refusal does not name the holder: {}", said); |
| 418 | |
| 419 | // The refusal left the database open and the record in it. |
| 420 | let storage = O3dbStorage::new(extra, setup::Uid::default()); |
| 421 | let back = res!(storage.get("identity", &rid(1))); |
| 422 | assert_eq!(back.map(|r| r.value), Some(b"kept".to_vec())); |
| 423 | res!(storage.close()); |
| 424 | Ok(()) |
| 425 | } |
| 426 | |
| 427 | #[test] |
| 428 | fn a_put_returned_before_close_reads_back_after_reopen_on_one_zone() -> Outcome<()> { |
| 429 | let _one = lock_mutex!(ONE_AT_A_TIME); |
| 430 | let root = res!(fresh_root("one_zone")); |
| 431 | // One zone, a barrier on every write, and a cache far smaller than what is stored. |
| 432 | let overrides = res!(mapdat!{ |
| 433 | 1u16 => mapdat!{ "dir" => "", "max_size" => 104_857_600u64 }, |
| 434 | }.get_map().ok_or_else(|| err!("The zone override map is not a map."; Test, Invalid))); |
| 435 | let cfg = OzoneConfig { |
| 436 | num_zones: 1, |
| 437 | sync_on_write: true, |
| 438 | cache_size_limit_bytes: 4_096, |
| 439 | zone_overrides: overrides, |
| 440 | ..OzoneConfig::default() |
| 441 | }; |
| 442 | let e = res!(open(&root, Some(cfg), res!(single(&["identity"])), LastVersionWins)); |
| 443 | let mut puts = Vec::new(); |
| 444 | for n in 1..=24u8 { |
| 445 | let value: Vec<u8> = (0..300u32).map(|i| (i as u8).wrapping_mul(n).wrapping_add(n)).collect(); |
| 446 | let out = res!(e.put_at(rec("identity", rid(n), &value), 1_000)); |
| 447 | assert!(out.local_persisted); |
| 448 | puts.push((rid(n), value)); |
| 449 | } |
| 450 | res!(e.close()); |
| 451 | |
| 452 | let e = res!(open(&root, None, res!(single(&["identity"])), LastVersionWins)); |
| 453 | for (id, value) in &puts { |
| 454 | assert_eq!(res!(held(&e, "identity", *id)).as_ref(), Some(value), |
| 455 | "{:?} did not come back after the reopen", id); |
| 456 | } |
| 457 | assert_eq!(res!(digests(&e, "identity")).len(), puts.len()); |
| 458 | res!(e.close()); |
| 459 | Ok(()) |
| 460 | } |