oxedyne/fe2o3/fe2o3_o3db_sync/tests/lane.rs
26.2 KiB, 1 run
created by r1870400018:74447, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! The sequence lane on a real Ozone store: rising sequence numbers, pages bounded by bytes that |
| 2 | //! answer `more`, holes left by `remove` and `trim_through`, lazy expiry, replayed appends, reader |
| 3 | //! cursors and `stable`, pages bounded by steps, lanes under their own locks side by side, and |
| 4 | //! the lane read back after a restart. |
| 5 | |
| 6 | use oxedyne_fe2o3_core::prelude::*; |
| 7 | use oxedyne_fe2o3_crypto::enc::EncryptionScheme; |
| 8 | use oxedyne_fe2o3_hash::{ |
| 9 | csum::ChecksumScheme, |
| 10 | hash::HashScheme, |
| 11 | }; |
| 12 | use oxedyne_fe2o3_iop_db::api::Database; |
| 13 | use oxedyne_fe2o3_jdat::prelude::*; |
| 14 | use oxedyne_fe2o3_o3db_sync::{ |
| 15 | data::core::RestSchemesInput, |
| 16 | lane::{ |
| 17 | Lane, |
| 18 | LaneLocks, |
| 19 | ReaderState, |
| 20 | }, |
| 21 | test::setup, |
| 22 | }; |
| 23 | |
| 24 | use std::{ |
| 25 | path::PathBuf, |
| 26 | sync::{ |
| 27 | Arc, |
| 28 | RwLock, |
| 29 | }, |
| 30 | thread, |
| 31 | }; |
| 32 | |
| 33 | |
| 34 | type TestDb = oxedyne_fe2o3_o3db_sync::O3db< |
| 35 | { setup::UID_LEN }, |
| 36 | setup::Uid, |
| 37 | EncryptionScheme, |
| 38 | HashScheme, |
| 39 | HashScheme, |
| 40 | ChecksumScheme, |
| 41 | >; |
| 42 | |
| 43 | type TestLane<'a> = Lane<'a, { setup::UID_LEN }, setup::Uid, EncryptionScheme, HashScheme, TestDb>; |
| 44 | |
| 45 | const NOW: u64 = 1_760_659_200_000; |
| 46 | |
| 47 | fn open(root: &PathBuf, wipe: bool) -> Outcome<TestDb> { |
| 48 | let schms = RestSchemesInput::new( |
| 49 | None::<EncryptionScheme>, |
| 50 | None::<HashScheme>, |
| 51 | None::<HashScheme>, |
| 52 | Some(ChecksumScheme::new_crc32()), |
| 53 | ); |
| 54 | let mut cfg = res!(setup::default_cfg()); |
| 55 | cfg.zone_overrides = DaticleMap::new(); |
| 56 | setup::start_db(root.clone(), Some(cfg), schms, None, false, wipe) |
| 57 | } |
| 58 | |
| 59 | fn val(i: u64, bytes: usize) -> Dat { |
| 60 | let mut v = vec![0u8; bytes]; |
| 61 | v[..8].copy_from_slice(&i.to_be_bytes()); |
| 62 | Dat::bytdat(v) |
| 63 | } |
| 64 | |
| 65 | fn seqs(p: &oxedyne_fe2o3_o3db_sync::lane::Page) -> Vec<u64> { |
| 66 | p.entries.iter().map(|e| e.seq).collect() |
| 67 | } |
| 68 | |
| 69 | fn check(ok: bool, what: String) -> Outcome<()> { |
| 70 | if ok { Ok(()) } else { Err(err!("{}", what; Test, Mismatch)) } |
| 71 | } |
| 72 | |
| 73 | #[test] |
| 74 | fn lane() -> Outcome<()> { |
| 75 | log_set_level!("error"); |
| 76 | let outcome = run(); |
| 77 | log_finish_wait!(); |
| 78 | outcome |
| 79 | } |
| 80 | |
| 81 | fn run() -> Outcome<()> { |
| 82 | let root = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join("test_db_lane"); |
| 83 | let _ = std::fs::remove_dir_all(&root); |
| 84 | res!(std::fs::create_dir_all(&root)); |
| 85 | let root = res!(root.canonicalize()); |
| 86 | let user = setup::Uid::default(); |
| 87 | { |
| 88 | let mut db = res!(open(&root, true)); |
| 89 | res!(append_rises(&db, user)); |
| 90 | res!(pages_honour_max_bytes(&db, user)); |
| 91 | res!(holes_are_skipped(&db, user)); |
| 92 | res!(expired_reads_absent_and_is_deleted(&db, user)); |
| 93 | res!(replayed_append_does_not_double(&db, user)); |
| 94 | res!(replay_after_a_torn_append_is_not_held(&db, user)); |
| 95 | res!(cursors_and_stable(&db, user)); |
| 96 | res!(trim_never_passes_the_ask(&db, user)); |
| 97 | res!(steps_bound_a_page(&db, user)); |
| 98 | res!(measured_pages(&db, user)); |
| 99 | res!(a_trim_keeps_live_ephemerals(&db, user)); |
| 100 | res!(the_sweep_is_bounded(&db, user)); |
| 101 | res!(db.shutdown()); |
| 102 | } |
| 103 | { |
| 104 | let mut db = res!(open(&root, false)); |
| 105 | res!(survives_a_restart(&db, user)); |
| 106 | res!(db.shutdown()); |
| 107 | } |
| 108 | res!(replay_under_the_lane_lock(&root, user)); |
| 109 | let _ = std::fs::remove_dir_all(&root); |
| 110 | Ok(()) |
| 111 | } |
| 112 | |
| 113 | fn append_rises(db: &TestDb, user: setup::Uid) -> Outcome<()> { |
| 114 | let locks = LaneLocks::default(); |
| 115 | let g = locks.lock("t/rise"); |
| 116 | let lane: TestLane = Lane::new(db, user, &g); |
| 117 | res!(check(res!(lane.head()) == 0, fmt!("a new lane has head {}", res!(lane.head())))); |
| 118 | for i in 1..=5 { |
| 119 | let a = res!(lane.append(val(i, 16), None, None)); |
| 120 | res!(check(a.seq == i && !a.replay, fmt!("append {} answered {:?}", i, a))); |
| 121 | } |
| 122 | let p = res!(lane.read_after(0, 1 << 20, usize::MAX, NOW)); |
| 123 | res!(check(seqs(&p) == vec![1, 2, 3, 4, 5] && p.through == 5 && !p.more, |
| 124 | fmt!("read all: {:?} through {} more {}", seqs(&p), p.through, p.more))); |
| 125 | res!(check(p.entries[2].val == val(3, 16), fmt!("entry 3 reads {:?}", p.entries[2].val))); |
| 126 | let p = res!(lane.read_after(3, 1 << 20, usize::MAX, NOW)); |
| 127 | res!(check(seqs(&p) == vec![4, 5], fmt!("read after 3: {:?}", seqs(&p)))); |
| 128 | let p = res!(lane.read_after(5, 1 << 20, usize::MAX, NOW)); |
| 129 | res!(check(p.entries.is_empty() && p.through == 5 && !p.more, fmt!("read after head: {:?}", p))); |
| 130 | // A cursor beyond the head answers the head. |
| 131 | let p = res!(lane.read_after(99, 1 << 20, usize::MAX, NOW)); |
| 132 | res!(check(p.entries.is_empty() && p.through == 5, fmt!("read after 99: through {}", p.through))); |
| 133 | Ok(()) |
| 134 | } |
| 135 | |
| 136 | fn pages_honour_max_bytes(db: &TestDb, user: setup::Uid) -> Outcome<()> { |
| 137 | let locks = LaneLocks::default(); |
| 138 | let g = locks.lock("t/page"); |
| 139 | let lane: TestLane = Lane::new(db, user, &g); |
| 140 | for i in 1..=10 { |
| 141 | res!(lane.append(val(i, 1_000), None, None)); |
| 142 | } |
| 143 | let size = match val(0, 1_000).byte_len() { |
| 144 | Some(n) => n, |
| 145 | None => return Err(err!("A byte value cannot be measured."; Test, Unexpected)), |
| 146 | }; |
| 147 | // Room for three and a half: three, and more. |
| 148 | let mut after = 0; |
| 149 | let mut got = Vec::new(); |
| 150 | let mut pages = 0; |
| 151 | loop { |
| 152 | let p = res!(lane.read_after(after, 3 * size + size / 2, usize::MAX, NOW)); |
| 153 | pages += 1; |
| 154 | let used: usize = p.entries.len() * size; |
| 155 | res!(check(used <= 3 * size + size / 2, fmt!("a page carried {} bytes", used))); |
| 156 | got.extend(seqs(&p)); |
| 157 | after = p.through; |
| 158 | if !p.more { |
| 159 | break; |
| 160 | } |
| 161 | res!(check(p.entries.len() == 3, fmt!("a full page held {} entries", p.entries.len()))); |
| 162 | } |
| 163 | res!(check(got == (1..=10).collect::<Vec<u64>>() && pages == 4, |
| 164 | fmt!("paged {:?} in {} pages", got, pages))); |
| 165 | // A first entry larger than the page is still answered. |
| 166 | let p = res!(lane.read_after(0, 10, usize::MAX, NOW)); |
| 167 | res!(check(seqs(&p) == vec![1] && p.more, fmt!("a tiny page: {:?} more {}", seqs(&p), p.more))); |
| 168 | Ok(()) |
| 169 | } |
| 170 | |
| 171 | fn holes_are_skipped(db: &TestDb, user: setup::Uid) -> Outcome<()> { |
| 172 | let locks = LaneLocks::default(); |
| 173 | let g = locks.lock("t/hole"); |
| 174 | let lane: TestLane = Lane::new(db, user, &g); |
| 175 | for i in 1..=8 { |
| 176 | res!(lane.append(val(i, 16), None, None)); |
| 177 | } |
| 178 | res!(check(res!(lane.remove(3)), fmt!("remove 3 found nothing"))); |
| 179 | res!(check(!res!(lane.remove(3)), fmt!("a second remove of 3 found it"))); |
| 180 | res!(check(res!(lane.remove(8)), fmt!("remove 8 found nothing"))); |
| 181 | res!(check(res!(lane.trim_through(2)) == 2, fmt!("trim through 2"))); |
| 182 | let p = res!(lane.read_after(0, 1 << 20, usize::MAX, NOW)); |
| 183 | res!(check(seqs(&p) == vec![4, 5, 6, 7] && p.through == 8 && !p.more, |
| 184 | fmt!("holes: {:?} through {}", seqs(&p), p.through))); |
| 185 | // The head is not lowered by removing its entry: the next append is 9. |
| 186 | let a = res!(lane.append(val(9, 16), None, None)); |
| 187 | res!(check(a.seq == 9, fmt!("after removing the top, append got {}", a.seq))); |
| 188 | res!(check(res!(lane.depth()) == 7 && res!(lane.floor()) == 2, |
| 189 | fmt!("depth {} floor {}", res!(lane.depth()), res!(lane.floor())))); |
| 190 | Ok(()) |
| 191 | } |
| 192 | |
| 193 | fn expired_reads_absent_and_is_deleted(db: &TestDb, user: setup::Uid) -> Outcome<()> { |
| 194 | let locks = LaneLocks::default(); |
| 195 | let g = locks.lock("t/exp"); |
| 196 | let lane: TestLane = Lane::new(db, user, &g); |
| 197 | res!(lane.append(val(1, 16), None, Some(NOW + 1_000))); |
| 198 | res!(lane.append(val(2, 16), Some(b"two"), Some(NOW + 5_000))); |
| 199 | res!(lane.append(val(3, 16), None, None)); |
| 200 | let p = res!(lane.read_after(0, 1 << 20, usize::MAX, NOW)); |
| 201 | res!(check(seqs(&p) == vec![1, 2, 3] && p.entries[0].exp_ms == Some(NOW + 1_000), |
| 202 | fmt!("before expiry: {:?}", p.entries))); |
| 203 | let p = res!(lane.read_after(0, 1 << 20, usize::MAX, NOW + 1_000)); |
| 204 | res!(check(seqs(&p) == vec![2, 3] && p.through == 3, fmt!("at entry 1's expiry: {:?}", seqs(&p)))); |
| 205 | // Deleted, not merely hidden: a read at an earlier time does not see it again. |
| 206 | let p = res!(lane.read_after(0, 1 << 20, usize::MAX, NOW)); |
| 207 | res!(check(seqs(&p) == vec![2, 3], fmt!("entry 1 came back: {:?}", seqs(&p)))); |
| 208 | let p = res!(lane.read_after(0, 1 << 20, usize::MAX, NOW + 9_000)); |
| 209 | res!(check(seqs(&p) == vec![3], fmt!("after both expiries: {:?}", seqs(&p)))); |
| 210 | // The expired entry's id went with it, so its replay appends anew. |
| 211 | let a = res!(lane.append(val(2, 16), Some(b"two"), None)); |
| 212 | res!(check(a.seq == 4 && !a.replay, fmt!("a replay after expiry answered {:?}", a))); |
| 213 | let held = res!(db.get(&dat!("t/exp/e/1"), None)).is_some() |
| 214 | || res!(db.get(&dat!("t/exp/e/2"), None)).is_some(); |
| 215 | res!(check(!held, fmt!("an expired entry is still stored"))); |
| 216 | Ok(()) |
| 217 | } |
| 218 | |
| 219 | fn replayed_append_does_not_double(db: &TestDb, user: setup::Uid) -> Outcome<()> { |
| 220 | let locks = LaneLocks::default(); |
| 221 | let g = locks.lock("t/replay"); |
| 222 | let lane: TestLane = Lane::new(db, user, &g); |
| 223 | let a = res!(lane.append(val(1, 16), Some(&[0xab, 0x01]), None)); |
| 224 | let b = res!(lane.append(val(2, 16), Some(&[0xab, 0x02]), None)); |
| 225 | let r = res!(lane.append(val(1, 16), Some(&[0xab, 0x01]), None)); |
| 226 | res!(check(a.seq == 1 && b.seq == 2 && r.seq == 1 && r.replay && res!(lane.head()) == 2, |
| 227 | fmt!("appends {:?} {:?}, replay {:?}, head {}", a, b, r, res!(lane.head())))); |
| 228 | // Removed, the id is forgotten with it. |
| 229 | res!(lane.remove(1)); |
| 230 | let n = res!(lane.append(val(1, 16), Some(&[0xab, 0x01]), None)); |
| 231 | res!(check(n.seq == 3 && !n.replay, fmt!("a replay after remove answered {:?}", n))); |
| 232 | Ok(()) |
| 233 | } |
| 234 | |
| 235 | // A crash between the id write and the head write leaves A's id naming a sequence the next |
| 236 | // append, B, takes. A replay of A must not answer B's entry as its own. |
| 237 | fn replay_after_a_torn_append_is_not_held(db: &TestDb, user: setup::Uid) -> Outcome<()> { |
| 238 | let a_id: &[u8] = &[0xcd, 0x01]; |
| 239 | res!(db.insert( |
| 240 | dat!("t/torn/e/1"), |
| 241 | Dat::List(vec![Dat::U64(0), Dat::bytdat(a_id.to_vec()), val(1, 16)]), |
| 242 | user, |
| 243 | None, |
| 244 | )); |
| 245 | res!(db.insert(dat!("t/torn/i/cd01"), Dat::U64(1), user, None)); |
| 246 | let locks = LaneLocks::default(); |
| 247 | let g = locks.lock("t/torn"); |
| 248 | let lane: TestLane = Lane::new(db, user, &g); |
| 249 | let b = res!(lane.append(val(2, 16), Some(&[0xcd, 0x02]), None)); |
| 250 | let r = res!(lane.append(val(1, 16), Some(a_id), None)); |
| 251 | res!(check(b.seq == 1 && r.seq == 2 && !r.replay, |
| 252 | fmt!("after a torn append, B {:?}, the replay of A {:?}", b, r))); |
| 253 | let p = res!(lane.read_after(0, 1 << 20, usize::MAX, NOW)); |
| 254 | res!(check(seqs(&p) == vec![1, 2] && p.entries[0].val == val(2, 16) && p.entries[1].val == val(1, 16), |
| 255 | fmt!("after a torn append the lane holds {:?}", p.entries))); |
| 256 | // The id now names A's own entry, so a second replay is held. |
| 257 | let r = res!(lane.append(val(1, 16), Some(a_id), None)); |
| 258 | res!(check(r.seq == 2 && r.replay, fmt!("a second replay of A answered {:?}", r))); |
| 259 | Ok(()) |
| 260 | } |
| 261 | |
| 262 | fn cursors_and_stable(db: &TestDb, user: setup::Uid) -> Outcome<()> { |
| 263 | let locks = LaneLocks::default(); |
| 264 | let g = locks.lock("t/cur"); |
| 265 | let lane: TestLane = Lane::new(db, user, &g); |
| 266 | for i in 1..=10 { |
| 267 | res!(lane.append(val(i, 16), None, None)); |
| 268 | } |
| 269 | res!(check(res!(lane.stable()) == 10, fmt!("with no reader stable is {}", res!(lane.stable())))); |
| 270 | res!(lane.ack("phone", 4)); |
| 271 | res!(lane.ack("tab", 7)); |
| 272 | res!(lane.ack("laptop", 2)); |
| 273 | res!(check(res!(lane.stable()) == 2, fmt!("stable {} over 4, 7, 2", res!(lane.stable())))); |
| 274 | // A dormant reader leaves stable, keeping its cursor; acknowledging brings it back. |
| 275 | res!(check(res!(lane.set_dormant("laptop")), fmt!("laptop unknown"))); |
| 276 | res!(check(res!(lane.stable()) == 4, fmt!("stable {} with laptop dormant", res!(lane.stable())))); |
| 277 | let rs = res!(lane.readers()); |
| 278 | let laptop = rs.iter().find(|r| r.name == "laptop"); |
| 279 | res!(check(matches!(laptop, Some(r) if r.cursor == 2 && r.state == ReaderState::Dormant), |
| 280 | fmt!("laptop reads {:?}", laptop))); |
| 281 | res!(lane.ack("laptop", 3)); |
| 282 | res!(check(res!(lane.stable()) == 3, fmt!("stable {} with laptop back at 3", res!(lane.stable())))); |
| 283 | // A forgotten reader leaves it too. |
| 284 | res!(check(res!(lane.forget("laptop")), fmt!("laptop not forgotten"))); |
| 285 | res!(check(!res!(lane.forget("laptop")), fmt!("laptop forgotten twice"))); |
| 286 | res!(check(res!(lane.stable()) == 4, fmt!("stable {} with laptop forgotten", res!(lane.stable())))); |
| 287 | // A cursor past the head is held at it, so stable never exceeds the head. |
| 288 | res!(check(res!(lane.ack("phone", 50)) == 10, fmt!("a cursor past the head was kept"))); |
| 289 | res!(lane.ack("tab", 99)); |
| 290 | res!(check(res!(lane.stable()) == 10, fmt!("stable {} with every reader at the head", res!(lane.stable())))); |
| 291 | // All dormant: the head. |
| 292 | res!(lane.ack("phone", 1)); |
| 293 | res!(lane.set_dormant("phone")); |
| 294 | res!(lane.set_dormant("tab")); |
| 295 | res!(check(res!(lane.stable()) == 10, fmt!("stable {} with no live reader", res!(lane.stable())))); |
| 296 | res!(check(!res!(lane.set_dormant("nobody")), fmt!("an unknown reader was marked dormant"))); |
| 297 | Ok(()) |
| 298 | } |
| 299 | |
| 300 | fn trim_never_passes_the_ask(db: &TestDb, user: setup::Uid) -> Outcome<()> { |
| 301 | let locks = LaneLocks::default(); |
| 302 | let g = locks.lock("t/trim"); |
| 303 | let lane: TestLane = Lane::new(db, user, &g); |
| 304 | for i in 1..=10 { |
| 305 | res!(lane.append(val(i, 16), None, None)); |
| 306 | } |
| 307 | res!(lane.ack("a", 6)); |
| 308 | res!(lane.ack("b", 4)); |
| 309 | let s = res!(lane.stable()); |
| 310 | let f = res!(lane.trim_through(s)); |
| 311 | let p = res!(lane.read_after(0, 1 << 20, usize::MAX, NOW)); |
| 312 | res!(check(f == 4 && seqs(&p) == vec![5, 6, 7, 8, 9, 10], |
| 313 | fmt!("trim through stable {}: floor {}, left {:?}", s, f, seqs(&p)))); |
| 314 | // A reader below the floor is told of the gap; one at or above it is not. |
| 315 | res!(check(p.gap && p.floor == 4, fmt!("from 0 over floor 4: gap {} floor {}", p.gap, p.floor))); |
| 316 | let p = res!(lane.read_after(4, 1 << 20, usize::MAX, NOW)); |
| 317 | res!(check(!p.gap && p.floor == 4 && seqs(&p) == vec![5, 6, 7, 8, 9, 10], |
| 318 | fmt!("from the floor: gap {} floor {} {:?}", p.gap, p.floor, seqs(&p)))); |
| 319 | // Never lowers the floor, and never passes the head. |
| 320 | res!(check(res!(lane.trim_through(2)) == 4, fmt!("a lower trim moved the floor"))); |
| 321 | res!(check(res!(lane.trim_through(1_000)) == 10, fmt!("a trim past the head"))); |
| 322 | let p = res!(lane.read_after(0, 1 << 20, usize::MAX, NOW)); |
| 323 | res!(check(p.entries.is_empty() && res!(lane.depth()) == 0, fmt!("after trimming all: {:?}", p))); |
| 324 | let a = res!(lane.append(val(11, 16), None, None)); |
| 325 | res!(check(a.seq == 11, fmt!("after trimming all, append got {}", a.seq))); |
| 326 | Ok(()) |
| 327 | } |
| 328 | |
| 329 | fn survives_a_restart(db: &TestDb, user: setup::Uid) -> Outcome<()> { |
| 330 | let locks = LaneLocks::default(); |
| 331 | let g = locks.lock("t/cur"); |
| 332 | let lane: TestLane = Lane::new(db, user, &g); |
| 333 | res!(check(res!(lane.head()) == 10, fmt!("after a restart head {}", res!(lane.head())))); |
| 334 | let rs = res!(lane.readers()); |
| 335 | res!(check(rs.len() == 2 && rs.iter().all(|r| r.state == ReaderState::Dormant), |
| 336 | fmt!("after a restart readers {:?}", rs))); |
| 337 | let locks = LaneLocks::default(); |
| 338 | let g = locks.lock("t/hole"); |
| 339 | let lane: TestLane = Lane::new(db, user, &g); |
| 340 | let p = res!(lane.read_after(0, 1 << 20, usize::MAX, NOW)); |
| 341 | res!(check(seqs(&p) == vec![4, 5, 6, 7, 9], fmt!("after a restart holes {:?}", seqs(&p)))); |
| 342 | let locks = LaneLocks::default(); |
| 343 | let g = locks.lock("t/replay"); |
| 344 | let r = res!(TestLane::new(db, user, &g).append(val(2, 16), Some(&[0xab, 0x02]), None)); |
| 345 | res!(check(r.seq == 2 && r.replay, fmt!("after a restart a replay answered {:?}", r))); |
| 346 | Ok(()) |
| 347 | } |
| 348 | |
| 349 | // Eight threads each append the same eight ids under the store's READ lock and the lane's own |
| 350 | // lock, as a gateway's request handlers do: every id lands once. |
| 351 | fn replay_under_the_lane_lock(root: &PathBuf, user: setup::Uid) -> Outcome<()> { |
| 352 | let db = Arc::new(RwLock::new(res!(open(root, false)))); |
| 353 | let locks = Arc::new(LaneLocks::default()); |
| 354 | let mut hs = Vec::new(); |
| 355 | for t in 0..8u64 { |
| 356 | let db = db.clone(); |
| 357 | let locks = locks.clone(); |
| 358 | hs.push(thread::spawn(move || -> Outcome<Vec<u64>> { |
| 359 | let mut got = Vec::new(); |
| 360 | for k in 0..8u64 { |
| 361 | let id = ((t + k) % 8).to_be_bytes(); |
| 362 | let shared = match db.read() { |
| 363 | Ok(g) => g, |
| 364 | Err(_) => return Err(err!("The store lock is poisoned."; Test, Lock)), |
| 365 | }; |
| 366 | let g = locks.lock("t/excl"); |
| 367 | let lane: TestLane = Lane::new(&*shared, user, &g); |
| 368 | got.push(res!(lane.append(val(k, 16), Some(&id), None)).seq); |
| 369 | } |
| 370 | Ok(got) |
| 371 | })); |
| 372 | } |
| 373 | for h in hs { |
| 374 | match h.join() { |
| 375 | Ok(Ok(_)) => {}, |
| 376 | Ok(Err(e)) => return Err(e), |
| 377 | Err(_) => return Err(err!("An append thread panicked."; Test, Thread)), |
| 378 | } |
| 379 | } |
| 380 | { |
| 381 | let shared = match db.read() { |
| 382 | Ok(g) => g, |
| 383 | Err(_) => return Err(err!("The store lock is poisoned."; Test, Lock)), |
| 384 | }; |
| 385 | let g = locks.lock("t/excl"); |
| 386 | let lane: TestLane = Lane::new(&*shared, user, &g); |
| 387 | let p = res!(lane.read_after(0, 1 << 20, usize::MAX, NOW)); |
| 388 | res!(check(res!(lane.head()) == 8 && seqs(&p) == (1..=8).collect::<Vec<u64>>(), |
| 389 | fmt!("64 appends of 8 ids: head {}, entries {:?}", res!(lane.head()), seqs(&p)))); |
| 390 | } |
| 391 | res!(lanes_run_side_by_side(&db, &locks, user)); |
| 392 | drop(locks); |
| 393 | let db = match Arc::try_unwrap(db) { |
| 394 | Ok(l) => l, |
| 395 | Err(_) => return Err(err!("The store is still shared."; Test, Unexpected)), |
| 396 | }; |
| 397 | let db = match db.into_inner() { |
| 398 | Ok(d) => d, |
| 399 | Err(_) => return Err(err!("The store lock is poisoned."; Test, Lock)), |
| 400 | }; |
| 401 | res!(db.shutdown()); |
| 402 | Ok(()) |
| 403 | } |
| 404 | |
| 405 | // While one lane's lock is held, that lane cannot be taken again, and another lane appends and |
| 406 | // reads on another thread without waiting for it (C8a check F1: one request's lane section must |
| 407 | // not hold up the rest of the store). |
| 408 | fn lanes_run_side_by_side(db: &Arc<RwLock<TestDb>>, locks: &Arc<LaneLocks>, user: setup::Uid) |
| 409 | -> Outcome<()> |
| 410 | { |
| 411 | let held = locks.lock("t/side/a"); |
| 412 | res!(check(locks.try_lock("t/side/a").is_none(), fmt!("a held lane was taken again"))); |
| 413 | // Another lane on another stripe (the stripe is drawn under a random key). |
| 414 | let mut other = None; |
| 415 | for i in 0..32 { |
| 416 | let p = fmt!("t/side/b{}", i); |
| 417 | if locks.try_lock(p.clone()).is_some() { |
| 418 | other = Some(p); |
| 419 | break; |
| 420 | } |
| 421 | } |
| 422 | let other = match other { |
| 423 | Some(p) => p, |
| 424 | None => return Err(err!("No lane off the held stripe in 32 tries."; Test, Unexpected)), |
| 425 | }; |
| 426 | let (tx, rx) = std::sync::mpsc::channel(); |
| 427 | let (db2, locks2) = (db.clone(), locks.clone()); |
| 428 | let h = thread::spawn(move || { |
| 429 | let run = || -> Outcome<u64> { |
| 430 | let shared = match db2.read() { |
| 431 | Ok(g) => g, |
| 432 | Err(_) => return Err(err!("The store lock is poisoned."; Test, Lock)), |
| 433 | }; |
| 434 | let g = locks2.lock(other); |
| 435 | let lane: TestLane = Lane::new(&*shared, user, &g); |
| 436 | res!(lane.append(val(1, 16), None, None)); |
| 437 | let p = res!(lane.read_after(0, 1 << 20, usize::MAX, NOW)); |
| 438 | Ok(p.through) |
| 439 | }; |
| 440 | let _ = tx.send(run()); |
| 441 | }); |
| 442 | let got = match rx.recv_timeout(std::time::Duration::from_secs(10)) { |
| 443 | Ok(r) => r, |
| 444 | Err(_) => return Err(err!("Another lane waited 10 s on a held lane."; Test, Unexpected)), |
| 445 | }; |
| 446 | drop(held); |
| 447 | if h.join().is_err() { |
| 448 | return Err(err!("The side lane's thread panicked."; Test, Thread)); |
| 449 | } |
| 450 | let through = res!(got); |
| 451 | check(through == 1, fmt!("the side lane read through {}", through)) |
| 452 | } |
| 453 | |
| 454 | // A page stops after `max_steps` slots, holes counting, with `more` set and `through` at the |
| 455 | // last slot visited (C8a check F1: a walk over thousands of holes held the lane for seconds). |
| 456 | fn steps_bound_a_page(db: &TestDb, user: setup::Uid) -> Outcome<()> { |
| 457 | let locks = LaneLocks::default(); |
| 458 | let g = locks.lock("t/step"); |
| 459 | let lane: TestLane = Lane::new(db, user, &g); |
| 460 | for i in 1..=10u64 { |
| 461 | res!(lane.append(val(i, 16), None, None)); |
| 462 | } |
| 463 | for s in 1..=6u64 { |
| 464 | res!(lane.remove(s)); |
| 465 | } |
| 466 | let p = res!(lane.read_after(0, 1 << 20, 4, NOW)); |
| 467 | res!(check(p.entries.is_empty() && p.through == 4 && p.more, |
| 468 | fmt!("four steps over six holes: {:?}", p))); |
| 469 | let p = res!(lane.read_after(p.through, 1 << 20, 4, NOW)); |
| 470 | res!(check(seqs(&p) == vec![7, 8] && p.through == 8 && p.more, |
| 471 | fmt!("the next four steps: {:?}", p))); |
| 472 | let p = res!(lane.read_after(p.through, 1 << 20, 4, NOW)); |
| 473 | res!(check(seqs(&p) == vec![9, 10] && p.through == 10 && !p.more, |
| 474 | fmt!("the last steps: {:?}", p))); |
| 475 | let p = res!(lane.read_after(0, 1 << 20, 0, NOW)); |
| 476 | res!(check(p.entries.is_empty() && p.through == 0 && p.more, |
| 477 | fmt!("no steps: {:?}", p))); |
| 478 | Ok(()) |
| 479 | } |
| 480 | |
| 481 | // A page measured by the caller's own sizes (a wire form, say) rather than stored bytes. |
| 482 | fn measured_pages(db: &TestDb, user: setup::Uid) -> Outcome<()> { |
| 483 | let locks = LaneLocks::default(); |
| 484 | let g = locks.lock("t/meas"); |
| 485 | let lane: TestLane = Lane::new(db, user, &g); |
| 486 | for i in 1..=5u64 { |
| 487 | res!(lane.append(val(i, 16), None, None)); |
| 488 | } |
| 489 | let p = res!(lane.read_after_measured(0, 250, usize::MAX, NOW, |_| Some(100))); |
| 490 | res!(check(seqs(&p) == vec![1, 2] && p.more, fmt!("250 of 100s: {:?}", p))); |
| 491 | let p = lane.read_after_measured(0, 250, usize::MAX, NOW, |_| None); |
| 492 | check(p.is_err(), fmt!("an unmeasurable entry answered {:?}", p)) |
| 493 | } |
| 494 | |
| 495 | // A trim drops durable entries only: an ephemeral at or below the floor (a Stop, a consent) stays |
| 496 | // at its own sequence and reads to a reader below it until it expires, before and after the |
| 497 | // sweep (Daimond C8 check F1, amendment 10). |
| 498 | fn a_trim_keeps_live_ephemerals(db: &TestDb, user: setup::Uid) -> Outcome<()> { |
| 499 | let locks = LaneLocks::default(); |
| 500 | let g = locks.lock("t/keep"); |
| 501 | let lane: TestLane = Lane::new(db, user, &g); |
| 502 | for i in 1..=10u64 { |
| 503 | let exp = if i == 4 || i == 7 { Some(NOW + 60_000) } else { None }; |
| 504 | res!(lane.append(val(i, 16), None, exp)); |
| 505 | } |
| 506 | res!(check(res!(lane.trim_through(8)) == 8, fmt!("trim through 8"))); |
| 507 | // Before any sweep: the floor is up, nothing deleted, durables below it read absent. |
| 508 | let p = res!(lane.read_after(3, 1 << 20, usize::MAX, NOW)); |
| 509 | res!(check(seqs(&p) == vec![4, 7, 9, 10] && p.gap && p.through == 10, |
| 510 | fmt!("unswept, from 3: {:?}", p))); |
| 511 | res!(check(res!(lane.trim_pending(NOW)), fmt!("nothing pending after a trim"))); |
| 512 | // Swept in steps of 3: three steps reach the floor, the two ephemerals listed. |
| 513 | let mut n = 0; |
| 514 | while res!(lane.sweep_trimmed(3, NOW)) { |
| 515 | n += 1; |
| 516 | res!(check(n < 10, fmt!("the sweep never ends"))); |
| 517 | } |
| 518 | res!(check(n == 2, fmt!("swept in {} pending steps", n))); |
| 519 | let held: Vec<u64> = (1..=8u64).filter(|s| matches!( |
| 520 | db.get(&dat!(fmt!("t/keep/e/{}", s)), None), Ok(Some(_)))).collect(); |
| 521 | res!(check(held == vec![4, 7], fmt!("held below the floor after the sweep: {:?}", held))); |
| 522 | let p = res!(lane.read_after(0, 1 << 20, usize::MAX, NOW)); |
| 523 | res!(check(seqs(&p) == vec![4, 7, 9, 10] && p.gap && p.steps == 4, |
| 524 | fmt!("swept, from 0: {:?}", p))); |
| 525 | let p = res!(lane.read_after(5, 1 << 20, usize::MAX, NOW)); |
| 526 | res!(check(seqs(&p) == vec![7, 9, 10], fmt!("swept, from 5: {:?}", p))); |
| 527 | let p = res!(lane.read_after(8, 1 << 20, usize::MAX, NOW)); |
| 528 | res!(check(seqs(&p) == vec![9, 10] && !p.gap, fmt!("from the floor: {:?}", p))); |
| 529 | // A page of one entry from 0 goes on from the kept one it stopped at. |
| 530 | let p = res!(lane.read_after(0, 1, usize::MAX, NOW)); |
| 531 | res!(check(seqs(&p) == vec![4] && p.more && p.through == 4, fmt!("a one-entry page: {:?}", p))); |
| 532 | let p = res!(lane.read_after(p.through, 1, usize::MAX, NOW)); |
| 533 | res!(check(seqs(&p) == vec![7] && p.more, fmt!("the next one-entry page: {:?}", p))); |
| 534 | // Expired: unread at once, deleted by the next sweep step, which then has nothing left. |
| 535 | let later = NOW + 60_000; |
| 536 | let p = res!(lane.read_after(0, 1 << 20, usize::MAX, later)); |
| 537 | res!(check(seqs(&p) == vec![9, 10] && p.dropped == 0, fmt!("after expiry: {:?}", p))); |
| 538 | res!(check(res!(lane.trim_pending(later)), fmt!("an expired kept entry is not pending"))); |
| 539 | res!(check(!res!(lane.sweep_trimmed(8, later)), fmt!("work left after dropping the kept"))); |
| 540 | let held = (1..=8u64).any(|s| matches!(db.get(&dat!(fmt!("t/keep/e/{}", s)), None), Ok(Some(_)))); |
| 541 | res!(check(!held && !res!(lane.trim_pending(later)), fmt!("an expired kept entry is still stored"))); |
| 542 | // An expired entry above the floor is reported in `dropped`. |
| 543 | res!(lane.append(val(11, 16), None, Some(NOW + 1))); |
| 544 | let p = res!(lane.read_after(8, 1 << 20, usize::MAX, later)); |
| 545 | res!(check(seqs(&p) == vec![9, 10] && p.dropped == val(11, 16).byte_len().unwrap_or(0) as u64, |
| 546 | fmt!("dropped above the floor: {:?}", p))); |
| 547 | Ok(()) |
| 548 | } |
| 549 | |
| 550 | // Each sweep step visits at most its budget, and a reader below an unswept floor walks the |
| 551 | // slots itself, bounded by its own steps. |
| 552 | fn the_sweep_is_bounded(db: &TestDb, user: setup::Uid) -> Outcome<()> { |
| 553 | let locks = LaneLocks::default(); |
| 554 | let g = locks.lock("t/sweep"); |
| 555 | let lane: TestLane = Lane::new(db, user, &g); |
| 556 | for i in 1..=100u64 { |
| 557 | res!(lane.append(val(i, 16), Some(&i.to_be_bytes()), None)); |
| 558 | } |
| 559 | res!(lane.trim_through(100)); |
| 560 | res!(check(res!(lane.sweep_trimmed(10, NOW)), fmt!("ten of a hundred left nothing"))); |
| 561 | let held = (1..=100u64).filter(|s| matches!( |
| 562 | db.get(&dat!(fmt!("t/sweep/e/{}", s)), None), Ok(Some(_)))).count(); |
| 563 | res!(check(held == 90, fmt!("one step of 10 left {} held", held))); |
| 564 | let p = res!(lane.read_after(0, 1 << 20, 25, NOW)); |
| 565 | // The swept ten are passed at no step (nothing is kept there), then 25 slots walked. |
| 566 | res!(check(p.entries.is_empty() && p.more && p.through == 35 && p.steps == 25, |
| 567 | fmt!("a reader below the unswept floor: {:?}", p))); |
| 568 | // A replay of a trimmed, unswept entry is held; once swept it appends anew. |
| 569 | let a = res!(lane.append(val(50, 16), Some(&50u64.to_be_bytes()), None)); |
| 570 | res!(check(a.replay && a.seq == 50, fmt!("unswept replay: {:?}", a))); |
| 571 | while res!(lane.sweep_trimmed(10, NOW)) {} |
| 572 | let a = res!(lane.append(val(5, 16), Some(&5u64.to_be_bytes()), None)); |
| 573 | res!(check(!a.replay && a.seq == 101, fmt!("swept replay: {:?}", a))); |
| 574 | let p = res!(lane.read_after(0, 1 << 20, 3, NOW)); |
| 575 | res!(check(seqs(&p) == vec![101] && p.steps == 1, fmt!("swept, from 0: {:?}", p))); |
| 576 | Ok(()) |
| 577 | } |