oxedyne/fe2o3/fe2o3_o3db_sync/tests/chunk_delete.rs
9.4 KiB, 37 runs
created by r1870400018:67595, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! A delete of a chunked value waits for the tombstones of its chunks and fails if one fails |
| 2 | //! (A3 R2, 2026-10-08). The chunk tombstones were sent with no responder, so `delete` returned on |
| 3 | //! the bunch key's answer alone, with the chunks possibly unwritten and a failure only logged. |
| 4 | //! Since fix A (2026-10-09) a delete is a store with no chunks: the chunks are retired only once |
| 5 | //! the head tombstone is durable, and a chunk tombstone that fails after that is left to the |
| 6 | //! orphan sweep rather than failing the delete. Since the durable mark (2026-10-10) the cache bot |
| 7 | //! sends the chunk tombstones once a barrier covers the head tombstone, and the delete does not |
| 8 | //! wait for them. |
| 9 | //! The hooks used are process-wide, so the tests here take a lock and put each back. |
| 10 | |
| 11 | mod gc_pair; |
| 12 | |
| 13 | use gc_pair::*; |
| 14 | |
| 15 | use oxedyne_fe2o3_core::prelude::*; |
| 16 | use oxedyne_fe2o3_iop_db::api::Database; |
| 17 | use oxedyne_fe2o3_jdat::{ |
| 18 | prelude::*, |
| 19 | chunk::PartKey, |
| 20 | }; |
| 21 | use oxedyne_fe2o3_o3db_sync::{ |
| 22 | base::cfg::OzoneConfig, |
| 23 | test::{ |
| 24 | hooks, |
| 25 | setup, |
| 26 | }, |
| 27 | }; |
| 28 | |
| 29 | use std::{ |
| 30 | collections::BTreeMap, |
| 31 | path::Path, |
| 32 | sync::Mutex, |
| 33 | time::{ |
| 34 | Duration, |
| 35 | Instant, |
| 36 | }, |
| 37 | }; |
| 38 | |
| 39 | static HOOKS: Mutex<()> = Mutex::new(()); |
| 40 | |
| 41 | const CHUNKS: u64 = 3; |
| 42 | |
| 43 | fn cfg() -> Outcome<OzoneConfig> { |
| 44 | let mut c = res!(setup::default_cfg()); |
| 45 | c.num_zones = 1; |
| 46 | c.num_cbots_per_zone = 4; |
| 47 | c.num_fbots_per_zone = 1; |
| 48 | c.num_wbots_per_zone = 2; |
| 49 | c.zone_overrides = BTreeMap::new(); |
| 50 | c.rest_chunk_threshold = 1_000; |
| 51 | c.rest_chunk_bytes = 400; |
| 52 | c.sync_on_write = true; |
| 53 | Ok(c) |
| 54 | } |
| 55 | |
| 56 | fn big(seed: u8) -> Dat { |
| 57 | Dat::BU32((0..1_100usize).map(|j| seed ^ (j as u8)).collect()) |
| 58 | } |
| 59 | |
| 60 | fn open(dir: &str) -> Outcome<TestDb> { |
| 61 | let _ = std::fs::remove_dir_all(dir); |
| 62 | res!(std::fs::create_dir_all(dir)); |
| 63 | let root = res!(Path::new(dir).canonicalize()); |
| 64 | setup::start_db(root, Some(res!(cfg())), schemes(), None, false, true) |
| 65 | } |
| 66 | |
| 67 | /// The chunk keys of the value held under `k`, read from its bunch key. |
| 68 | fn chunk_keys(db: &TestDb, k: &Dat) -> Outcome<Vec<Dat>> { |
| 69 | let resp = res!(db.api().fetch_using_schemes(k, None)); |
| 70 | let pkey = match res!(resp.recv_daticle(db.api().schemes().encrypter(), None)) { |
| 71 | (Some((Dat::Tup5u64(tup), _)), _) => PartKey(tup), |
| 72 | other => return Err(err!("Key {:?} holds {:?}, not a bunch key.", k, other; Test, Invalid)), |
| 73 | }; |
| 74 | assert_eq!(pkey.num_parts(), CHUNKS, "the value did not split into {} chunks", CHUNKS); |
| 75 | Ok((1..(pkey.num_parts() + 1)).map(|i| Dat::Tup5u64([ |
| 76 | pkey.set_id(), i, pkey.data_len(), pkey.num_parts(), pkey.part_size(), |
| 77 | ])).collect()) |
| 78 | } |
| 79 | |
| 80 | /// Does the newest record at this chunk key retire the chunk? |
| 81 | fn retired(db: &TestDb, ck: &Dat) -> Outcome<bool> { |
| 82 | let resp = res!(db.api().fetch_using_schemes(ck, None)); |
| 83 | let (found, _) = res!(resp.recv_daticle(db.api().schemes().encrypter(), None)); |
| 84 | Ok(found.is_none()) |
| 85 | } |
| 86 | |
| 87 | fn cbot_of(db: &TestDb, k: &Dat) -> Outcome<usize> { |
| 88 | let (_, wind, _) = res!(db.api().ozone_key_dat(k, None)); |
| 89 | Ok(**wind.bpind()) |
| 90 | } |
| 91 | |
| 92 | #[test] |
| 93 | fn chunked_delete_retires_its_chunks_after_it_returns() -> Outcome<()> { |
| 94 | log_set_level!("error"); |
| 95 | let _lock = match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() }; |
| 96 | let db = res!(open("./test_db_chunk_delete_wait")); |
| 97 | let user = setup::Uid::default(); |
| 98 | |
| 99 | // A key whose bunch key shares no cache bot with any of its chunks, so that the bunch key's |
| 100 | // answer is not queued behind the held chunk tombstones. |
| 101 | let mut found = None; |
| 102 | for n in 0..200 { |
| 103 | let k = dat!(fmt!("chunk delete wait {:03}", n)); |
| 104 | res!(db.insert(k.clone(), big(n as u8), user, None)); |
| 105 | let cks = res!(chunk_keys(&db, &k)); |
| 106 | let bunch = res!(cbot_of(&db, &k)); |
| 107 | let mut apart = true; |
| 108 | for ck in &cks { |
| 109 | if res!(cbot_of(&db, ck)) == bunch { apart = false; } |
| 110 | } |
| 111 | if apart { found = Some((k, cks)); break; } |
| 112 | res!(db.delete(&k, user, None)); |
| 113 | } |
| 114 | let (k, cks) = match found { |
| 115 | Some(f) => f, |
| 116 | None => return Err(err!("No key kept its bunch key apart from its chunks."; Test, Missing)), |
| 117 | }; |
| 118 | for ck in &cks { |
| 119 | assert!(!res!(retired(&db, ck)), "positive control: chunk {:?} is not live before the delete", ck); |
| 120 | } |
| 121 | |
| 122 | hooks::set_chunk_tombstone_delay(Duration::from_millis(300)); |
| 123 | let t0 = Instant::now(); |
| 124 | let existed = res!(db.delete(&k, user, None)); |
| 125 | let took = t0.elapsed(); |
| 126 | assert!(existed, "the delete did not find the value"); |
| 127 | msg!("The delete returned after {:?}.", took); |
| 128 | // The head tombstone is answered once it is entered; its chunks' tombstones follow it from |
| 129 | // the cache bot, after its barrier, and no one waits on them. |
| 130 | assert!(took < Duration::from_millis(300), "delete waited {:?} for its chunk tombstones", took); |
| 131 | let gone = wait_until(Duration::from_secs(10), || cks.iter().all(|ck| matches!(retired(&db, ck), Ok(true)))); |
| 132 | hooks::set_chunk_tombstone_delay(Duration::ZERO); |
| 133 | assert!(gone, "the chunks of the deleted value were never retired"); |
| 134 | res!(db.close()); |
| 135 | Ok(()) |
| 136 | } |
| 137 | |
| 138 | // A delete whose head tombstone is not confirmed retires nothing: the value's chunks stay live, |
| 139 | // so a bunch key that did not land still reads whole. Every sync fails, so no tombstone is |
| 140 | // confirmed; the old order sent the chunks' beside the head's and took them all. |
| 141 | #[test] |
| 142 | fn head_tombstone_failure_retires_nothing() -> Outcome<()> { |
| 143 | log_set_level!("error"); |
| 144 | let _lock = match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() }; |
| 145 | let db = res!(open("./test_db_chunk_delete_fail")); |
| 146 | let user = setup::Uid::default(); |
| 147 | let k = dat!("chunk delete fail"); |
| 148 | res!(db.insert(k.clone(), big(9), user, None)); |
| 149 | let cks = res!(chunk_keys(&db, &k)); |
| 150 | |
| 151 | hooks::set_barrier_failure(true); |
| 152 | let result = db.delete(&k, user, None); |
| 153 | hooks::set_barrier_failure(false); |
| 154 | if let Ok(b) = result { |
| 155 | return Err(err!("A delete whose head tombstone failed returned Ok({}).", b; Test, Invalid)); |
| 156 | } |
| 157 | for (i, ck) in cks.iter().enumerate() { |
| 158 | assert!(!res!(retired(&db, ck)), "chunk {} was retired although the head tombstone failed", i + 1); |
| 159 | } |
| 160 | res!(db.close()); |
| 161 | Ok(()) |
| 162 | } |
| 163 | |
| 164 | // A delete removes what existed when it began. Its tombstones are stamped once, at the start: |
| 165 | // stamped as each is sent, an overwrite of the same length begun while the chunk tombstones are |
| 166 | // becoming durable has chunks newer than theirs and a bunch key older than the bunch tombstone, |
| 167 | // so the delete takes the value and leaves its chunks live for the orphan sweep (QA A2-6). |
| 168 | #[test] |
| 169 | fn delete_does_not_erase_a_later_overwrite() -> Outcome<()> { |
| 170 | log_set_level!("error"); |
| 171 | let _lock = match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() }; |
| 172 | let db = res!(open("./test_db_chunk_delete_race")); |
| 173 | let user = setup::Uid::default(); |
| 174 | let k = dat!("chunk delete race"); |
| 175 | res!(db.insert(k.clone(), big(1), user, None)); |
| 176 | let _ = res!(chunk_keys(&db, &k)); |
| 177 | |
| 178 | hooks::set_barrier_delay(Duration::from_millis(1_000)); |
| 179 | let del = { |
| 180 | let (db, k) = (db.clone(), k.clone()); |
| 181 | std::thread::spawn(move || db.delete(&k, user, None)) |
| 182 | }; |
| 183 | std::thread::sleep(Duration::from_millis(300)); |
| 184 | let put = { |
| 185 | let (db, k) = (db.clone(), k.clone()); |
| 186 | std::thread::spawn(move || db.insert(k, big(2), user, None).map(|_| ())) |
| 187 | }; |
| 188 | let del = del.join(); |
| 189 | let put = put.join(); |
| 190 | hooks::set_barrier_delay(Duration::ZERO); |
| 191 | match (del, put) { |
| 192 | (Ok(a), Ok(b)) => { res!(a); res!(b); }, |
| 193 | _ => return Err(err!("A thread panicked."; Test, Invalid)), |
| 194 | } |
| 195 | match db.get(&k, None) { |
| 196 | Ok(Some((v, _))) => assert_eq!(v, big(2), "the overwrite reads back changed"), |
| 197 | other => { |
| 198 | let t: String = fmt!("{:?}", other).chars().take(300).collect(); |
| 199 | panic!("a value begun after the delete began does not read back whole: {}", t); |
| 200 | }, |
| 201 | } |
| 202 | res!(db.close()); |
| 203 | Ok(()) |
| 204 | } |
| 205 | |
| 206 | // A chunked delete sends its head tombstone, and the cache bot sends the tombstones of the chunks only |
| 207 | // once a barrier covers it: sent together, the chunks' could land first and a reader find a chunk |
| 208 | // gone under a current bunch key (fix A). Counted from the order of the steps, not timed. |
| 209 | #[test] |
| 210 | fn chunked_delete_retires_after_its_head_is_durable() -> Outcome<()> { |
| 211 | log_set_level!("error"); |
| 212 | let _lock = match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() }; |
| 213 | let db = res!(open("./test_db_chunk_delete_round")); |
| 214 | let user = setup::Uid::default(); |
| 215 | let k = dat!("chunk delete round"); |
| 216 | res!(db.insert(k.clone(), big(3), user, None)); |
| 217 | let cks = res!(chunk_keys(&db, &k)); |
| 218 | |
| 219 | hooks::set_trace(true); |
| 220 | let existed = res!(db.delete(&k, user, None)); |
| 221 | let all = wait_until(Duration::from_secs(10), |
| 222 | || hooks::trace_steps().iter().filter(|s| **s == hooks::Step::Sent).count() > cks.len()); |
| 223 | let steps = hooks::trace_steps(); |
| 224 | hooks::set_trace(false); |
| 225 | assert!(existed, "the delete did not find the value"); |
| 226 | assert!(all, "expected a tombstone for each chunk and the bunch key: {:?}", steps); |
| 227 | let mut want = vec![hooks::Step::Sent, hooks::Step::Durable]; |
| 228 | want.extend(cks.iter().map(|_| hooks::Step::Sent)); |
| 229 | assert_eq!(steps, want, "the chunks were not retired after the head tombstone was durable"); |
| 230 | let gone = wait_until(Duration::from_secs(10), || cks.iter().all(|ck| matches!(retired(&db, ck), Ok(true)))); |
| 231 | assert!(gone, "a chunk of the deleted value is still live"); |
| 232 | res!(db.close()); |
| 233 | Ok(()) |
| 234 | } |