oxedyne/fe2o3/fe2o3_o3db_sync/src/api/pending.rs
17.8 KiB, 125 runs
created by r1870400018:69615, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! A store in progress. A chunked value is stored in two steps, its chunks and then its bunch |
| 2 | //! key, and the second waits on the first being written, readable and durable. The server bot |
| 3 | //! that began the store must not sit through it, or the bots of a database hold one chunked put |
| 4 | //! each and every other put queues behind them (A3 round 2a, 2026-10-09: twelve concurrent |
| 5 | //! chunked puts timed out against two server bots). The store is therefore held here as a state |
| 6 | //! that is advanced by one answer at a time, and the bot goes on with other requests between |
| 7 | //! them. A caller that has nothing else to do runs it to the end with `run`, which is how |
| 8 | //! `OzoneApi::store_dat_using_responder` waits. |
| 9 | //! |
| 10 | //! Since 2026-10-09 (B1, B2) every chunked write has a chunk set of its own, held in the |
| 11 | //! database's pending sets from the moment its id is drawn until its head is in the cache, so |
| 12 | //! the orphan sweep leaves it alone. Since 2026-10-10 (durable mark, R1) the head goes out only |
| 13 | //! once a barrier covers every chunk, which may sit in another writer's files: each writer that |
| 14 | //! took a chunk is asked for a barrier at once, so the wait is one fsync. A head is never |
| 15 | //! durable before its chunks. The cache bot that enters the head retires the set it displaced, |
| 16 | //! or the store's own when a newer record was already there, once the winner is durable; the |
| 17 | //! store does not wait for it. A crash before then leaves orphans for the sweep, never a loss. |
| 18 | //! |
| 19 | //! A delete is a store with no chunks (fix A, 2026-10-09). Its tombstone is the head, and |
| 20 | //! retires the set of the head it displaces. |
| 21 | |
| 22 | use crate::{ |
| 23 | prelude::*, |
| 24 | base::{ |
| 25 | constant, |
| 26 | index::ZoneInd, |
| 27 | }, |
| 28 | comm::{ |
| 29 | channels::PendingSets, |
| 30 | msg::{ |
| 31 | HeadTicket, |
| 32 | OzoneMsg, |
| 33 | StoreRole, |
| 34 | }, |
| 35 | response::{ |
| 36 | AckWait, |
| 37 | Responder, |
| 38 | }, |
| 39 | }, |
| 40 | test::hooks, |
| 41 | }; |
| 42 | |
| 43 | use oxedyne_fe2o3_core::channels::{ |
| 44 | self, |
| 45 | Receiver, |
| 46 | Recv, |
| 47 | }; |
| 48 | use oxedyne_fe2o3_iop_db::api::{ |
| 49 | Meta, |
| 50 | RestSchemesOverride, |
| 51 | }; |
| 52 | use oxedyne_fe2o3_jdat::{ |
| 53 | prelude::*, |
| 54 | chunk::PartKey, |
| 55 | id::NumIdDat, |
| 56 | }; |
| 57 | |
| 58 | use std::{ |
| 59 | mem, |
| 60 | time::Duration, |
| 61 | }; |
| 62 | |
| 63 | // Which answers the store is waiting for. |
| 64 | #[derive(Clone, Copy, Debug, Eq, PartialEq)] |
| 65 | enum Step { |
| 66 | Chunks, // every chunk written, readable and durable |
| 67 | Head, // the bunch key |
| 68 | } |
| 69 | |
| 70 | // Holds a chunk set in the database's pending sets for as long as it lives, or until it is handed |
| 71 | // over with the head. |
| 72 | pub(crate) struct SetHold { |
| 73 | sets: PendingSets, |
| 74 | id: Option<u64>, |
| 75 | } |
| 76 | |
| 77 | impl SetHold { |
| 78 | pub(crate) fn new(sets: &PendingSets, id: Option<u64>) -> Outcome<Self> { |
| 79 | if let Some(id) = id { |
| 80 | let mut held = lock_mutex!(sets); |
| 81 | held.hold(id); |
| 82 | } |
| 83 | Ok(Self { sets: sets.clone(), id }) |
| 84 | } |
| 85 | |
| 86 | // The id, for the head to carry; whoever answers the head releases it. |
| 87 | fn hand_over(&mut self) -> Option<u64> { self.id.take() } |
| 88 | |
| 89 | fn release(&mut self) { |
| 90 | if let Some(id) = self.id.take() { |
| 91 | if let Ok(mut held) = self.sets.lock() { |
| 92 | held.release(&id); |
| 93 | } |
| 94 | } |
| 95 | } |
| 96 | } |
| 97 | |
| 98 | impl Drop for SetHold { |
| 99 | fn drop(&mut self) { self.release(); } |
| 100 | } |
| 101 | |
| 102 | type Msgs<const UIDL: usize, UID, ENC, KH> = Vec<(OzoneMsg<UIDL, UID, ENC, KH>, ZoneInd)>; |
| 103 | |
| 104 | pub struct PendingStore< |
| 105 | const UIDL: usize, |
| 106 | UID: NumIdDat<UIDL>, |
| 107 | ENC: Encrypter, |
| 108 | KH: Hasher, |
| 109 | > { |
| 110 | k: Dat, |
| 111 | pred: Option<(PartKey, Meta<UIDL, UID>)>, // the chunked value replaced, with its bunch key's metadata |
| 112 | set: Option<PartKey>, // the new value's own chunks |
| 113 | hold: SetHold, |
| 114 | ticketed: bool, // is the head answered with what it displaced? |
| 115 | meta: Meta<UIDL, UID>, |
| 116 | schms2: Option<RestSchemesOverride<ENC, KH>>, |
| 117 | resp: Responder<UIDL, UID, ENC, KH>, // the caller's |
| 118 | nchunks: usize, // records in all, as the caller is told |
| 119 | head: Option<(OzoneMsg<UIDL, UID, ENC, KH>, ZoneInd)>, |
| 120 | chunks: Msgs<UIDL, UID, ENC, KH>, |
| 121 | step: Step, |
| 122 | own: Responder<UIDL, UID, ENC, KH>, // answers to the step in hand |
| 123 | wait: AckWait<UIDL, UID, ENC, KH>, |
| 124 | held: Vec<OzoneMsg<UIDL, UID, ENC, KH>>, // the bunch key's answers, told last |
| 125 | promised: bool, // has the caller been told the bunch key is written? |
| 126 | } |
| 127 | |
| 128 | impl< |
| 129 | const UIDL: usize, |
| 130 | UID: NumIdDat<UIDL> + 'static, |
| 131 | ENC: Encrypter + 'static, |
| 132 | KH: Hasher + 'static, |
| 133 | > |
| 134 | PendingStore<UIDL, UID, ENC, KH> |
| 135 | { |
| 136 | /// Holds the prepared write `head` and `chunks`, whose keys beyond the main key are |
| 137 | /// `datkeys`, the bunch key first, and `hold` on the new value's chunk set. Nothing is sent |
| 138 | /// until `start`. An unticketed store, the forcing path's, retires nothing. |
| 139 | pub(crate) fn new<PR: Hasher, CS: Checksummer>( |
| 140 | api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>, |
| 141 | k: Dat, |
| 142 | pred: Option<(PartKey, Meta<UIDL, UID>)>, |
| 143 | datkeys: &[Dat], |
| 144 | hold: SetHold, |
| 145 | ticketed: bool, |
| 146 | meta: Meta<UIDL, UID>, |
| 147 | schms2: Option<&RestSchemesOverride<ENC, KH>>, |
| 148 | resp: Responder<UIDL, UID, ENC, KH>, |
| 149 | head: (OzoneMsg<UIDL, UID, ENC, KH>, ZoneInd), |
| 150 | chunks: Msgs<UIDL, UID, ENC, KH>, |
| 151 | ) |
| 152 | -> Outcome<Self> |
| 153 | { |
| 154 | let set = match datkeys.first() { |
| 155 | Some(Dat::Tup5u64(tup)) => Some(PartKey(*tup)), |
| 156 | Some(other) => return Err(err!( |
| 157 | "The bunch key of a prepared write of {:?} is {:?}, not a part key.", k, other; |
| 158 | Bug, Unexpected)), |
| 159 | None => None, |
| 160 | }; |
| 161 | if ticketed && hold.id != set.as_ref().map(|p| p.set_id()) { |
| 162 | return Err(err!( |
| 163 | "The chunk set held for a prepared write of {:?} is {:?}, but its bunch key \ |
| 164 | names {:?}.", k, hold.id, set; |
| 165 | Bug, Mismatch)); |
| 166 | } |
| 167 | Ok(Self { |
| 168 | k, |
| 169 | pred, |
| 170 | set, |
| 171 | hold, |
| 172 | ticketed, |
| 173 | meta, |
| 174 | schms2: schms2.cloned(), |
| 175 | resp, |
| 176 | nchunks: chunks.len() + 1, |
| 177 | head: Some(head), |
| 178 | chunks, |
| 179 | step: Step::Chunks, |
| 180 | own: api.responder(), |
| 181 | wait: Self::wait_for(0), |
| 182 | held: Vec::new(), |
| 183 | promised: false, |
| 184 | }) |
| 185 | } |
| 186 | |
| 187 | fn wait_for(n: usize) -> AckWait<UIDL, UID, ENC, KH> { |
| 188 | let durability = match hooks::durability_timeout() { |
| 189 | Some(d) => d, |
| 190 | None => constant::DURABILITY_TIMEOUT, |
| 191 | }; |
| 192 | AckWait::new(n, constant::USER_REQUEST_TIMEOUT, durability) |
| 193 | } |
| 194 | |
| 195 | pub fn nchunks(&self) -> usize { self.nchunks } |
| 196 | |
| 197 | /// The channel the answers to the step in hand arrive on. |
| 198 | pub fn rx(&self) -> Outcome<&Receiver<OzoneMsg<UIDL, UID, ENC, KH>>> { |
| 199 | match self.own.channel() { |
| 200 | Some(chan) => Ok(chan.rx()), |
| 201 | None => Err(err!("The responder of a store in progress has no channel."; Bug, Missing)), |
| 202 | } |
| 203 | } |
| 204 | |
| 205 | /// The time left before the silence that fails the step in hand. |
| 206 | pub fn left(&self) -> Duration { self.wait.left() } |
| 207 | |
| 208 | /// Sends the chunks, each answering to the store's own responder; the caller hears of them |
| 209 | /// when all are in. A value without chunks goes straight to its bunch key. A store tells its |
| 210 | /// caller the count of records before this, and a delete, whose caller expects one record's |
| 211 | /// answers, does not. Returns whether the store is already complete. |
| 212 | pub fn start<PR: Hasher, CS: Checksummer>( |
| 213 | &mut self, |
| 214 | api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>, |
| 215 | ) |
| 216 | -> Outcome<bool> |
| 217 | { |
| 218 | let mut chunks = mem::take(&mut self.chunks); |
| 219 | if chunks.is_empty() { |
| 220 | return self.publish(api); |
| 221 | } |
| 222 | // The bunch key is not sent until the chunks are readable, which waits on the disk, and |
| 223 | // a caller counts the silence before the last record is written against a short |
| 224 | // deadline. The store answers for the bunch key's liveness itself, as its own wait for |
| 225 | // it is held to the same deadline, and the caller waits on the disk as it would have. |
| 226 | if self.resp.is_some() { |
| 227 | res!(self.resp.send(OzoneMsg::Written)); |
| 228 | self.promised = true; |
| 229 | } |
| 230 | for (msg, _) in chunks.iter_mut() { |
| 231 | if let OzoneMsg::Write { resp, role, .. } = msg { |
| 232 | *resp = self.own.clone(); |
| 233 | *role = Some(StoreRole::Chunk); |
| 234 | } |
| 235 | } |
| 236 | self.wait = Self::wait_for(chunks.len()); |
| 237 | let took = res!(api.store_bytes(chunks)); |
| 238 | res!(api.nudge(&took)); |
| 239 | Ok(false) |
| 240 | } |
| 241 | |
| 242 | /// Takes in one answer to the step in hand. Returns whether the store is complete, in which |
| 243 | /// case the caller has been told everything it is to be told. An error comes back only when |
| 244 | /// the caller has no responder to be told it on. |
| 245 | pub fn advance<PR: Hasher, CS: Checksummer>( |
| 246 | &mut self, |
| 247 | api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>, |
| 248 | msg: OzoneMsg<UIDL, UID, ENC, KH>, |
| 249 | ) |
| 250 | -> Outcome<bool> |
| 251 | { |
| 252 | // A failure to send the next step, the bunch key among them, is the caller's to hear: |
| 253 | // left to the bot's log, a caller already told `Written` waited out the durability |
| 254 | // deadline and was then told its records were in the files (A3 QA m1, 2026-10-09). |
| 255 | match self.take_in(api, msg) { |
| 256 | Err(e) => self.fail(api, e), |
| 257 | done => done, |
| 258 | } |
| 259 | } |
| 260 | |
| 261 | fn take_in<PR: Hasher, CS: Checksummer>( |
| 262 | &mut self, |
| 263 | api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>, |
| 264 | msg: OzoneMsg<UIDL, UID, ENC, KH>, |
| 265 | ) |
| 266 | -> Outcome<bool> |
| 267 | { |
| 268 | // The chunks' answers are passed to the caller as they come, and not when the last is |
| 269 | // in: its deadlines measure silence, and a chunk that is written but not yet readable |
| 270 | // is not silence. The bunch key's follow from its own writer. |
| 271 | if self.step == Step::Head && !self.promised && self.resp.is_some() { |
| 272 | // A value without chunks over one that has them: its record is written, which is all |
| 273 | // the caller's short deadline asks, and what follows waits on the disk. |
| 274 | if matches!(&msg, OzoneMsg::Written) { |
| 275 | res!(self.resp.send(OzoneMsg::Written)); |
| 276 | self.promised = true; |
| 277 | } |
| 278 | } |
| 279 | if self.step == Step::Chunks && self.resp.is_some() { |
| 280 | match &msg { |
| 281 | OzoneMsg::Written => res!(self.resp.send(OzoneMsg::Written)), |
| 282 | OzoneMsg::KeyExists(b) => res!(self.resp.send(OzoneMsg::KeyExists(*b))), |
| 283 | OzoneMsg::KeyChunkExists(b, i) => res!(self.resp.send(OzoneMsg::KeyChunkExists(*b, *i))), |
| 284 | _ => (), |
| 285 | } |
| 286 | } |
| 287 | match self.wait.feed(msg) { |
| 288 | Err(e) => self.fail(api, e), |
| 289 | Ok(false) => Ok(false), |
| 290 | Ok(true) => self.complete(api), |
| 291 | } |
| 292 | } |
| 293 | |
| 294 | /// The silence the step in hand is allowed has run out. |
| 295 | pub fn expire<PR: Hasher, CS: Checksummer>( |
| 296 | &mut self, |
| 297 | api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>, |
| 298 | ) |
| 299 | -> Outcome<bool> |
| 300 | { |
| 301 | let e = self.wait.expired(); |
| 302 | self.fail(api, e) |
| 303 | } |
| 304 | |
| 305 | /// Waits on the store's answers, one step after another, until it is complete. |
| 306 | pub fn run<PR: Hasher, CS: Checksummer>( |
| 307 | mut self, |
| 308 | api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>, |
| 309 | ) |
| 310 | -> Outcome<()> |
| 311 | { |
| 312 | loop { |
| 313 | let left = self.left(); |
| 314 | if left.is_zero() { |
| 315 | if res!(self.expire(api)) { return Ok(()); } |
| 316 | continue; |
| 317 | } |
| 318 | let got = match self.own.channel() { |
| 319 | Some(chan) => chan.recv_timeout(left), |
| 320 | None => return Err(err!( |
| 321 | "The responder of a store in progress has no channel."; Bug, Missing)), |
| 322 | }; |
| 323 | match got { |
| 324 | Recv::Empty => (), // Out of time, which the next pass reports. |
| 325 | Recv::Result(Err(e)) => return Err(err!(e, |
| 326 | "{}: Waiting on the answers to a store of {:?}.", api.ozid(), self.k; |
| 327 | Channel, Read)), |
| 328 | Recv::Result(Ok(msg)) => if res!(self.advance(api, msg)) { return Ok(()); }, |
| 329 | } |
| 330 | } |
| 331 | } |
| 332 | |
| 333 | // The step in hand has all its answers. |
| 334 | fn complete<PR: Hasher, CS: Checksummer>( |
| 335 | &mut self, |
| 336 | api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>, |
| 337 | ) |
| 338 | -> Outcome<bool> |
| 339 | { |
| 340 | let acks = mem::replace(&mut self.wait, Self::wait_for(0)).into_acks(); |
| 341 | match self.step { |
| 342 | Step::Chunks => self.publish(api), |
| 343 | Step::Head => { |
| 344 | self.held = acks; |
| 345 | self.answer() |
| 346 | }, |
| 347 | } |
| 348 | } |
| 349 | |
| 350 | // The bunch key goes out once every chunk of the value is written, readable and durable, so |
| 351 | // that a reader finds the value it replaces, or none, until the value that names its chunks |
| 352 | // has all of them, and no restart finds the head without them. The head carries the set's |
| 353 | // hold, handed over once it is sent, and what the cache bot retires by; until then a failure |
| 354 | // here leaves the hold with the store, which releases it. |
| 355 | fn publish<PR: Hasher, CS: Checksummer>( |
| 356 | &mut self, |
| 357 | api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>, |
| 358 | ) |
| 359 | -> Outcome<bool> |
| 360 | { |
| 361 | if self.set.is_some() { |
| 362 | hooks::head_hold(); |
| 363 | } |
| 364 | if hooks::head_drops() { |
| 365 | return Err(err!("A store's head was dropped after its chunks (test::hooks)."; Data, Write)); |
| 366 | } |
| 367 | let mut head = match self.head.take() { |
| 368 | Some(head) => head, |
| 369 | None => return Err(err!("A store in progress has no bunch key left to send."; Bug, Missing)), |
| 370 | }; |
| 371 | self.own = api.responder(); |
| 372 | if let (OzoneMsg::Write { resp, role: ticket, .. }, _) = &mut head { |
| 373 | *resp = self.own.clone(); |
| 374 | if self.ticketed { |
| 375 | *ticket = Some(StoreRole::Head(HeadTicket { |
| 376 | set: self.hold.id, |
| 377 | own: self.set.clone(), |
| 378 | pred: self.pred.take(), |
| 379 | schms2: self.schms2.clone(), |
| 380 | tmeta: self.meta.clone(), |
| 381 | })); |
| 382 | } |
| 383 | } |
| 384 | self.step = Step::Head; |
| 385 | self.wait = Self::wait_for(1); |
| 386 | res!(api.store_bytes(vec![head])); |
| 387 | if self.ticketed { |
| 388 | self.hold.hand_over(); |
| 389 | } |
| 390 | Ok(false) |
| 391 | } |
| 392 | |
| 393 | // Tells the caller the bunch key is stored. |
| 394 | fn answer(&mut self) -> Outcome<bool> { |
| 395 | self.hold.release(); |
| 396 | if self.resp.is_some() { |
| 397 | if !self.promised { |
| 398 | res!(self.resp.send(OzoneMsg::Written)); |
| 399 | } |
| 400 | for ack in mem::take(&mut self.held) { |
| 401 | res!(self.resp.send(ack)); |
| 402 | } |
| 403 | } |
| 404 | Ok(true) |
| 405 | } |
| 406 | |
| 407 | // A step has failed. Over the chunks that is the end of the store, with no bunch key |
| 408 | // written; over the bunch key it is the writer's error, as it stands. |
| 409 | fn fail<PR: Hasher, CS: Checksummer>( |
| 410 | &mut self, |
| 411 | api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>, |
| 412 | e: Error<ErrTag>, |
| 413 | ) |
| 414 | -> Outcome<bool> |
| 415 | { |
| 416 | self.hold.release(); |
| 417 | let e = match self.step { |
| 418 | Step::Head => e, |
| 419 | // A new error, not a wrapper: the tags of a chain are all reported, and the |
| 420 | // `Unconfirmed` of a chunk that was written but not confirmed would say a write that |
| 421 | // publishes nothing may yet land. |
| 422 | Step::Chunks => if e.tags().contains(&ErrTag::Timeout) { |
| 423 | err!("{}: Storing a chunked value, its {} chunks were not all written and \ |
| 424 | readable in time, so its bunch key was not written and nothing was \ |
| 425 | stored. Chunks that landed wait for the orphan sweep: {}", |
| 426 | api.ozid(), self.nchunks - 1, e; |
| 427 | Write, Timeout) |
| 428 | } else { |
| 429 | err!("{}: Storing a chunked value, a chunk failed, so its bunch key was not \ |
| 430 | written and nothing was stored. Chunks that landed wait for the orphan \ |
| 431 | sweep: {}", api.ozid(), e; |
| 432 | Write) |
| 433 | }, |
| 434 | }; |
| 435 | if self.resp.is_none() { |
| 436 | return Err(e); |
| 437 | } |
| 438 | res!(self.resp.send(OzoneMsg::Error(e))); |
| 439 | Ok(true) |
| 440 | } |
| 441 | } |
| 442 | |
| 443 | /// Waits on the channels of the stores in progress, and on `inbox`, for the next thing to do. |
| 444 | /// Returns the position of the store that was answered, with the answer, or `None` for a message |
| 445 | /// from `inbox`, or `None` altogether when the time ran out. |
| 446 | pub fn next_event<const UIDL: usize, UID, ENC, KH>( |
| 447 | inbox: &Receiver<OzoneMsg<UIDL, UID, ENC, KH>>, |
| 448 | pending: &[PendingStore<UIDL, UID, ENC, KH>], |
| 449 | ) |
| 450 | -> Outcome<Event<UIDL, UID, ENC, KH>> |
| 451 | where |
| 452 | UID: NumIdDat<UIDL> + 'static, |
| 453 | ENC: Encrypter + 'static, |
| 454 | KH: Hasher + 'static, |
| 455 | { |
| 456 | let mut rxs = vec![inbox]; |
| 457 | let mut timeout = None; |
| 458 | for p in pending { |
| 459 | rxs.push(res!(p.rx())); |
| 460 | let left = p.left(); |
| 461 | timeout = Some(match timeout { |
| 462 | Some(t) if t < left => t, |
| 463 | _ => left, |
| 464 | }); |
| 465 | } |
| 466 | Ok(match res!(channels::recv_any(&rxs, timeout)) { |
| 467 | Some((0, msg)) => Event::Inbox(msg), |
| 468 | Some((i, msg)) => Event::Answer(i - 1, msg), |
| 469 | None => Event::Quiet, |
| 470 | }) |
| 471 | } |
| 472 | |
| 473 | pub enum Event<const UIDL: usize, UID: NumIdDat<UIDL>, ENC: Encrypter, KH: Hasher> { |
| 474 | Inbox(OzoneMsg<UIDL, UID, ENC, KH>), // a request for the bot |
| 475 | Answer(usize, OzoneMsg<UIDL, UID, ENC, KH>), // to the store at this position |
| 476 | Quiet, // a deadline has passed, somewhere |
| 477 | } |