Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/src/api/pending.rs

17.8 KiB, 125 runs

created by r1870400018:69615, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! A store in progress. A chunked value is stored in two steps, its chunks and then its bunch
2//! key, and the second waits on the first being written, readable and durable. The server bot
3//! that began the store must not sit through it, or the bots of a database hold one chunked put
4//! each and every other put queues behind them (A3 round 2a, 2026-10-09: twelve concurrent
5//! chunked puts timed out against two server bots). The store is therefore held here as a state
6//! that is advanced by one answer at a time, and the bot goes on with other requests between
7//! them. A caller that has nothing else to do runs it to the end with `run`, which is how
8//! `OzoneApi::store_dat_using_responder` waits.
9//!
10//! Since 2026-10-09 (B1, B2) every chunked write has a chunk set of its own, held in the
11//! database's pending sets from the moment its id is drawn until its head is in the cache, so
12//! the orphan sweep leaves it alone. Since 2026-10-10 (durable mark, R1) the head goes out only
13//! once a barrier covers every chunk, which may sit in another writer's files: each writer that
14//! took a chunk is asked for a barrier at once, so the wait is one fsync. A head is never
15//! durable before its chunks. The cache bot that enters the head retires the set it displaced,
16//! or the store's own when a newer record was already there, once the winner is durable; the
17//! store does not wait for it. A crash before then leaves orphans for the sweep, never a loss.
18//!
19//! A delete is a store with no chunks (fix A, 2026-10-09). Its tombstone is the head, and
20//! retires the set of the head it displaces.
21
22use crate::{
23 prelude::*,
24 base::{
25 constant,
26 index::ZoneInd,
27 },
28 comm::{
29 channels::PendingSets,
30 msg::{
31 HeadTicket,
32 OzoneMsg,
33 StoreRole,
34 },
35 response::{
36 AckWait,
37 Responder,
38 },
39 },
40 test::hooks,
41};
42
43use oxedyne_fe2o3_core::channels::{
44 self,
45 Receiver,
46 Recv,
47};
48use oxedyne_fe2o3_iop_db::api::{
49 Meta,
50 RestSchemesOverride,
51};
52use oxedyne_fe2o3_jdat::{
53 prelude::*,
54 chunk::PartKey,
55 id::NumIdDat,
56};
57
58use std::{
59 mem,
60 time::Duration,
61};
62
63// Which answers the store is waiting for.
64#[derive(Clone, Copy, Debug, Eq, PartialEq)]
65enum Step {
66 Chunks, // every chunk written, readable and durable
67 Head, // the bunch key
68}
69
70// Holds a chunk set in the database's pending sets for as long as it lives, or until it is handed
71// over with the head.
72pub(crate) struct SetHold {
73 sets: PendingSets,
74 id: Option<u64>,
75}
76
77impl SetHold {
78 pub(crate) fn new(sets: &PendingSets, id: Option<u64>) -> Outcome<Self> {
79 if let Some(id) = id {
80 let mut held = lock_mutex!(sets);
81 held.hold(id);
82 }
83 Ok(Self { sets: sets.clone(), id })
84 }
85
86 // The id, for the head to carry; whoever answers the head releases it.
87 fn hand_over(&mut self) -> Option<u64> { self.id.take() }
88
89 fn release(&mut self) {
90 if let Some(id) = self.id.take() {
91 if let Ok(mut held) = self.sets.lock() {
92 held.release(&id);
93 }
94 }
95 }
96}
97
98impl Drop for SetHold {
99 fn drop(&mut self) { self.release(); }
100}
101
102type Msgs<const UIDL: usize, UID, ENC, KH> = Vec<(OzoneMsg<UIDL, UID, ENC, KH>, ZoneInd)>;
103
104pub struct PendingStore<
105 const UIDL: usize,
106 UID: NumIdDat<UIDL>,
107 ENC: Encrypter,
108 KH: Hasher,
109> {
110 k: Dat,
111 pred: Option<(PartKey, Meta<UIDL, UID>)>, // the chunked value replaced, with its bunch key's metadata
112 set: Option<PartKey>, // the new value's own chunks
113 hold: SetHold,
114 ticketed: bool, // is the head answered with what it displaced?
115 meta: Meta<UIDL, UID>,
116 schms2: Option<RestSchemesOverride<ENC, KH>>,
117 resp: Responder<UIDL, UID, ENC, KH>, // the caller's
118 nchunks: usize, // records in all, as the caller is told
119 head: Option<(OzoneMsg<UIDL, UID, ENC, KH>, ZoneInd)>,
120 chunks: Msgs<UIDL, UID, ENC, KH>,
121 step: Step,
122 own: Responder<UIDL, UID, ENC, KH>, // answers to the step in hand
123 wait: AckWait<UIDL, UID, ENC, KH>,
124 held: Vec<OzoneMsg<UIDL, UID, ENC, KH>>, // the bunch key's answers, told last
125 promised: bool, // has the caller been told the bunch key is written?
126}
127
128impl<
129 const UIDL: usize,
130 UID: NumIdDat<UIDL> + 'static,
131 ENC: Encrypter + 'static,
132 KH: Hasher + 'static,
133>
134 PendingStore<UIDL, UID, ENC, KH>
135{
136 /// Holds the prepared write `head` and `chunks`, whose keys beyond the main key are
137 /// `datkeys`, the bunch key first, and `hold` on the new value's chunk set. Nothing is sent
138 /// until `start`. An unticketed store, the forcing path's, retires nothing.
139 pub(crate) fn new<PR: Hasher, CS: Checksummer>(
140 api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>,
141 k: Dat,
142 pred: Option<(PartKey, Meta<UIDL, UID>)>,
143 datkeys: &[Dat],
144 hold: SetHold,
145 ticketed: bool,
146 meta: Meta<UIDL, UID>,
147 schms2: Option<&RestSchemesOverride<ENC, KH>>,
148 resp: Responder<UIDL, UID, ENC, KH>,
149 head: (OzoneMsg<UIDL, UID, ENC, KH>, ZoneInd),
150 chunks: Msgs<UIDL, UID, ENC, KH>,
151 )
152 -> Outcome<Self>
153 {
154 let set = match datkeys.first() {
155 Some(Dat::Tup5u64(tup)) => Some(PartKey(*tup)),
156 Some(other) => return Err(err!(
157 "The bunch key of a prepared write of {:?} is {:?}, not a part key.", k, other;
158 Bug, Unexpected)),
159 None => None,
160 };
161 if ticketed && hold.id != set.as_ref().map(|p| p.set_id()) {
162 return Err(err!(
163 "The chunk set held for a prepared write of {:?} is {:?}, but its bunch key \
164 names {:?}.", k, hold.id, set;
165 Bug, Mismatch));
166 }
167 Ok(Self {
168 k,
169 pred,
170 set,
171 hold,
172 ticketed,
173 meta,
174 schms2: schms2.cloned(),
175 resp,
176 nchunks: chunks.len() + 1,
177 head: Some(head),
178 chunks,
179 step: Step::Chunks,
180 own: api.responder(),
181 wait: Self::wait_for(0),
182 held: Vec::new(),
183 promised: false,
184 })
185 }
186
187 fn wait_for(n: usize) -> AckWait<UIDL, UID, ENC, KH> {
188 let durability = match hooks::durability_timeout() {
189 Some(d) => d,
190 None => constant::DURABILITY_TIMEOUT,
191 };
192 AckWait::new(n, constant::USER_REQUEST_TIMEOUT, durability)
193 }
194
195 pub fn nchunks(&self) -> usize { self.nchunks }
196
197 /// The channel the answers to the step in hand arrive on.
198 pub fn rx(&self) -> Outcome<&Receiver<OzoneMsg<UIDL, UID, ENC, KH>>> {
199 match self.own.channel() {
200 Some(chan) => Ok(chan.rx()),
201 None => Err(err!("The responder of a store in progress has no channel."; Bug, Missing)),
202 }
203 }
204
205 /// The time left before the silence that fails the step in hand.
206 pub fn left(&self) -> Duration { self.wait.left() }
207
208 /// Sends the chunks, each answering to the store's own responder; the caller hears of them
209 /// when all are in. A value without chunks goes straight to its bunch key. A store tells its
210 /// caller the count of records before this, and a delete, whose caller expects one record's
211 /// answers, does not. Returns whether the store is already complete.
212 pub fn start<PR: Hasher, CS: Checksummer>(
213 &mut self,
214 api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>,
215 )
216 -> Outcome<bool>
217 {
218 let mut chunks = mem::take(&mut self.chunks);
219 if chunks.is_empty() {
220 return self.publish(api);
221 }
222 // The bunch key is not sent until the chunks are readable, which waits on the disk, and
223 // a caller counts the silence before the last record is written against a short
224 // deadline. The store answers for the bunch key's liveness itself, as its own wait for
225 // it is held to the same deadline, and the caller waits on the disk as it would have.
226 if self.resp.is_some() {
227 res!(self.resp.send(OzoneMsg::Written));
228 self.promised = true;
229 }
230 for (msg, _) in chunks.iter_mut() {
231 if let OzoneMsg::Write { resp, role, .. } = msg {
232 *resp = self.own.clone();
233 *role = Some(StoreRole::Chunk);
234 }
235 }
236 self.wait = Self::wait_for(chunks.len());
237 let took = res!(api.store_bytes(chunks));
238 res!(api.nudge(&took));
239 Ok(false)
240 }
241
242 /// Takes in one answer to the step in hand. Returns whether the store is complete, in which
243 /// case the caller has been told everything it is to be told. An error comes back only when
244 /// the caller has no responder to be told it on.
245 pub fn advance<PR: Hasher, CS: Checksummer>(
246 &mut self,
247 api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>,
248 msg: OzoneMsg<UIDL, UID, ENC, KH>,
249 )
250 -> Outcome<bool>
251 {
252 // A failure to send the next step, the bunch key among them, is the caller's to hear:
253 // left to the bot's log, a caller already told `Written` waited out the durability
254 // deadline and was then told its records were in the files (A3 QA m1, 2026-10-09).
255 match self.take_in(api, msg) {
256 Err(e) => self.fail(api, e),
257 done => done,
258 }
259 }
260
261 fn take_in<PR: Hasher, CS: Checksummer>(
262 &mut self,
263 api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>,
264 msg: OzoneMsg<UIDL, UID, ENC, KH>,
265 )
266 -> Outcome<bool>
267 {
268 // The chunks' answers are passed to the caller as they come, and not when the last is
269 // in: its deadlines measure silence, and a chunk that is written but not yet readable
270 // is not silence. The bunch key's follow from its own writer.
271 if self.step == Step::Head && !self.promised && self.resp.is_some() {
272 // A value without chunks over one that has them: its record is written, which is all
273 // the caller's short deadline asks, and what follows waits on the disk.
274 if matches!(&msg, OzoneMsg::Written) {
275 res!(self.resp.send(OzoneMsg::Written));
276 self.promised = true;
277 }
278 }
279 if self.step == Step::Chunks && self.resp.is_some() {
280 match &msg {
281 OzoneMsg::Written => res!(self.resp.send(OzoneMsg::Written)),
282 OzoneMsg::KeyExists(b) => res!(self.resp.send(OzoneMsg::KeyExists(*b))),
283 OzoneMsg::KeyChunkExists(b, i) => res!(self.resp.send(OzoneMsg::KeyChunkExists(*b, *i))),
284 _ => (),
285 }
286 }
287 match self.wait.feed(msg) {
288 Err(e) => self.fail(api, e),
289 Ok(false) => Ok(false),
290 Ok(true) => self.complete(api),
291 }
292 }
293
294 /// The silence the step in hand is allowed has run out.
295 pub fn expire<PR: Hasher, CS: Checksummer>(
296 &mut self,
297 api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>,
298 )
299 -> Outcome<bool>
300 {
301 let e = self.wait.expired();
302 self.fail(api, e)
303 }
304
305 /// Waits on the store's answers, one step after another, until it is complete.
306 pub fn run<PR: Hasher, CS: Checksummer>(
307 mut self,
308 api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>,
309 )
310 -> Outcome<()>
311 {
312 loop {
313 let left = self.left();
314 if left.is_zero() {
315 if res!(self.expire(api)) { return Ok(()); }
316 continue;
317 }
318 let got = match self.own.channel() {
319 Some(chan) => chan.recv_timeout(left),
320 None => return Err(err!(
321 "The responder of a store in progress has no channel."; Bug, Missing)),
322 };
323 match got {
324 Recv::Empty => (), // Out of time, which the next pass reports.
325 Recv::Result(Err(e)) => return Err(err!(e,
326 "{}: Waiting on the answers to a store of {:?}.", api.ozid(), self.k;
327 Channel, Read)),
328 Recv::Result(Ok(msg)) => if res!(self.advance(api, msg)) { return Ok(()); },
329 }
330 }
331 }
332
333 // The step in hand has all its answers.
334 fn complete<PR: Hasher, CS: Checksummer>(
335 &mut self,
336 api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>,
337 )
338 -> Outcome<bool>
339 {
340 let acks = mem::replace(&mut self.wait, Self::wait_for(0)).into_acks();
341 match self.step {
342 Step::Chunks => self.publish(api),
343 Step::Head => {
344 self.held = acks;
345 self.answer()
346 },
347 }
348 }
349
350 // The bunch key goes out once every chunk of the value is written, readable and durable, so
351 // that a reader finds the value it replaces, or none, until the value that names its chunks
352 // has all of them, and no restart finds the head without them. The head carries the set's
353 // hold, handed over once it is sent, and what the cache bot retires by; until then a failure
354 // here leaves the hold with the store, which releases it.
355 fn publish<PR: Hasher, CS: Checksummer>(
356 &mut self,
357 api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>,
358 )
359 -> Outcome<bool>
360 {
361 if self.set.is_some() {
362 hooks::head_hold();
363 }
364 if hooks::head_drops() {
365 return Err(err!("A store's head was dropped after its chunks (test::hooks)."; Data, Write));
366 }
367 let mut head = match self.head.take() {
368 Some(head) => head,
369 None => return Err(err!("A store in progress has no bunch key left to send."; Bug, Missing)),
370 };
371 self.own = api.responder();
372 if let (OzoneMsg::Write { resp, role: ticket, .. }, _) = &mut head {
373 *resp = self.own.clone();
374 if self.ticketed {
375 *ticket = Some(StoreRole::Head(HeadTicket {
376 set: self.hold.id,
377 own: self.set.clone(),
378 pred: self.pred.take(),
379 schms2: self.schms2.clone(),
380 tmeta: self.meta.clone(),
381 }));
382 }
383 }
384 self.step = Step::Head;
385 self.wait = Self::wait_for(1);
386 res!(api.store_bytes(vec![head]));
387 if self.ticketed {
388 self.hold.hand_over();
389 }
390 Ok(false)
391 }
392
393 // Tells the caller the bunch key is stored.
394 fn answer(&mut self) -> Outcome<bool> {
395 self.hold.release();
396 if self.resp.is_some() {
397 if !self.promised {
398 res!(self.resp.send(OzoneMsg::Written));
399 }
400 for ack in mem::take(&mut self.held) {
401 res!(self.resp.send(ack));
402 }
403 }
404 Ok(true)
405 }
406
407 // A step has failed. Over the chunks that is the end of the store, with no bunch key
408 // written; over the bunch key it is the writer's error, as it stands.
409 fn fail<PR: Hasher, CS: Checksummer>(
410 &mut self,
411 api: &OzoneApi<UIDL, UID, ENC, KH, PR, CS>,
412 e: Error<ErrTag>,
413 )
414 -> Outcome<bool>
415 {
416 self.hold.release();
417 let e = match self.step {
418 Step::Head => e,
419 // A new error, not a wrapper: the tags of a chain are all reported, and the
420 // `Unconfirmed` of a chunk that was written but not confirmed would say a write that
421 // publishes nothing may yet land.
422 Step::Chunks => if e.tags().contains(&ErrTag::Timeout) {
423 err!("{}: Storing a chunked value, its {} chunks were not all written and \
424 readable in time, so its bunch key was not written and nothing was \
425 stored. Chunks that landed wait for the orphan sweep: {}",
426 api.ozid(), self.nchunks - 1, e;
427 Write, Timeout)
428 } else {
429 err!("{}: Storing a chunked value, a chunk failed, so its bunch key was not \
430 written and nothing was stored. Chunks that landed wait for the orphan \
431 sweep: {}", api.ozid(), e;
432 Write)
433 },
434 };
435 if self.resp.is_none() {
436 return Err(e);
437 }
438 res!(self.resp.send(OzoneMsg::Error(e)));
439 Ok(true)
440 }
441}
442
443/// Waits on the channels of the stores in progress, and on `inbox`, for the next thing to do.
444/// Returns the position of the store that was answered, with the answer, or `None` for a message
445/// from `inbox`, or `None` altogether when the time ran out.
446pub fn next_event<const UIDL: usize, UID, ENC, KH>(
447 inbox: &Receiver<OzoneMsg<UIDL, UID, ENC, KH>>,
448 pending: &[PendingStore<UIDL, UID, ENC, KH>],
449)
450 -> Outcome<Event<UIDL, UID, ENC, KH>>
451where
452 UID: NumIdDat<UIDL> + 'static,
453 ENC: Encrypter + 'static,
454 KH: Hasher + 'static,
455{
456 let mut rxs = vec![inbox];
457 let mut timeout = None;
458 for p in pending {
459 rxs.push(res!(p.rx()));
460 let left = p.left();
461 timeout = Some(match timeout {
462 Some(t) if t < left => t,
463 _ => left,
464 });
465 }
466 Ok(match res!(channels::recv_any(&rxs, timeout)) {
467 Some((0, msg)) => Event::Inbox(msg),
468 Some((i, msg)) => Event::Answer(i - 1, msg),
469 None => Event::Quiet,
470 })
471}
472
473pub enum Event<const UIDL: usize, UID: NumIdDat<UIDL>, ENC: Encrypter, KH: Hasher> {
474 Inbox(OzoneMsg<UIDL, UID, ENC, KH>), // a request for the bot
475 Answer(usize, OzoneMsg<UIDL, UID, ENC, KH>), // to the store at this position
476 Quiet, // a deadline has passed, somewhere
477}