Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/chunk_publish.rs

11.8 KiB, 1 run

created by r1870400018:69617, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! A chunked value's bunch key is published after its chunks (A3 round 2a, 2026-10-09). The
2//! records of a store go to writers that make them readable one at a time, each when its own disk
3//! barrier completes, so a bunch key sent beside its chunks was readable for as long as the
4//! slowest chunk's fsync took: a reader found a key whose chunks were "not found". The chunks go
5//! first now and the bunch key follows once every chunk is written and readable. The hooks used
6//! are process-wide, so every test takes one lock and puts each hook back.
7
8mod gc_pair;
9
10use gc_pair::*;
11
12use oxedyne_fe2o3_core::{
13 channels::Recv,
14 prelude::*,
15};
16use oxedyne_fe2o3_iop_db::api::Database;
17use oxedyne_fe2o3_jdat::prelude::*;
18use oxedyne_fe2o3_o3db_sync::{
19 base::cfg::OzoneConfig,
20 comm::msg::OzoneMsg,
21 test::{
22 hooks,
23 setup::{
24 self,
25 Uid,
26 },
27 },
28};
29
30use std::{
31 collections::BTreeMap,
32 path::Path,
33 sync::{
34 Arc,
35 Mutex,
36 MutexGuard,
37 atomic::{
38 AtomicBool,
39 Ordering,
40 },
41 },
42 thread,
43 time::{
44 Duration,
45 Instant,
46 },
47};
48
49static HOOKS: Mutex<()> = Mutex::new(());
50
51fn lock() -> MutexGuard<'static, ()> {
52 match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() }
53}
54
55const HOLD: Duration = Duration::from_millis(250); // a slow chunk
56const VALUES: usize = 6; // stored in turn
57
58fn cfg(cbots: u16) -> Outcome<OzoneConfig> {
59 let mut c = res!(setup::default_cfg());
60 c.num_zones = 1;
61 c.num_cbots_per_zone = cbots; // so that a bunch key often has a cache bot to itself
62 c.num_fbots_per_zone = 1;
63 c.num_wbots_per_zone = 2;
64 c.zone_overrides = BTreeMap::new();
65 c.rest_chunk_threshold = 1_000;
66 c.rest_chunk_bytes = 400;
67 c.sync_on_write = true;
68 Ok(c)
69}
70
71fn big(seed: u8) -> Dat {
72 Dat::BU32((0..1_100usize).map(|j| seed ^ (j as u8)).collect()) // three chunks
73}
74
75fn open(dir: &str) -> Outcome<TestDb> {
76 open_with(dir, 8)
77}
78
79fn open_with(dir: &str, cbots: u16) -> Outcome<TestDb> {
80 log_set_level!("error");
81 let _ = std::fs::remove_dir_all(dir);
82 res!(std::fs::create_dir_all(dir));
83 let root = res!(Path::new(dir).canonicalize());
84 setup::start_db(root, Some(res!(cfg(cbots))), schemes(), None, false, true)
85}
86
87// A reader that comes while the chunks of a new value are slow to become readable finds no value
88// yet, or the whole one, and never a bunch key over chunks that are not there. A cache bot
89// enters its records in turn, so a bunch key queued behind a held chunk at the same cache bot is
90// held with it; several values are stored over many cache bots so that some bunch key has one to
91// itself. They are stored in turn here; `concurrent_chunked_puts_do_not_queue` stores them at once.
92#[test]
93fn bunch_key_is_not_readable_before_its_chunks() -> Outcome<()> {
94 let _lock = lock();
95 let dir = "./test_db_chunk_publish_order";
96 let db = res!(open(dir));
97 let keys: Vec<Dat> = (0..VALUES).map(|i| dat!(fmt!("chunks first {}", i))).collect();
98
99 let done = Arc::new(AtomicBool::new(false));
100 let seen = {
101 let (db, keys, done) = (db.clone(), keys.clone(), done.clone());
102 thread::spawn(move || {
103 let (mut n_none, mut n_whole, mut bad) = (0usize, 0usize, 0usize);
104 while !done.load(Ordering::Relaxed) {
105 for (i, k) in keys.iter().enumerate() {
106 match db.get(k, None) {
107 Ok(None) => n_none += 1,
108 Ok(Some((v, _))) if v == big(i as u8) => n_whole += 1,
109 other => {
110 bad += 1;
111 let s: String = fmt!("{:?}", other).chars().take(300).collect();
112 msg!("a reader found neither no value nor the whole one: {}", s);
113 },
114 }
115 }
116 thread::sleep(Duration::from_millis(2));
117 }
118 (n_none, n_whole, bad)
119 })
120 };
121 hooks::set_chunk_insert_delay(HOLD);
122 let begun = Instant::now();
123 let mut put = Ok(());
124 for (i, k) in keys.iter().enumerate() {
125 if let Err(e) = db.insert(k.clone(), big(i as u8), Uid::default(), None) {
126 put = Err(e);
127 }
128 }
129 let took = begun.elapsed();
130 hooks::set_chunk_insert_delay(Duration::ZERO);
131 done.store(true, Ordering::Relaxed);
132 let (n_none, n_whole, bad) = match seen.join() {
133 Ok(t) => t,
134 Err(_) => return Err(err!("The reader thread panicked."; Test, Invalid)),
135 };
136 res!(put);
137 msg!("reader saw no value {} times, a whole one {} times, {} broken; the stores took {:?}.",
138 n_none, n_whole, bad, took);
139 // Positive control: the chunks were held, so a bunch key sent beside them had a window.
140 assert!(took >= HOLD, "the hook did not hold the chunks: the stores took {:?}", took);
141 assert_eq!(bad, 0, "a reader found a bunch key whose chunks were not readable");
142 for (i, k) in keys.iter().enumerate() {
143 match res!(db.get(k, None)) {
144 Some((v, _)) => assert_eq!(v, big(i as u8), "the stored value reads back changed"),
145 None => return Err(err!("A stored value is gone."; Test, Missing)),
146 }
147 }
148 res!(db.close());
149 let _ = std::fs::remove_dir_all(dir);
150 Ok(())
151}
152
153// Chunks that are not confirmed durable are not published: the caller is told the write failed,
154// untagged `Unconfirmed` because nothing of the value is readable, and the key holds no value
155// rather than a bunch key over chunks of doubtful standing.
156#[test]
157fn unconfirmed_chunks_publish_no_bunch_key() -> Outcome<()> {
158 let _lock = lock();
159 let dir = "./test_db_chunk_publish_failed";
160 let db = res!(open(dir));
161 let k = dat!("chunks that fail");
162 // A value beside it that stays, so the check is of this key and not of a dead store.
163 res!(db.insert(key(0), value(0, 9), Uid::default(), None));
164
165 hooks::set_barrier_failure(true);
166 let told = db.insert(k.clone(), big(3), Uid::default(), None);
167 hooks::set_barrier_failure(false);
168 match told {
169 Ok(_) => return Err(err!(
170 "A chunked store whose chunks failed their barrier was confirmed.";
171 Test, Unexpected)),
172 Err(e) => {
173 if e.tags().contains(&ErrTag::Unconfirmed) {
174 return Err(err!(
175 "A chunked store that published nothing was told it was written but \
176 unconfirmed: {}", e;
177 Test, Mismatch));
178 }
179 },
180 }
181 match db.get(&k, None) {
182 Ok(None) => (),
183 other => return Err(err!(
184 "A chunked store whose chunks failed left {:?} at its key, where it should hold no \
185 value.", other;
186 Test, Mismatch)),
187 }
188 match res!(db.get(&key(0), None)) {
189 Some((v, _)) => assert_eq!(v, value(0, 9), "the value beside it changed"),
190 None => return Err(err!("The value beside it is gone."; Test, Missing)),
191 }
192 res!(db.close());
193 let _ = std::fs::remove_dir_all(dir);
194 Ok(())
195}
196
197// A caller is told its records are written without waiting for the chunks to become readable. The
198// answers of a write are counted against deadlines of silence, 6 s for each record to be written
199// and far longer for the disk to make them durable. The chunks' answers, and the bunch key's
200// that waits behind them, were first reported only once the chunks were readable, so a disk
201// slower than 6 s (an fsync on a loaded machine) failed a put with "0 of 8 records were confirmed
202// written" although nothing was wrong (A3 round 2a, `chunk_leak` under load).
203#[test]
204fn chunks_are_reported_written_while_they_are_held() -> Outcome<()> {
205 let _lock = lock();
206 let dir = "./test_db_chunk_publish_told";
207 let db = res!(open(dir));
208 let k = dat!("chunks told early");
209 let hold = Duration::from_millis(2_500);
210
211 hooks::set_chunk_insert_delay(hold);
212 let begun = Instant::now();
213 let resp = res!(db.api().put(k.clone(), big(5), Uid::default(), None));
214 let chan = match resp.channel() {
215 Some(chan) => chan,
216 None => return Err(err!("A put returned a responder without a channel."; Test, Missing)),
217 };
218 let (mut records, mut written, mut told) = (0usize, 0usize, None);
219 while told.is_none() && begun.elapsed() < Duration::from_secs(10) {
220 match chan.recv_timeout(Duration::from_millis(50)) {
221 Recv::Result(Ok(OzoneMsg::Chunks(n))) => records = n,
222 Recv::Result(Ok(OzoneMsg::Written)) => {
223 written += 1;
224 if records > 0 && written == records { // every record, the bunch key included
225 told = Some(begun.elapsed());
226 }
227 },
228 _ => (),
229 }
230 }
231 hooks::set_chunk_insert_delay(Duration::ZERO);
232 let told = match told {
233 Some(t) => t,
234 None => return Err(err!(
235 "The caller of a put of {} records was told {} written in 10 s.", records, written;
236 Test, Missing)),
237 };
238 msg!("the caller was told {} records written after {:?}; the hook holds each chunk for {:?}.",
239 written, told, hold);
240 // The chunks are held for 2.5 s each, so a caller told only when they are readable hears
241 // nothing for at least that long.
242 assert!(told < hold, "the caller was told its records were written only after {:?}, when the \
243 chunks became readable", told);
244 // The put goes on to complete.
245 let end = Instant::now();
246 loop {
247 match res!(db.get(&k, None)) {
248 Some((v, _)) => { assert_eq!(v, big(5), "the stored value reads back changed"); break; },
249 None if end.elapsed() < Duration::from_secs(30) => thread::sleep(Duration::from_millis(50)),
250 None => return Err(err!("The put never completed."; Test, Missing)),
251 }
252 }
253 res!(db.close());
254 let _ = std::fs::remove_dir_all(dir);
255 Ok(())
256}
257
258// Chunked puts do not queue behind one another for the disk. A server bot that waited through
259// its put's chunks held one put each, so twelve at once against the two server bots made the
260// last of a queue wait, with chunks held for 1.5 s, past the 6 s a request is given to be
261// counted (A3 round 2a). Twelve over two bots leave one with six, whatever the choice: at
262// 1.5 s a put, the last would be reached after 7.5 s.
263#[test]
264fn concurrent_chunked_puts_do_not_queue() -> Outcome<()> {
265 let _lock = lock();
266 let dir = "./test_db_chunk_publish_many";
267 let db = res!(open_with(dir, 32));
268 let nputs = 12usize;
269 let hold = Duration::from_millis(1_500);
270
271 hooks::set_chunk_insert_delay(hold);
272 let begun = Instant::now();
273 let workers: Vec<_> = (0..nputs).map(|i| {
274 let db = db.clone();
275 thread::spawn(move || {
276 db.insert(dat!(fmt!("many {}", i)), big(i as u8), Uid::default(), None)
277 .map(|_| ()).map_err(|e| fmt!("{}", e))
278 })
279 }).collect();
280 let mut failed = Vec::new();
281 for (i, w) in workers.into_iter().enumerate() {
282 match w.join() {
283 Ok(Ok(())) => (),
284 Ok(Err(e)) => failed.push((i, e.chars().take(200).collect::<String>())),
285 Err(_) => failed.push((i, fmt!("the thread panicked"))),
286 }
287 }
288 let took = begun.elapsed();
289 hooks::set_chunk_insert_delay(Duration::ZERO);
290 msg!("{} chunked puts at once took {:?}; {} failed.", nputs, took, failed.len());
291 assert!(failed.is_empty(), "concurrent chunked puts failed: {:?}", failed);
292 for i in 0..nputs {
293 match res!(db.get(&dat!(fmt!("many {}", i)), None)) {
294 Some((v, _)) => assert_eq!(v, big(i as u8), "a stored value reads back changed"),
295 None => return Err(err!("Chunked put {} reported success but its value is gone.", i;
296 Test, Missing)),
297 }
298 }
299 res!(db.close());
300 let _ = std::fs::remove_dir_all(dir);
301 Ok(())
302}