Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/chunk_barriers.rs

9.5 KiB, 1 run

created by r1870400018:74443, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! Barriers on a chunked put's path (Ozone chunk barriers, 2026-10-11). A store's chunks are
2//! never answered before a barrier covers them, and the store asks each writer it used for one
3//! behind them, so a chunk must not trip the timed or every-N barrier as well: that barrier would
4//! cover only the chunks taken so far, and the rest would wait on a second one, in series (M1).
5//! Nor does one put spread its chunks over every writer of a zone (M3). Barriers are counted
6//! with `hooks::barriers_by_writer`, not timed. The chunk phase is read at the head hold, once
7//! every chunk is answered and before the head is sent. The hooks are process-wide, so the
8//! tests run one at a time.
9
10mod gc_pair;
11
12use gc_pair::*;
13
14use oxedyne_fe2o3_core::prelude::*;
15use oxedyne_fe2o3_iop_db::api::Database;
16use oxedyne_fe2o3_jdat::prelude::*;
17use oxedyne_fe2o3_o3db_sync::{
18 base::{
19 cfg::OzoneConfig,
20 constant,
21 },
22 comm::response::Wait,
23 test::{
24 hooks,
25 setup::{
26 self,
27 Uid,
28 },
29 },
30};
31
32use std::{
33 collections::{
34 BTreeMap,
35 BTreeSet,
36 },
37 path::Path,
38 sync::{
39 Mutex,
40 MutexGuard,
41 },
42 thread,
43 time::{
44 Duration,
45 Instant,
46 },
47};
48
49static HOOKS: Mutex<()> = Mutex::new(());
50
51const CHUNK: u64 = 400;
52const LENS: [usize; 4] = [350, 1_100, 3_000, 6_400]; // 1, 3, 8 and 16 chunks
53
54#[derive(Clone, Copy, Debug)]
55enum Policy { EveryN(u32), EveryWrite, Interval(u64), Never }
56
57fn lock() -> MutexGuard<'static, ()> {
58 let g = match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() };
59 hooks::set_barrier_delay(Duration::ZERO);
60 hooks::set_head_hold(false);
61 g
62}
63
64fn cfg(p: Policy, zones: u16, wbots: u16, file_max: u64) -> Outcome<OzoneConfig> {
65 let mut c = res!(setup::default_cfg());
66 c.num_zones = zones;
67 c.num_cbots_per_zone = 2;
68 c.num_fbots_per_zone = 1;
69 c.num_wbots_per_zone = wbots;
70 c.zone_overrides = BTreeMap::new();
71 c.rest_chunk_threshold = 300;
72 c.rest_chunk_bytes = CHUNK;
73 c.data_file_max_bytes = file_max;
74 c.sync_on_write = false;
75 c.sync_every_n_writes = 0;
76 c.sync_interval_ms = 0;
77 match p {
78 Policy::EveryN(n) => c.sync_every_n_writes = n,
79 Policy::EveryWrite => c.sync_on_write = true,
80 Policy::Interval(t) => c.sync_interval_ms = t,
81 Policy::Never => (),
82 }
83 Ok(c)
84}
85
86fn start(dir: &str, c: &OzoneConfig) -> Outcome<TestDb> {
87 log_set_level!("error");
88 let _ = std::fs::remove_dir_all(dir);
89 res!(std::fs::create_dir_all(dir));
90 let root = res!(Path::new(dir).canonicalize());
91 setup::start_db(root, Some(c.clone()), schemes(), None, false, true)
92}
93
94fn val(len: usize, seed: u8) -> Dat {
95 Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect())
96}
97
98fn wait() -> Wait {
99 Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL }
100}
101
102fn wait_until(limit: Duration, mut cond: impl FnMut() -> bool) -> bool {
103 let deadline = Instant::now() + limit;
104 loop {
105 if cond() {
106 return true;
107 }
108 if Instant::now() >= deadline {
109 return false;
110 }
111 thread::sleep(Duration::from_millis(5));
112 }
113}
114
115// The barriers each writer made between two readings of the gauge.
116fn since(before: &BTreeMap<(usize, usize), u64>) -> BTreeMap<(usize, usize), u64> {
117 hooks::barriers_by_writer().into_iter()
118 .map(|(w, n)| (w, n - before.get(&w).copied().unwrap_or(0)))
119 .filter(|(_, n)| *n > 0)
120 .collect()
121}
122
123// Puts a fresh key and returns the barriers made, by writer, before its head was sent.
124fn chunk_phase(db: &TestDb, k: Dat, v: Dat) -> Outcome<BTreeMap<(usize, usize), u64>> {
125 let held = hooks::heads_held();
126 let before = hooks::barriers_by_writer();
127 hooks::set_head_hold(true);
128 let put = { let db = db.clone(); thread::spawn(move || db.insert(k, v, Uid::default(), None).map(|_| ())) };
129 let reached = wait_until(Duration::from_secs(30), || hooks::heads_held() > held);
130 let phase = since(&before);
131 hooks::set_head_hold(false);
132 let put = match put.join() { Ok(r) => r, Err(_) => Err(err!("The put panicked."; Test)) };
133 res!(put);
134 if !reached {
135 return Err(err!("The put never reached the head hold."; Test));
136 }
137 Ok(phase)
138}
139
140// Each layout puts values of 1 to 16 chunks to fresh keys, each after the writers have been
141// idle past the period, so that a chunk would find the timed barrier due.
142fn run_layouts(p: Policy, settle: Duration, tag: &str) -> Outcome<Vec<String>> {
143 let mut bad = Vec::new();
144 for (z, w) in [(1u16, 1u16), (1, 4), (2, 2)] {
145 let c = res!(cfg(p, z, w, 16 * 1_048_576));
146 let dir = fmt!("./test_db_chunk_barriers_{}_{}x{}", tag, z, w);
147 let db = res!(start(&dir, &c));
148 res!(db.insert(dat!("warm"), dat!("up"), Uid::default(), None));
149 let mut counts = Vec::new();
150 for (i, len) in LENS.iter().enumerate() {
151 res!(db.api().sync_now(wait()));
152 thread::sleep(settle);
153 let phase = res!(chunk_phase(&db, dat!(fmt!("chunked {}", i)), val(*len, i as u8)));
154 let total: u64 = phase.values().sum();
155 let worst = phase.values().copied().max().unwrap_or(0);
156 counts.push(fmt!("{}B:{}/{}w", len, total, phase.len()));
157 // R1: the chunks are durable before the head goes out.
158 if total == 0 {
159 bad.push(fmt!("{:?} {}x{} {} bytes: no barrier before the head", p, z, w, len));
160 }
161 if let Policy::EveryWrite = p {
162 continue;
163 }
164 // M1: one barrier per writer touched.
165 if worst > 1 {
166 bad.push(fmt!("{:?} {}x{} {} bytes: a writer made {} barriers in the chunk phase {:?}",
167 p, z, w, len, worst, phase));
168 }
169 // M3: one writer per zone per put, each writer touched in a zone of its own.
170 let zones: BTreeSet<usize> = phase.keys().map(|(zone, _)| *zone).collect();
171 if zones.len() != phase.len() || phase.len() > z as usize {
172 bad.push(fmt!("{:?} {}x{} {} bytes: {} writers barriered in {} distinct zones of {} {:?}",
173 p, z, w, len, phase.len(), zones.len(), z, phase));
174 }
175 }
176 println!("BARRIERS {:?} {}x{}: {}", p, z, w, counts.join(" "));
177 res!(db.close());
178 let _ = std::fs::remove_dir_all(&dir);
179 }
180 Ok(bad)
181}
182
183#[test]
184fn chunk_phase_interval() -> Outcome<()> {
185 let _lock = lock();
186 let bad = res!(run_layouts(Policy::Interval(200), Duration::from_millis(400), "int"));
187 if !bad.is_empty() {
188 return Err(err!("{:?}", bad; Test));
189 }
190 Ok(())
191}
192
193// The chunks do not count towards every n writes either, so a put of more chunks than n still
194// takes one barrier per writer touched.
195#[test]
196fn chunk_phase_every_n() -> Outcome<()> {
197 let _lock = lock();
198 let bad = res!(run_layouts(Policy::EveryN(3), Duration::ZERO, "everyn"));
199 if !bad.is_empty() {
200 return Err(err!("{:?}", bad; Test));
201 }
202 Ok(())
203}
204
205#[test]
206fn chunk_phase_never() -> Outcome<()> {
207 let _lock = lock();
208 let bad = res!(run_layouts(Policy::Never, Duration::ZERO, "never"));
209 if !bad.is_empty() {
210 return Err(err!("{:?}", bad; Test));
211 }
212 Ok(())
213}
214
215// Every write waits on a barrier of its own, as before: counted, and held to R1 only.
216#[test]
217fn chunk_phase_every_write() -> Outcome<()> {
218 let _lock = lock();
219 let bad = res!(run_layouts(Policy::EveryWrite, Duration::ZERO, "every"));
220 if !bad.is_empty() {
221 return Err(err!("{:?}", bad; Test));
222 }
223 Ok(())
224}
225
226// The interval still bounds the loss of acknowledged writes: a stream of small inserts over a
227// second, one every 50 ms, barriers about once per 200 ms period.
228#[test]
229fn interval_floor_holds_for_plain_writes() -> Outcome<()> {
230 let _lock = lock();
231 let c = res!(cfg(Policy::Interval(200), 1, 1, 16 * 1_048_576));
232 let dir = "./test_db_chunk_barriers_floor";
233 let db = res!(start(dir, &c));
234 res!(db.api().sync_now(wait()));
235 let before = hooks::barriers_by_writer();
236 for i in 0..20u32 {
237 res!(db.insert(dat!(fmt!("small {}", i)), dat!(i), Uid::default(), None));
238 thread::sleep(Duration::from_millis(50));
239 }
240 thread::sleep(Duration::from_millis(300));
241 let made: u64 = since(&before).values().sum();
242 println!("FLOOR Interval(200) 20 inserts over 1 s: {} barriers", made);
243 res!(db.close());
244 let _ = std::fs::remove_dir_all(dir);
245 if !(3..=10).contains(&made) {
246 return Err(err!("{} barriers for a second of writes under a 200 ms interval.", made; Test));
247 }
248 Ok(())
249}
250
251// A chunk that does not fit rolls the live file, and the dirty old file is sealed with a barrier
252// of its own (M2). Documented, not bounded: the count is one plus the rollovers.
253#[test]
254fn rollover_seals_counted() -> Outcome<()> {
255 let _lock = lock();
256 let c = res!(cfg(Policy::Never, 1, 1, 2_500));
257 let dir = "./test_db_chunk_barriers_roll";
258 let db = res!(start(dir, &c));
259 let mut counts = Vec::new();
260 for i in 0..6u8 {
261 let phase = res!(chunk_phase(&db, dat!(fmt!("roll {}", i)), val(3_000, i)));
262 counts.push(phase.values().sum::<u64>());
263 }
264 println!("ROLLOVER Never 1x1 2500-byte files, 8-chunk puts: {:?}", counts);
265 res!(db.close());
266 let _ = std::fs::remove_dir_all(dir);
267 if counts.iter().any(|n| *n == 0) {
268 return Err(err!("A put's head went out with no barrier over its chunks: {:?}", counts; Test));
269 }
270 Ok(())
271}