Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/unconfirmed_winner.rs

9.0 KiB, 58 runs

created by r1870400018:70431, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! No record is ever beaten or displaced by an unconfirmed winner (durable mark, commit A).
2//! Before it, a record U whose sync failed could stand in the cache over an older chunked put P,
3//! and P had to be held from retiring, collection and the sweep until a restart (R2-1). Now the
4//! failed sync stops the store: U is refused, a store in flight is refused, a read of what no
5//! mark covers is refused, compaction and the sweep are refused, and the restart cuts U from the
6//! disk. Two orders, as before: P's head waits while U fails (P would be beaten), and U fails
7//! after P (P would be displaced). The hooks are process-wide, so the two run one at a time.
8
9mod gc_pair;
10
11use gc_pair::*;
12
13use oxedyne_fe2o3_core::prelude::*;
14use oxedyne_fe2o3_iop_db::api::{
15 Database,
16 Meta,
17};
18use oxedyne_fe2o3_jdat::prelude::*;
19use oxedyne_fe2o3_o3db_sync::{
20 base::{
21 cfg::OzoneConfig,
22 constant,
23 },
24 comm::response::Wait,
25 file::{
26 core::FileType,
27 refusal::Refusal,
28 },
29 sweep,
30 test::{
31 hooks,
32 setup::{
33 self,
34 Uid,
35 UID_LEN,
36 },
37 },
38 verify,
39};
40
41use std::{
42 collections::BTreeMap,
43 path::{
44 Path,
45 PathBuf,
46 },
47 sync::Mutex,
48 thread,
49 time::Duration,
50};
51
52fn cfg() -> Outcome<OzoneConfig> {
53 let mut c = res!(setup::default_cfg());
54 c.num_zones = 1;
55 c.num_cbots_per_zone = 4;
56 c.num_fbots_per_zone = 1;
57 c.num_wbots_per_zone = 1;
58 c.zone_overrides = BTreeMap::new();
59 c.rest_chunk_threshold = 1_000;
60 c.rest_chunk_bytes = 400;
61 c.sync_on_write = true;
62 Ok(c)
63}
64
65fn val(len: usize, seed: u8) -> Dat {
66 Dat::BU32((0..len).map(|j| seed ^ (j as u8)).collect())
67}
68
69fn start(root: &Path, wipe: bool) -> Outcome<TestDb> {
70 log_set_level!("error");
71 setup::start_db(root.to_path_buf(), Some(res!(cfg())), schemes(), None, false, wipe)
72}
73
74fn scan_wait() -> Wait {
75 Wait { max_wait: Duration::from_secs(60), check_interval: constant::CHECK_INTERVAL }
76}
77
78// "L/S" for a whole value of length L and seed S as `val` writes it, else what was read.
79fn describe(r: &Outcome<Option<(Dat, Meta<{ UID_LEN }, Uid>)>>) -> String {
80 match r {
81 Ok(None) => fmt!("none"),
82 Ok(Some((Dat::BU32(b), _))) | Ok(Some((Dat::BU8(b), _))) | Ok(Some((Dat::BU16(b), _))) |
83 Ok(Some((Dat::BU64(b), _))) if !b.is_empty()
84 && b.iter().enumerate().all(|(j, x)| *x == b[0] ^ (j as u8)) => fmt!("{}/{}", b.len(), b[0]),
85 Ok(Some((d, _))) => fmt!("TORN:{:?}", d).chars().take(100).collect(),
86 Err(e) => fmt!("ERR:{}", fmt!("{}", e).chars().take(200).collect::<String>()),
87 }
88}
89
90static SERIAL: Mutex<()> = Mutex::new(());
91
92// The highest numbered data file, the live one just after `new_live_files`.
93fn live_fnum(root: &Path, c: &OzoneConfig) -> u32 {
94 let mut n = 1;
95 while file(root, c, FileType::Data, n + 1).exists() {
96 n += 1;
97 }
98 n
99}
100
101// Writes U, unchunked, into a live file of its own whose barrier fails. The roll after it is
102// refused once the store has stopped, and that is not what is tested here.
103fn write_unconfirmed(db: &TestDb, root: &Path, k: &Dat) -> Outcome<Outcome<()>> {
104 let c = res!(cfg());
105 res!(db.api().new_live_files());
106 hooks::set_barrier_failure_for(Some(live_fnum(root, &c)));
107 let u = db.insert(k.clone(), val(50, 3), Uid::default(), None).map(|_| ());
108 let _ = db.api().new_live_files();
109 hooks::set_barrier_failure_for(None);
110 Ok(u)
111}
112
113// What the stopped store still does: a read of the key, compaction and the sweep.
114fn stopped(db: &TestDb, k: &Dat) -> Outcome<(bool, String, Outcome<()>, Outcome<sweep::SweepReport>)> {
115 res!(db.api().settle_for_test(Duration::from_secs(10)));
116 let faulted = db.api().fault().is_some();
117 let live = describe(&db.get(k, None));
118 let compact = db.compact_now(Duration::from_secs(60)).map(|_| ());
119 let swept = sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait());
120 Ok((faulted, live, compact, swept))
121}
122
123// The restart cuts U. Returns the key's value, whether the fault is gone, and whether the store
124// verifies clean after a sweep.
125fn restart(root: &Path, k: &Dat) -> Outcome<(String, bool, bool)> {
126 let db = res!(start(root, false));
127 let after = describe(&db.get(k, None));
128 let cleared = db.api().fault().is_none();
129 res!(sweep::sweep_orphans(db.api(), Uid::default(), None, scan_wait()));
130 let v = res!(verify::verify_live_set(db.api(), None, scan_wait()));
131 res!(db.close());
132 Ok((after, cleared, v.clean()))
133}
134
135fn fresh(dir: &str) -> Outcome<PathBuf> {
136 let _ = std::fs::remove_dir_all(dir);
137 res!(std::fs::create_dir_all(dir));
138 Ok(res!(Path::new(dir).canonicalize()))
139}
140
141fn assert_stopped(
142 faulted: bool,
143 live: &str,
144 compact: &Outcome<()>,
145 swept: &Outcome<sweep::SweepReport>,
146)
147 -> Outcome<()>
148{
149 assert!(faulted, "a failed sync did not stop the store");
150 assert!(live.starts_with("ERR:"), "a read of U, which no mark covers, was served: {}", live);
151 assert!(compact.is_err(), "compaction ran on a stopped store");
152 match swept {
153 Ok(r) => return Err(err!("the sweep ran on a stopped store: {:?}", r; Test)),
154 Err(e) => assert!(fmt!("{}", e).contains("Restart the store"),
155 "the sweep's refusal does not say what to do: {}", e),
156 }
157 Ok(())
158}
159
160/// P's head waits while U's sync fails. The store stops, so P's head is refused rather than
161/// beaten, and after the restart neither U nor P is the key's value.
162#[test]
163fn a_store_in_flight_when_a_sync_fails_is_refused() -> Outcome<()> {
164 let _serial = SERIAL.lock();
165 let dir = "./test_db_unconfirmed_winner_beaten";
166 let root = res!(fresh(dir));
167 let db = res!(start(&root, true));
168 let k = dat!("unconfirmed winner beaten");
169 // P: chunked, held with its chunks in until U is in. Stored through the API, its store
170 // runs in this thread rather than a server bot.
171 hooks::set_head_hold(true);
172 let p = { let (db, k) = (db.clone(), k.clone());
173 thread::spawn(move || -> Outcome<()> {
174 let resp = db.api().responder();
175 res!(db.api().store_dat_using_responder(k, val(1_500, 2), Uid::default(), None, resp.clone()));
176 resp.recv_store_ack().map(|_| ())
177 }) };
178 if !wait_until(Duration::from_secs(30), || hooks::heads_held() >= 1) {
179 hooks::set_head_hold(false);
180 return Err(err!("P's store never reached its head"; Test));
181 }
182 let u = res!(write_unconfirmed(&db, &root, &k));
183 hooks::set_head_hold(false);
184 let pr = match p.join() { Ok(r) => r, Err(_) => Err(err!("P panicked"; Test)) };
185 let (faulted, live, compact, swept) = res!(stopped(&db, &k));
186 let _ = db.close();
187 println!("beaten: p={:?} u={:?} faulted={} live={} compact={:?} swept={:?}",
188 pr.as_ref().map_err(|e| fmt!("{}", e)), u.as_ref().map_err(|e| fmt!("{}", e)),
189 faulted, live, compact.as_ref().map_err(|e| fmt!("{}", e)),
190 swept.as_ref().map_err(|e| fmt!("{}", e)));
191 assert!(u.is_err(), "U's store succeeded although its sync was failed");
192 assert!(pr.is_err(), "P's head was written on a stopped store");
193 res!(assert_stopped(faulted, &live, &compact, &swept));
194 res!(Refusal::clear(&root)); // as the operator would
195 let (after, cleared, clean) = res!(restart(&root, &k));
196 let _ = std::fs::remove_dir_all(dir);
197 assert_eq!(after, "none", "after the restart's cut the key holds neither U nor P");
198 assert!(cleared, "the restart left the fault standing");
199 assert!(clean, "the store does not verify clean after the restart and a sweep");
200 Ok(())
201}
202
203/// U's sync fails after P has landed. U never displaces P: the store stops, U is not served, and
204/// the restart's cut leaves P as the key's value.
205#[test]
206fn a_failed_sync_never_displaces_a_durable_head() -> Outcome<()> {
207 let _serial = SERIAL.lock();
208 let dir = "./test_db_unconfirmed_winner_displaced";
209 let root = res!(fresh(dir));
210 let db = res!(start(&root, true));
211 let k = dat!("unconfirmed winner displaced");
212 res!(db.insert(k.clone(), val(1_500, 2), Uid::default(), None));
213 res!(db.api().sync_now(scan_wait()));
214 let u = res!(write_unconfirmed(&db, &root, &k));
215 let (faulted, live, compact, swept) = res!(stopped(&db, &k));
216 let _ = db.close();
217 println!("displaced: u={:?} faulted={} live={} compact={:?} swept={:?}",
218 u.as_ref().map_err(|e| e.tags().to_vec()), faulted, live,
219 compact.as_ref().map_err(|e| fmt!("{}", e)), swept.as_ref().map_err(|e| fmt!("{}", e)));
220 assert!(u.is_err(), "U's store succeeded although its sync was failed");
221 res!(assert_stopped(faulted, &live, &compact, &swept));
222 res!(Refusal::clear(&root)); // as the operator would
223 let (after, cleared, clean) = res!(restart(&root, &k));
224 let _ = std::fs::remove_dir_all(dir);
225 assert_eq!(after, "1500/2", "after the restart's cut P is the key's value");
226 assert!(cleared, "the restart left the fault standing");
227 assert!(clean, "the store does not verify clean after the restart and a sweep");
228 Ok(())
229}