Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/append_rewind.rs

9.7 KiB, 1 run

created by r1870400018:70846, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! An append to a live file that fails part-way (2026-10-11, QA round 1 F4). The live files are
2//! opened for appending, so a seek back has no effect on where the next write lands: the torn
3//! bytes stayed in the file, the next record went in after them, and the writer's idea of the
4//! file's length, from which the next record's location is taken, fell behind the file. The torn
5//! append is now cut back off, both files of the pair return to their last good lengths, and a
6//! cut that fails stops the store through its fault. The hooks are process-wide, so the tests
7//! run one at a time.
8
9mod gc_pair;
10
11use gc_pair::*;
12
13use oxedyne_fe2o3_core::prelude::*;
14use oxedyne_fe2o3_iop_db::api::Database;
15use oxedyne_fe2o3_jdat::prelude::*;
16use oxedyne_fe2o3_o3db_sync::{
17 base::cfg::OzoneConfig,
18 file::{
19 core::FileType,
20 refusal::Refusal,
21 },
22 test::{
23 hooks,
24 setup::{
25 self,
26 Uid,
27 },
28 },
29};
30
31use std::{
32 collections::BTreeMap,
33 fs,
34 path::{
35 Path,
36 PathBuf,
37 },
38 sync::Mutex,
39};
40
41static SERIAL: Mutex<()> = Mutex::new(());
42
43fn lock() -> std::sync::MutexGuard<'static, ()> {
44 let g = match SERIAL.lock() { Ok(g) => g, Err(p) => p.into_inner() };
45 hooks::set_write_tear(FileType::Data, None);
46 hooks::set_write_tear(FileType::Index, None);
47 hooks::set_cut_back_failure(false);
48 g
49}
50
51// One zone of one writer, syncing on every write.
52fn cfg() -> Outcome<OzoneConfig> {
53 let mut c = res!(setup::default_cfg());
54 c.num_zones = 1;
55 c.num_wbots_per_zone = 1;
56 c.zone_overrides = BTreeMap::new();
57 c.sync_on_write = true;
58 c.sync_every_n_writes = 0;
59 c.sync_interval_ms = 0;
60 c.data_file_max_bytes = 1_000_000;
61 Ok(c)
62}
63
64fn fresh(dir: &str) -> Outcome<PathBuf> {
65 let _ = fs::remove_dir_all(dir);
66 res!(fs::create_dir_all(dir));
67 Ok(res!(Path::new(dir).canonicalize()))
68}
69
70fn open(root: &Path, c: &OzoneConfig, wipe: bool) -> Outcome<TestDb> {
71 log_set_level!("error");
72 setup::start_db(root.to_path_buf(), Some(c.clone()), schemes(), None, false, wipe)
73}
74
75fn val(seed: u8) -> Dat {
76 Dat::BU32((0..100).map(|j| seed ^ (j as u8)).collect())
77}
78
79fn put(db: &TestDb, k: &str, seed: u8) -> Outcome<()> {
80 db.insert(dat!(k), val(seed), Uid::default(), None).map(|_| ())
81}
82
83fn reads(db: &TestDb, k: &str) -> Outcome<Option<Dat>> {
84 Ok(res!(db.get(&dat!(k), None)).map(|(v, _)| v))
85}
86
87// The live pair, the highest numbered data file and its index.
88fn live(root: &Path, c: &OzoneConfig) -> Outcome<(PathBuf, PathBuf)> {
89 let n = res!((1..10_000u32).rev().find(|n| file(root, c, FileType::Data, *n).exists())
90 .ok_or_else(|| err!("No data file is live."; Test)));
91 Ok((file(root, c, FileType::Data, n), file(root, c, FileType::Index, n)))
92}
93
94// Two good records, "k0" and "k1", and the pair's lengths before and after the second.
95struct Start {
96 root: PathBuf,
97 c: OzoneConfig,
98 db: TestDb,
99 dat: PathBuf,
100 ind: PathBuf,
101 rec: u64, // data bytes in one record
102 dlen: u64, // data length after "k1"
103 ilen: u64, // index length after "k1"
104}
105
106fn start(dir: &str) -> Outcome<Start> {
107 let root = res!(fresh(dir));
108 let c = res!(cfg());
109 let db = res!(open(&root, &c, true));
110 res!(put(&db, "k0", 1));
111 let (dat, ind) = res!(live(&root, &c));
112 let before = size(&dat);
113 res!(put(&db, "k1", 2));
114 let dlen = size(&dat);
115 let ilen = size(&ind);
116 Ok(Start { root, c, db, dat, ind, rec: dlen - before, dlen, ilen })
117}
118
119// After the next record, and again after a reopen, every key reads its value and the failed one
120// is absent.
121fn check_after(s: Start, failed: &str) -> Outcome<()> {
122 let Start { root, c, db, dat, ind, rec, dlen, ilen } = s;
123 res!(put(&db, "k3", 4));
124 assert_eq!(size(&dat), dlen + rec,
125 "the record after the failed one did not land at the last good length {}", dlen);
126 assert!(size(&ind) > ilen, "the record after the failed one is not in the index");
127 let want = [("k0", Some(val(1))), ("k1", Some(val(2))), (failed, None), ("k3", Some(val(4)))];
128 for (k, v) in &want {
129 assert_eq!(&res!(reads(&db, k)), v, "{} reads wrong before the reopen", k);
130 }
131 res!(db.close());
132 let db = res!(open(&root, &c, false));
133 for (k, v) in &want {
134 assert_eq!(&res!(reads(&db, k)), v, "{} reads wrong after the reopen", k);
135 }
136 res!(db.close());
137 let _ = fs::remove_dir_all(&root);
138 Ok(())
139}
140
141/// A data append torn part-way is cut back off, and the next record lands where it began.
142#[test]
143fn a_torn_data_append_is_cut_back() -> Outcome<()> {
144 let _g = lock();
145 let s = res!(start("./test_db_append_rewind_data"));
146 let torn = hooks::writes_torn();
147 hooks::set_write_tear(FileType::Data, Some(s.rec / 2));
148 let failed = put(&s.db, "k2", 3);
149 assert_eq!(hooks::writes_torn(), torn + 1, "the tear was not reached");
150 assert!(failed.is_err(), "a torn write was confirmed");
151 assert_eq!(size(&s.dat), s.dlen, "the torn bytes were left in the data file");
152 assert_eq!(size(&s.ind), s.ilen, "the index changed for a record that failed");
153 assert!(s.db.api().fault().is_none(), "a torn write cut back cleanly stopped the store");
154 check_after(s, "k2")
155}
156
157/// An index append torn part-way withdraws the whole record, its data as well, so a write
158/// reported failed does not come back at the next open.
159#[test]
160fn a_torn_index_append_withdraws_the_record() -> Outcome<()> {
161 let _g = lock();
162 let s = res!(start("./test_db_append_rewind_index"));
163 let torn = hooks::writes_torn();
164 hooks::set_write_tear(FileType::Index, Some(3));
165 let failed = put(&s.db, "k2", 3);
166 assert_eq!(hooks::writes_torn(), torn + 1, "the tear was not reached");
167 assert!(failed.is_err(), "a torn write was confirmed");
168 assert_eq!(size(&s.ind), s.ilen, "the torn bytes were left in the index file");
169 assert_eq!(size(&s.dat), s.dlen, "the data of a record whose index failed was left");
170 assert!(s.db.api().fault().is_none(), "a torn write cut back cleanly stopped the store");
171 check_after(s, "k2")
172}
173
174/// A torn append that cannot be cut back off stops the store through its fault, which names the
175/// last good lengths, and the next open cuts the pair back to them.
176#[test]
177fn a_failed_cut_back_stops_the_store() -> Outcome<()> {
178 let _g = lock();
179 let Start { root, c, db, dat, ind, rec, dlen, ilen } =
180 res!(start("./test_db_append_rewind_stop"));
181 hooks::set_cut_back_failure(true);
182 hooks::set_write_tear(FileType::Data, Some(rec / 2));
183 let failed = put(&db, "k2", 3);
184 hooks::set_cut_back_failure(false);
185 assert!(failed.is_err(), "a torn write was confirmed");
186 let report = res!(db.api().fault().ok_or_else(|| err!(
187 "A torn append that could not be cut back did not stop the store."; Test)));
188 assert!(report.recorded(), "the stop was not written down");
189 assert!(report.cuts.iter().any(|cut| cut.data == dat && cut.data_upto == dlen
190 && cut.index == ind && cut.index_upto == ilen),
191 "the fault does not cut the pair to ({}, {}): {:?}", dlen, ilen, report.cuts);
192 assert!(put(&db, "k3", 4).is_err(), "a stopped store took a write");
193 let _ = db.close();
194 assert!(res!(Refusal::clear(&root)));
195 let db = res!(open(&root, &c, false));
196 assert_eq!(size(&dat), dlen, "the open did not cut the torn bytes off the data file");
197 assert_eq!(res!(reads(&db, "k0")), Some(val(1)));
198 assert_eq!(res!(reads(&db, "k1")), Some(val(2)));
199 assert_eq!(res!(reads(&db, "k2")), None, "the torn record was served");
200 assert_eq!(res!(reads(&db, "k3")), None, "the refused record was served");
201 res!(db.close());
202 let _ = fs::remove_dir_all(&root);
203 Ok(())
204}
205
206/// A failed cut back whose fault record cannot be written is still recorded, by the refusal
207/// marker, which carries the cut; clearing the marker folds it into the record, and the next open
208/// cuts the pair back (2026-10-11, rebased onto durable-mark re-QA round 2 F1, F2).
209#[test]
210fn a_failed_cut_back_is_held_by_the_marker_alone() -> Outcome<()> {
211 let _g = lock();
212 let Start { root, c, db, dat, ind, rec, dlen, ilen } =
213 res!(start("./test_db_append_rewind_marker"));
214 hooks::set_fault_record_fails_at(1);
215 hooks::set_cut_back_failure(true);
216 hooks::set_write_tear(FileType::Data, Some(rec / 2));
217 let failed = put(&db, "k2", 3);
218 hooks::set_cut_back_failure(false);
219 hooks::set_fault_record_fails_at(0);
220 assert!(failed.is_err(), "a torn write was confirmed");
221 let report = res!(db.api().fault().ok_or_else(|| err!(
222 "A torn append that could not be cut back did not stop the store."; Test)));
223 assert!(report.record.is_none(), "a record that failed to be written is named");
224 assert!(report.refusal.is_some(), "the refusal marker was not written");
225 assert!(report.recorded(), "a stop held by the marker says it is not recorded");
226 let cut = res!(report.cuts.iter().find(|cut| cut.data == dat && cut.data_upto == dlen
227 && cut.index == ind && cut.index_upto == ilen).ok_or_else(|| err!(
228 "The fault does not cut the pair to ({}, {}): {:?}", dlen, ilen, report.cuts; Test)));
229 let marker = res!(fs::read_to_string(Refusal::path(&root)));
230 assert!(marker.lines().any(|l| l.starts_with(&cut.line())),
231 "the refusal marker does not carry the cut {:?}: {:?}", cut.line(), marker);
232 let _ = db.close();
233 assert!(res!(Refusal::clear(&root)));
234 let db = res!(open(&root, &c, false));
235 assert_eq!(size(&dat), dlen, "the open did not cut the torn bytes off the data file");
236 assert_eq!(res!(reads(&db, "k0")), Some(val(1)));
237 assert_eq!(res!(reads(&db, "k1")), Some(val(2)));
238 assert_eq!(res!(reads(&db, "k2")), None, "the torn record was served");
239 res!(db.close());
240 let _ = fs::remove_dir_all(&root);
241 Ok(())
242}