Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/gc_torn_pair.rs

7.6 KiB, 1 run

created by r1870400018:62339, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! A collection whose cache bot is slow to answer must still leave its data and index files a
2//! matching pair. Until 2026-09-24 the collector renamed the rebuilt index over the old one,
3//! waited five seconds for the cache, gave up, and never renamed the data file: the next start read
4//! the new index beside the old data file and filled the cache with offsets from the one for the
5//! other. Before o3i that returned other keys' values for a whole session, and after it the same
6//! reads failed. Found by QA of lane o3i (D-C, 2026-09-24). `test::hooks` holds the one cache
7//! bot and the collection, and the hooks are process-wide, which is why this is a test binary of
8//! its own with a single test.
9
10use oxedyne_fe2o3_core::prelude::*;
11use oxedyne_fe2o3_hash::{
12 csum::ChecksumScheme,
13 hash::HashScheme,
14};
15use oxedyne_fe2o3_iop_db::api::Database;
16use oxedyne_fe2o3_jdat::prelude::*;
17use oxedyne_fe2o3_o3db_sync::{
18 O3db,
19 base::cfg::OzoneConfig,
20 data::core::RestSchemesInput,
21 file::{
22 core::FileType,
23 zdir::ZoneDir,
24 },
25 test::{
26 hooks,
27 setup::{
28 self,
29 Uid,
30 UID_LEN,
31 },
32 },
33};
34
35use std::{
36 collections::BTreeMap,
37 path::{
38 Path,
39 PathBuf,
40 },
41 thread,
42 time::Duration,
43};
44
45type TestDb = O3db<
46 { UID_LEN },
47 Uid,
48 (),
49 HashScheme,
50 HashScheme,
51 ChecksumScheme,
52>;
53
54const NKEYS: usize = 12;
55
56fn config(max: u64) -> Outcome<OzoneConfig> {
57 let mut cfg = res!(setup::default_cfg());
58 cfg.num_zones = 1;
59 cfg.num_cbots_per_zone = 1;
60 cfg.num_fbots_per_zone = 1;
61 cfg.num_igbots_per_zone = 1;
62 cfg.num_rbots_per_zone = 1;
63 cfg.num_wbots_per_zone = 1;
64 cfg.data_file_max_bytes = max;
65 cfg.rest_chunk_threshold = max * 7 / 10;
66 cfg.zone_overrides = BTreeMap::new();
67 cfg.sync_on_write = true;
68 Ok(cfg)
69}
70
71fn schemes() -> RestSchemesInput<(), HashScheme, HashScheme, ChecksumScheme> {
72 RestSchemesInput::new(
73 None::<()>,
74 None::<HashScheme>,
75 None::<HashScheme>,
76 Some(ChecksumScheme::new_crc32()),
77 )
78}
79
80fn key(i: usize) -> Dat { dat!(fmt!("torn pair key {:03}", i)) }
81
82/// Every value is 100 bytes and carries its key's number and version, so a value read back names
83/// the record it came from.
84fn value(i: usize, ver: u8) -> Dat {
85 let mut v = vec![0u8; 100];
86 v[0] = i as u8;
87 v[1] = ver;
88 for j in 2..100 {
89 v[j] = (i as u8) ^ ver ^ (j as u8);
90 }
91 Dat::BU32(v)
92}
93
94fn size(p: &Path) -> u64 {
95 match std::fs::metadata(p) { Ok(m) => m.len(), Err(_) => 0 }
96}
97
98fn file(root: &Path, cfg: &OzoneConfig, typ: FileType, n: u32) -> PathBuf {
99 cfg.zone_root(root).join("zone_001").join(ZoneDir::relative_file_path(&typ, n))
100}
101
102/// Reads every key and counts the answers that are not the version last written.
103fn judge(db: &TestDb, want: &[u8], label: &str) -> usize {
104 let mut bad = 0;
105 for i in 0..NKEYS {
106 match db.get(&key(i), None) {
107 Ok(Some((v, _))) if v == value(i, want[i]) => (),
108 Ok(Some((v, _))) => {
109 bad += 1;
110 let head = match v.bytes_ref() {
111 Some(b) if b.len() >= 2 => fmt!("key {} version {}", b[0], b[1]),
112 _ => fmt!("{:?}", v),
113 };
114 msg!("{}: key {} read back {}, not version {}.", label, i, head, want[i]);
115 },
116 Ok(None) => {
117 bad += 1;
118 msg!("{}: key {} is missing.", label, i);
119 },
120 Err(e) => {
121 bad += 1;
122 let s: String = fmt!("{}", e).chars().take(300).collect();
123 msg!("{}: key {} could not be read: {}", label, i, s);
124 },
125 }
126 }
127 msg!("{}: {} of {} keys wrong.", label, bad, NKEYS);
128 bad
129}
130
131#[test]
132fn main() -> Outcome<()> {
133 log_set_level!("error");
134 let dir = "./test_db_gc_torn_pair";
135 let _ = std::fs::remove_dir_all(dir);
136 res!(std::fs::create_dir_all(dir));
137 let root = res!(Path::new(dir).canonicalize());
138
139 // The length of one record, all being one length.
140 let pdir = "./test_db_gc_torn_pair_probe";
141 let _ = std::fs::remove_dir_all(pdir);
142 res!(std::fs::create_dir_all(pdir));
143 let probe = res!(Path::new(pdir).canonicalize());
144 let pcfg = res!(config(4_000));
145 let pdb: TestDb = res!(setup::start_db(probe.clone(), Some(pcfg.clone()), schemes(), None, false, true));
146 res!(pdb.insert(key(0), value(0, 1), Uid::default(), None));
147 res!(pdb.close());
148 let len = size(&file(&probe, &pcfg, FileType::Data, 1));
149 assert!(len > 100, "the probe record is {} bytes, shorter than its own value", len);
150
151 // Five records to a file, so that superseding two of file 1's crosses the collection trigger.
152 let cfg = res!(config(5 * len + len / 2));
153 let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, true, true));
154 let mut want = vec![1u8; NKEYS];
155 for i in 0..NKEYS {
156 res!(db.insert(key(i), value(i, 1), Uid::default(), None));
157 }
158 let d1 = file(&root, &cfg, FileType::Data, 1);
159 let full = size(&d1);
160 assert_eq!(full, 5 * len, "file 1 does not hold the five records it was sized for");
161
162 // The collection of file 1 starts a second after keys 0 and 1 are superseded, and its cache
163 // update then waits behind an insert held eight seconds, past the five a collector used to
164 // wait for it.
165 hooks::set_collect_delay(Duration::from_millis(1_000));
166 res!(db.insert(key(0), value(0, 2), Uid::default(), None));
167 want[0] = 2;
168 res!(db.insert(key(1), value(1, 2), Uid::default(), None));
169 want[1] = 2;
170 hooks::set_insert_delay(Duration::from_millis(8_000));
171 let writing = db.clone();
172 let slow = res!(thread::Builder::new().name(fmt!("slow insert")).spawn(move || {
173 let _ = writing.insert(key(NKEYS - 1), value(NKEYS - 1, 3), Uid::default(), None);
174 }));
175 want[NKEYS - 1] = 3;
176 thread::sleep(Duration::from_millis(9_000));
177 hooks::set_insert_delay(Duration::ZERO);
178 hooks::set_collect_delay(Duration::ZERO);
179 let _ = slow.join();
180 let collected = size(&d1);
181 let same = judge(&db, &want, "Same process");
182 res!(db.close());
183
184 let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, true, false));
185 let reopened = judge(&db, &want, "After reopening");
186
187 // Churn so that file 1 is collected again with whatever state the start built.
188 for round in 4..7u8 {
189 for i in 2..6 {
190 res!(db.insert(key(i), value(i, round), Uid::default(), None));
191 want[i] = round;
192 }
193 }
194 thread::sleep(Duration::from_secs(3));
195 let churned = judge(&db, &want, "After more collection");
196 res!(db.close());
197
198 let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, true, false));
199 let second = judge(&db, &want, "After a second reopening");
200 res!(db.close());
201 log_finish_wait!();
202
203 assert_eq!(same, 0, "{} keys read wrong in the process that ran the collection", same);
204 assert_eq!(reopened, 0, "{} keys read wrong after the first reopening", reopened);
205 assert_eq!(churned, 0, "{} keys read wrong after file 1 was collected again", churned);
206 assert_eq!(second, 0, "{} keys read wrong after the second reopening", second);
207 // The collection has to have happened for any of this to mean anything.
208 assert_eq!(collected, 3 * len,
209 "file 1 should have lost the two superseded records to its collection, and is {} bytes \
210 where it was {}", collected, full);
211 let _ = std::fs::remove_dir_all(dir);
212 let _ = std::fs::remove_dir_all(pdir);
213 Ok(())
214}