Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/tests/scan_after_delete.rs

6.3 KiB, 1 run

created by r1870400018:73817, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! A scan lists the keys with a live value, and a deleted key is not one of them.
2//!
3//! A delete writes a tombstone under the key, and the scan walks the index files, which name
4//! every key's newest record without its value. So the scan listed every deleted key as though
5//! it were live, while every read of it came back empty: a pass that counted by the keys alone
6//! miscounted (Daimond gateway, G2b session migration, 2026-10-12). These tests delete keys and
7//! scan, in a live store, after a restart, after a rebuild of the index from the data files, and
8//! after a compaction.
9
10use oxedyne_fe2o3_core::prelude::*;
11use oxedyne_fe2o3_crypto::enc::EncryptionScheme;
12use oxedyne_fe2o3_hash::{
13 csum::ChecksumScheme,
14 hash::HashScheme,
15};
16use oxedyne_fe2o3_iop_db::api::{
17 Database,
18 ScanOpts,
19};
20use oxedyne_fe2o3_jdat::prelude::*;
21use oxedyne_fe2o3_o3db_sync::{
22 base::cfg::OzoneConfig,
23 data::core::RestSchemesInput,
24 test::{
25 file::delete_all_index_files,
26 setup,
27 },
28};
29
30use std::{
31 collections::BTreeSet,
32 path::{
33 Path,
34 PathBuf,
35 },
36 time::Duration,
37};
38
39const N: usize = 24;
40
41type TestDb = oxedyne_fe2o3_o3db_sync::O3db<
42 { setup::UID_LEN },
43 setup::Uid,
44 EncryptionScheme,
45 HashScheme,
46 HashScheme,
47 ChecksumScheme,
48>;
49
50// Unencrypted, so that short values are as short as the tombstone itself.
51fn schemes() -> RestSchemesInput<EncryptionScheme, HashScheme, HashScheme, ChecksumScheme> {
52 RestSchemesInput::new(
53 None::<EncryptionScheme>,
54 None::<HashScheme>,
55 None::<HashScheme>,
56 Some(ChecksumScheme::new_crc32()),
57 )
58}
59
60fn root(name: &str) -> Outcome<PathBuf> {
61 let dir = fmt!("./test_db_scan_after_delete_{}", name);
62 let _ = std::fs::remove_dir_all(&dir);
63 res!(std::fs::create_dir_all(&dir));
64 Ok(res!(Path::new(&dir).canonicalize()))
65}
66
67fn config(file_max: u64) -> Outcome<OzoneConfig> {
68 let mut cfg = res!(setup::default_cfg());
69 cfg.sync_on_write = true;
70 cfg.data_file_max_bytes = file_max;
71 // Every zone inside this test's own root.
72 cfg.zone_overrides = DaticleMap::new();
73 Ok(cfg)
74}
75
76fn key(i: usize) -> String { fmt!("k:{:03}", i) }
77
78// Strings of every length from empty to eleven, so that some live value is stored with exactly
79// the tombstone's length, and every fifth value large enough to be chunked.
80fn val(i: usize) -> Dat {
81 if i % 5 == 4 {
82 dat!("c".repeat(2_000 + i))
83 } else {
84 dat!("v".repeat(i % 12))
85 }
86}
87
88// Every third key, chunked ones among them.
89fn doomed(i: usize) -> bool { i % 3 == 1 }
90
91fn fill(db: &TestDb) -> Outcome<BTreeSet<String>> {
92 let user = setup::Uid::default();
93 for i in 0..N {
94 res!(db.insert(dat!(key(i)), val(i), user, None));
95 }
96 let mut live = BTreeSet::new();
97 for i in 0..N {
98 if doomed(i) {
99 let found = res!(db.delete(&dat!(key(i)), user, None));
100 assert!(found, "the delete of {} found nothing", key(i));
101 } else {
102 live.insert(key(i));
103 }
104 }
105 res!(db.api().settle_for_test(Duration::from_secs(10)));
106 Ok(live)
107}
108
109// The string keys listed, and any other key, which can only be a retired chunk key.
110fn scanned(db: &TestDb, opts: &ScanOpts) -> Outcome<(BTreeSet<String>, Vec<Dat>)> {
111 let mut out = BTreeSet::new();
112 let mut odd = Vec::new();
113 for (k, _, _) in res!(db.scan(opts, None)) {
114 match k {
115 Dat::Str(s) => { out.insert(s); },
116 other => odd.push(other),
117 }
118 }
119 Ok((out, odd))
120}
121
122fn check(db: &TestDb, live: &BTreeSet<String>, phase: &str) -> Outcome<()> {
123 let (all, odd) = res!(scanned(db, &ScanOpts::all()));
124 let extra: Vec<&String> = all.difference(live).collect();
125 let missing: Vec<&String> = live.difference(&all).collect();
126 assert!(extra.is_empty() && missing.is_empty() && odd.is_empty(),
127 "{}: the scan must list exactly the {} live keys; it listed {} deleted key(s) {:?}, \
128 {} retired chunk key(s) {:?}, and left out {} live key(s) {:?}", phase, live.len(),
129 extra.len(), extra, odd.len(), odd, missing.len(), missing);
130 let (pre, _) = res!(scanned(db, &ScanOpts::with_str_prefix("k:".to_string())));
131 assert_eq!(&pre, live, "{}: the prefix scan differs from the live keys", phase);
132 // Every key listed reads, which is the contract a caller counting by keys relies on.
133 for k in &all {
134 assert!(res!(db.get(&dat!(k.clone()), None)).is_some(), "{}: {} was listed but reads absent", phase, k);
135 }
136 Ok(())
137}
138
139#[test]
140fn scan_leaves_deleted_keys_out_live_and_after_restart() -> Outcome<()> {
141 log_set_level!("error");
142 let root = res!(root("restart"));
143 let cfg = res!(config(1_000_000));
144
145 let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, true, true));
146 let live = res!(fill(&db));
147 res!(check(&db, &live, "live store"));
148 let zdirs = res!(db.api().get_zone_dirs());
149 res!(db.shutdown());
150
151 // Restarted on the index files.
152 let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, true, false));
153 res!(check(&db, &live, "after restart"));
154 res!(db.shutdown());
155
156 // Restarted with the index rebuilt by replaying the data files.
157 res!(delete_all_index_files(&zdirs));
158 let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg), schemes(), None, true, false));
159 res!(check(&db, &live, "after replay"));
160 res!(db.shutdown());
161
162 let _ = std::fs::remove_dir_all(&root);
163 Ok(())
164}
165
166#[test]
167fn scan_leaves_deleted_keys_out_after_compaction() -> Outcome<()> {
168 log_set_level!("error");
169 let root = res!(root("compact"));
170 // Small files, so that the deleted records sit in sealed files a compaction collects.
171 let cfg = res!(config(4_000));
172
173 let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, true, true));
174 let live = res!(fill(&db));
175 res!(check(&db, &live, "before compaction"));
176 let report = res!(db.compact_now(Duration::from_secs(30)));
177 msg!("{:?}", report);
178 res!(check(&db, &live, "after compaction"));
179 res!(db.shutdown());
180
181 let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg), schemes(), None, true, false));
182 res!(check(&db, &live, "after compaction and restart"));
183 res!(db.shutdown());
184
185 let _ = std::fs::remove_dir_all(&root);
186 Ok(())
187}