oxedyne/fe2o3/fe2o3_o3db_sync/tests/size_bytes.rs
7.6 KiB, 1 run
created by r1870400018:67603, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! `O3db::size_bytes` against the files themselves (A3 R1, 2026-10-08). A cap on disk use needs |
| 2 | //! the length the zone directories hold now, not the file bots' accounted figure, which is pushed |
| 3 | //! periodically and lags any write still draining. Every figure here is checked against a walk of |
| 4 | //! the directories made by the test with nothing of the store's own (`zone_files_len`). |
| 5 | |
| 6 | mod gc_pair; |
| 7 | |
| 8 | use gc_pair::*; |
| 9 | |
| 10 | use oxedyne_fe2o3_core::prelude::*; |
| 11 | use oxedyne_fe2o3_iop_db::api::Database; |
| 12 | use oxedyne_fe2o3_o3db_sync::{ |
| 13 | file::core::FileType, |
| 14 | test::setup::{ |
| 15 | self, |
| 16 | Uid, |
| 17 | }, |
| 18 | }; |
| 19 | |
| 20 | use std::{ |
| 21 | path::Path, |
| 22 | sync::{ |
| 23 | Arc, |
| 24 | atomic::{ |
| 25 | AtomicBool, |
| 26 | Ordering, |
| 27 | }, |
| 28 | }, |
| 29 | thread, |
| 30 | }; |
| 31 | |
| 32 | fn fresh(dir: &str) -> Outcome<std::path::PathBuf> { |
| 33 | let _ = std::fs::remove_dir_all(dir); |
| 34 | res!(std::fs::create_dir_all(dir)); |
| 35 | Ok(res!(Path::new(dir).canonicalize())) |
| 36 | } |
| 37 | |
| 38 | #[test] |
| 39 | fn size_equals_zone_files_after_known_writes() -> Outcome<()> { |
| 40 | log_set_level!("error"); |
| 41 | let dir = "./test_db_size_bytes_known"; |
| 42 | let root = res!(fresh(dir)); |
| 43 | |
| 44 | let probe = res!(probe("./test_db_size_bytes_known_probe")); |
| 45 | let len = probe.len; |
| 46 | let cfg = res!(config(5 * len + len / 2)); |
| 47 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, false, true)); |
| 48 | res!(fill(&db)); |
| 49 | |
| 50 | // Twelve records of one length, five to a data file: files 1 and 2 hold five, file 3 holds two. |
| 51 | let (full, part) = (5 * len, 2 * len); |
| 52 | let (ifull, ipart) = (probe.ind[5], probe.ind[2]); |
| 53 | for (n, d, i) in [(1, full, ifull), (2, full, ifull), (3, part, ipart)] { |
| 54 | assert_eq!(size(&file(&root, &cfg, FileType::Data, n)), d, "data file {}", n); |
| 55 | assert_eq!(size(&file(&root, &cfg, FileType::Index, n)), i, "index file {}", n); |
| 56 | } |
| 57 | let want = 2 * full + part + 2 * ifull + ipart; |
| 58 | |
| 59 | let got = res!(db.size_bytes()); |
| 60 | let walked = zone_files_len(&root, &cfg); |
| 61 | msg!("size_bytes {}, walked {}, probe says {}.", got, walked, want); |
| 62 | assert_eq!(walked, want, "the files are not the probe's twelve records"); |
| 63 | assert_eq!(got, walked, "size_bytes is not the length of the files"); |
| 64 | |
| 65 | res!(db.close()); |
| 66 | log_finish_wait!(); |
| 67 | let _ = std::fs::remove_dir_all(dir); |
| 68 | Ok(()) |
| 69 | } |
| 70 | |
| 71 | #[test] |
| 72 | fn size_counts_every_zone() -> Outcome<()> { |
| 73 | log_set_level!("error"); |
| 74 | let dir = "./test_db_size_bytes_zones"; |
| 75 | let root = res!(fresh(dir)); |
| 76 | |
| 77 | let probe = res!(probe("./test_db_size_bytes_zones_probe")); |
| 78 | let mut cfg = res!(config(20 * probe.len)); |
| 79 | cfg.num_zones = 3; |
| 80 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, false, true)); |
| 81 | for i in 0..30 { |
| 82 | res!(db.insert(key(i), value(i, 1), Uid::default(), None)); |
| 83 | } |
| 84 | |
| 85 | let zones = cfg.zone_root(&root); |
| 86 | let per_zone: Vec<u64> = (1..=3).map(|z| dir_len(&zones.join(fmt!("zone_{:03}", z)))).collect(); |
| 87 | let got = res!(db.size_bytes()); |
| 88 | msg!("size_bytes {}, zone directories {:?}.", got, per_zone); |
| 89 | assert_eq!(per_zone.iter().sum::<u64>(), zone_files_len(&root, &cfg)); |
| 90 | assert!(per_zone.iter().all(|n| *n > 0), "a zone holds nothing: {:?}", per_zone); |
| 91 | assert_eq!(got, zone_files_len(&root, &cfg), "size_bytes is not the length of all three zones"); |
| 92 | assert!(got > per_zone[0], "size_bytes {} counts no more than zone 1's {}", got, per_zone[0]); |
| 93 | |
| 94 | res!(db.close()); |
| 95 | log_finish_wait!(); |
| 96 | let _ = std::fs::remove_dir_all(dir); |
| 97 | Ok(()) |
| 98 | } |
| 99 | |
| 100 | #[test] |
| 101 | fn size_tracks_delete_and_reopen() -> Outcome<()> { |
| 102 | log_set_level!("error"); |
| 103 | let dir = "./test_db_size_bytes_tracks"; |
| 104 | let root = res!(fresh(dir)); |
| 105 | |
| 106 | let probe = res!(probe("./test_db_size_bytes_tracks_probe")); |
| 107 | let cfg = res!(config(5 * probe.len + probe.len / 2)); |
| 108 | let mut seen = Vec::new(); |
| 109 | let mut check = |db: &TestDb, label: &str| -> Outcome<()> { |
| 110 | let got = res!(db.size_bytes()); |
| 111 | let walked = zone_files_len(&root, &cfg); |
| 112 | msg!("{}: size_bytes {}, walked {}.", label, got, walked); |
| 113 | assert_eq!(got, walked, "{}: size_bytes is not the length of the files", label); |
| 114 | seen.push(got); |
| 115 | Ok(()) |
| 116 | }; |
| 117 | |
| 118 | // 1. Written, then one record deleted: both leave the files longer, with collection off. |
| 119 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, false, true)); |
| 120 | res!(fill(&db)); |
| 121 | res!(check(&db, "after the writes")); |
| 122 | assert!(res!(db.delete(&key(0), Uid::default(), None))); |
| 123 | res!(check(&db, "after the delete")); |
| 124 | |
| 125 | // 2. Closed and reopened: the reopened store starts a new live file. |
| 126 | res!(db.close()); |
| 127 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, false, false)); |
| 128 | res!(check(&db, "after reopening")); |
| 129 | res!(db.close()); |
| 130 | |
| 131 | // 3. Reopened with collection on and overwritten three times: files are collected and |
| 132 | // deleted, so the length falls as well as grows. Measured once the files have stopped |
| 133 | // changing. |
| 134 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, true, false)); |
| 135 | for ver in 2..5u8 { |
| 136 | for i in 0..NKEYS { |
| 137 | res!(db.insert(key(i), value(i, ver), Uid::default(), None)); |
| 138 | } |
| 139 | } |
| 140 | let settled = settled_len(&root, &cfg); |
| 141 | res!(check(&db, "after the overwrites")); |
| 142 | assert_eq!(*seen.last().unwrap_or(&0), settled, "the files moved while they were measured"); |
| 143 | |
| 144 | // The figure moved with the files rather than being the first answer given. |
| 145 | let first = seen[0]; |
| 146 | assert!(seen.iter().any(|n| *n != first), "size_bytes never changed: {:?}", seen); |
| 147 | |
| 148 | res!(db.close()); |
| 149 | log_finish_wait!(); |
| 150 | let _ = std::fs::remove_dir_all(dir); |
| 151 | Ok(()) |
| 152 | } |
| 153 | |
| 154 | #[test] |
| 155 | fn size_survives_files_vanishing() -> Outcome<()> { |
| 156 | log_set_level!("error"); |
| 157 | let dir = "./test_db_size_bytes_vanish"; |
| 158 | let root = res!(fresh(dir)); |
| 159 | |
| 160 | let probe = res!(probe("./test_db_size_bytes_vanish_probe")); |
| 161 | let cfg = res!(config(5 * probe.len + probe.len / 2)); |
| 162 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg.clone()), schemes(), None, true, true)); |
| 163 | res!(fill(&db)); |
| 164 | |
| 165 | // Overwrites keep collections renaming and deleting files while another thread asks for the |
| 166 | // size, at least 200 times and until the overwrites are done. |
| 167 | let done = Arc::new(AtomicBool::new(false)); |
| 168 | let reader = { |
| 169 | let (db, done) = (db.clone(), done.clone()); |
| 170 | thread::spawn(move || -> Outcome<u64> { |
| 171 | let mut calls = 0u64; |
| 172 | loop { |
| 173 | let n = res!(db.size_bytes()); |
| 174 | assert!(n > 0, "size_bytes gave 0 for a store that holds files"); |
| 175 | calls += 1; |
| 176 | if calls >= 200 && done.load(Ordering::Relaxed) { |
| 177 | return Ok(calls); |
| 178 | } |
| 179 | } |
| 180 | }) |
| 181 | }; |
| 182 | for round in 0..40u8 { |
| 183 | for i in 0..NKEYS { |
| 184 | res!(db.insert(key(i), value(i, 2 + round), Uid::default(), None)); |
| 185 | } |
| 186 | } |
| 187 | done.store(true, Ordering::Relaxed); |
| 188 | let calls = match reader.join() { |
| 189 | Ok(r) => res!(r), |
| 190 | Err(_) => return Err(err!("The size_bytes reader thread panicked."; Test, Thread)), |
| 191 | }; |
| 192 | msg!("{} size_bytes calls answered while 40 rounds of overwrites drove collections.", calls); |
| 193 | assert!(calls >= 200); |
| 194 | |
| 195 | let settled = settled_len(&root, &cfg); |
| 196 | let got = res!(db.size_bytes()); |
| 197 | msg!("size_bytes {} with the files settled at {}.", got, settled); |
| 198 | assert_eq!(got, settled, "size_bytes is not the length of the settled files"); |
| 199 | assert_eq!(got, zone_files_len(&root, &cfg)); |
| 200 | |
| 201 | res!(db.close()); |
| 202 | log_finish_wait!(); |
| 203 | let _ = std::fs::remove_dir_all(dir); |
| 204 | Ok(()) |
| 205 | } |