oxedyne/fe2o3/fe2o3_o3db_sync/archive/gc.rs
11.8 KiB, 49 runs
created by r1870400018:655, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | use oxedyne_fe2o3_core::prelude::*; |
| 2 | use oxedyne_fe2o3_crypto::enc::EncryptionScheme; |
| 3 | use oxedyne_fe2o3_jdat::prelude::*; |
| 4 | use oxedyne_fe2o3_o3db_sync2::{ |
| 5 | base::{ |
| 6 | cfg_db::ConfigInit, |
| 7 | }, |
| 8 | data::{ |
| 9 | cache::Meta, |
| 10 | core::StoreSpec, |
| 11 | }, |
| 12 | test::{ |
| 13 | data::{ |
| 14 | find_unique, |
| 15 | }, |
| 16 | dbapi::{ |
| 17 | store, |
| 18 | fetch, |
| 19 | }, |
| 20 | file::save, |
| 21 | setup::{ |
| 22 | default_cfg, |
| 23 | start_db, |
| 24 | }, |
| 25 | }, |
| 26 | }; |
| 27 | use oxedyne_fe2o3_test::{ |
| 28 | data::{ |
| 29 | DataArrangement, |
| 30 | DataFill, |
| 31 | DataSize, |
| 32 | DataSpec, |
| 33 | }, |
| 34 | error::delayed_error, |
| 35 | }; |
| 36 | |
| 37 | use std::{ |
| 38 | path::PathBuf, |
| 39 | thread, |
| 40 | time::Duration, |
| 41 | }; |
| 42 | |
| 43 | use rand_core::{ |
| 44 | RngCore, |
| 45 | OsRng, |
| 46 | }; |
| 47 | |
| 48 | //#[test] |
| 49 | //fn test_gc() -> Result<()> { |
| 50 | // |
| 51 | // let root_path = fmt!("./test_db"); |
| 52 | // let mut enckey = [0u8; 32]; |
| 53 | // OsRng.fill_bytes(&mut enckey); |
| 54 | // let aes_gcm = EncryptionScheme::new_aes_gcm_256(&enckey); |
| 55 | // let store_spec = StoreSpec::new().set_encrypter(Some(aes_gcm)); |
| 56 | // //let store_spec = StoreSpec::new(); |
| 57 | // let mut meta: Meta; |
| 58 | // let user1_name = fmt!("Alice83"); |
| 59 | // let user1_email = fmt!("alice83@gmail.com"); |
| 60 | // let user1_pass = fmt!("alice_pass"); |
| 61 | // |
| 62 | // let error_delay = 2; |
| 63 | // |
| 64 | // { |
| 65 | // for case in &[ |
| 66 | // 0, |
| 67 | // 50, |
| 68 | // 100, |
| 69 | // 200, |
| 70 | // 300, |
| 71 | // ] { |
| 72 | // let (note, cfg, keyspec, valsize) = match case { |
| 73 | // 0 => { |
| 74 | // let mut cfg = res!(default_cfg(&root_path)); |
| 75 | // cfg.bytes_before_chunking = 500; |
| 76 | // cfg.num_zones = 1; |
| 77 | // ( |
| 78 | // fmt!("Small sample, all-unique keys."), |
| 79 | // cfg, |
| 80 | // DataSpec { |
| 81 | // size: DataSize::Const(10), |
| 82 | // fill: DataFill::Random, |
| 83 | // arr: DataArrangement::PlainFill(15), |
| 84 | // }, |
| 85 | // DataSize::Const(10), |
| 86 | // ) |
| 87 | // }, |
| 88 | // 50 => { |
| 89 | // let mut cfg = res!(default_cfg(&root_path)); |
| 90 | // cfg.cache_size_limit_bytes = 3_000; |
| 91 | // cfg.bytes_before_chunking = 500; |
| 92 | // cfg.num_zones = 1; |
| 93 | // ( |
| 94 | // fmt!("Small sample, all-unique keys, with chunking and cache size management."), |
| 95 | // cfg, |
| 96 | // DataSpec { |
| 97 | // size: DataSize::Const(10), |
| 98 | // fill: DataFill::Random, |
| 99 | // arr: DataArrangement::PlainFill(3), |
| 100 | // }, |
| 101 | // DataSize::Const(600), |
| 102 | // ) |
| 103 | // }, |
| 104 | // 100 => { |
| 105 | // let mut cfg = res!(default_cfg(&root_path)); |
| 106 | // cfg.bytes_before_chunking = 500; |
| 107 | // ( |
| 108 | // fmt!("The same key repeated."), |
| 109 | // cfg, |
| 110 | // DataSpec { |
| 111 | // size: DataSize::Const(10), |
| 112 | // fill: DataFill::Const(42), |
| 113 | // arr: DataArrangement::PlainFill(500), |
| 114 | // }, |
| 115 | // DataSize::Const(10), |
| 116 | // ) |
| 117 | // }, |
| 118 | // 200 => { |
| 119 | // let mut cfg = res!(default_cfg(&root_path)); |
| 120 | // cfg.bytes_before_chunking = 500; |
| 121 | // cfg.num_zones = 2; |
| 122 | // ( |
| 123 | // fmt!("Generate lots of files with no gc."), |
| 124 | // cfg, |
| 125 | // DataSpec { |
| 126 | // size: DataSize::Const(10), |
| 127 | // fill: DataFill::Random, |
| 128 | // arr: DataArrangement::PlainFill(500), |
| 129 | // }, |
| 130 | // DataSize::Const(600), |
| 131 | // ) |
| 132 | // }, |
| 133 | // 300 => { |
| 134 | // let mut cfg = res!(default_cfg(&root_path)); |
| 135 | // cfg.cache_size_limit_bytes = 20_000_000; |
| 136 | // cfg.data_file_max_bytes = 1_000_000; |
| 137 | // cfg.bytes_before_chunking = 1_024; |
| 138 | // cfg.chunk_byte_size = 1_024; |
| 139 | // ( |
| 140 | // fmt!("Performance run with 2 zones."), |
| 141 | // cfg, |
| 142 | // DataSpec { |
| 143 | // size: DataSize::RandUniform { lo: 30, hi: 3000 }, |
| 144 | // fill: DataFill::Random, |
| 145 | // arr: DataArrangement::RepeatFillAndSeq{ |
| 146 | // n: 60, |
| 147 | // rep: 100, |
| 148 | // specbox: Box::new( |
| 149 | // DataSpec { |
| 150 | // size: DataSize::RandUniform { lo: 30, hi: 3000 }, |
| 151 | // fill: DataFill::Random, |
| 152 | // arr: DataArrangement::PlainFill(15), |
| 153 | // } |
| 154 | // ), |
| 155 | // }, |
| 156 | // }, |
| 157 | // DataSize::RandUniform { lo: 30, hi: 10_000 }, |
| 158 | // ) |
| 159 | // }, |
| 160 | // _ => unimplemented!(), |
| 161 | // }; |
| 162 | // msg!("+---------------------------------------------+"); |
| 163 | // msg!("| NEW OZONE SESSION |"); |
| 164 | // msg!("| Wipe all traces of previous test. |"); |
| 165 | // msg!("| Start database. |"); |
| 166 | // msg!("| Create a new user. |"); |
| 167 | // msg!("| Store and fetch some simple data. |"); |
| 168 | // msg!("| Gracefully shut down the database. |"); |
| 169 | // msg!("+---------------------------------------------+"); |
| 170 | // let mut db = match start_db( |
| 171 | // ConfigInit::Data(cfg.clone()), |
| 172 | // &root_path, |
| 173 | // fmt!("./test_db_zone_container"), |
| 174 | // true, |
| 175 | // ) { |
| 176 | // // These pauses on errors are needed to capture tardy messages from logging to help |
| 177 | // // with debugging. |
| 178 | // Err(e) => return Err(delayed_error(e, error_delay)), |
| 179 | // Ok(db) => db, |
| 180 | // }; |
| 181 | // |
| 182 | // let user_id = match db.create_user( |
| 183 | // user1_name.clone(), |
| 184 | // vec![user1_email.clone()], |
| 185 | // user1_pass.clone(), |
| 186 | // ) { |
| 187 | // Err(e) => return Err(delayed_error(e, error_delay)), |
| 188 | // Ok(user) => *user.id(), |
| 189 | // }; |
| 190 | // meta = Meta::new(user_id); |
| 191 | // |
| 192 | // thread::sleep(Duration::from_secs(1)); |
| 193 | // info!(sync_log::stream(), "Begin..."); |
| 194 | // let n = res!(keyspec.len()); |
| 195 | // let valspec = DataSpec { |
| 196 | // size: valsize, |
| 197 | // fill: DataFill::Random, |
| 198 | // arr: DataArrangement::PlainFill(n), |
| 199 | // }; |
| 200 | // |
| 201 | // info!(sync_log::stream(), "Generating {} data pairs using:", n); |
| 202 | // info!(sync_log::stream(), " keyspec: {:?}", keyspec); |
| 203 | // info!(sync_log::stream(), " valspec: {:?}", valspec); |
| 204 | // let k = res!(keyspec.generate()); |
| 205 | // let v = res!(valspec.generate()); |
| 206 | // //let n = k.len(); |
| 207 | // let mut kmaxlen = 0; |
| 208 | // let mut kminlen = usize::MAX; |
| 209 | // let mut vmaxlen = 0; |
| 210 | // let mut vminlen = usize::MAX; |
| 211 | // let mut kbyts = 0; |
| 212 | // let mut vbyts = 0; |
| 213 | // let mut kdats = Vec::new(); |
| 214 | // let mut vdats = Vec::new(); |
| 215 | // for i in 0..k.len() { |
| 216 | // let x = &k[i]; |
| 217 | // if x.len() > kmaxlen { kmaxlen = x.len() } |
| 218 | // if x.len() < kminlen { kminlen = x.len() } |
| 219 | // kbyts += x.len(); |
| 220 | // kdats.push(Dat::Byt(k[i].clone())); |
| 221 | // |
| 222 | // let x = &v[i]; |
| 223 | // if x.len() > vmaxlen { vmaxlen = x.len() } |
| 224 | // if x.len() < vminlen { vminlen = x.len() } |
| 225 | // vbyts += x.len(); |
| 226 | // vdats.push(Dat::Byt(v[i].clone())); |
| 227 | // } |
| 228 | // info!(sync_log::stream(), "Completed generation of {} pairs with metrics:", n); |
| 229 | // info!(sync_log::stream(), " Key sizes: mean {} range {}..{}", kbyts / n, kminlen, kmaxlen); |
| 230 | // info!(sync_log::stream(), " Val sizes: mean {} range {}..{}", vbyts / n, vminlen, vmaxlen); |
| 231 | // info!(sync_log::stream(), "ozone_keys:"); |
| 232 | // let mut kbufs = Vec::new(); |
| 233 | // let mut zinds = Vec::new(); |
| 234 | // for kdat in &kdats { |
| 235 | // let (kbuf, zind, _) = res!(db.ozone_key_dat(&kdat, &store_spec)); |
| 236 | // kbufs.push(kbuf); |
| 237 | // zinds.push(zind); |
| 238 | // } |
| 239 | // let mask = find_unique(&kbufs); |
| 240 | // |
| 241 | // //for (i, kbuf) in kbufs.iter().enumerate() { |
| 242 | // // info!(sync_log::stream(), " z={} len={} u={} kbuf={:02x?}", zinds[i], kbuf.len(), mask[i], kbuf); |
| 243 | // //} |
| 244 | // //info!(sync_log::stream(), "value dats:"); |
| 245 | // //for vdat in &vdats { |
| 246 | // // info!(sync_log::stream(), " vdat={:?}", vdat); |
| 247 | // //} |
| 248 | // // |
| 249 | // |
| 250 | // if *case == 550 { |
| 251 | // |
| 252 | // let mut path = PathBuf::from(&root_path); |
| 253 | // path.push("control"); |
| 254 | // info!(sync_log::stream(), "Creating {:?}", path); |
| 255 | // res!(std::fs::create_dir(&path)); |
| 256 | // match save( |
| 257 | // path, |
| 258 | // res!(cfg.data_file_max_bytes.try_into()), |
| 259 | // &mut db, |
| 260 | // &meta, |
| 261 | // &store_spec, |
| 262 | // kdats.clone(), |
| 263 | // vdats.clone(), |
| 264 | // kbyts + vbyts, |
| 265 | // ) { |
| 266 | // Err(e) => return Err(delayed_error(e, error_delay)), |
| 267 | // _ => (), |
| 268 | // } |
| 269 | // |
| 270 | // } |
| 271 | // |
| 272 | // info!(sync_log::stream(), "Test {} {}", case, note); |
| 273 | // thread::sleep(Duration::from_secs(3)); |
| 274 | // |
| 275 | // match store( |
| 276 | // &mut db, |
| 277 | // &meta, |
| 278 | // &store_spec, |
| 279 | // kdats.clone(), |
| 280 | // vdats.clone(), |
| 281 | // kbyts + vbyts, |
| 282 | // ) { |
| 283 | // Err(e) => return Err(delayed_error(e, error_delay)), |
| 284 | // _ => (), |
| 285 | // } |
| 286 | // |
| 287 | // thread::sleep(Duration::from_secs(5)); |
| 288 | // info!(sync_log::stream(), "Ozone state:"); |
| 289 | // let zstats = res!(db.ozone_state(db.default_wait())); |
| 290 | // for (i, zstat) in zstats.iter().enumerate() { |
| 291 | // info!(sync_log::stream(), "Zone {}", i+1); |
| 292 | // for line in oxedyne_fe2o3_text::string::to_lines(fmt!("{:?}", zstat), " ") { |
| 293 | // info!(sync_log::stream(), "{}", line); |
| 294 | // } |
| 295 | // } |
| 296 | // res!(db.list_files()); |
| 297 | // |
| 298 | // //res!(db.dump_caches(db.default_wait())); |
| 299 | // thread::sleep(Duration::from_secs(3)); |
| 300 | // |
| 301 | // match fetch( |
| 302 | // &mut db, |
| 303 | // &store_spec, |
| 304 | // &kdats, |
| 305 | // &mask, |
| 306 | // &vdats, |
| 307 | // kbyts + vbyts, |
| 308 | // ) { |
| 309 | // Err(e) => return Err(delayed_error(e, error_delay)), |
| 310 | // _ => (), |
| 311 | // } |
| 312 | // |
| 313 | // thread::sleep(Duration::from_secs(3)); |
| 314 | // |
| 315 | // info!(sync_log::stream(), "Shutting db down..."); |
| 316 | // res!(db.shutdown()); |
| 317 | // } |
| 318 | // } |
| 319 | // |
| 320 | // Ok(()) |
| 321 | //} |