oxedyne/fe2o3/fe2o3_o3db_sync/tests/settle.rs
5.7 KiB, 1 run
created by r1870400018:67601, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! The accounting barrier (A3 R2, 2026-10-08): once `settle` returns, every supersession caused by |
| 2 | //! a write acknowledged before the call is in the file bots' states. A cache bot answers its |
| 3 | //! caller before it tells the file bot of the new record's file, and that file bot tells the file |
| 4 | //! bot of the old record's file, so a request that merely follows the answer can overtake the |
| 5 | //! supersession. `set_forward_delay` holds each forward and `set_schedule_delay` holds each |
| 6 | //! receipt. The hooks are process-wide, so the tests here take a lock and put each back. |
| 7 | |
| 8 | mod gc_pair; |
| 9 | |
| 10 | use gc_pair::*; |
| 11 | |
| 12 | use oxedyne_fe2o3_core::prelude::*; |
| 13 | use oxedyne_fe2o3_iop_db::api::Database; |
| 14 | use oxedyne_fe2o3_o3db_sync::{ |
| 15 | base::constant, |
| 16 | test::{ |
| 17 | hooks, |
| 18 | setup, |
| 19 | }, |
| 20 | }; |
| 21 | |
| 22 | use std::{ |
| 23 | path::Path, |
| 24 | sync::Mutex, |
| 25 | time::Duration, |
| 26 | }; |
| 27 | |
| 28 | static HOOKS: Mutex<()> = Mutex::new(()); |
| 29 | |
| 30 | const N: usize = 20; |
| 31 | |
| 32 | fn old_sum(db: &TestDb) -> Outcome<u64> { |
| 33 | let states = res!(db.api().collect_file_states(constant::USER_REQUEST_WAIT)); |
| 34 | let mut sum = 0; |
| 35 | for (_wind, fmap) in states { |
| 36 | for (_fnum, fstat) in fmap.map() { |
| 37 | sum += fstat.get_old_sum(); |
| 38 | } |
| 39 | } |
| 40 | Ok(sum) |
| 41 | } |
| 42 | |
| 43 | #[test] |
| 44 | fn settle_orders_supersessions() -> Outcome<()> { |
| 45 | let _lock = match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() }; |
| 46 | log_set_level!("error"); |
| 47 | let dir = "./test_db_settle_order"; |
| 48 | let _ = std::fs::remove_dir_all(dir); |
| 49 | res!(std::fs::create_dir_all(dir)); |
| 50 | let root = res!(Path::new(dir).canonicalize()); |
| 51 | let probe = res!(probe("./test_db_settle_order_probe")); |
| 52 | let mut cfg = res!(config(5 * probe.len + probe.len / 2)); |
| 53 | cfg.num_cbots_per_zone = 2; |
| 54 | cfg.num_fbots_per_zone = 2; |
| 55 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg), schemes(), None, false, true)); |
| 56 | let user = setup::Uid::default(); |
| 57 | for i in 0..N { |
| 58 | res!(db.insert(key(i), value(i, 1), user, None)); |
| 59 | } |
| 60 | res!(db.api().settle_for_test(Duration::from_secs(10))); |
| 61 | assert_eq!(res!(old_sum(&db)), 0, "positive control: nothing is old before the overwrite"); |
| 62 | |
| 63 | // Two more records put the overwrites a file out of step with the records they replace. File |
| 64 | // k belongs to file bot k % 2, and a supersession forwards only when its two files belong to |
| 65 | // different bots, which five records to a file and twenty keys would never arrange. |
| 66 | for i in N..(N + 2) { |
| 67 | res!(db.insert(key(i), value(i, 1), user, None)); |
| 68 | } |
| 69 | hooks::set_forward_delay(Duration::from_millis(700)); |
| 70 | for i in 0..N { |
| 71 | res!(db.insert(key(i), value(i, 2), user, None)); |
| 72 | } |
| 73 | res!(db.api().settle_for_test(Duration::from_secs(60))); |
| 74 | let at_settle = res!(old_sum(&db)); |
| 75 | hooks::set_forward_delay(Duration::ZERO); |
| 76 | std::thread::sleep(Duration::from_millis(1500)); |
| 77 | let quiet = res!(old_sum(&db)); |
| 78 | msg!("Old bytes registered at settle {}, once quiet {}, records {} of {} bytes.", at_settle, quiet, N, probe.len); |
| 79 | assert_eq!(quiet, N as u64 * probe.len, "the quiet total is not the {} superseded records", N); |
| 80 | assert_eq!(at_settle, quiet, "supersessions were still travelling when settle returned"); |
| 81 | res!(db.close()); |
| 82 | Ok(()) |
| 83 | } |
| 84 | |
| 85 | // The echo stage: a file bot that has forwarded a supersession has not finished with it, since the |
| 86 | // file bot it went to may be behind. Every later request queues behind the supersession, so what |
| 87 | // tells a barrier that waits for the receiver from one that stops at the sender is when it returns, |
| 88 | // counted here in the supersessions the receivers had handled by then. The forwards are slowed as |
| 89 | // well, so that some arrive after the receiver has reached its own request, which it answers. |
| 90 | #[test] |
| 91 | fn settle_waits_for_the_receiving_file_bot() -> Outcome<()> { |
| 92 | let _lock = match HOOKS.lock() { Ok(g) => g, Err(p) => p.into_inner() }; |
| 93 | log_set_level!("error"); |
| 94 | let dir = "./test_db_settle_receiver"; |
| 95 | let _ = std::fs::remove_dir_all(dir); |
| 96 | res!(std::fs::create_dir_all(dir)); |
| 97 | let root = res!(Path::new(dir).canonicalize()); |
| 98 | let probe = res!(probe("./test_db_settle_receiver_probe")); |
| 99 | let mut cfg = res!(config(5 * probe.len + probe.len / 2)); |
| 100 | cfg.num_cbots_per_zone = 2; |
| 101 | cfg.num_fbots_per_zone = 2; |
| 102 | let db: TestDb = res!(setup::start_db(root.clone(), Some(cfg), schemes(), None, false, true)); |
| 103 | let user = setup::Uid::default(); |
| 104 | for i in 0..(N + 2) { |
| 105 | res!(db.insert(key(i), value(i, 1), user, None)); |
| 106 | } |
| 107 | res!(db.api().settle_for_test(Duration::from_secs(10))); |
| 108 | assert_eq!(res!(old_sum(&db)), 0, "positive control: nothing is old before the overwrite"); |
| 109 | |
| 110 | let base = hooks::schedules_held(); |
| 111 | hooks::set_forward_delay(Duration::from_millis(150)); |
| 112 | hooks::set_schedule_delay(Duration::from_millis(250)); |
| 113 | for i in 0..N { |
| 114 | res!(db.insert(key(i), value(i, 2), user, None)); |
| 115 | } |
| 116 | let settled = db.api().settle_for_test(Duration::from_secs(60)); |
| 117 | let at_settle = hooks::schedules_held() - base; |
| 118 | // The hold stays on until the receivers have caught up, or the count would stop short. |
| 119 | let mut last = hooks::schedules_held(); |
| 120 | loop { |
| 121 | std::thread::sleep(Duration::from_millis(800)); |
| 122 | let now = hooks::schedules_held(); |
| 123 | if now == last { break; } |
| 124 | last = now; |
| 125 | } |
| 126 | hooks::set_forward_delay(Duration::ZERO); |
| 127 | hooks::set_schedule_delay(Duration::ZERO); |
| 128 | let held = last - base; |
| 129 | res!(settled); |
| 130 | msg!("Supersessions held {}, handled when settle returned {}.", held, at_settle); |
| 131 | assert!(held > 0, "positive control: some supersession was forwarded to another file bot"); |
| 132 | assert_eq!(at_settle, held, "a file bot was still holding a received supersession when settle returned"); |
| 133 | assert_eq!(res!(old_sum(&db)), N as u64 * probe.len, "the superseded records are not all registered"); |
| 134 | res!(db.close()); |
| 135 | Ok(()) |
| 136 | } |