oxedyne/fe2o3/fe2o3_steel/tests/addr_guard.rs
17.5 KiB, 1 run
created by r1870400018:35389, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! The address guard, asked by a real Steel over a real socket. |
| 2 | //! |
| 3 | //! # Why this is not a unit test |
| 4 | //! |
| 5 | //! `guard.rs` builds the guard, `cfg.rs` parses its settings, and |
| 6 | //! `fe2o3_net::guard::addr` holds the state machine, and every one of those has |
| 7 | //! tests of its own. None of them can say whether a running Steel *asks* it. That |
| 8 | //! question has had a wrong answer here before: `RingTimer::update` wrote its |
| 9 | //! timestamp into a `Copy` of the ring, so the rate was always zero and a guard |
| 10 | //! that read as configured refused nothing whatever. A limiter that records |
| 11 | //! nothing is indistinguishable from one that works, right up until the day it |
| 12 | //! matters. |
| 13 | //! |
| 14 | //! So a Steel is started, and connections are made to it, and what is asserted is |
| 15 | //! what a caller on the far side of the socket can see. |
| 16 | //! |
| 17 | //! # Where the guard sits, and what that means |
| 18 | //! |
| 19 | //! [`srv::server`] checks it in the TCP accept loop -- before the TLS handshake, |
| 20 | //! before the SNI is read, before any vhost is chosen. Two things follow, and |
| 21 | //! both matter to an operator: |
| 22 | //! |
| 23 | //! - It covers **every** vhost, proxied ones included. There is no per-vhost |
| 24 | //! configuration of it and no way for a vhost to be missed. |
| 25 | //! - It counts **connections**, not requests. A client that keeps one connection |
| 26 | //! alive and sends a thousand requests down it is one event to this guard. What |
| 27 | //! that is worth depends on the route: Steel answers a proxied route with |
| 28 | //! `Connection: close`, so there the two are the same number, while a static |
| 29 | //! vhost keeps the connection and there they are not. |
| 30 | //! |
| 31 | //! The second point is why a per-request limiter still earns its place upstream |
| 32 | //! of an application, and why `auth_path_prefixes` exists as a second tier that |
| 33 | //! *is* consulted per request. |
| 34 | //! |
| 35 | //! # A guard that refuses a browser is worse than none |
| 36 | //! |
| 37 | //! One test here floods and one does not, and the second is the one that would |
| 38 | //! cost more if it were wrong: a page load opens several connections at once, and |
| 39 | //! a threshold that treats that as an attack breaks every visitor rather than |
| 40 | //! stopping one. |
| 41 | //! |
| 42 | //! Unix only, for the same reason as `stopping_signal.rs`: the fixture runs a |
| 43 | //! real server as a child process. |
| 44 | //! |
| 45 | //! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\ |
| 46 | //! Anthropic Claude |
| 47 | |
| 48 | #![cfg(unix)] |
| 49 | |
| 50 | use oxedyne_fe2o3_core::prelude::*; |
| 51 | use oxedyne_fe2o3_crypto::keystore::{ |
| 52 | DEFAULT_WALLET_KDF_NAME, |
| 53 | Wallet, |
| 54 | }; |
| 55 | use oxedyne_fe2o3_jdat::{ |
| 56 | prelude::*, |
| 57 | file::JdatFile, |
| 58 | string::enc::EncoderConfig, |
| 59 | }; |
| 60 | use oxedyne_fe2o3_steel::{ |
| 61 | app::constant as app_const, |
| 62 | srv::admin::guard::{ |
| 63 | DEFAULT_RPS_MAX, |
| 64 | GUARD_RING, |
| 65 | }, |
| 66 | }; |
| 67 | |
| 68 | use std::{ |
| 69 | collections::BTreeMap, |
| 70 | io::Read, |
| 71 | net::{ |
| 72 | IpAddr, |
| 73 | Ipv4Addr, |
| 74 | SocketAddr, |
| 75 | TcpListener, |
| 76 | TcpStream, |
| 77 | }, |
| 78 | path::{ |
| 79 | Path, |
| 80 | PathBuf, |
| 81 | }, |
| 82 | process::{ |
| 83 | Child, |
| 84 | Command, |
| 85 | Stdio, |
| 86 | }, |
| 87 | time::{ |
| 88 | Duration, |
| 89 | Instant, |
| 90 | }, |
| 91 | }; |
| 92 | |
| 93 | // The wallet passphrase, in the clear and on purpose: it protects a wallet that |
| 94 | // exists for a few seconds in a scratch directory and holds one key to one empty |
| 95 | // database. `stopping_signal.rs` beside this says the same thing for the same |
| 96 | // reason. |
| 97 | const PASS: &str = "steel-addr-guard-test-passphrase-not-a-secret"; // allowlist secret |
| 98 | |
| 99 | const APP: &str = "addrguard"; |
| 100 | |
| 101 | const START_SECS: u64 = 120; |
| 102 | |
| 103 | // How long the whole burst is given to settle before any of it is read. Every |
| 104 | // connection the guard refused has had its close queued by then, so a read that |
| 105 | // blocks after this is a connection the server is still holding. |
| 106 | const SETTLE_MS: u64 = 500; |
| 107 | |
| 108 | // How long one settled connection is given to say something. Short on purpose: |
| 109 | // the server has already decided by now, so this is the cost of asking and not a |
| 110 | // wait for an answer. It must not be long enough to matter, because a burst read |
| 111 | // one socket at a time at 400 ms each was what made the first version of this |
| 112 | // test open its connections at two and a half a second and conclude, wrongly, |
| 113 | // that a guard set to fifty a second had never fired. |
| 114 | const ASK_MS: u64 = 40; |
| 115 | |
| 116 | // A browser's worst honest moment: a page whose assets open this many |
| 117 | // connections at once. Under `GUARD_RING`, so the rate machine has not even |
| 118 | // filled its ring, which is the arithmetic the assertion rests on. |
| 119 | const BROWSERS_BURST: usize = 24; |
| 120 | |
| 121 | |
| 122 | /// One connection's fate, as the far end sees it. |
| 123 | #[derive(Clone, Copy, Debug, Eq, PartialEq)] |
| 124 | enum Fate { |
| 125 | Kept, // the server is holding it open, waiting for a handshake |
| 126 | Dropped, // closed with nothing said, which is what the guard does |
| 127 | Refused, // the connection never opened at all |
| 128 | } |
| 129 | |
| 130 | |
| 131 | #[test] |
| 132 | fn test_the_guard_refuses_a_flood_of_connections_00() -> Outcome<()> { |
| 133 | let dir = res!(scratch("flood")); |
| 134 | let port = res!(free_port()); |
| 135 | res!(lay_out(&dir, port)); |
| 136 | let (child, probes) = res!(serve_fixture(&dir, port)); |
| 137 | |
| 138 | // Far past the ring, as fast as loopback allows, which is thousands a second |
| 139 | // against a ceiling of `DEFAULT_RPS_MAX`. |
| 140 | let fates = res!(burst(port, GUARD_RING * 4)); |
| 141 | put_down(child); |
| 142 | |
| 143 | let dropped = fates.iter().filter(|f| **f == Fate::Dropped).count(); |
| 144 | let kept = fates.iter().filter(|f| **f == Fate::Kept).count(); |
| 145 | // Asserted as a count rather than as "no error", because a guard that |
| 146 | // recorded nothing would also raise no error. The number is what tells a |
| 147 | // working guard from an ornamental one. |
| 148 | assert!(dropped > 0, |
| 149 | "{} connections were opened as fast as a socket allows, against a ceiling \ |
| 150 | of {} a second, and every one of them was served. The guard is either not \ |
| 151 | consulted in the accept path or is not counting.", |
| 152 | fates.len(), DEFAULT_RPS_MAX); |
| 153 | // And it did not start refusing before it had anything to go on. `avg_rps` |
| 154 | // reports zero until the ring is full, so a ring's worth must be served |
| 155 | // first; a guard refusing sooner than that is refusing on no evidence. |
| 156 | // |
| 157 | // The probes count. Waiting for the server to bind opens a connection each |
| 158 | // time round, and those are connections from this address like any other -- |
| 159 | // which is how this assertion first failed, at 63 served against a ring of |
| 160 | // 64, and it was right to. |
| 161 | assert!(kept + probes >= GUARD_RING, |
| 162 | "{} connections were served, after {} while waiting for the port, and the \ |
| 163 | guard's ring holds {}. It cannot have measured a rate on fewer than a \ |
| 164 | ring, so it is refusing on nothing.", kept, probes, GUARD_RING); |
| 165 | Ok(()) |
| 166 | } |
| 167 | |
| 168 | #[test] |
| 169 | fn test_a_page_load_is_not_a_flood_01() -> Outcome<()> { |
| 170 | let dir = res!(scratch("browser")); |
| 171 | let port = res!(free_port()); |
| 172 | res!(lay_out(&dir, port)); |
| 173 | let (child, _probes) = res!(serve_fixture(&dir, port)); |
| 174 | |
| 175 | // A browser opening every connection a page load would, back to back and with |
| 176 | // no pause, which is faster than any browser really manages. |
| 177 | let fates = res!(burst(port, BROWSERS_BURST)); |
| 178 | put_down(child); |
| 179 | |
| 180 | let dropped = fates.iter().filter(|f| **f == Fate::Dropped).count(); |
| 181 | assert_eq!(dropped, 0, |
| 182 | "{} of a page load's {} connections were dropped by the address guard. A \ |
| 183 | limit that refuses a browser fetching one page has cost more than any \ |
| 184 | attacker would.", dropped, BROWSERS_BURST); |
| 185 | Ok(()) |
| 186 | } |
| 187 | |
| 188 | |
| 189 | /// Opens `n` connections as fast as the socket allows, then reports what the |
| 190 | /// server did with each. |
| 191 | /// |
| 192 | /// The two halves are apart for a reason worth keeping. The whole test is about a |
| 193 | /// rate, so the connections have to be opened at a rate a rate limiter would |
| 194 | /// object to; reading each one as it is opened puts a timeout between every pair |
| 195 | /// of them and turns a flood into a trickle. The first version of this test did |
| 196 | /// exactly that, offered two and a half connections a second against a ceiling of |
| 197 | /// fifty, and reported that the guard never fired. |
| 198 | /// |
| 199 | /// Saying nothing on the connections is the method. Steel peeks for the first |
| 200 | /// bytes of a TLS record before it decides anything, so a connection it means to |
| 201 | /// serve simply waits; one the guard refused is closed at once with nothing sent. |
| 202 | /// The two are told apart by whether a read returns end-of-file or nothing at |
| 203 | /// all. |
| 204 | fn burst(port: u16, n: usize) -> Outcome<Vec<Fate>> { |
| 205 | let here = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port); |
| 206 | let mut held = Vec::with_capacity(n); |
| 207 | let began = Instant::now(); |
| 208 | for _ in 0..n { |
| 209 | held.push(TcpStream::connect_timeout(&here, Duration::from_secs(2))); |
| 210 | } |
| 211 | let took = began.elapsed(); |
| 212 | // The claim rests on the offered rate, so it is checked rather than assumed. |
| 213 | // A machine slow enough to have offered less than the ceiling would fail the |
| 214 | // assertions below for a reason that has nothing to do with the guard. |
| 215 | let rate = (n as f64) / took.as_secs_f64().max(f64::MIN_POSITIVE); |
| 216 | if rate < (DEFAULT_RPS_MAX as f64) { |
| 217 | return Err(err!( |
| 218 | "{} connections took {:?}, which is {:.0} a second and below the \ |
| 219 | guard's ceiling of {}. Nothing was offered that it should have \ |
| 220 | refused, so neither answer would mean anything.", |
| 221 | n, took, rate, DEFAULT_RPS_MAX; |
| 222 | Test, Timeout)); |
| 223 | } |
| 224 | std::thread::sleep(Duration::from_millis(SETTLE_MS)); |
| 225 | let mut fates = Vec::with_capacity(n); |
| 226 | for opened in held.iter_mut() { |
| 227 | fates.push(match opened { |
| 228 | Err(_) => Fate::Refused, |
| 229 | Ok(stream) => { |
| 230 | if stream.set_read_timeout( |
| 231 | Some(Duration::from_millis(ASK_MS))).is_err() |
| 232 | { |
| 233 | Fate::Refused |
| 234 | } else { |
| 235 | let mut buf = [0u8; 1]; |
| 236 | match stream.read(&mut buf) { |
| 237 | Ok(0) => Fate::Dropped, |
| 238 | Ok(_) => Fate::Kept, // it spoke, so it is serving us |
| 239 | Err(_) => Fate::Kept, // silent and open: waiting |
| 240 | } |
| 241 | } |
| 242 | }, |
| 243 | }); |
| 244 | } |
| 245 | Ok(fates) |
| 246 | } |
| 247 | |
| 248 | |
| 249 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 250 | // │ THE FIXTURE │ |
| 251 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 252 | // |
| 253 | // The same shape as `stopping_signal.rs`, and deliberately not shared with it: a |
| 254 | // fixture that two test files steer is a fixture neither of them can change. |
| 255 | |
| 256 | /// A name per fixture, because the tests in this file run at the same time. |
| 257 | /// Under `~/.cache` rather than `/tmp`, which is a tmpfs on this machine and has |
| 258 | /// brought it down before. |
| 259 | fn scratch(name: &str) -> Outcome<PathBuf> { |
| 260 | let base = match std::env::var("HOME") { |
| 261 | Ok(home) => PathBuf::from(home).join(".cache"), |
| 262 | Err(_) => std::env::temp_dir(), |
| 263 | }; |
| 264 | let dir = base.join(fmt!("steel_addrguard_{}", name)); |
| 265 | let _ = std::fs::remove_dir_all(&dir); |
| 266 | res!(std::fs::create_dir_all(&dir), IO, File); |
| 267 | Ok(dir) |
| 268 | } |
| 269 | |
| 270 | /// A port nothing is listening on, asked of the operating system rather than |
| 271 | /// picked. |
| 272 | fn free_port() -> Outcome<u16> { |
| 273 | let probe = res!(TcpListener::bind("127.0.0.1:0"), IO, Network); |
| 274 | let port = res!(probe.local_addr(), IO, Network).port(); |
| 275 | drop(probe); |
| 276 | Ok(port) |
| 277 | } |
| 278 | |
| 279 | /// Everything a Steel needs to start. |
| 280 | /// |
| 281 | /// The `addr_guard` block is left empty on purpose. That is the deployed |
| 282 | /// configuration on the host this was written for, and an empty block is not an |
| 283 | /// absent guard: every field falls back to the `DEFAULT_*` constants, which is |
| 284 | /// exactly what these tests are asserting about. |
| 285 | fn lay_out(dir: &Path, port: u16) -> Outcome<()> { |
| 286 | for sub in ["www/public", "www/src/styles", "www/logs"] { |
| 287 | res!(std::fs::create_dir_all(dir.join(sub)), IO, File); |
| 288 | } |
| 289 | res!(std::fs::write( |
| 290 | dir.join("www").join("public").join("index.html"), |
| 291 | "<!doctype html><title>guard</title><p>here.\n", |
| 292 | ), IO, File); |
| 293 | |
| 294 | let cfg = fmt!("{{ |
| 295 | \"app_description\": \"Address guard fixture\", |
| 296 | \"app_human_name\": \"Addrguard\", |
| 297 | \"app_log_level\": \"info\", |
| 298 | \"app_name\": \"{app}\", |
| 299 | \"app_root\": \"current\", |
| 300 | \"enc_name\": \"AES-256-GCM\", |
| 301 | \"kdf_name\": \"Argon2id_v0x13\", |
| 302 | \"dev_cfg\": {{ |
| 303 | \"src_path_rel\": \"./www/src\", |
| 304 | \"js_bundles_rel\": {{}}, |
| 305 | \"js_import_aliases_rel\": {{}}, |
| 306 | \"css_source_dir_rel\": \"./www/src/styles\", |
| 307 | \"css_bundle_rel\": \"./www/public/styles.css\" |
| 308 | }}, |
| 309 | \"server_cfg\": {{ |
| 310 | \"log_level\": \"info\", |
| 311 | \"num_server_bots\": (u16|1), |
| 312 | \"server_address\": \"127.0.0.1\", |
| 313 | \"server_port_tcp\": (u16|{port}), |
| 314 | \"server_port_tcp_plaintext\": (u16|0), |
| 315 | \"hsts_max_age_secs\": (u32|0), |
| 316 | \"admin_local_port\": (u16|0), |
| 317 | \"session_expiry_default_secs\": (u32|604800), |
| 318 | \"ws_ping_interval_secs\": (u8|30), |
| 319 | \"server_max_errors_allowed\": (u8|30), |
| 320 | \"allow_anonymous_sessions\": (true), |
| 321 | \"http_max_header_bytes\": (u64|16384), |
| 322 | \"http_max_body_bytes\": (u64|8388608), |
| 323 | \"http_header_read_timeout_ms\": (u64|15000), |
| 324 | \"security_headers_enabled\": (true), |
| 325 | \"content_security_policy\": \"\", |
| 326 | \"addr_guard\": {{}}, |
| 327 | \"auth_path_prefixes\": (vek|[\"/admin/login\"]), |
| 328 | \"auth_rps_max\": (u64|100), |
| 329 | \"tls_dir_rel\": \"./tls\", |
| 330 | \"acme\": {{ |
| 331 | \"enabled\": (false), |
| 332 | \"contact_email\": \"\", |
| 333 | \"directory_url\": \"https://acme-staging-v02.api.letsencrypt.org/directory\", |
| 334 | \"cache_dir_rel\": \"./tls/acme\" |
| 335 | }}, |
| 336 | \"mail\": {{}}, |
| 337 | \"vhosts\": [ |
| 338 | {{ |
| 339 | \"hostnames\": (vek|[\"localhost\",\"localhost.\"]), |
| 340 | \"public_dir_rel\": \"./www/public\", |
| 341 | \"static_route_paths_rel\": {{}}, |
| 342 | \"default_index_files\": (vek|[\"index.html\",\"index.htm\"]), |
| 343 | \"redirects\": [], |
| 344 | \"db_dir_rel\": \"./o3db\" |
| 345 | }} |
| 346 | ] |
| 347 | }} |
| 348 | }} |
| 349 | ", app = APP, port = port); |
| 350 | res!(std::fs::write(dir.join("config.jdat"), cfg), IO, File); |
| 351 | |
| 352 | let mut metadata = BTreeMap::new(); |
| 353 | metadata.insert(dat!("app_name"), dat!(APP)); |
| 354 | let (wallet, _unlocked) = res!(Wallet::create_with_first_admin( |
| 355 | metadata, |
| 356 | fmt!("operator"), |
| 357 | PASS.as_bytes(), |
| 358 | DEFAULT_WALLET_KDF_NAME, |
| 359 | )); |
| 360 | res!(wallet.save( |
| 361 | &dir.join(app_const::WALLET_NAME), |
| 362 | " ", |
| 363 | Some(EncoderConfig::<(), ()>::default()), |
| 364 | )); |
| 365 | Ok(()) |
| 366 | } |
| 367 | |
| 368 | /// Everything the server has said, wherever it said it. |
| 369 | fn said(dir: &Path) -> String { |
| 370 | let mut all = String::new(); |
| 371 | for path in [ |
| 372 | dir.join("console.log"), |
| 373 | dir.join("www").join("logs").join(fmt!("{}.log", APP)), |
| 374 | ] { |
| 375 | if let Ok(text) = std::fs::read_to_string(&path) { |
| 376 | all.push_str(&text); |
| 377 | } |
| 378 | } |
| 379 | all |
| 380 | } |
| 381 | |
| 382 | /// Starts a Steel and waits until it is answering. |
| 383 | /// |
| 384 | /// It must be *unsealed* before the tests run, because the guard is reached |
| 385 | /// through the admin state and a Steel with none skips it entirely. Waiting for |
| 386 | /// the port alone would start the flood against a server that had bound and not |
| 387 | /// yet built one. |
| 388 | fn serve_fixture(dir: &Path, port: u16) -> Outcome<(Child, usize)> { |
| 389 | let exe = PathBuf::from(env!("CARGO_BIN_EXE_steel")); |
| 390 | let console = res!(std::fs::File::create(dir.join("console.log")), IO, File); |
| 391 | let errs = res!(console.try_clone(), IO, File); |
| 392 | let child = res!(Command::new(&exe) |
| 393 | .current_dir(dir) |
| 394 | .arg("server") |
| 395 | // `-d` or nothing: without it a first run refuses production mode and |
| 396 | // exits 0 in silence. Dev mode also generates the self-signed |
| 397 | // certificate this fixture would otherwise have to carry. |
| 398 | .arg("-d") |
| 399 | .env("STEEL_ADMIN_PASS", PASS) |
| 400 | .stdin(Stdio::piped()) |
| 401 | .stdout(Stdio::from(console)) |
| 402 | .stderr(Stdio::from(errs)) |
| 403 | .spawn(), IO, File); |
| 404 | |
| 405 | let here = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port); |
| 406 | let began = Instant::now(); |
| 407 | let mut bound = false; |
| 408 | // Every one of these is a connection from this address, and the guard counts |
| 409 | // it like any other. The count is handed back so a test measuring the ring |
| 410 | // can allow for what waiting cost it. |
| 411 | let mut probes = 0; |
| 412 | while began.elapsed() < Duration::from_secs(START_SECS) { |
| 413 | if !bound { |
| 414 | probes += 1; |
| 415 | if TcpStream::connect_timeout(&here, Duration::from_millis(500)).is_ok() { |
| 416 | bound = true; |
| 417 | } |
| 418 | } |
| 419 | if bound && said(dir).contains("database(s) open and attached") { |
| 420 | return Ok((child, probes)); |
| 421 | } |
| 422 | std::thread::sleep(Duration::from_millis(250)); |
| 423 | } |
| 424 | let log = said(dir); |
| 425 | put_down(child); |
| 426 | Err(err!( |
| 427 | "The server did not bind {} and open its database within {} seconds \ |
| 428 | (bound: {}). It said:\n{}", |
| 429 | here, START_SECS, bound, tail_of(&log); |
| 430 | Test, Timeout)) |
| 431 | } |
| 432 | |
| 433 | /// The last of what a server said, for a message that has to carry a reason. |
| 434 | fn tail_of(log: &str) -> String { |
| 435 | let lines: Vec<&str> = log.lines().collect(); |
| 436 | let from = lines.len().saturating_sub(40); |
| 437 | lines[from..].join("\n") |
| 438 | } |
| 439 | |
| 440 | fn put_down(mut child: Child) { |
| 441 | let _ = child.kill(); |
| 442 | let _ = child.wait(); |
| 443 | } |