oxedyne/fe2o3/fe2o3_net/tests/http_forward.rs
42.2 KiB, 128 runs
created by r3497093317:77381, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | #![cfg(feature = "async")] |
| 2 | //! `http::forward_one`: one request carried to a TLS upstream with named fields replaced, driven |
| 3 | //! from a real Unix-socket client through to a TLS stand-in upstream on loopback, asserting on the |
| 4 | //! bytes each end received. Every key here is fake. |
| 5 | |
| 6 | use oxedyne_fe2o3_core::prelude::*; |
| 7 | use oxedyne_fe2o3_net::{ |
| 8 | http::{ |
| 9 | forward_one, |
| 10 | ForwardBounds, |
| 11 | ForwardTally, |
| 12 | RequestHead, |
| 13 | }, |
| 14 | tls::ensure_crypto_provider, |
| 15 | }; |
| 16 | |
| 17 | use std::{ |
| 18 | sync::{ |
| 19 | atomic::Ordering, |
| 20 | Arc, |
| 21 | }, |
| 22 | time::{ |
| 23 | Duration, |
| 24 | Instant, |
| 25 | }, |
| 26 | }; |
| 27 | |
| 28 | use tokio::{ |
| 29 | io::{ |
| 30 | AsyncReadExt, |
| 31 | AsyncWriteExt, |
| 32 | }, |
| 33 | net::{ |
| 34 | TcpListener, |
| 35 | UnixStream, |
| 36 | }, |
| 37 | task::JoinHandle, |
| 38 | }; |
| 39 | use tokio_rustls::{ |
| 40 | rustls::{ |
| 41 | client::ClientConfig, |
| 42 | pki_types::{ |
| 43 | CertificateDer, |
| 44 | PrivateKeyDer, |
| 45 | PrivatePkcs8KeyDer, |
| 46 | }, |
| 47 | server::ServerConfig, |
| 48 | RootCertStore, |
| 49 | }, |
| 50 | TlsAcceptor, |
| 51 | }; |
| 52 | |
| 53 | |
| 54 | const HOST: &str = "localhost"; |
| 55 | const STRIP: [&str; 3] = ["authorization", "x-api-key", "proxy-*"]; |
| 56 | const ADD: [(&str, &str); 1] = [("x-api-key", "sk-fake-secret-0000")]; |
| 57 | const BOUNDS: ForwardBounds = ForwardBounds { |
| 58 | head_cap: 8_192, |
| 59 | head_wait: Duration::from_millis(400), |
| 60 | connect: Duration::from_millis(400), |
| 61 | answer_wait: Duration::from_millis(500), |
| 62 | idle: Duration::from_millis(200), |
| 63 | body_cap: 1_024, |
| 64 | }; |
| 65 | const SLACK: Duration = Duration::from_millis(300); |
| 66 | |
| 67 | /// The memory test reads the process's resident set, so it runs alone; the others share. |
| 68 | static SERIAL: tokio::sync::RwLock<()> = tokio::sync::RwLock::const_new(()); |
| 69 | |
| 70 | /// What the stand-in upstream does once the request is in. |
| 71 | enum Reply { |
| 72 | Bytes(&'static [u8]), // Sent whole, then the stand-in closes. |
| 73 | Pieces(&'static [u8], usize, Duration), // A chunked head, then n pieces apart, then the end. |
| 74 | Big(u64), // A Content-Length answer of this many bytes. |
| 75 | Silent, // Never answers. |
| 76 | Steps(&'static [(u64, &'static [u8])]), // Each after its wait in ms, reading nothing meanwhile. |
| 77 | } |
| 78 | |
| 79 | /// A TLS stand-in upstream for `name` on loopback, and a client config trusting it. The task |
| 80 | /// returns every byte it received once the forwarder lets go of the connection. |
| 81 | async fn stand_in(name: &str, done: fn(&[u8]) -> bool, reply: Reply) |
| 82 | -> Outcome<(u16, Arc<ClientConfig>, JoinHandle<Vec<u8>>)> |
| 83 | { |
| 84 | ensure_crypto_provider(); |
| 85 | let cert = res!(rcgen::generate_simple_self_signed(vec![name.to_string()]), Init); |
| 86 | let der = CertificateDer::from(res!(cert.serialize_der(), Init)); |
| 87 | let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(cert.serialize_private_key_der())); |
| 88 | let mut roots = RootCertStore::empty(); |
| 89 | res!(roots.add(der.clone()), Init); |
| 90 | let client = ClientConfig::builder().with_root_certificates(roots).with_no_client_auth(); |
| 91 | let server = res!(ServerConfig::builder().with_no_client_auth().with_single_cert(vec![der], key), Init); |
| 92 | let listener = res!(TcpListener::bind("127.0.0.1:0").await, IO, Network); |
| 93 | let port = res!(listener.local_addr(), IO, Network).port(); |
| 94 | let task = tokio::spawn(async move { |
| 95 | let mut got = Vec::new(); |
| 96 | let tcp = match listener.accept().await { Ok((s, _)) => s, Err(_) => return got }; |
| 97 | let mut s = match TlsAcceptor::from(Arc::new(server)).accept(tcp).await { |
| 98 | Ok(s) => s, |
| 99 | Err(_) => return got, |
| 100 | }; |
| 101 | let mut buf = vec![0u8; 64 * 1_024]; |
| 102 | while !done(&got) { |
| 103 | match s.read(&mut buf).await { Ok(0) | Err(_) => return got, Ok(n) => got.extend_from_slice(&buf[..n]) } |
| 104 | } |
| 105 | let wrote = match reply { |
| 106 | Reply::Bytes(b) => s.write_all(b).await.is_ok() && s.shutdown().await.is_ok(), |
| 107 | Reply::Pieces(h, n, d) => { |
| 108 | let mut ok = s.write_all(h).await.is_ok(); |
| 109 | for i in 0..n { |
| 110 | if !ok { break; } |
| 111 | tokio::time::sleep(d).await; |
| 112 | let p = fmt!("data: {:02}\n\n", i); |
| 113 | ok = ok && s.write_all(fmt!("{:x}\r\n{}\r\n", p.len(), p).as_bytes()).await.is_ok(); |
| 114 | } |
| 115 | ok && s.write_all(b"0\r\n\r\n").await.is_ok() |
| 116 | }, |
| 117 | Reply::Big(n) => { |
| 118 | let mut ok = s.write_all(fmt!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", n).as_bytes()).await.is_ok(); |
| 119 | let mut left = n; |
| 120 | while ok && left > 0 { |
| 121 | let k = std::cmp::min(left, buf.len() as u64) as usize; |
| 122 | ok = s.write_all(&buf[..k]).await.is_ok(); |
| 123 | left -= k as u64; |
| 124 | } |
| 125 | ok |
| 126 | }, |
| 127 | Reply::Silent => true, |
| 128 | Reply::Steps(steps) => { |
| 129 | let mut ok = true; |
| 130 | for (ms, b) in steps { |
| 131 | tokio::time::sleep(Duration::from_millis(*ms)).await; |
| 132 | ok = ok && s.write_all(b).await.is_ok(); |
| 133 | } |
| 134 | ok |
| 135 | }, |
| 136 | }; |
| 137 | // Hold on until the forwarder closes: this task ending is the proof that it did. |
| 138 | while wrote { match s.read(&mut buf).await { Ok(0) | Err(_) => break, Ok(n) => got.extend_from_slice(&buf[..n]) } } |
| 139 | got |
| 140 | }); |
| 141 | Ok((port, Arc::new(client), task)) |
| 142 | } |
| 143 | |
| 144 | /// The keeper's side: read the head, forward it, and report. |
| 145 | async fn forwarder(s: UnixStream, port: u16, tls: Arc<ClientConfig>) -> (Outcome<()>, ForwardTally) { |
| 146 | forwarder_with(s, port, tls, BOUNDS, &ADD).await |
| 147 | } |
| 148 | |
| 149 | async fn forwarder_with( |
| 150 | mut s: UnixStream, |
| 151 | port: u16, |
| 152 | tls: Arc<ClientConfig>, |
| 153 | b: ForwardBounds, |
| 154 | add: &'static [(&'static str, &'static str)], |
| 155 | ) |
| 156 | -> (Outcome<()>, ForwardTally) |
| 157 | { |
| 158 | let tally = ForwardTally::default(); |
| 159 | let r = match RequestHead::read(&mut s, &b).await { |
| 160 | Ok(h) => forward_one(&mut s, h, &STRIP, add, HOST, port, tls, &b, &tally).await, |
| 161 | Err(e) => Err(e), |
| 162 | }; |
| 163 | (r, tally) |
| 164 | } |
| 165 | |
| 166 | /// Does the error, shown either way, hold a fake key's word `secret`, as text, as decimal bytes |
| 167 | /// or as hex bytes? Every fake key in this file holds that word. |
| 168 | fn leaks(e: &Error<ErrTag>) -> bool { |
| 169 | let w = b"secret"; |
| 170 | let dec = w.iter().map(|b| b.to_string()).collect::<Vec<_>>().join(", "); |
| 171 | let hex = w.iter().map(|b| fmt!("{:x}", b)).collect::<Vec<_>>(); |
| 172 | let forms = ["secret".to_string(), dec, hex.join(", "), hex.concat()]; |
| 173 | [fmt!("{}", e), fmt!("{:?}", e)].iter().any(|t| forms.iter().any(|f| t.contains(f.as_str()))) |
| 174 | } |
| 175 | |
| 176 | /// Sends `req` through a forwarder to a stand-in and reads the answer to its end. |
| 177 | async fn drive(req: &[u8], done: fn(&[u8]) -> bool, reply: Reply) |
| 178 | -> Outcome<(Outcome<()>, ForwardTally, Vec<u8>, Vec<u8>)> |
| 179 | { |
| 180 | drive_with(BOUNDS, req, done, reply).await |
| 181 | } |
| 182 | |
| 183 | async fn drive_with(b: ForwardBounds, req: &[u8], done: fn(&[u8]) -> bool, reply: Reply) |
| 184 | -> Outcome<(Outcome<()>, ForwardTally, Vec<u8>, Vec<u8>)> |
| 185 | { |
| 186 | drive_adding(b, &ADD, req, done, reply).await |
| 187 | } |
| 188 | |
| 189 | async fn drive_adding( |
| 190 | b: ForwardBounds, |
| 191 | add: &'static [(&'static str, &'static str)], |
| 192 | req: &[u8], |
| 193 | done: fn(&[u8]) -> bool, |
| 194 | reply: Reply, |
| 195 | ) |
| 196 | -> Outcome<(Outcome<()>, ForwardTally, Vec<u8>, Vec<u8>)> |
| 197 | { |
| 198 | let (port, tls, up) = res!(stand_in(HOST, done, reply).await); |
| 199 | let (mut c, s) = res!(UnixStream::pair(), IO); |
| 200 | let fwd = tokio::spawn(forwarder_with(s, port, tls, b, add)); |
| 201 | res!(c.write_all(req).await, IO); |
| 202 | let mut answer = Vec::new(); |
| 203 | res!(c.read_to_end(&mut answer).await, IO); |
| 204 | let (r, tally) = res!(fwd.await, IO); |
| 205 | let got = res!(tokio::time::timeout(SLACK, up).await, IO, Timeout); |
| 206 | Ok((r, tally, answer, res!(got, IO))) |
| 207 | } |
| 208 | |
| 209 | fn head_in(b: &[u8]) -> bool { b.windows(4).any(|w| w == b"\r\n\r\n") } |
| 210 | fn hello_in(b: &[u8]) -> bool { b.ends_with(b"hello") } |
| 211 | |
| 212 | fn has(e: &Error<ErrTag>, t: ErrTag) -> bool { e.tags().contains(&t) } |
| 213 | |
| 214 | fn ended<T>(r: Result<T, tokio::time::error::Elapsed>) -> bool { r.is_ok() } |
| 215 | |
| 216 | /// Fault caught: a client's key, in any case, repeated or oddly spaced, reaching the provider; the |
| 217 | /// keeper's key arriving twice or not at all; the client's `Host` or `Connection` going through. |
| 218 | #[tokio::test] |
| 219 | async fn test_secret_fields_never_reach_the_upstream_and_the_added_one_arrives_once() -> Outcome<()> { |
| 220 | let _one = SERIAL.read().await; |
| 221 | let req = b"POST /v1/messages HTTP/1.1\r\nHost: evil.example\r\n\ |
| 222 | Authorization: Bearer client-secret-1\r\nauthorization: Bearer client-secret-2\r\n\ |
| 223 | X-API-KEY:client-secret-3\r\nx-api-key: \t client-secret-4 \t\r\n\ |
| 224 | Proxy-Authorization: Basic client-secret-5\r\nPROXY-connection: client-secret-6\r\n\ |
| 225 | Connection: keep-alive\r\nAccept: */*\r\nContent-Length: 5\r\n\r\nhello"; |
| 226 | let (r, tally, answer, got) = res!(drive(req, hello_in, |
| 227 | Reply::Bytes(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")).await); |
| 228 | res!(r); |
| 229 | let up = String::from_utf8_lossy(&got).to_string(); |
| 230 | let lines: Vec<&str> = up.split("\r\n").collect(); |
| 231 | let count = |p: &str| lines.iter().filter(|l| l.to_ascii_lowercase().starts_with(p)).count(); |
| 232 | req!(false, up.contains("client-secret"), "a client's key reached the upstream:\n{}", up); |
| 233 | req!(false, up.contains("evil"), "the client's Host reached the upstream"); |
| 234 | req!(1, count("x-api-key:"), "the added field did not arrive exactly once"); |
| 235 | req!(true, up.contains("\r\nx-api-key: sk-fake-secret-0000\r\n")); |
| 236 | req!(1, count("host:")); |
| 237 | req!(true, up.starts_with("POST /v1/messages HTTP/1.1\r\nHost: localhost:"), "the upstream saw another Host"); |
| 238 | req!(1, count("connection:")); |
| 239 | req!(true, up.contains("\r\nConnection: close\r\n\r\nhello")); |
| 240 | req!(true, up.contains("\r\nAccept: */*\r\n"), "an ordinary field was lost"); |
| 241 | req!(true, answer.ends_with(b"\r\n\r\nok")); |
| 242 | req!(200, tally.status.load(Ordering::Relaxed)); |
| 243 | req!(got.len() as u64, tally.sent.load(Ordering::Relaxed)); |
| 244 | req!(answer.len() as u64, tally.got.load(Ordering::Relaxed)); |
| 245 | // An added name replaces its own field only, in any case, and never a longer one. |
| 246 | let req = b"GET / HTTP/1.1\r\nx-provider-key: client-secret-7\r\nX-Other: kept-7\r\n\ |
| 247 | X-Provider-Keyring: kept-8\r\n\r\n"; |
| 248 | let (r, _, _, got) = res!(drive_adding(BOUNDS, &[("X-Provider-Key", "sk-fake-secret-1111")], req, |
| 249 | head_in, Reply::Bytes(b"HTTP/1.1 204 No Content\r\n\r\n")).await); |
| 250 | res!(r); |
| 251 | let up = String::from_utf8_lossy(&got).to_string(); |
| 252 | req!(false, up.contains("client-secret"), "a client's key reached the upstream:\n{}", up); |
| 253 | req!(1, up.matches("\r\nX-Provider-Key: sk-fake-secret-1111\r\n").count(), "{}", up); |
| 254 | req!(true, up.contains("\r\nX-Other: kept-7\r\n") && up.contains("\r\nX-Provider-Keyring: kept-8\r\n"), |
| 255 | "a client field another name begins was lost:\n{}", up); |
| 256 | Ok(()) |
| 257 | } |
| 258 | |
| 259 | /// Fault caught: a head the keeper cannot read as one unambiguous request still being dialled out, |
| 260 | /// a slow head holding the keeper, or a key in the head echoed in the error. |
| 261 | #[tokio::test] |
| 262 | async fn test_a_bad_head_is_refused_within_the_bound_and_nothing_is_dialled() -> Outcome<()> { |
| 263 | let _one = SERIAL.read().await; |
| 264 | let long = fmt!("GET / HTTP/1.1\r\nX-Pad: {}\r\n\r\n", "a".repeat(9_000)); |
| 265 | let cases: [(&str, &[u8], ErrTag); 21] = [ |
| 266 | ("over the cap", long.as_bytes(), ErrTag::TooBig), |
| 267 | ("not HTTP", b"SSH-2.0-OpenSSH_9.6 client-secret\r\n\r\n", ErrTag::Invalid), |
| 268 | ("not UTF-8", b"GET / HTTP/1.1\r\nAuthorization: Bearer client-secret\r\nX: \xff\r\n\r\n", ErrTag::Invalid), |
| 269 | ("absolute form", b"GET http://evil.example/ HTTP/1.1\r\nHost: evil.example\r\n\r\n", ErrTag::Invalid), |
| 270 | ("asterisk form", b"OPTIONS * HTTP/1.1\r\n\r\n", ErrTag::Invalid), |
| 271 | ("folded", b"GET / HTTP/1.1\r\nAuthorization: Bearer\r\n client-secret\r\n\r\n", ErrTag::Invalid), |
| 272 | ("space before colon", b"GET / HTTP/1.1\r\nAuthorization : client-secret\r\n\r\n", ErrTag::Invalid), |
| 273 | ("bare LF", b"GET / HTTP/1.1\r\nX: y\nAuthorization: client-secret\r\n\r\n", ErrTag::Invalid), |
| 274 | ("two lengths", b"POST / HTTP/1.1\r\nContent-Length: 5\r\nContent-Length: 5\r\n\r\nhello", ErrTag::Invalid), |
| 275 | ("length not digits", b"POST / HTTP/1.1\r\nContent-Length: 5 client-secret\r\n\r\nhello", ErrTag::Invalid), |
| 276 | ("body over the cap", b"POST / HTTP/1.1\r\nContent-Length: 1025\r\n\r\nhello", ErrTag::TooBig), |
| 277 | ("TE and CL", b"POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nContent-Length: 5\r\n\r\n0\r\n\r\n", ErrTag::Invalid), |
| 278 | ("TE not chunked", b"POST / HTTP/1.1\r\nTransfer-Encoding: client-secret, chunked\r\n\r\n0\r\n\r\n", ErrTag::Invalid), |
| 279 | ("TRACE", b"TRACE / HTTP/1.1\r\nHost: x\r\n\r\n", ErrTag::Invalid), |
| 280 | ("trace", b"trace / HTTP/1.1\r\nHost: x\r\n\r\n", ErrTag::Invalid), |
| 281 | ("TrAcE", b"TrAcE / HTTP/1.1\r\nHost: x\r\n\r\n", ErrTag::Invalid), |
| 282 | ("TRACK", b"TRACK / HTTP/1.1\r\nHost: x\r\n\r\n", ErrTag::Invalid), |
| 283 | ("CONNECT origin form", b"CONNECT /x HTTP/1.1\r\nHost: x\r\n\r\n", ErrTag::Invalid), |
| 284 | ("CONNECT", b"CONNECT evil.example:443 HTTP/1.1\r\nHost: evil.example:443\r\n\r\n", ErrTag::Invalid), |
| 285 | ("HTTP/1.0", b"GET / HTTP/1.0\r\n\r\n", ErrTag::Invalid), |
| 286 | ("never ends", b"GET / HTTP/1.1\r\nAuthorization: client-secret", ErrTag::Timeout), |
| 287 | ]; |
| 288 | let listener = res!(TcpListener::bind("127.0.0.1:0").await, IO); |
| 289 | let port = res!(listener.local_addr(), IO).port(); |
| 290 | ensure_crypto_provider(); |
| 291 | let tls = Arc::new(ClientConfig::builder().with_root_certificates(RootCertStore::empty()).with_no_client_auth()); |
| 292 | for (what, req, tag) in cases { |
| 293 | let (mut c, s) = res!(UnixStream::pair(), IO); |
| 294 | let t0 = Instant::now(); |
| 295 | let fwd = tokio::spawn(forwarder(s, port, tls.clone())); |
| 296 | let _ = c.write_all(req).await; |
| 297 | let (r, _) = res!(fwd.await, IO); |
| 298 | let took = t0.elapsed(); |
| 299 | match r { |
| 300 | Ok(()) => return Err(err!("A head {} was forwarded.", what; Test, Unreachable)), |
| 301 | Err(e) => { |
| 302 | let ok = has(&e, tag.clone()); |
| 303 | req!(true, ok, "{}: {:?} lacks {:?}: {}", what, e.tags(), tag, e); |
| 304 | req!(false, leaks(&e), "{}: the error shows a value: {:?}", what, e); |
| 305 | }, |
| 306 | } |
| 307 | req!(true, took < BOUNDS.head_wait + SLACK, "{}: refused after {:?}", what, took); |
| 308 | req!(true, tokio::time::timeout(Duration::from_millis(20), listener.accept()).await.is_err(), |
| 309 | "{}: the upstream was dialled", what); |
| 310 | drop(c); |
| 311 | } |
| 312 | // An added value that would split the head is refused before the dial. |
| 313 | // A `*` would read as a strip pattern, so an added name holding one is refused too. |
| 314 | let bad: [&'static [(&'static str, &'static str)]; 9] = [ |
| 315 | &[("x-api-key", "sk-fake-secret\r\nX-Evil: 1")], |
| 316 | &[("x-api-key", "sk-fake-secret\nX-Evil: 1")], |
| 317 | &[("x-api-key", "sk-fake-secret\r")], |
| 318 | &[("x-api-key", "sk-fake\0secret")], |
| 319 | &[("x-api key", "sk-fake-secret")], |
| 320 | &[("x-*", "sk-fake-secret")], |
| 321 | &[("*", "sk-fake-secret")], |
| 322 | &[("x-api:key", "sk-fake-secret")], |
| 323 | &[("", "sk-fake-secret")], |
| 324 | ]; |
| 325 | for add in bad { |
| 326 | let (mut c, s) = res!(UnixStream::pair(), IO); |
| 327 | let fwd = tokio::spawn(forwarder_with(s, port, tls.clone(), BOUNDS, add)); |
| 328 | res!(c.write_all(b"GET / HTTP/1.1\r\nX-Other: client-secret\r\n\r\n").await, IO); |
| 329 | match res!(fwd.await, IO).0 { |
| 330 | Ok(()) => return Err(err!("An added field {:?} was forwarded.", add[0].0; Test, Unreachable)), |
| 331 | Err(e) => req!(true, has(&e, ErrTag::Invalid) && !leaks(&e), "add {:?}: {:?}", add[0].0, e), |
| 332 | } |
| 333 | req!(true, tokio::time::timeout(Duration::from_millis(20), listener.accept()).await.is_err(), |
| 334 | "add {:?}: the upstream was dialled", add[0].0); |
| 335 | } |
| 336 | Ok(()) |
| 337 | } |
| 338 | |
| 339 | /// Fault caught: a keeper choosing the upstream from a head with no `Host`, or from one of two. |
| 340 | #[tokio::test] |
| 341 | async fn test_host_is_the_one_host_field() -> Outcome<()> { |
| 342 | let _one = SERIAL.read().await; |
| 343 | let cases: [(&[u8], Option<&str>); 8] = [ |
| 344 | (b"GET / HTTP/1.1\r\nHost: api.example\r\n\r\n", Some("api.example")), |
| 345 | (b"GET / HTTP/1.1\r\nhOsT:api.example:8443\r\n\r\n", Some("api.example:8443")), |
| 346 | (b"GET / HTTP/1.1\r\nHost: [::1]:8443\r\n\r\n", Some("[::1]:8443")), |
| 347 | (b"GET / HTTP/1.1\r\nAccept: */*\r\n\r\n", None), |
| 348 | (b"GET / HTTP/1.1\r\nHost: api.one\r\nHost: api.two\r\n\r\n", None), |
| 349 | (b"GET / HTTP/1.1\r\nHost: api.one\r\nHOST: api.one\r\n\r\n", None), |
| 350 | (b"GET / HTTP/1.1\r\nHost: \r\n\r\n", None), |
| 351 | (b"GET / HTTP/1.1\r\nHost: api.one api.two\r\n\r\n", None), |
| 352 | ]; |
| 353 | for (head, want) in cases { |
| 354 | let (mut c, mut s) = res!(UnixStream::pair(), IO); |
| 355 | res!(c.write_all(head).await, IO); |
| 356 | let h = res!(RequestHead::read(&mut s, &BOUNDS).await); |
| 357 | match (h.host(), want) { |
| 358 | (Ok(v), Some(w)) => req!(w, v), |
| 359 | (Err(e), None) => req!(true, has(&e, ErrTag::Invalid) && !leaks(&e)), |
| 360 | (got, _) => return Err(err!("Host of {:?}: {:?}", |
| 361 | String::from_utf8_lossy(head), got.map(|v| v.to_string()); Test, Unreachable)), |
| 362 | } |
| 363 | } |
| 364 | Ok(()) |
| 365 | } |
| 366 | |
| 367 | /// Fault caught: a head's end missed where CRLF CRLF straddles reads, a head of exactly the cap |
| 368 | /// refused, or one byte over it let through. |
| 369 | #[tokio::test] |
| 370 | async fn test_a_head_ends_wherever_the_reads_split_it() -> Outcome<()> { |
| 371 | let _one = SERIAL.read().await; |
| 372 | let at = |len: usize| { |
| 373 | let pad = len - b"GET / HTTP/1.1\r\nX: \r\n\r\n".len(); |
| 374 | fmt!("GET / HTTP/1.1\r\nX: {}\r\n\r\nBODY", "a".repeat(pad)).into_bytes() |
| 375 | }; |
| 376 | let cases = [(at(BOUNDS.head_cap), 1, true), (at(BOUNDS.head_cap), 3, true), |
| 377 | (at(BOUNDS.head_cap + 1), 1, false), (at(40), 1, true), (at(40), 2, true), (at(41), 2, true), |
| 378 | (at(42), 3, true), (at(43), 3, true)]; |
| 379 | for (wire, per, fits) in cases { |
| 380 | // A pipe of `per` bytes hands over at most that many a read. |
| 381 | let (mut c, mut s) = tokio::io::duplex(per); |
| 382 | let w = tokio::spawn(async move { let _ = c.write_all(&wire).await; c }); |
| 383 | match (RequestHead::read(&mut s, &BOUNDS).await, fits) { |
| 384 | (Ok(h), true) => req!("/", h.target()), |
| 385 | (Err(e), false) => req!(true, has(&e, ErrTag::TooBig), "{:?}", e), |
| 386 | (Ok(_), _) => return Err(err!("A head, {} a read, over the cap passed.", per; Test)), |
| 387 | (Err(e), _) => return Err(err!(e, "A head, {} a read, within the cap failed.", per; Test)), |
| 388 | } |
| 389 | w.abort(); |
| 390 | } |
| 391 | // The cost of a head grows with its length, not its square: 64 KiB a byte a read took 7 s. |
| 392 | let big = ForwardBounds { head_cap: 65_536, head_wait: Duration::from_secs(3), ..BOUNDS }; |
| 393 | let (mut c, mut s) = tokio::io::duplex(1); |
| 394 | let wire = at(big.head_cap); |
| 395 | let w = tokio::spawn(async move { let _ = c.write_all(&wire).await; c }); |
| 396 | let t0 = Instant::now(); |
| 397 | let r = RequestHead::read(&mut s, &big).await; |
| 398 | eprintln!("64 KiB head a byte a read: {:?}", t0.elapsed()); |
| 399 | w.abort(); |
| 400 | let h = res!(r); |
| 401 | req!("/", h.target()); |
| 402 | Ok(()) |
| 403 | } |
| 404 | |
| 405 | /// Fault caught: a body altered or cut on the way, a chunked body decoded or re-framed, and a |
| 406 | /// second request smuggled behind the first reaching the upstream with the keeper's key. |
| 407 | #[tokio::test] |
| 408 | async fn test_bodies_go_verbatim_and_only_one_request_is_forwarded() -> Outcome<()> { |
| 409 | let _one = SERIAL.read().await; |
| 410 | let chunked = b"POST /c HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n2;x=1\r\nhe\r\n3\r\nllo\r\n0\r\n\r\n"; |
| 411 | let (r, _, _, got) = res!(drive(chunked, |b| b.ends_with(b"llo\r\n0\r\n\r\n"), |
| 412 | Reply::Bytes(b"HTTP/1.1 204 No Content\r\n\r\n")).await); |
| 413 | res!(r); |
| 414 | req!(true, got.ends_with(b"\r\n\r\n2;x=1\r\nhe\r\n3\r\nllo\r\n0\r\n\r\n"), "the chunked body changed"); |
| 415 | let piped = b"POST /a HTTP/1.1\r\nContent-Length: 5\r\n\r\nhelloGET /second HTTP/1.1\r\nHost: x\r\n\r\n"; |
| 416 | let (r, tally, answer, got) = res!(drive(piped, hello_in, |
| 417 | Reply::Bytes(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")).await); |
| 418 | res!(r); |
| 419 | req!(false, String::from_utf8_lossy(&got).contains("second"), "a pipelined request was forwarded"); |
| 420 | req!(true, got.ends_with(b"hello")); |
| 421 | req!(true, answer.ends_with(b"ok")); |
| 422 | req!(got.len() as u64, tally.sent.load(Ordering::Relaxed)); |
| 423 | // A field in a request's trailers would pass the strip, so the request is refused there. |
| 424 | let trailer = b"POST /t HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n0\r\nAuthorization: client-secret\r\n\r\n"; |
| 425 | let (r, _, _, got) = res!(drive(trailer, |b| b.ends_with(b"0\r\n"), Reply::Silent).await); |
| 426 | match r { Err(e) => req!(false, leaks(&e), "the error shows a value: {:?}", e), Ok(()) => return Err(err!("A request trailer was forwarded."; Test)) } |
| 427 | req!(false, String::from_utf8_lossy(&got).contains("client-secret"), "a trailer field reached the upstream"); |
| 428 | // A bad chunk-size line is body text, so its error names the place and the reason only. |
| 429 | let lines: [&'static [u8]; 4] = [b"5 client-secret\r\nhello\r\n", b"client-secret\r\n", |
| 430 | b"736563726574 x\r\n", b"5;115, 101, 99, 114, 101, 116\x01\r\n"]; |
| 431 | for line in lines { |
| 432 | let wire = [&b"POST /t HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"[..], line].concat(); |
| 433 | let (r, _, _, _) = res!(drive(&wire, head_in, Reply::Silent).await); |
| 434 | let shown = String::from_utf8_lossy(line); |
| 435 | match r { Err(e) => req!(true, has(&e, ErrTag::Invalid) && !leaks(&e), "{:?}: {:?}", shown, e), Ok(()) => return Err(err!("{:?}: Ok", shown; Test)) } |
| 436 | } |
| 437 | // A chunk-size line of 1 MB with no CRLF is refused at the line cap. |
| 438 | let (port, tls, _up) = res!(stand_in(HOST, head_in, Reply::Silent).await); |
| 439 | let (mut c, s) = res!(UnixStream::pair(), IO); |
| 440 | let fwd = tokio::spawn(forwarder(s, port, tls)); |
| 441 | let w = tokio::spawn(async move { |
| 442 | let wire = [&b"POST /t HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"[..], &[b'1'; 1_000_000]].concat(); |
| 443 | let _ = c.write_all(&wire).await; |
| 444 | c |
| 445 | }); |
| 446 | match res!(fwd.await, IO).0 { Err(e) => req!(true, has(&e, ErrTag::TooBig) && !leaks(&e), "long chunk line: {:?}", e), Ok(()) => return Err(err!("long chunk line: Ok"; Test)) } |
| 447 | w.abort(); |
| 448 | // Exactly the cap passes; so does no body, and a body of 0, under a cap of 0. |
| 449 | let full = [&b"POST /cap HTTP/1.1\r\nContent-Length: 1024\r\n\r\n"[..], &[b'a'; 1_023], b"z"].concat(); |
| 450 | let ok = || Reply::Bytes(b"HTTP/1.1 204 No Content\r\n\r\n"); |
| 451 | let (r, _, _, got) = res!(drive(&full, |b| b.ends_with(b"aaz"), ok()).await); |
| 452 | res!(r); |
| 453 | req!(true, got.ends_with(&full[full.len() - 1_024..])); |
| 454 | let none = ForwardBounds { body_cap: 0, ..BOUNDS }; |
| 455 | res!(res!(drive_with(none, b"GET / HTTP/1.1\r\n\r\n", head_in, ok()).await).0); |
| 456 | res!(res!(drive_with(none, b"POST / HTTP/1.1\r\nContent-Length: 0\r\n\r\n", head_in, ok()).await).0); |
| 457 | // A chunked body is cut where it passes the cap: the upstream has the head and the chunk's |
| 458 | // size line by then, and is closed (`drive` fails if it stays open). |
| 459 | let over = [&b"POST /c HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n400\r\n"[..], &[b'a'; 1_024], b"\r\n0\r\n\r\n"].concat(); |
| 460 | let (r, _, _, got) = res!(drive(&over, |b| b.ends_with(b"\r\n400\r\n"), Reply::Silent).await); |
| 461 | match r { Err(e) => req!(true, has(&e, ErrTag::TooBig), "chunked over the cap: {:?}", e), Ok(()) => return Err(err!("A chunked body over the cap was forwarded."; Test)) } |
| 462 | req!(false, got.ends_with(b"aaa"), "a chunk over the cap reached the upstream"); |
| 463 | Ok(()) |
| 464 | } |
| 465 | |
| 466 | /// Fault caught: a streamed answer held until its end (a daimon's reply appearing all at once). |
| 467 | #[tokio::test] |
| 468 | async fn test_a_streamed_answer_arrives_as_it_is_sent() -> Outcome<()> { |
| 469 | let _one = SERIAL.read().await; |
| 470 | let gap = Duration::from_millis(100); // Half of `idle`, so a loaded run still passes. |
| 471 | let (port, tls, up) = res!(stand_in(HOST, head_in, Reply::Pieces( |
| 472 | b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n", 6, gap)).await); |
| 473 | let (mut c, s) = res!(UnixStream::pair(), IO); |
| 474 | let fwd = tokio::spawn(forwarder(s, port, tls)); |
| 475 | res!(c.write_all(b"GET /sse HTTP/1.1\r\n\r\n").await, IO); |
| 476 | let (t0, mut seen, mut buf, mut answer) = (Instant::now(), Vec::new(), vec![0u8; 4_096], Vec::new()); |
| 477 | loop { |
| 478 | let n = res!(c.read(&mut buf).await, IO); |
| 479 | if n == 0 { break; } |
| 480 | answer.extend_from_slice(&buf[..n]); |
| 481 | let pieces = String::from_utf8_lossy(&answer).matches("data: ").count(); |
| 482 | while seen.len() < pieces { seen.push(t0.elapsed()); } |
| 483 | } |
| 484 | let (r, tally) = res!(fwd.await, IO); |
| 485 | res!(r); |
| 486 | let _ = up.await; |
| 487 | req!(6, seen.len()); |
| 488 | for (i, t) in seen.iter().enumerate() { |
| 489 | // Piece i is sent at about (i + 1) gaps; held to the end, every piece would come at 6. |
| 490 | req!(true, *t < gap * (i as u32 + 2), "piece {} came at {:?}", i, t); |
| 491 | } |
| 492 | req!(answer.len() as u64, tally.got.load(Ordering::Relaxed)); |
| 493 | Ok(()) |
| 494 | } |
| 495 | |
| 496 | fn rss_kib() -> u64 { |
| 497 | std::fs::read_to_string("/proc/self/status").ok() |
| 498 | .and_then(|s| s.lines().find(|l| l.starts_with("VmRSS:")) |
| 499 | .and_then(|l| l.split_whitespace().nth(1)).and_then(|v| v.parse().ok())) |
| 500 | .unwrap_or(0) |
| 501 | } |
| 502 | |
| 503 | /// Fault caught: an answer held in memory on its way through (a 64 MB answer is 64 MB of keeper). |
| 504 | #[tokio::test] |
| 505 | async fn test_a_64_mb_answer_passes_in_flat_memory() -> Outcome<()> { |
| 506 | let _one = SERIAL.write().await; |
| 507 | const SIZE: u64 = 64 * 1_024 * 1_024; |
| 508 | let (port, tls, up) = res!(stand_in(HOST, head_in, Reply::Big(SIZE)).await); |
| 509 | let (mut c, s) = res!(UnixStream::pair(), IO); |
| 510 | let fwd = tokio::spawn(forwarder(s, port, tls)); |
| 511 | res!(c.write_all(b"GET /big HTTP/1.1\r\n\r\n").await, IO); |
| 512 | let (base, mut peak, mut total, mut buf) = (rss_kib(), 0, 0u64, vec![0u8; 64 * 1_024]); |
| 513 | loop { |
| 514 | let n = res!(c.read(&mut buf).await, IO); |
| 515 | if n == 0 { break; } |
| 516 | total += n as u64; |
| 517 | if total % (8 << 20) < n as u64 { peak = std::cmp::max(peak, rss_kib()); } |
| 518 | } |
| 519 | let (r, tally) = res!(fwd.await, IO); |
| 520 | res!(r); |
| 521 | let _ = up.await; |
| 522 | req!(true, total > SIZE); |
| 523 | req!(total, tally.got.load(Ordering::Relaxed)); |
| 524 | req!(true, peak.saturating_sub(base) < 16 * 1_024, "resident set grew {} KiB", peak.saturating_sub(base)); |
| 525 | Ok(()) |
| 526 | } |
| 527 | |
| 528 | /// Fault caught: upstream faults that hang the keeper past its bound, or that the caller cannot |
| 529 | /// tell apart (a refusal, a bad certificate, silence, a cut answer), and a status lost. |
| 530 | #[tokio::test] |
| 531 | async fn test_upstream_faults_end_within_the_bound_and_are_told_apart() -> Outcome<()> { |
| 532 | let _one = SERIAL.read().await; |
| 533 | let req = b"GET / HTTP/1.1\r\n\r\n"; |
| 534 | // Refused: a port nobody listens on. |
| 535 | let port = res!(res!(TcpListener::bind("127.0.0.1:0").await, IO).local_addr(), IO).port(); |
| 536 | let (_, tls, _up) = res!(stand_in(HOST, head_in, Reply::Silent).await); |
| 537 | let (mut c, s) = res!(UnixStream::pair(), IO); |
| 538 | let fwd = tokio::spawn(forwarder(s, port, tls)); |
| 539 | res!(c.write_all(req).await, IO); |
| 540 | let (r, t) = res!(fwd.await, IO); |
| 541 | match r { Err(e) => req!(true, has(&e, ErrTag::Init) && !has(&e, ErrTag::Timeout), "refused: {}", e), Ok(()) => return Err(err!("refused upstream: Ok"; Test)) } |
| 542 | req!(0, t.status.load(Ordering::Relaxed)); |
| 543 | // Wrong name: a certificate for another host. |
| 544 | let (port, tls, _up) = res!(stand_in("other.example", head_in, Reply::Silent).await); |
| 545 | let (mut c, s) = res!(UnixStream::pair(), IO); |
| 546 | let fwd = tokio::spawn(forwarder(s, port, tls)); |
| 547 | res!(c.write_all(req).await, IO); |
| 548 | match res!(fwd.await, IO).0 { Err(e) => req!(true, has(&e, ErrTag::Security), "wrong name: {}", e), Ok(()) => return Err(err!("wrong name: Ok"; Test)) } |
| 549 | // Silent, then cut short, then 401 and 500. |
| 550 | let t0 = Instant::now(); |
| 551 | let (r, t, _, _) = res!(drive(req, head_in, Reply::Silent).await); |
| 552 | match r { Err(e) => req!(true, has(&e, ErrTag::Timeout), "silent: {}", e), Ok(()) => return Err(err!("silent: Ok"; Test)) } |
| 553 | req!(true, t0.elapsed() >= BOUNDS.answer_wait && t0.elapsed() < BOUNDS.answer_wait + SLACK, |
| 554 | "silent took {:?}", t0.elapsed()); |
| 555 | req!(0, t.status.load(Ordering::Relaxed)); |
| 556 | let (r, t, answer, _) = res!(drive(req, head_in, |
| 557 | Reply::Bytes(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n0123456789")).await); |
| 558 | match r { Err(e) => req!(true, has(&e, ErrTag::Missing), "cut: {}", e), Ok(()) => return Err(err!("cut: Ok"; Test)) } |
| 559 | req!(200, t.status.load(Ordering::Relaxed)); |
| 560 | req!(true, answer.ends_with(b"0123456789")); |
| 561 | // A reply head that is not UTF-8 (an upstream's cookie) is refused without its bytes. |
| 562 | let (r, _, answer, _) = res!(drive(req, head_in, |
| 563 | Reply::Bytes(b"HTTP/1.1 200 OK\r\nSet-Cookie: s=upstream-secret\xff\r\n\r\n")).await); |
| 564 | match r { Err(e) => req!(true, has(&e, ErrTag::Invalid) && !leaks(&e), "reply not UTF-8: {:?}", e), Ok(()) => return Err(err!("reply not UTF-8: Ok"; Test)) } |
| 565 | req!(true, answer.is_empty(), "a reply head that is not UTF-8 reached the client"); |
| 566 | // A reply's bad chunk-size line, holding what the upstream echoed, stays out of the error. |
| 567 | let (r, _, _, _) = res!(drive(req, head_in, Reply::Bytes( |
| 568 | b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5 sk-fake-secret-0000\r\nhello\r\n0\r\n\r\n")).await); |
| 569 | match r { Err(e) => req!(true, has(&e, ErrTag::Invalid) && !leaks(&e), "reply chunk line: {:?}", e), Ok(()) => return Err(err!("reply chunk line: Ok"; Test)) } |
| 570 | // A status line other than HTTP/1.1, three digits from 100 to 599, and a blank or the end. |
| 571 | let bad: [&'static [u8]; 9] = [b"HTTP/1.1 99 X\r\n\r\n", b"HTTP/1.1 099 X\r\n\r\n", |
| 572 | b"HTTP/1.1 600 X\r\n\r\n", b"HTTP/1.0 200 OK\r\n\r\n", b"HTTP/1.1\r\n\r\n", b"HTTP/1.1 2x0 OK\r\n\r\n", |
| 573 | b"HTTP/1.1 2000 OK\r\n\r\n", b"HTTP/1.1 200secret\r\n\r\n", "HTTP/1.1 2\u{e9}0 secret\r\n\r\n".as_bytes()]; |
| 574 | for wire in bad { |
| 575 | let (r, t, answer, _) = res!(drive(req, head_in, Reply::Bytes(wire)).await); |
| 576 | let shown = String::from_utf8_lossy(wire); |
| 577 | match r { Err(e) => req!(true, has(&e, ErrTag::Invalid) && !leaks(&e), "{:?}: {:?}", shown, e), Ok(()) => return Err(err!("{:?}: Ok", shown; Test)) } |
| 578 | req!(true, answer.is_empty(), "{:?} reached the client", shown); |
| 579 | req!(0, t.status.load(Ordering::Relaxed)); |
| 580 | } |
| 581 | for (code, wire) in [(401u16, &b"HTTP/1.1 401 Unauthorized\r\nContent-Length: 3\r\n\r\nno!"[..]), |
| 582 | (500, &b"HTTP/1.1 500 Internal Server Error\r\n\r\nboom"[..]), |
| 583 | (200, &b"HTTP/1.1 200\r\nContent-Length: 2\r\n\r\nok"[..])] { |
| 584 | let (r, t, answer, _) = res!(drive(req, head_in, Reply::Bytes(wire)).await); |
| 585 | res!(r); |
| 586 | req!(code, t.status.load(Ordering::Relaxed)); |
| 587 | req!(wire, &answer[..]); |
| 588 | } |
| 589 | Ok(()) |
| 590 | } |
| 591 | |
| 592 | /// Fault caught: a reply head slower than `idle` cut short, a trickling reply body or an upstream |
| 593 | /// that stops taking the body holding the keeper past its waits. |
| 594 | #[tokio::test] |
| 595 | async fn test_the_upstream_waits_end_each_stall() -> Outcome<()> { |
| 596 | let _one = SERIAL.read().await; |
| 597 | let req = b"GET / HTTP/1.1\r\n\r\n"; |
| 598 | // A head later than `idle` but within `answer_wait` is waited for. |
| 599 | let (r, _, answer, _) = res!(drive(req, head_in, |
| 600 | Reply::Steps(&[(300, b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")])).await); |
| 601 | res!(r); |
| 602 | req!(true, answer.ends_with(b"\r\n\r\nok")); |
| 603 | // A body that stops between bytes ends on `idle`. |
| 604 | let t0 = Instant::now(); |
| 605 | let (r, t, answer, _) = res!(drive(req, head_in, Reply::Steps(&[ |
| 606 | (0, b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n01234"), (350, b"56789")])).await); |
| 607 | match r { Err(e) => req!(true, has(&e, ErrTag::Timeout) && has(&e, ErrTag::Read), "trickle: {:?}", e), Ok(()) => return Err(err!("trickle: Ok"; Test)) } |
| 608 | req!(true, t0.elapsed() < BOUNDS.idle + SLACK, "trickle took {:?}", t0.elapsed()); |
| 609 | req!(200, t.status.load(Ordering::Relaxed)); |
| 610 | req!(true, answer.ends_with(b"01234")); |
| 611 | // An upstream that stops reading the body: the write ends on `idle`, then the answer has |
| 612 | // `answer_wait` to say why. |
| 613 | const BIG: usize = 16 << 20; |
| 614 | let big = ForwardBounds { body_cap: 2 * BIG as u64, ..BOUNDS }; |
| 615 | let (port, tls, up) = res!(stand_in(HOST, head_in, Reply::Steps(&[(5_000, b"")])).await); |
| 616 | let (mut c, s) = res!(UnixStream::pair(), IO); |
| 617 | let t0 = Instant::now(); |
| 618 | let fwd = tokio::spawn(forwarder_with(s, port, tls, big, &ADD)); |
| 619 | let w = tokio::spawn(async move { |
| 620 | let _ = c.write_all(fmt!("POST / HTTP/1.1\r\nContent-Length: {}\r\n\r\n", BIG).as_bytes()).await; |
| 621 | let _ = c.write_all(&vec![b'a'; BIG]).await; |
| 622 | }); |
| 623 | let (r, t) = res!(fwd.await, IO); |
| 624 | let took = t0.elapsed(); |
| 625 | up.abort(); |
| 626 | w.abort(); |
| 627 | match r { Err(e) => req!(true, has(&e, ErrTag::Timeout) && !has(&e, ErrTag::Write), "deaf: {:?}", e), Ok(()) => return Err(err!("deaf: Ok"; Test)) } |
| 628 | req!(true, t.sent.load(Ordering::Relaxed) < BIG as u64, "deaf: the whole body went up"); |
| 629 | req!(true, took >= BOUNDS.idle + BOUNDS.answer_wait && took < BOUNDS.idle + BOUNDS.answer_wait + 2 * SLACK, |
| 630 | "deaf took {:?}", took); |
| 631 | Ok(()) |
| 632 | } |
| 633 | |
| 634 | /// Fault caught: a client stalling mid-body or not reading the answer holding the keeper, a healthy |
| 635 | /// upload longer than `answer_wait` cut short because the answer's wait began at the dial, and an |
| 636 | /// early answer held back until the body is up. |
| 637 | #[tokio::test] |
| 638 | async fn test_the_client_waits_end_a_stall_but_not_a_slow_upload() -> Outcome<()> { |
| 639 | let _one = SERIAL.read().await; |
| 640 | // Stalled: five of ten bytes, then nothing, the client still there. |
| 641 | let (port, tls, up) = res!(stand_in(HOST, |b| b.ends_with(b"01234"), Reply::Silent).await); |
| 642 | let (mut c, s) = res!(UnixStream::pair(), IO); |
| 643 | let t0 = Instant::now(); |
| 644 | let fwd = tokio::spawn(forwarder(s, port, tls)); |
| 645 | res!(c.write_all(b"POST / HTTP/1.1\r\nContent-Length: 10\r\n\r\n01234").await, IO); |
| 646 | match res!(fwd.await, IO).0 { Err(e) => req!(true, has(&e, ErrTag::Timeout) && has(&e, ErrTag::Read), "stall: {:?}", e), Ok(()) => return Err(err!("stall: Ok"; Test)) } |
| 647 | req!(true, t0.elapsed() < BOUNDS.idle + SLACK, "stall took {:?}", t0.elapsed()); |
| 648 | let closed = ended(tokio::time::timeout(SLACK, up).await); |
| 649 | req!(true, closed, "stall: the upstream stayed open"); |
| 650 | // Slow but steady: five pieces 150 ms apart, 750 ms in all. |
| 651 | let (port, tls, up) = res!(stand_in(HOST, |b| b.ends_with(b"0123456789"), |
| 652 | Reply::Bytes(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")).await); |
| 653 | let (mut c, s) = res!(UnixStream::pair(), IO); |
| 654 | let fwd = tokio::spawn(forwarder(s, port, tls)); |
| 655 | res!(c.write_all(b"POST / HTTP/1.1\r\nContent-Length: 10\r\n\r\n").await, IO); |
| 656 | for p in [b"01", b"23", b"45", b"67", b"89"] { |
| 657 | tokio::time::sleep(Duration::from_millis(150)).await; |
| 658 | res!(c.write_all(p).await, IO); |
| 659 | } |
| 660 | let mut answer = Vec::new(); |
| 661 | res!(c.read_to_end(&mut answer).await, IO); |
| 662 | res!(res!(fwd.await, IO).0); |
| 663 | req!(true, answer.ends_with(b"\r\n\r\nok")); |
| 664 | let _ = up.await; |
| 665 | // An upstream that answers before the body is all up: the client has the answer, and the |
| 666 | // forward ends at once, with the rest of the body never read. |
| 667 | let (port, tls, up) = res!(stand_in(HOST, head_in, |
| 668 | Reply::Bytes(b"HTTP/1.1 413 Content Too Large\r\nContent-Length: 3\r\n\r\nno!")).await); |
| 669 | let (mut c, s) = res!(UnixStream::pair(), IO); |
| 670 | let t0 = Instant::now(); |
| 671 | let fwd = tokio::spawn(forwarder(s, port, tls)); |
| 672 | res!(c.write_all(b"POST / HTTP/1.1\r\nContent-Length: 10\r\n\r\n01234").await, IO); |
| 673 | let mut answer = Vec::new(); |
| 674 | res!(c.read_to_end(&mut answer).await, IO); |
| 675 | let (r, t) = res!(fwd.await, IO); |
| 676 | res!(r); |
| 677 | req!(true, answer.ends_with(b"\r\n\r\nno!")); |
| 678 | req!(413, t.status.load(Ordering::Relaxed)); |
| 679 | req!(true, t0.elapsed() < BOUNDS.idle, "early answer took {:?}", t0.elapsed()); |
| 680 | let closed = ended(tokio::time::timeout(SLACK, up).await); |
| 681 | req!(true, closed, "early answer: the upstream stayed open"); |
| 682 | // A client that stops reading the answer: the write to it ends on `idle`, as a timeout. |
| 683 | const BIG: u64 = 4 << 20; // Past every buffer between the two. |
| 684 | let (port, tls, up) = res!(stand_in(HOST, head_in, Reply::Big(BIG)).await); |
| 685 | let (mut c, s) = res!(UnixStream::pair(), IO); |
| 686 | let t0 = Instant::now(); |
| 687 | let fwd = tokio::spawn(forwarder(s, port, tls)); |
| 688 | res!(c.write_all(b"GET / HTTP/1.1\r\n\r\n").await, IO); |
| 689 | let (r, t) = res!(fwd.await, IO); |
| 690 | let took = t0.elapsed(); |
| 691 | up.abort(); |
| 692 | match r { Err(e) => req!(true, has(&e, ErrTag::Timeout) && has(&e, ErrTag::Write), "deaf client: {:?}", e), Ok(()) => return Err(err!("deaf client: Ok"; Test)) } |
| 693 | req!(true, t.got.load(Ordering::Relaxed) < BIG, "deaf client: the whole answer went"); |
| 694 | req!(true, took >= BOUNDS.idle && took < BOUNDS.idle + SLACK, "deaf client took {:?}", took); |
| 695 | drop(c); |
| 696 | Ok(()) |
| 697 | } |
| 698 | |
| 699 | /// Fault caught: a run of `1xx` heads ended by `answer_wait` (each restarts it), or not ended by |
| 700 | /// the caller's total bound. |
| 701 | #[tokio::test] |
| 702 | async fn test_a_run_of_1xx_heads_is_bounded_by_the_caller() -> Outcome<()> { |
| 703 | let _one = SERIAL.read().await; |
| 704 | const EARLY: &[u8] = b"HTTP/1.1 103 Early Hints\r\n\r\n"; |
| 705 | let (port, tls, up) = res!(stand_in(HOST, head_in, |
| 706 | Reply::Steps(&[(300, EARLY), (300, EARLY), (300, EARLY), (300, EARLY)])).await); |
| 707 | let (mut c, s) = res!(UnixStream::pair(), IO); |
| 708 | let fwd = tokio::spawn(tokio::time::timeout(Duration::from_millis(1_000), forwarder(s, port, tls))); |
| 709 | res!(c.write_all(b"GET / HTTP/1.1\r\n\r\n").await, IO); |
| 710 | let mut answer = Vec::new(); |
| 711 | res!(c.read_to_end(&mut answer).await, IO); |
| 712 | let cut = res!(fwd.await, IO).is_err(); |
| 713 | req!(true, cut, "the caller's bound did not end the forward"); |
| 714 | req!(3, String::from_utf8_lossy(&answer).matches("103 Early").count()); |
| 715 | up.abort(); |
| 716 | Ok(()) |
| 717 | } |
| 718 | |
| 719 | /// Fault caught: an upstream connection (holding the keeper's key in its stream) left open after |
| 720 | /// the client went away or the keeper stopped the forward. |
| 721 | #[tokio::test] |
| 722 | async fn test_a_client_gone_or_a_stop_closes_the_upstream() -> Outcome<()> { |
| 723 | let _one = SERIAL.read().await; |
| 724 | // Mid-request: ten of a hundred bytes, then the client leaves. |
| 725 | let (port, tls, up) = res!(stand_in(HOST, |b| b.ends_with(b"0123456789"), Reply::Silent).await); |
| 726 | let (mut c, s) = res!(UnixStream::pair(), IO); |
| 727 | let fwd = tokio::spawn(forwarder(s, port, tls)); |
| 728 | res!(c.write_all(b"POST / HTTP/1.1\r\nContent-Length: 100\r\n\r\n0123456789").await, IO); |
| 729 | tokio::time::sleep(Duration::from_millis(50)).await; |
| 730 | drop(c); |
| 731 | match res!(fwd.await, IO).0 { Err(e) => req!(true, has(&e, ErrTag::Missing), "mid-request: {}", e), Ok(()) => return Err(err!("mid-request: Ok"; Test)) } |
| 732 | let closed = ended(tokio::time::timeout(SLACK, up).await); |
| 733 | req!(true, closed, "mid-request: the upstream stayed open"); |
| 734 | // Mid-answer: the client leaves after the first piece. |
| 735 | let (port, tls, up) = res!(stand_in(HOST, head_in, Reply::Pieces( |
| 736 | b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n", 40, Duration::from_millis(50))).await); |
| 737 | let (mut c, s) = res!(UnixStream::pair(), IO); |
| 738 | let fwd = tokio::spawn(forwarder(s, port, tls)); |
| 739 | res!(c.write_all(b"GET / HTTP/1.1\r\n\r\n").await, IO); |
| 740 | let mut buf = vec![0u8; 4_096]; |
| 741 | let mut got = Vec::new(); |
| 742 | while !String::from_utf8_lossy(&got).contains("data: 00") { |
| 743 | let n = res!(c.read(&mut buf).await, IO); |
| 744 | got.extend_from_slice(&buf[..n]); |
| 745 | } |
| 746 | drop(c); |
| 747 | let t0 = Instant::now(); |
| 748 | match res!(fwd.await, IO).0 { Err(e) => req!(true, has(&e, ErrTag::Write), "mid-answer: {}", e), Ok(()) => return Err(err!("mid-answer: Ok"; Test)) } |
| 749 | req!(true, t0.elapsed() < SLACK, "mid-answer: noticed after {:?}", t0.elapsed()); |
| 750 | let closed = ended(tokio::time::timeout(SLACK, up).await); |
| 751 | req!(true, closed, "mid-answer: the upstream stayed open"); |
| 752 | // A stop: the keeper drops the forward while the upstream is silent. |
| 753 | let (port, tls, up) = res!(stand_in(HOST, head_in, Reply::Silent).await); |
| 754 | let (mut c, s) = res!(UnixStream::pair(), IO); |
| 755 | let fwd = tokio::spawn(forwarder(s, port, tls)); |
| 756 | res!(c.write_all(b"GET / HTTP/1.1\r\n\r\n").await, IO); |
| 757 | tokio::time::sleep(Duration::from_millis(100)).await; |
| 758 | fwd.abort(); |
| 759 | let t0 = Instant::now(); |
| 760 | let closed = ended(tokio::time::timeout(Duration::from_millis(100), up).await); |
| 761 | req!(true, closed, "stop: the upstream stayed open"); |
| 762 | req!(true, t0.elapsed() < BOUNDS.idle, "stop: closed only by the idle wait"); |
| 763 | Ok(()) |
| 764 | } |
| 765 | |
| 766 | /// Fault caught: a doc that says a dropped forward closes the client too. The future borrows the |
| 767 | /// client, so a stop closes the upstream and leaves the client's stream open for the caller. |
| 768 | #[tokio::test] |
| 769 | async fn test_a_dropped_forward_closes_the_upstream_and_leaves_the_client_to_the_caller() -> Outcome<()> { |
| 770 | let _one = SERIAL.read().await; |
| 771 | let (port, tls, up) = res!(stand_in(HOST, head_in, Reply::Silent).await); |
| 772 | let (mut c, mut s) = res!(UnixStream::pair(), IO); |
| 773 | res!(c.write_all(b"GET / HTTP/1.1\r\n\r\n").await, IO); |
| 774 | let tally = ForwardTally::default(); |
| 775 | let fwd = async { |
| 776 | let h = res!(RequestHead::read(&mut s, &BOUNDS).await); |
| 777 | forward_one(&mut s, h, &STRIP, &ADD, HOST, port, tls, &BOUNDS, &tally).await |
| 778 | }; |
| 779 | let stopped = tokio::time::timeout(Duration::from_millis(100), fwd).await.is_err(); |
| 780 | req!(true, stopped, "the forward ended before the caller's stop"); |
| 781 | let closed = ended(tokio::time::timeout(SLACK, up).await); |
| 782 | req!(true, closed, "the upstream stayed open"); |
| 783 | let late = b"HTTP/1.1 504 Gateway Timeout\r\n\r\n"; |
| 784 | res!(s.write_all(late).await, IO); |
| 785 | let mut buf = [0u8; 64]; |
| 786 | let n = res!(c.read(&mut buf).await, IO); |
| 787 | req!(&late[..], &buf[..n], "the client's stream did not stay the caller's"); |
| 788 | drop(s); |
| 789 | req!(0, res!(c.read(&mut buf).await, IO), "the client's end did not close with the caller's"); |
| 790 | Ok(()) |
| 791 | } |