Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_net/tests/http_forward.rs

42.2 KiB, 128 runs

created by r3497093317:77381, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1#![cfg(feature = "async")]
2//! `http::forward_one`: one request carried to a TLS upstream with named fields replaced, driven
3//! from a real Unix-socket client through to a TLS stand-in upstream on loopback, asserting on the
4//! bytes each end received. Every key here is fake.
5
6use oxedyne_fe2o3_core::prelude::*;
7use oxedyne_fe2o3_net::{
8 http::{
9 forward_one,
10 ForwardBounds,
11 ForwardTally,
12 RequestHead,
13 },
14 tls::ensure_crypto_provider,
15};
16
17use std::{
18 sync::{
19 atomic::Ordering,
20 Arc,
21 },
22 time::{
23 Duration,
24 Instant,
25 },
26};
27
28use tokio::{
29 io::{
30 AsyncReadExt,
31 AsyncWriteExt,
32 },
33 net::{
34 TcpListener,
35 UnixStream,
36 },
37 task::JoinHandle,
38};
39use tokio_rustls::{
40 rustls::{
41 client::ClientConfig,
42 pki_types::{
43 CertificateDer,
44 PrivateKeyDer,
45 PrivatePkcs8KeyDer,
46 },
47 server::ServerConfig,
48 RootCertStore,
49 },
50 TlsAcceptor,
51};
52
53
54const HOST: &str = "localhost";
55const STRIP: [&str; 3] = ["authorization", "x-api-key", "proxy-*"];
56const ADD: [(&str, &str); 1] = [("x-api-key", "sk-fake-secret-0000")];
57const BOUNDS: ForwardBounds = ForwardBounds {
58 head_cap: 8_192,
59 head_wait: Duration::from_millis(400),
60 connect: Duration::from_millis(400),
61 answer_wait: Duration::from_millis(500),
62 idle: Duration::from_millis(200),
63 body_cap: 1_024,
64};
65const SLACK: Duration = Duration::from_millis(300);
66
67/// The memory test reads the process's resident set, so it runs alone; the others share.
68static SERIAL: tokio::sync::RwLock<()> = tokio::sync::RwLock::const_new(());
69
70/// What the stand-in upstream does once the request is in.
71enum Reply {
72 Bytes(&'static [u8]), // Sent whole, then the stand-in closes.
73 Pieces(&'static [u8], usize, Duration), // A chunked head, then n pieces apart, then the end.
74 Big(u64), // A Content-Length answer of this many bytes.
75 Silent, // Never answers.
76 Steps(&'static [(u64, &'static [u8])]), // Each after its wait in ms, reading nothing meanwhile.
77}
78
79/// A TLS stand-in upstream for `name` on loopback, and a client config trusting it. The task
80/// returns every byte it received once the forwarder lets go of the connection.
81async fn stand_in(name: &str, done: fn(&[u8]) -> bool, reply: Reply)
82 -> Outcome<(u16, Arc<ClientConfig>, JoinHandle<Vec<u8>>)>
83{
84 ensure_crypto_provider();
85 let cert = res!(rcgen::generate_simple_self_signed(vec![name.to_string()]), Init);
86 let der = CertificateDer::from(res!(cert.serialize_der(), Init));
87 let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(cert.serialize_private_key_der()));
88 let mut roots = RootCertStore::empty();
89 res!(roots.add(der.clone()), Init);
90 let client = ClientConfig::builder().with_root_certificates(roots).with_no_client_auth();
91 let server = res!(ServerConfig::builder().with_no_client_auth().with_single_cert(vec![der], key), Init);
92 let listener = res!(TcpListener::bind("127.0.0.1:0").await, IO, Network);
93 let port = res!(listener.local_addr(), IO, Network).port();
94 let task = tokio::spawn(async move {
95 let mut got = Vec::new();
96 let tcp = match listener.accept().await { Ok((s, _)) => s, Err(_) => return got };
97 let mut s = match TlsAcceptor::from(Arc::new(server)).accept(tcp).await {
98 Ok(s) => s,
99 Err(_) => return got,
100 };
101 let mut buf = vec![0u8; 64 * 1_024];
102 while !done(&got) {
103 match s.read(&mut buf).await { Ok(0) | Err(_) => return got, Ok(n) => got.extend_from_slice(&buf[..n]) }
104 }
105 let wrote = match reply {
106 Reply::Bytes(b) => s.write_all(b).await.is_ok() && s.shutdown().await.is_ok(),
107 Reply::Pieces(h, n, d) => {
108 let mut ok = s.write_all(h).await.is_ok();
109 for i in 0..n {
110 if !ok { break; }
111 tokio::time::sleep(d).await;
112 let p = fmt!("data: {:02}\n\n", i);
113 ok = ok && s.write_all(fmt!("{:x}\r\n{}\r\n", p.len(), p).as_bytes()).await.is_ok();
114 }
115 ok && s.write_all(b"0\r\n\r\n").await.is_ok()
116 },
117 Reply::Big(n) => {
118 let mut ok = s.write_all(fmt!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", n).as_bytes()).await.is_ok();
119 let mut left = n;
120 while ok && left > 0 {
121 let k = std::cmp::min(left, buf.len() as u64) as usize;
122 ok = s.write_all(&buf[..k]).await.is_ok();
123 left -= k as u64;
124 }
125 ok
126 },
127 Reply::Silent => true,
128 Reply::Steps(steps) => {
129 let mut ok = true;
130 for (ms, b) in steps {
131 tokio::time::sleep(Duration::from_millis(*ms)).await;
132 ok = ok && s.write_all(b).await.is_ok();
133 }
134 ok
135 },
136 };
137 // Hold on until the forwarder closes: this task ending is the proof that it did.
138 while wrote { match s.read(&mut buf).await { Ok(0) | Err(_) => break, Ok(n) => got.extend_from_slice(&buf[..n]) } }
139 got
140 });
141 Ok((port, Arc::new(client), task))
142}
143
144/// The keeper's side: read the head, forward it, and report.
145async fn forwarder(s: UnixStream, port: u16, tls: Arc<ClientConfig>) -> (Outcome<()>, ForwardTally) {
146 forwarder_with(s, port, tls, BOUNDS, &ADD).await
147}
148
149async fn forwarder_with(
150 mut s: UnixStream,
151 port: u16,
152 tls: Arc<ClientConfig>,
153 b: ForwardBounds,
154 add: &'static [(&'static str, &'static str)],
155)
156 -> (Outcome<()>, ForwardTally)
157{
158 let tally = ForwardTally::default();
159 let r = match RequestHead::read(&mut s, &b).await {
160 Ok(h) => forward_one(&mut s, h, &STRIP, add, HOST, port, tls, &b, &tally).await,
161 Err(e) => Err(e),
162 };
163 (r, tally)
164}
165
166/// Does the error, shown either way, hold a fake key's word `secret`, as text, as decimal bytes
167/// or as hex bytes? Every fake key in this file holds that word.
168fn leaks(e: &Error<ErrTag>) -> bool {
169 let w = b"secret";
170 let dec = w.iter().map(|b| b.to_string()).collect::<Vec<_>>().join(", ");
171 let hex = w.iter().map(|b| fmt!("{:x}", b)).collect::<Vec<_>>();
172 let forms = ["secret".to_string(), dec, hex.join(", "), hex.concat()];
173 [fmt!("{}", e), fmt!("{:?}", e)].iter().any(|t| forms.iter().any(|f| t.contains(f.as_str())))
174}
175
176/// Sends `req` through a forwarder to a stand-in and reads the answer to its end.
177async fn drive(req: &[u8], done: fn(&[u8]) -> bool, reply: Reply)
178 -> Outcome<(Outcome<()>, ForwardTally, Vec<u8>, Vec<u8>)>
179{
180 drive_with(BOUNDS, req, done, reply).await
181}
182
183async fn drive_with(b: ForwardBounds, req: &[u8], done: fn(&[u8]) -> bool, reply: Reply)
184 -> Outcome<(Outcome<()>, ForwardTally, Vec<u8>, Vec<u8>)>
185{
186 drive_adding(b, &ADD, req, done, reply).await
187}
188
189async fn drive_adding(
190 b: ForwardBounds,
191 add: &'static [(&'static str, &'static str)],
192 req: &[u8],
193 done: fn(&[u8]) -> bool,
194 reply: Reply,
195)
196 -> Outcome<(Outcome<()>, ForwardTally, Vec<u8>, Vec<u8>)>
197{
198 let (port, tls, up) = res!(stand_in(HOST, done, reply).await);
199 let (mut c, s) = res!(UnixStream::pair(), IO);
200 let fwd = tokio::spawn(forwarder_with(s, port, tls, b, add));
201 res!(c.write_all(req).await, IO);
202 let mut answer = Vec::new();
203 res!(c.read_to_end(&mut answer).await, IO);
204 let (r, tally) = res!(fwd.await, IO);
205 let got = res!(tokio::time::timeout(SLACK, up).await, IO, Timeout);
206 Ok((r, tally, answer, res!(got, IO)))
207}
208
209fn head_in(b: &[u8]) -> bool { b.windows(4).any(|w| w == b"\r\n\r\n") }
210fn hello_in(b: &[u8]) -> bool { b.ends_with(b"hello") }
211
212fn has(e: &Error<ErrTag>, t: ErrTag) -> bool { e.tags().contains(&t) }
213
214fn ended<T>(r: Result<T, tokio::time::error::Elapsed>) -> bool { r.is_ok() }
215
216/// Fault caught: a client's key, in any case, repeated or oddly spaced, reaching the provider; the
217/// keeper's key arriving twice or not at all; the client's `Host` or `Connection` going through.
218#[tokio::test]
219async fn test_secret_fields_never_reach_the_upstream_and_the_added_one_arrives_once() -> Outcome<()> {
220 let _one = SERIAL.read().await;
221 let req = b"POST /v1/messages HTTP/1.1\r\nHost: evil.example\r\n\
222 Authorization: Bearer client-secret-1\r\nauthorization: Bearer client-secret-2\r\n\
223 X-API-KEY:client-secret-3\r\nx-api-key: \t client-secret-4 \t\r\n\
224 Proxy-Authorization: Basic client-secret-5\r\nPROXY-connection: client-secret-6\r\n\
225 Connection: keep-alive\r\nAccept: */*\r\nContent-Length: 5\r\n\r\nhello";
226 let (r, tally, answer, got) = res!(drive(req, hello_in,
227 Reply::Bytes(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")).await);
228 res!(r);
229 let up = String::from_utf8_lossy(&got).to_string();
230 let lines: Vec<&str> = up.split("\r\n").collect();
231 let count = |p: &str| lines.iter().filter(|l| l.to_ascii_lowercase().starts_with(p)).count();
232 req!(false, up.contains("client-secret"), "a client's key reached the upstream:\n{}", up);
233 req!(false, up.contains("evil"), "the client's Host reached the upstream");
234 req!(1, count("x-api-key:"), "the added field did not arrive exactly once");
235 req!(true, up.contains("\r\nx-api-key: sk-fake-secret-0000\r\n"));
236 req!(1, count("host:"));
237 req!(true, up.starts_with("POST /v1/messages HTTP/1.1\r\nHost: localhost:"), "the upstream saw another Host");
238 req!(1, count("connection:"));
239 req!(true, up.contains("\r\nConnection: close\r\n\r\nhello"));
240 req!(true, up.contains("\r\nAccept: */*\r\n"), "an ordinary field was lost");
241 req!(true, answer.ends_with(b"\r\n\r\nok"));
242 req!(200, tally.status.load(Ordering::Relaxed));
243 req!(got.len() as u64, tally.sent.load(Ordering::Relaxed));
244 req!(answer.len() as u64, tally.got.load(Ordering::Relaxed));
245 // An added name replaces its own field only, in any case, and never a longer one.
246 let req = b"GET / HTTP/1.1\r\nx-provider-key: client-secret-7\r\nX-Other: kept-7\r\n\
247 X-Provider-Keyring: kept-8\r\n\r\n";
248 let (r, _, _, got) = res!(drive_adding(BOUNDS, &[("X-Provider-Key", "sk-fake-secret-1111")], req,
249 head_in, Reply::Bytes(b"HTTP/1.1 204 No Content\r\n\r\n")).await);
250 res!(r);
251 let up = String::from_utf8_lossy(&got).to_string();
252 req!(false, up.contains("client-secret"), "a client's key reached the upstream:\n{}", up);
253 req!(1, up.matches("\r\nX-Provider-Key: sk-fake-secret-1111\r\n").count(), "{}", up);
254 req!(true, up.contains("\r\nX-Other: kept-7\r\n") && up.contains("\r\nX-Provider-Keyring: kept-8\r\n"),
255 "a client field another name begins was lost:\n{}", up);
256 Ok(())
257}
258
259/// Fault caught: a head the keeper cannot read as one unambiguous request still being dialled out,
260/// a slow head holding the keeper, or a key in the head echoed in the error.
261#[tokio::test]
262async fn test_a_bad_head_is_refused_within_the_bound_and_nothing_is_dialled() -> Outcome<()> {
263 let _one = SERIAL.read().await;
264 let long = fmt!("GET / HTTP/1.1\r\nX-Pad: {}\r\n\r\n", "a".repeat(9_000));
265 let cases: [(&str, &[u8], ErrTag); 21] = [
266 ("over the cap", long.as_bytes(), ErrTag::TooBig),
267 ("not HTTP", b"SSH-2.0-OpenSSH_9.6 client-secret\r\n\r\n", ErrTag::Invalid),
268 ("not UTF-8", b"GET / HTTP/1.1\r\nAuthorization: Bearer client-secret\r\nX: \xff\r\n\r\n", ErrTag::Invalid),
269 ("absolute form", b"GET http://evil.example/ HTTP/1.1\r\nHost: evil.example\r\n\r\n", ErrTag::Invalid),
270 ("asterisk form", b"OPTIONS * HTTP/1.1\r\n\r\n", ErrTag::Invalid),
271 ("folded", b"GET / HTTP/1.1\r\nAuthorization: Bearer\r\n client-secret\r\n\r\n", ErrTag::Invalid),
272 ("space before colon", b"GET / HTTP/1.1\r\nAuthorization : client-secret\r\n\r\n", ErrTag::Invalid),
273 ("bare LF", b"GET / HTTP/1.1\r\nX: y\nAuthorization: client-secret\r\n\r\n", ErrTag::Invalid),
274 ("two lengths", b"POST / HTTP/1.1\r\nContent-Length: 5\r\nContent-Length: 5\r\n\r\nhello", ErrTag::Invalid),
275 ("length not digits", b"POST / HTTP/1.1\r\nContent-Length: 5 client-secret\r\n\r\nhello", ErrTag::Invalid),
276 ("body over the cap", b"POST / HTTP/1.1\r\nContent-Length: 1025\r\n\r\nhello", ErrTag::TooBig),
277 ("TE and CL", b"POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nContent-Length: 5\r\n\r\n0\r\n\r\n", ErrTag::Invalid),
278 ("TE not chunked", b"POST / HTTP/1.1\r\nTransfer-Encoding: client-secret, chunked\r\n\r\n0\r\n\r\n", ErrTag::Invalid),
279 ("TRACE", b"TRACE / HTTP/1.1\r\nHost: x\r\n\r\n", ErrTag::Invalid),
280 ("trace", b"trace / HTTP/1.1\r\nHost: x\r\n\r\n", ErrTag::Invalid),
281 ("TrAcE", b"TrAcE / HTTP/1.1\r\nHost: x\r\n\r\n", ErrTag::Invalid),
282 ("TRACK", b"TRACK / HTTP/1.1\r\nHost: x\r\n\r\n", ErrTag::Invalid),
283 ("CONNECT origin form", b"CONNECT /x HTTP/1.1\r\nHost: x\r\n\r\n", ErrTag::Invalid),
284 ("CONNECT", b"CONNECT evil.example:443 HTTP/1.1\r\nHost: evil.example:443\r\n\r\n", ErrTag::Invalid),
285 ("HTTP/1.0", b"GET / HTTP/1.0\r\n\r\n", ErrTag::Invalid),
286 ("never ends", b"GET / HTTP/1.1\r\nAuthorization: client-secret", ErrTag::Timeout),
287 ];
288 let listener = res!(TcpListener::bind("127.0.0.1:0").await, IO);
289 let port = res!(listener.local_addr(), IO).port();
290 ensure_crypto_provider();
291 let tls = Arc::new(ClientConfig::builder().with_root_certificates(RootCertStore::empty()).with_no_client_auth());
292 for (what, req, tag) in cases {
293 let (mut c, s) = res!(UnixStream::pair(), IO);
294 let t0 = Instant::now();
295 let fwd = tokio::spawn(forwarder(s, port, tls.clone()));
296 let _ = c.write_all(req).await;
297 let (r, _) = res!(fwd.await, IO);
298 let took = t0.elapsed();
299 match r {
300 Ok(()) => return Err(err!("A head {} was forwarded.", what; Test, Unreachable)),
301 Err(e) => {
302 let ok = has(&e, tag.clone());
303 req!(true, ok, "{}: {:?} lacks {:?}: {}", what, e.tags(), tag, e);
304 req!(false, leaks(&e), "{}: the error shows a value: {:?}", what, e);
305 },
306 }
307 req!(true, took < BOUNDS.head_wait + SLACK, "{}: refused after {:?}", what, took);
308 req!(true, tokio::time::timeout(Duration::from_millis(20), listener.accept()).await.is_err(),
309 "{}: the upstream was dialled", what);
310 drop(c);
311 }
312 // An added value that would split the head is refused before the dial.
313 // A `*` would read as a strip pattern, so an added name holding one is refused too.
314 let bad: [&'static [(&'static str, &'static str)]; 9] = [
315 &[("x-api-key", "sk-fake-secret\r\nX-Evil: 1")],
316 &[("x-api-key", "sk-fake-secret\nX-Evil: 1")],
317 &[("x-api-key", "sk-fake-secret\r")],
318 &[("x-api-key", "sk-fake\0secret")],
319 &[("x-api key", "sk-fake-secret")],
320 &[("x-*", "sk-fake-secret")],
321 &[("*", "sk-fake-secret")],
322 &[("x-api:key", "sk-fake-secret")],
323 &[("", "sk-fake-secret")],
324 ];
325 for add in bad {
326 let (mut c, s) = res!(UnixStream::pair(), IO);
327 let fwd = tokio::spawn(forwarder_with(s, port, tls.clone(), BOUNDS, add));
328 res!(c.write_all(b"GET / HTTP/1.1\r\nX-Other: client-secret\r\n\r\n").await, IO);
329 match res!(fwd.await, IO).0 {
330 Ok(()) => return Err(err!("An added field {:?} was forwarded.", add[0].0; Test, Unreachable)),
331 Err(e) => req!(true, has(&e, ErrTag::Invalid) && !leaks(&e), "add {:?}: {:?}", add[0].0, e),
332 }
333 req!(true, tokio::time::timeout(Duration::from_millis(20), listener.accept()).await.is_err(),
334 "add {:?}: the upstream was dialled", add[0].0);
335 }
336 Ok(())
337}
338
339/// Fault caught: a keeper choosing the upstream from a head with no `Host`, or from one of two.
340#[tokio::test]
341async fn test_host_is_the_one_host_field() -> Outcome<()> {
342 let _one = SERIAL.read().await;
343 let cases: [(&[u8], Option<&str>); 8] = [
344 (b"GET / HTTP/1.1\r\nHost: api.example\r\n\r\n", Some("api.example")),
345 (b"GET / HTTP/1.1\r\nhOsT:api.example:8443\r\n\r\n", Some("api.example:8443")),
346 (b"GET / HTTP/1.1\r\nHost: [::1]:8443\r\n\r\n", Some("[::1]:8443")),
347 (b"GET / HTTP/1.1\r\nAccept: */*\r\n\r\n", None),
348 (b"GET / HTTP/1.1\r\nHost: api.one\r\nHost: api.two\r\n\r\n", None),
349 (b"GET / HTTP/1.1\r\nHost: api.one\r\nHOST: api.one\r\n\r\n", None),
350 (b"GET / HTTP/1.1\r\nHost: \r\n\r\n", None),
351 (b"GET / HTTP/1.1\r\nHost: api.one api.two\r\n\r\n", None),
352 ];
353 for (head, want) in cases {
354 let (mut c, mut s) = res!(UnixStream::pair(), IO);
355 res!(c.write_all(head).await, IO);
356 let h = res!(RequestHead::read(&mut s, &BOUNDS).await);
357 match (h.host(), want) {
358 (Ok(v), Some(w)) => req!(w, v),
359 (Err(e), None) => req!(true, has(&e, ErrTag::Invalid) && !leaks(&e)),
360 (got, _) => return Err(err!("Host of {:?}: {:?}",
361 String::from_utf8_lossy(head), got.map(|v| v.to_string()); Test, Unreachable)),
362 }
363 }
364 Ok(())
365}
366
367/// Fault caught: a head's end missed where CRLF CRLF straddles reads, a head of exactly the cap
368/// refused, or one byte over it let through.
369#[tokio::test]
370async fn test_a_head_ends_wherever_the_reads_split_it() -> Outcome<()> {
371 let _one = SERIAL.read().await;
372 let at = |len: usize| {
373 let pad = len - b"GET / HTTP/1.1\r\nX: \r\n\r\n".len();
374 fmt!("GET / HTTP/1.1\r\nX: {}\r\n\r\nBODY", "a".repeat(pad)).into_bytes()
375 };
376 let cases = [(at(BOUNDS.head_cap), 1, true), (at(BOUNDS.head_cap), 3, true),
377 (at(BOUNDS.head_cap + 1), 1, false), (at(40), 1, true), (at(40), 2, true), (at(41), 2, true),
378 (at(42), 3, true), (at(43), 3, true)];
379 for (wire, per, fits) in cases {
380 // A pipe of `per` bytes hands over at most that many a read.
381 let (mut c, mut s) = tokio::io::duplex(per);
382 let w = tokio::spawn(async move { let _ = c.write_all(&wire).await; c });
383 match (RequestHead::read(&mut s, &BOUNDS).await, fits) {
384 (Ok(h), true) => req!("/", h.target()),
385 (Err(e), false) => req!(true, has(&e, ErrTag::TooBig), "{:?}", e),
386 (Ok(_), _) => return Err(err!("A head, {} a read, over the cap passed.", per; Test)),
387 (Err(e), _) => return Err(err!(e, "A head, {} a read, within the cap failed.", per; Test)),
388 }
389 w.abort();
390 }
391 // The cost of a head grows with its length, not its square: 64 KiB a byte a read took 7 s.
392 let big = ForwardBounds { head_cap: 65_536, head_wait: Duration::from_secs(3), ..BOUNDS };
393 let (mut c, mut s) = tokio::io::duplex(1);
394 let wire = at(big.head_cap);
395 let w = tokio::spawn(async move { let _ = c.write_all(&wire).await; c });
396 let t0 = Instant::now();
397 let r = RequestHead::read(&mut s, &big).await;
398 eprintln!("64 KiB head a byte a read: {:?}", t0.elapsed());
399 w.abort();
400 let h = res!(r);
401 req!("/", h.target());
402 Ok(())
403}
404
405/// Fault caught: a body altered or cut on the way, a chunked body decoded or re-framed, and a
406/// second request smuggled behind the first reaching the upstream with the keeper's key.
407#[tokio::test]
408async fn test_bodies_go_verbatim_and_only_one_request_is_forwarded() -> Outcome<()> {
409 let _one = SERIAL.read().await;
410 let chunked = b"POST /c HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n2;x=1\r\nhe\r\n3\r\nllo\r\n0\r\n\r\n";
411 let (r, _, _, got) = res!(drive(chunked, |b| b.ends_with(b"llo\r\n0\r\n\r\n"),
412 Reply::Bytes(b"HTTP/1.1 204 No Content\r\n\r\n")).await);
413 res!(r);
414 req!(true, got.ends_with(b"\r\n\r\n2;x=1\r\nhe\r\n3\r\nllo\r\n0\r\n\r\n"), "the chunked body changed");
415 let piped = b"POST /a HTTP/1.1\r\nContent-Length: 5\r\n\r\nhelloGET /second HTTP/1.1\r\nHost: x\r\n\r\n";
416 let (r, tally, answer, got) = res!(drive(piped, hello_in,
417 Reply::Bytes(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")).await);
418 res!(r);
419 req!(false, String::from_utf8_lossy(&got).contains("second"), "a pipelined request was forwarded");
420 req!(true, got.ends_with(b"hello"));
421 req!(true, answer.ends_with(b"ok"));
422 req!(got.len() as u64, tally.sent.load(Ordering::Relaxed));
423 // A field in a request's trailers would pass the strip, so the request is refused there.
424 let trailer = b"POST /t HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n0\r\nAuthorization: client-secret\r\n\r\n";
425 let (r, _, _, got) = res!(drive(trailer, |b| b.ends_with(b"0\r\n"), Reply::Silent).await);
426 match r { Err(e) => req!(false, leaks(&e), "the error shows a value: {:?}", e), Ok(()) => return Err(err!("A request trailer was forwarded."; Test)) }
427 req!(false, String::from_utf8_lossy(&got).contains("client-secret"), "a trailer field reached the upstream");
428 // A bad chunk-size line is body text, so its error names the place and the reason only.
429 let lines: [&'static [u8]; 4] = [b"5 client-secret\r\nhello\r\n", b"client-secret\r\n",
430 b"736563726574 x\r\n", b"5;115, 101, 99, 114, 101, 116\x01\r\n"];
431 for line in lines {
432 let wire = [&b"POST /t HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"[..], line].concat();
433 let (r, _, _, _) = res!(drive(&wire, head_in, Reply::Silent).await);
434 let shown = String::from_utf8_lossy(line);
435 match r { Err(e) => req!(true, has(&e, ErrTag::Invalid) && !leaks(&e), "{:?}: {:?}", shown, e), Ok(()) => return Err(err!("{:?}: Ok", shown; Test)) }
436 }
437 // A chunk-size line of 1 MB with no CRLF is refused at the line cap.
438 let (port, tls, _up) = res!(stand_in(HOST, head_in, Reply::Silent).await);
439 let (mut c, s) = res!(UnixStream::pair(), IO);
440 let fwd = tokio::spawn(forwarder(s, port, tls));
441 let w = tokio::spawn(async move {
442 let wire = [&b"POST /t HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"[..], &[b'1'; 1_000_000]].concat();
443 let _ = c.write_all(&wire).await;
444 c
445 });
446 match res!(fwd.await, IO).0 { Err(e) => req!(true, has(&e, ErrTag::TooBig) && !leaks(&e), "long chunk line: {:?}", e), Ok(()) => return Err(err!("long chunk line: Ok"; Test)) }
447 w.abort();
448 // Exactly the cap passes; so does no body, and a body of 0, under a cap of 0.
449 let full = [&b"POST /cap HTTP/1.1\r\nContent-Length: 1024\r\n\r\n"[..], &[b'a'; 1_023], b"z"].concat();
450 let ok = || Reply::Bytes(b"HTTP/1.1 204 No Content\r\n\r\n");
451 let (r, _, _, got) = res!(drive(&full, |b| b.ends_with(b"aaz"), ok()).await);
452 res!(r);
453 req!(true, got.ends_with(&full[full.len() - 1_024..]));
454 let none = ForwardBounds { body_cap: 0, ..BOUNDS };
455 res!(res!(drive_with(none, b"GET / HTTP/1.1\r\n\r\n", head_in, ok()).await).0);
456 res!(res!(drive_with(none, b"POST / HTTP/1.1\r\nContent-Length: 0\r\n\r\n", head_in, ok()).await).0);
457 // A chunked body is cut where it passes the cap: the upstream has the head and the chunk's
458 // size line by then, and is closed (`drive` fails if it stays open).
459 let over = [&b"POST /c HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n400\r\n"[..], &[b'a'; 1_024], b"\r\n0\r\n\r\n"].concat();
460 let (r, _, _, got) = res!(drive(&over, |b| b.ends_with(b"\r\n400\r\n"), Reply::Silent).await);
461 match r { Err(e) => req!(true, has(&e, ErrTag::TooBig), "chunked over the cap: {:?}", e), Ok(()) => return Err(err!("A chunked body over the cap was forwarded."; Test)) }
462 req!(false, got.ends_with(b"aaa"), "a chunk over the cap reached the upstream");
463 Ok(())
464}
465
466/// Fault caught: a streamed answer held until its end (a daimon's reply appearing all at once).
467#[tokio::test]
468async fn test_a_streamed_answer_arrives_as_it_is_sent() -> Outcome<()> {
469 let _one = SERIAL.read().await;
470 let gap = Duration::from_millis(100); // Half of `idle`, so a loaded run still passes.
471 let (port, tls, up) = res!(stand_in(HOST, head_in, Reply::Pieces(
472 b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n", 6, gap)).await);
473 let (mut c, s) = res!(UnixStream::pair(), IO);
474 let fwd = tokio::spawn(forwarder(s, port, tls));
475 res!(c.write_all(b"GET /sse HTTP/1.1\r\n\r\n").await, IO);
476 let (t0, mut seen, mut buf, mut answer) = (Instant::now(), Vec::new(), vec![0u8; 4_096], Vec::new());
477 loop {
478 let n = res!(c.read(&mut buf).await, IO);
479 if n == 0 { break; }
480 answer.extend_from_slice(&buf[..n]);
481 let pieces = String::from_utf8_lossy(&answer).matches("data: ").count();
482 while seen.len() < pieces { seen.push(t0.elapsed()); }
483 }
484 let (r, tally) = res!(fwd.await, IO);
485 res!(r);
486 let _ = up.await;
487 req!(6, seen.len());
488 for (i, t) in seen.iter().enumerate() {
489 // Piece i is sent at about (i + 1) gaps; held to the end, every piece would come at 6.
490 req!(true, *t < gap * (i as u32 + 2), "piece {} came at {:?}", i, t);
491 }
492 req!(answer.len() as u64, tally.got.load(Ordering::Relaxed));
493 Ok(())
494}
495
496fn rss_kib() -> u64 {
497 std::fs::read_to_string("/proc/self/status").ok()
498 .and_then(|s| s.lines().find(|l| l.starts_with("VmRSS:"))
499 .and_then(|l| l.split_whitespace().nth(1)).and_then(|v| v.parse().ok()))
500 .unwrap_or(0)
501}
502
503/// Fault caught: an answer held in memory on its way through (a 64 MB answer is 64 MB of keeper).
504#[tokio::test]
505async fn test_a_64_mb_answer_passes_in_flat_memory() -> Outcome<()> {
506 let _one = SERIAL.write().await;
507 const SIZE: u64 = 64 * 1_024 * 1_024;
508 let (port, tls, up) = res!(stand_in(HOST, head_in, Reply::Big(SIZE)).await);
509 let (mut c, s) = res!(UnixStream::pair(), IO);
510 let fwd = tokio::spawn(forwarder(s, port, tls));
511 res!(c.write_all(b"GET /big HTTP/1.1\r\n\r\n").await, IO);
512 let (base, mut peak, mut total, mut buf) = (rss_kib(), 0, 0u64, vec![0u8; 64 * 1_024]);
513 loop {
514 let n = res!(c.read(&mut buf).await, IO);
515 if n == 0 { break; }
516 total += n as u64;
517 if total % (8 << 20) < n as u64 { peak = std::cmp::max(peak, rss_kib()); }
518 }
519 let (r, tally) = res!(fwd.await, IO);
520 res!(r);
521 let _ = up.await;
522 req!(true, total > SIZE);
523 req!(total, tally.got.load(Ordering::Relaxed));
524 req!(true, peak.saturating_sub(base) < 16 * 1_024, "resident set grew {} KiB", peak.saturating_sub(base));
525 Ok(())
526}
527
528/// Fault caught: upstream faults that hang the keeper past its bound, or that the caller cannot
529/// tell apart (a refusal, a bad certificate, silence, a cut answer), and a status lost.
530#[tokio::test]
531async fn test_upstream_faults_end_within_the_bound_and_are_told_apart() -> Outcome<()> {
532 let _one = SERIAL.read().await;
533 let req = b"GET / HTTP/1.1\r\n\r\n";
534 // Refused: a port nobody listens on.
535 let port = res!(res!(TcpListener::bind("127.0.0.1:0").await, IO).local_addr(), IO).port();
536 let (_, tls, _up) = res!(stand_in(HOST, head_in, Reply::Silent).await);
537 let (mut c, s) = res!(UnixStream::pair(), IO);
538 let fwd = tokio::spawn(forwarder(s, port, tls));
539 res!(c.write_all(req).await, IO);
540 let (r, t) = res!(fwd.await, IO);
541 match r { Err(e) => req!(true, has(&e, ErrTag::Init) && !has(&e, ErrTag::Timeout), "refused: {}", e), Ok(()) => return Err(err!("refused upstream: Ok"; Test)) }
542 req!(0, t.status.load(Ordering::Relaxed));
543 // Wrong name: a certificate for another host.
544 let (port, tls, _up) = res!(stand_in("other.example", head_in, Reply::Silent).await);
545 let (mut c, s) = res!(UnixStream::pair(), IO);
546 let fwd = tokio::spawn(forwarder(s, port, tls));
547 res!(c.write_all(req).await, IO);
548 match res!(fwd.await, IO).0 { Err(e) => req!(true, has(&e, ErrTag::Security), "wrong name: {}", e), Ok(()) => return Err(err!("wrong name: Ok"; Test)) }
549 // Silent, then cut short, then 401 and 500.
550 let t0 = Instant::now();
551 let (r, t, _, _) = res!(drive(req, head_in, Reply::Silent).await);
552 match r { Err(e) => req!(true, has(&e, ErrTag::Timeout), "silent: {}", e), Ok(()) => return Err(err!("silent: Ok"; Test)) }
553 req!(true, t0.elapsed() >= BOUNDS.answer_wait && t0.elapsed() < BOUNDS.answer_wait + SLACK,
554 "silent took {:?}", t0.elapsed());
555 req!(0, t.status.load(Ordering::Relaxed));
556 let (r, t, answer, _) = res!(drive(req, head_in,
557 Reply::Bytes(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n0123456789")).await);
558 match r { Err(e) => req!(true, has(&e, ErrTag::Missing), "cut: {}", e), Ok(()) => return Err(err!("cut: Ok"; Test)) }
559 req!(200, t.status.load(Ordering::Relaxed));
560 req!(true, answer.ends_with(b"0123456789"));
561 // A reply head that is not UTF-8 (an upstream's cookie) is refused without its bytes.
562 let (r, _, answer, _) = res!(drive(req, head_in,
563 Reply::Bytes(b"HTTP/1.1 200 OK\r\nSet-Cookie: s=upstream-secret\xff\r\n\r\n")).await);
564 match r { Err(e) => req!(true, has(&e, ErrTag::Invalid) && !leaks(&e), "reply not UTF-8: {:?}", e), Ok(()) => return Err(err!("reply not UTF-8: Ok"; Test)) }
565 req!(true, answer.is_empty(), "a reply head that is not UTF-8 reached the client");
566 // A reply's bad chunk-size line, holding what the upstream echoed, stays out of the error.
567 let (r, _, _, _) = res!(drive(req, head_in, Reply::Bytes(
568 b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5 sk-fake-secret-0000\r\nhello\r\n0\r\n\r\n")).await);
569 match r { Err(e) => req!(true, has(&e, ErrTag::Invalid) && !leaks(&e), "reply chunk line: {:?}", e), Ok(()) => return Err(err!("reply chunk line: Ok"; Test)) }
570 // A status line other than HTTP/1.1, three digits from 100 to 599, and a blank or the end.
571 let bad: [&'static [u8]; 9] = [b"HTTP/1.1 99 X\r\n\r\n", b"HTTP/1.1 099 X\r\n\r\n",
572 b"HTTP/1.1 600 X\r\n\r\n", b"HTTP/1.0 200 OK\r\n\r\n", b"HTTP/1.1\r\n\r\n", b"HTTP/1.1 2x0 OK\r\n\r\n",
573 b"HTTP/1.1 2000 OK\r\n\r\n", b"HTTP/1.1 200secret\r\n\r\n", "HTTP/1.1 2\u{e9}0 secret\r\n\r\n".as_bytes()];
574 for wire in bad {
575 let (r, t, answer, _) = res!(drive(req, head_in, Reply::Bytes(wire)).await);
576 let shown = String::from_utf8_lossy(wire);
577 match r { Err(e) => req!(true, has(&e, ErrTag::Invalid) && !leaks(&e), "{:?}: {:?}", shown, e), Ok(()) => return Err(err!("{:?}: Ok", shown; Test)) }
578 req!(true, answer.is_empty(), "{:?} reached the client", shown);
579 req!(0, t.status.load(Ordering::Relaxed));
580 }
581 for (code, wire) in [(401u16, &b"HTTP/1.1 401 Unauthorized\r\nContent-Length: 3\r\n\r\nno!"[..]),
582 (500, &b"HTTP/1.1 500 Internal Server Error\r\n\r\nboom"[..]),
583 (200, &b"HTTP/1.1 200\r\nContent-Length: 2\r\n\r\nok"[..])] {
584 let (r, t, answer, _) = res!(drive(req, head_in, Reply::Bytes(wire)).await);
585 res!(r);
586 req!(code, t.status.load(Ordering::Relaxed));
587 req!(wire, &answer[..]);
588 }
589 Ok(())
590}
591
592/// Fault caught: a reply head slower than `idle` cut short, a trickling reply body or an upstream
593/// that stops taking the body holding the keeper past its waits.
594#[tokio::test]
595async fn test_the_upstream_waits_end_each_stall() -> Outcome<()> {
596 let _one = SERIAL.read().await;
597 let req = b"GET / HTTP/1.1\r\n\r\n";
598 // A head later than `idle` but within `answer_wait` is waited for.
599 let (r, _, answer, _) = res!(drive(req, head_in,
600 Reply::Steps(&[(300, b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")])).await);
601 res!(r);
602 req!(true, answer.ends_with(b"\r\n\r\nok"));
603 // A body that stops between bytes ends on `idle`.
604 let t0 = Instant::now();
605 let (r, t, answer, _) = res!(drive(req, head_in, Reply::Steps(&[
606 (0, b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n01234"), (350, b"56789")])).await);
607 match r { Err(e) => req!(true, has(&e, ErrTag::Timeout) && has(&e, ErrTag::Read), "trickle: {:?}", e), Ok(()) => return Err(err!("trickle: Ok"; Test)) }
608 req!(true, t0.elapsed() < BOUNDS.idle + SLACK, "trickle took {:?}", t0.elapsed());
609 req!(200, t.status.load(Ordering::Relaxed));
610 req!(true, answer.ends_with(b"01234"));
611 // An upstream that stops reading the body: the write ends on `idle`, then the answer has
612 // `answer_wait` to say why.
613 const BIG: usize = 16 << 20;
614 let big = ForwardBounds { body_cap: 2 * BIG as u64, ..BOUNDS };
615 let (port, tls, up) = res!(stand_in(HOST, head_in, Reply::Steps(&[(5_000, b"")])).await);
616 let (mut c, s) = res!(UnixStream::pair(), IO);
617 let t0 = Instant::now();
618 let fwd = tokio::spawn(forwarder_with(s, port, tls, big, &ADD));
619 let w = tokio::spawn(async move {
620 let _ = c.write_all(fmt!("POST / HTTP/1.1\r\nContent-Length: {}\r\n\r\n", BIG).as_bytes()).await;
621 let _ = c.write_all(&vec![b'a'; BIG]).await;
622 });
623 let (r, t) = res!(fwd.await, IO);
624 let took = t0.elapsed();
625 up.abort();
626 w.abort();
627 match r { Err(e) => req!(true, has(&e, ErrTag::Timeout) && !has(&e, ErrTag::Write), "deaf: {:?}", e), Ok(()) => return Err(err!("deaf: Ok"; Test)) }
628 req!(true, t.sent.load(Ordering::Relaxed) < BIG as u64, "deaf: the whole body went up");
629 req!(true, took >= BOUNDS.idle + BOUNDS.answer_wait && took < BOUNDS.idle + BOUNDS.answer_wait + 2 * SLACK,
630 "deaf took {:?}", took);
631 Ok(())
632}
633
634/// Fault caught: a client stalling mid-body or not reading the answer holding the keeper, a healthy
635/// upload longer than `answer_wait` cut short because the answer's wait began at the dial, and an
636/// early answer held back until the body is up.
637#[tokio::test]
638async fn test_the_client_waits_end_a_stall_but_not_a_slow_upload() -> Outcome<()> {
639 let _one = SERIAL.read().await;
640 // Stalled: five of ten bytes, then nothing, the client still there.
641 let (port, tls, up) = res!(stand_in(HOST, |b| b.ends_with(b"01234"), Reply::Silent).await);
642 let (mut c, s) = res!(UnixStream::pair(), IO);
643 let t0 = Instant::now();
644 let fwd = tokio::spawn(forwarder(s, port, tls));
645 res!(c.write_all(b"POST / HTTP/1.1\r\nContent-Length: 10\r\n\r\n01234").await, IO);
646 match res!(fwd.await, IO).0 { Err(e) => req!(true, has(&e, ErrTag::Timeout) && has(&e, ErrTag::Read), "stall: {:?}", e), Ok(()) => return Err(err!("stall: Ok"; Test)) }
647 req!(true, t0.elapsed() < BOUNDS.idle + SLACK, "stall took {:?}", t0.elapsed());
648 let closed = ended(tokio::time::timeout(SLACK, up).await);
649 req!(true, closed, "stall: the upstream stayed open");
650 // Slow but steady: five pieces 150 ms apart, 750 ms in all.
651 let (port, tls, up) = res!(stand_in(HOST, |b| b.ends_with(b"0123456789"),
652 Reply::Bytes(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")).await);
653 let (mut c, s) = res!(UnixStream::pair(), IO);
654 let fwd = tokio::spawn(forwarder(s, port, tls));
655 res!(c.write_all(b"POST / HTTP/1.1\r\nContent-Length: 10\r\n\r\n").await, IO);
656 for p in [b"01", b"23", b"45", b"67", b"89"] {
657 tokio::time::sleep(Duration::from_millis(150)).await;
658 res!(c.write_all(p).await, IO);
659 }
660 let mut answer = Vec::new();
661 res!(c.read_to_end(&mut answer).await, IO);
662 res!(res!(fwd.await, IO).0);
663 req!(true, answer.ends_with(b"\r\n\r\nok"));
664 let _ = up.await;
665 // An upstream that answers before the body is all up: the client has the answer, and the
666 // forward ends at once, with the rest of the body never read.
667 let (port, tls, up) = res!(stand_in(HOST, head_in,
668 Reply::Bytes(b"HTTP/1.1 413 Content Too Large\r\nContent-Length: 3\r\n\r\nno!")).await);
669 let (mut c, s) = res!(UnixStream::pair(), IO);
670 let t0 = Instant::now();
671 let fwd = tokio::spawn(forwarder(s, port, tls));
672 res!(c.write_all(b"POST / HTTP/1.1\r\nContent-Length: 10\r\n\r\n01234").await, IO);
673 let mut answer = Vec::new();
674 res!(c.read_to_end(&mut answer).await, IO);
675 let (r, t) = res!(fwd.await, IO);
676 res!(r);
677 req!(true, answer.ends_with(b"\r\n\r\nno!"));
678 req!(413, t.status.load(Ordering::Relaxed));
679 req!(true, t0.elapsed() < BOUNDS.idle, "early answer took {:?}", t0.elapsed());
680 let closed = ended(tokio::time::timeout(SLACK, up).await);
681 req!(true, closed, "early answer: the upstream stayed open");
682 // A client that stops reading the answer: the write to it ends on `idle`, as a timeout.
683 const BIG: u64 = 4 << 20; // Past every buffer between the two.
684 let (port, tls, up) = res!(stand_in(HOST, head_in, Reply::Big(BIG)).await);
685 let (mut c, s) = res!(UnixStream::pair(), IO);
686 let t0 = Instant::now();
687 let fwd = tokio::spawn(forwarder(s, port, tls));
688 res!(c.write_all(b"GET / HTTP/1.1\r\n\r\n").await, IO);
689 let (r, t) = res!(fwd.await, IO);
690 let took = t0.elapsed();
691 up.abort();
692 match r { Err(e) => req!(true, has(&e, ErrTag::Timeout) && has(&e, ErrTag::Write), "deaf client: {:?}", e), Ok(()) => return Err(err!("deaf client: Ok"; Test)) }
693 req!(true, t.got.load(Ordering::Relaxed) < BIG, "deaf client: the whole answer went");
694 req!(true, took >= BOUNDS.idle && took < BOUNDS.idle + SLACK, "deaf client took {:?}", took);
695 drop(c);
696 Ok(())
697}
698
699/// Fault caught: a run of `1xx` heads ended by `answer_wait` (each restarts it), or not ended by
700/// the caller's total bound.
701#[tokio::test]
702async fn test_a_run_of_1xx_heads_is_bounded_by_the_caller() -> Outcome<()> {
703 let _one = SERIAL.read().await;
704 const EARLY: &[u8] = b"HTTP/1.1 103 Early Hints\r\n\r\n";
705 let (port, tls, up) = res!(stand_in(HOST, head_in,
706 Reply::Steps(&[(300, EARLY), (300, EARLY), (300, EARLY), (300, EARLY)])).await);
707 let (mut c, s) = res!(UnixStream::pair(), IO);
708 let fwd = tokio::spawn(tokio::time::timeout(Duration::from_millis(1_000), forwarder(s, port, tls)));
709 res!(c.write_all(b"GET / HTTP/1.1\r\n\r\n").await, IO);
710 let mut answer = Vec::new();
711 res!(c.read_to_end(&mut answer).await, IO);
712 let cut = res!(fwd.await, IO).is_err();
713 req!(true, cut, "the caller's bound did not end the forward");
714 req!(3, String::from_utf8_lossy(&answer).matches("103 Early").count());
715 up.abort();
716 Ok(())
717}
718
719/// Fault caught: an upstream connection (holding the keeper's key in its stream) left open after
720/// the client went away or the keeper stopped the forward.
721#[tokio::test]
722async fn test_a_client_gone_or_a_stop_closes_the_upstream() -> Outcome<()> {
723 let _one = SERIAL.read().await;
724 // Mid-request: ten of a hundred bytes, then the client leaves.
725 let (port, tls, up) = res!(stand_in(HOST, |b| b.ends_with(b"0123456789"), Reply::Silent).await);
726 let (mut c, s) = res!(UnixStream::pair(), IO);
727 let fwd = tokio::spawn(forwarder(s, port, tls));
728 res!(c.write_all(b"POST / HTTP/1.1\r\nContent-Length: 100\r\n\r\n0123456789").await, IO);
729 tokio::time::sleep(Duration::from_millis(50)).await;
730 drop(c);
731 match res!(fwd.await, IO).0 { Err(e) => req!(true, has(&e, ErrTag::Missing), "mid-request: {}", e), Ok(()) => return Err(err!("mid-request: Ok"; Test)) }
732 let closed = ended(tokio::time::timeout(SLACK, up).await);
733 req!(true, closed, "mid-request: the upstream stayed open");
734 // Mid-answer: the client leaves after the first piece.
735 let (port, tls, up) = res!(stand_in(HOST, head_in, Reply::Pieces(
736 b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n", 40, Duration::from_millis(50))).await);
737 let (mut c, s) = res!(UnixStream::pair(), IO);
738 let fwd = tokio::spawn(forwarder(s, port, tls));
739 res!(c.write_all(b"GET / HTTP/1.1\r\n\r\n").await, IO);
740 let mut buf = vec![0u8; 4_096];
741 let mut got = Vec::new();
742 while !String::from_utf8_lossy(&got).contains("data: 00") {
743 let n = res!(c.read(&mut buf).await, IO);
744 got.extend_from_slice(&buf[..n]);
745 }
746 drop(c);
747 let t0 = Instant::now();
748 match res!(fwd.await, IO).0 { Err(e) => req!(true, has(&e, ErrTag::Write), "mid-answer: {}", e), Ok(()) => return Err(err!("mid-answer: Ok"; Test)) }
749 req!(true, t0.elapsed() < SLACK, "mid-answer: noticed after {:?}", t0.elapsed());
750 let closed = ended(tokio::time::timeout(SLACK, up).await);
751 req!(true, closed, "mid-answer: the upstream stayed open");
752 // A stop: the keeper drops the forward while the upstream is silent.
753 let (port, tls, up) = res!(stand_in(HOST, head_in, Reply::Silent).await);
754 let (mut c, s) = res!(UnixStream::pair(), IO);
755 let fwd = tokio::spawn(forwarder(s, port, tls));
756 res!(c.write_all(b"GET / HTTP/1.1\r\n\r\n").await, IO);
757 tokio::time::sleep(Duration::from_millis(100)).await;
758 fwd.abort();
759 let t0 = Instant::now();
760 let closed = ended(tokio::time::timeout(Duration::from_millis(100), up).await);
761 req!(true, closed, "stop: the upstream stayed open");
762 req!(true, t0.elapsed() < BOUNDS.idle, "stop: closed only by the idle wait");
763 Ok(())
764}
765
766/// Fault caught: a doc that says a dropped forward closes the client too. The future borrows the
767/// client, so a stop closes the upstream and leaves the client's stream open for the caller.
768#[tokio::test]
769async fn test_a_dropped_forward_closes_the_upstream_and_leaves_the_client_to_the_caller() -> Outcome<()> {
770 let _one = SERIAL.read().await;
771 let (port, tls, up) = res!(stand_in(HOST, head_in, Reply::Silent).await);
772 let (mut c, mut s) = res!(UnixStream::pair(), IO);
773 res!(c.write_all(b"GET / HTTP/1.1\r\n\r\n").await, IO);
774 let tally = ForwardTally::default();
775 let fwd = async {
776 let h = res!(RequestHead::read(&mut s, &BOUNDS).await);
777 forward_one(&mut s, h, &STRIP, &ADD, HOST, port, tls, &BOUNDS, &tally).await
778 };
779 let stopped = tokio::time::timeout(Duration::from_millis(100), fwd).await.is_err();
780 req!(true, stopped, "the forward ended before the caller's stop");
781 let closed = ended(tokio::time::timeout(SLACK, up).await);
782 req!(true, closed, "the upstream stayed open");
783 let late = b"HTTP/1.1 504 Gateway Timeout\r\n\r\n";
784 res!(s.write_all(late).await, IO);
785 let mut buf = [0u8; 64];
786 let n = res!(c.read(&mut buf).await, IO);
787 req!(&late[..], &buf[..n], "the client's stream did not stay the caller's");
788 drop(s);
789 req!(0, res!(c.read(&mut buf).await, IO), "the client's end did not close with the caller's");
790 Ok(())
791}