Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_net/tests/http_relay.rs

14.8 KiB, 11 runs

created by r1870400018:74703, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1#![cfg(feature = "async")]
2//! `http::relay`: framing decided strictly from the raw header lines, and one body streamed
3//! byte for byte, leaving the next message's bytes to the caller, in bounded memory.
4
5use oxedyne_fe2o3_core::prelude::*;
6use oxedyne_fe2o3_net::http::relay::{
7 framing,
8 relay,
9 Framing,
10 RelayLimits,
11 Subject,
12 RELAY_BUF_SIZE,
13};
14
15use std::{
16 io::Cursor,
17 pin::Pin,
18 sync::atomic::{
19 AtomicU64,
20 Ordering,
21 },
22 task::{
23 Context,
24 Poll,
25 },
26};
27
28use tokio::io::{
29 AsyncRead,
30 AsyncReadExt,
31 AsyncWrite,
32 ReadBuf,
33};
34
35
36/// Held shared by every test that allocates, and alone by the memory test, so that no other
37/// test's buffers show in its resident set.
38static SERIAL: std::sync::RwLock<()> = std::sync::RwLock::new(());
39
40const REPLY: Subject<'static> = Subject::Reply { status: 200, method: "GET" };
41
42/// Relays `wire` (after `rest`) to a vector, returning what was written and what is left over.
43async fn run(
44 wire: &[u8],
45 rest: &[u8],
46 f: Framing,
47 bound: Option<u64>,
48 trickle: bool,
49)
50 -> Outcome<(Vec<u8>, Vec<u8>, u64)>
51{
52 let _shared = match SERIAL.read() {
53 Ok(g) => g,
54 Err(e) => e.into_inner(),
55 };
56 run_unlocked(wire, rest, f, bound, trickle).await
57}
58
59async fn run_unlocked(
60 wire: &[u8],
61 rest: &[u8],
62 f: Framing,
63 bound: Option<u64>,
64 trickle: bool,
65)
66 -> Outcome<(Vec<u8>, Vec<u8>, u64)>
67{
68 let count = AtomicU64::new(0);
69 let mut out: Vec<u8> = Vec::new();
70 let left = if trickle {
71 let mut r = Trickle(Cursor::new(wire.to_vec()));
72 res!(relay(&mut r, &mut out, f, &count, rest.to_vec(), RelayLimits { bound, ..RelayLimits::default() }).await)
73 } else {
74 let mut r = Cursor::new(wire.to_vec());
75 res!(relay(&mut r, &mut out, f, &count, rest.to_vec(), RelayLimits { bound, ..RelayLimits::default() }).await)
76 };
77 Ok((out, left, count.load(Ordering::Relaxed)))
78}
79
80/// A reader that hands over one byte per read, so every line straddles reads.
81struct Trickle(Cursor<Vec<u8>>);
82
83impl AsyncRead for Trickle {
84 fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>)
85 -> Poll<std::io::Result<()>>
86 {
87 let mut one = [0u8; 1];
88 let mut b = ReadBuf::new(&mut one);
89 match Pin::new(&mut self.0).poll_read(cx, &mut b) {
90 Poll::Ready(Ok(())) => {
91 buf.put_slice(b.filled());
92 Poll::Ready(Ok(()))
93 },
94 other => other,
95 }
96 }
97}
98
99// ┌───────────────────────────────────────────────────────────────────────────┐
100// │ FRAMING │
101// └───────────────────────────────────────────────────────────────────────────┘
102
103#[test]
104fn test_framing_is_read_from_the_raw_lines() -> Outcome<()> {
105 let none: [(&str, &str); 0] = [];
106 assert_eq!(res!(framing(REPLY, [("Content-Length", " 42 ")])), Framing::Length(42));
107 assert_eq!(res!(framing(REPLY, [("content-length", "0")])), Framing::Length(0));
108 assert_eq!(res!(framing(REPLY, [("Transfer-Encoding", "Chunked")])), Framing::Chunked);
109 assert_eq!(res!(framing(Subject::Request, [("transfer-encoding", "chunked")])),
110 Framing::Chunked);
111 assert_eq!(res!(framing(REPLY, none)), Framing::Close);
112 assert_eq!(res!(framing(Subject::Request, none)), Framing::None);
113 let head = Subject::Reply { status: 200, method: "HEAD" };
114 assert_eq!(res!(framing(head, [("Content-Length", "42")])), Framing::None);
115 for status in [101, 204, 304] {
116 let s = Subject::Reply { status, method: "GET" };
117 assert_eq!(res!(framing(s, [("Content-Length", "42")])), Framing::None);
118 }
119 let connect = Subject::Reply { status: 200, method: "CONNECT" };
120 assert_eq!(res!(framing(connect, none)), Framing::None);
121 Ok(())
122}
123
124#[test]
125fn test_framing_refuses_every_ambiguous_form() {
126 let bad: Vec<Vec<(&str, &str)>> = vec![
127 vec![("Content-Length", "5"), ("Transfer-Encoding", "chunked")],
128 vec![("Transfer-Encoding", "chunked"), ("Content-Length", "5")],
129 vec![("Content-Length", "5"), ("Content-Length", "5")],
130 vec![("Content-Length", "5, 5")],
131 vec![("Content-Length", "+5")],
132 vec![("Content-Length", "-1")],
133 vec![("Content-Length", "0x10")],
134 vec![("Content-Length", "")],
135 vec![("Content-Length", "99999999999999999999")],
136 vec![("Transfer-Encoding", "gzip, chunked")],
137 vec![("Transfer-Encoding", "chunked, chunked")],
138 vec![("Transfer-Encoding", "chunked"), ("Transfer-Encoding", "chunked")],
139 vec![("Transfer-Encoding", "identity")],
140 vec![("Transfer-Encoding", "")],
141 ];
142 for f in bad {
143 assert!(framing(REPLY, f.clone()).is_err(), "framing accepted {:?}", f);
144 assert!(framing(Subject::Request, f.clone()).is_err(), "framing accepted {:?}", f);
145 // Still checked where the reply has no body.
146 let head = Subject::Reply { status: 204, method: "GET" };
147 assert!(framing(head, f.clone()).is_err(), "framing accepted {:?} on a 204", f);
148 }
149}
150
151// ┌───────────────────────────────────────────────────────────────────────────┐
152// │ RELAY │
153// └───────────────────────────────────────────────────────────────────────────┘
154
155const CHUNKED: &[u8] = b"4;name=val;q=\"x y\"\r\nWiki\r\n5\r\npedia\r\n\
156 A ; ext\r\n in chunks\r\n0;last\r\nX-Trailer: yes\r\nOther:\t2\r\n\r\n";
157
158#[tokio::test]
159async fn test_a_chunked_body_is_copied_byte_for_byte() -> Outcome<()> {
160 let next = b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n";
161 let mut wire = CHUNKED.to_vec();
162 wire.extend_from_slice(next);
163 for trickle in [false, true] {
164 for split in [0, 3, 20, CHUNKED.len()] {
165 let (out, left, n) = res!(run(&wire[split..], &wire[..split], Framing::Chunked,
166 None, trickle).await);
167 assert_eq!(out, CHUNKED, "trickle {} split {}", trickle, split);
168 assert_eq!(n, CHUNKED.len() as u64);
169 if !trickle && split < CHUNKED.len() {
170 assert_eq!(left, next, "trickle {} split {}", trickle, split);
171 } else {
172 // Nothing past the final CRLF has been read: the reader gives one byte at a
173 // time, or the whole body was already in `rest`.
174 assert!(left.is_empty(), "trickle {} split {}", trickle, split);
175 }
176 }
177 }
178 Ok(())
179}
180
181#[tokio::test]
182async fn test_a_length_body_leaves_the_next_response_as_rest() -> Outcome<()> {
183 let body = b"0123456789";
184 let next = b"HTTP/1.1 204 No Content\r\n\r\n";
185 let mut wire = body[3..].to_vec();
186 wire.extend_from_slice(next);
187 let (out, left, n) = res!(run(&wire, &body[..3], Framing::Length(10), None, false).await);
188 assert_eq!(out, body);
189 assert_eq!(left, next);
190 assert_eq!(n, 10);
191 // All of the body, and the next message, already in `rest`.
192 let mut rest = body.to_vec();
193 rest.extend_from_slice(next);
194 let (out, left, _) = res!(run(b"", &rest, Framing::Length(10), None, false).await);
195 assert_eq!(out, body);
196 assert_eq!(left, next);
197 // No body leaves `rest` untouched.
198 let (out, left, n) = res!(run(b"more", next, Framing::None, None, false).await);
199 assert!(out.is_empty());
200 assert_eq!(left, next);
201 assert_eq!(n, 0);
202 // A body cut short is an error.
203 assert!(run(b"0123", b"", Framing::Length(10), None, false).await.is_err());
204 Ok(())
205}
206
207#[tokio::test]
208async fn test_a_close_body_runs_to_the_end_of_the_connection() -> Outcome<()> {
209 let _shared = match SERIAL.read() {
210 Ok(g) => g,
211 Err(e) => e.into_inner(),
212 };
213 let body: Vec<u8> = (0..200_000u32).map(|i| (i % 251) as u8).collect();
214 let (out, left, n) = res!(run_unlocked(&body[5..], &body[..5], Framing::Close, None, false).await);
215 assert_eq!(out, body);
216 assert!(left.is_empty());
217 assert_eq!(n, body.len() as u64);
218 Ok(())
219}
220
221#[tokio::test]
222async fn test_a_bad_chunk_line_is_an_error() {
223 let bad: [&[u8]; 16] = [
224 b"g\r\nx\r\n0\r\n\r\n", // Not hex.
225 b"\r\n", // No size.
226 b" 4\r\nWiki\r\n0\r\n\r\n", // Leading blank.
227 b"4 \r\nWiki\r\n0\r\n\r\n", // Trailing blank and no extension.
228 b"4x\r\nWiki\r\n0\r\n\r\n", // Junk after the size.
229 b"00000000000000004\r\nWiki\r\n0\r\n\r\n", // 17 digits.
230 b"-4\r\nWiki\r\n0\r\n\r\n",
231 b"4\nWiki\r\n0\r\n\r\n", // Bare LF.
232 b"4\r;x\r\nWiki\r\n0\r\n\r\n", // Bare CR.
233 b"4;a\x01b\r\nWiki\r\n0\r\n\r\n", // Control in an extension.
234 b"4\r\nWikiXY0\r\n\r\n", // No CRLF after the data.
235 b"4\r\nWiki\r\n0\r\nnot a field\r\n\r\n", // A trailer that is not a field.
236 b"4\r\nWiki\r\n0\r\n: v\r\n\r\n", // A trailer without a name.
237 b"4\r\nWiki\r\n0\r\nX\x00: v\r\n\r\n", // A trailer name with a control.
238 b"4\r\nWi", // Cut short in the data.
239 b"4\r\nWiki\r\n0\r\nX: y\r\n", // Cut short in the trailers.
240 ];
241 for wire in bad {
242 assert!(run(wire, b"", Framing::Chunked, None, false).await.is_err(),
243 "the relay accepted {:?}", String::from_utf8_lossy(wire));
244 }
245 // A chunk-size line without end, kept to the line limit.
246 let long = vec![b'0'; 10_000];
247 assert!(run(&long, b"", Framing::Chunked, None, false).await.is_err());
248 // Sixteen digits are allowed.
249 let ok = b"0000000000000004\r\nWiki\r\n0\r\n\r\n";
250 assert!(run(ok, b"", Framing::Chunked, None, false).await.is_ok());
251}
252
253#[tokio::test]
254async fn test_a_body_past_the_bound_is_an_error() -> Outcome<()> {
255 assert!(run(b"0123456789", b"", Framing::Length(10), Some(9), false).await.is_err());
256 res!(run(b"0123456789", b"", Framing::Length(10), Some(10), false).await);
257 let n = CHUNKED.len() as u64;
258 assert!(run(CHUNKED, b"", Framing::Chunked, Some(n - 1), false).await.is_err());
259 res!(run(CHUNKED, b"", Framing::Chunked, Some(n), false).await);
260 assert!(run(&[0u8; 100], b"", Framing::Close, Some(99), false).await.is_err());
261 Ok(())
262}
263
264/// Fault caught: a stalled write tagged only `Write`, so a caller cannot tell it from a peer gone.
265#[tokio::test]
266async fn test_the_idle_wait_ends_a_stalled_read_or_write_as_a_timeout() -> Outcome<()> {
267 let wait = std::time::Duration::from_millis(100);
268 let lim = RelayLimits { idle: Some(wait), ..RelayLimits::default() };
269 let count = AtomicU64::new(0);
270 // A reader that sends nothing, its writer held open.
271 let (mut r, _w) = tokio::io::duplex(64);
272 let t0 = std::time::Instant::now();
273 match relay(&mut r, &mut Vec::new(), Framing::Length(10), &count, Vec::new(), lim).await {
274 Err(e) => assert!(e.tags().contains(&ErrTag::Timeout) && e.tags().contains(&ErrTag::Read), "{:?}", e),
275 Ok(_) => return Err(err!("A stalled read relayed."; Test)),
276 }
277 assert!(t0.elapsed() < 3 * wait, "the read took {:?}", t0.elapsed());
278 // A writer whose reader never reads, past its 16 byte buffer.
279 let (mut w, _r) = tokio::io::duplex(16);
280 let t0 = std::time::Instant::now();
281 match relay(&mut Cursor::new(vec![0u8; 1_024]), &mut w, Framing::Length(1_024), &count, Vec::new(), lim).await {
282 Err(e) => assert!(e.tags().contains(&ErrTag::Timeout) && e.tags().contains(&ErrTag::Write), "{:?}", e),
283 Ok(_) => return Err(err!("A stalled write relayed."; Test)),
284 }
285 assert!(t0.elapsed() < 3 * wait, "the write took {:?}", t0.elapsed());
286 Ok(())
287}
288
289// ┌───────────────────────────────────────────────────────────────────────────┐
290// │ BOUNDED MEMORY │
291// └───────────────────────────────────────────────────────────────────────────┘
292
293/// The resident set in bytes, from `/proc/self/status`.
294fn rss() -> u64 {
295 let s = std::fs::read_to_string("/proc/self/status").unwrap_or_default();
296 s.lines()
297 .find(|l| l.starts_with("VmRSS:"))
298 .and_then(|l| l.split_whitespace().nth(1))
299 .and_then(|k| k.parse::<u64>().ok())
300 .map(|k| k * 1_024)
301 .unwrap_or(0)
302}
303
304/// A writer that keeps nothing, and samples the resident set every 8 MB.
305struct Gauge {
306 n: u64,
307 next: u64,
308 peak: u64,
309}
310
311impl AsyncWrite for Gauge {
312 fn poll_write(mut self: Pin<&mut Self>, _: &mut Context<'_>, buf: &[u8])
313 -> Poll<std::io::Result<usize>>
314 {
315 self.n += buf.len() as u64;
316 if self.n >= self.next {
317 self.next += 8 << 20;
318 let r = rss();
319 if r > self.peak {
320 self.peak = r;
321 }
322 }
323 Poll::Ready(Ok(buf.len()))
324 }
325 fn poll_flush(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<std::io::Result<()>> {
326 Poll::Ready(Ok(()))
327 }
328 fn poll_shutdown(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<std::io::Result<()>> {
329 Poll::Ready(Ok(()))
330 }
331}
332
333#[tokio::test]
334async fn test_a_256_mb_body_streams_in_bounded_memory() -> Outcome<()> {
335 const SIZE: u64 = 256 << 20;
336 let _alone = match SERIAL.write() {
337 Ok(g) => g,
338 Err(e) => e.into_inner(),
339 };
340 let mut r = tokio::io::repeat(7).take(SIZE).chain(Cursor::new(b"NEXT".to_vec()));
341 let start = rss();
342 let mut g = Gauge { n: 0, next: 0, peak: start };
343 let count = AtomicU64::new(0);
344 let left = res!(relay(&mut r, &mut g, Framing::Length(SIZE), &count, Vec::new(), RelayLimits::default()).await);
345 assert_eq!(g.n, SIZE);
346 assert_eq!(count.load(Ordering::Relaxed), SIZE);
347 assert!(left.is_empty() || left == b"NEXT");
348 let growth = g.peak.saturating_sub(start);
349 eprintln!("256 MB relayed: peak RSS growth {} KiB (buffer {} KiB)", growth / 1_024,
350 RELAY_BUF_SIZE / 1_024);
351 assert!(growth < (1 << 20) + RELAY_BUF_SIZE as u64,
352 "peak RSS grew by {} bytes relaying 256 MB", growth);
353 Ok(())
354}