oxedyne/fe2o3/fe2o3_net/tests/http_relay.rs
14.8 KiB, 11 runs
created by r1870400018:74703, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | #![cfg(feature = "async")] |
| 2 | //! `http::relay`: framing decided strictly from the raw header lines, and one body streamed |
| 3 | //! byte for byte, leaving the next message's bytes to the caller, in bounded memory. |
| 4 | |
| 5 | use oxedyne_fe2o3_core::prelude::*; |
| 6 | use oxedyne_fe2o3_net::http::relay::{ |
| 7 | framing, |
| 8 | relay, |
| 9 | Framing, |
| 10 | RelayLimits, |
| 11 | Subject, |
| 12 | RELAY_BUF_SIZE, |
| 13 | }; |
| 14 | |
| 15 | use std::{ |
| 16 | io::Cursor, |
| 17 | pin::Pin, |
| 18 | sync::atomic::{ |
| 19 | AtomicU64, |
| 20 | Ordering, |
| 21 | }, |
| 22 | task::{ |
| 23 | Context, |
| 24 | Poll, |
| 25 | }, |
| 26 | }; |
| 27 | |
| 28 | use tokio::io::{ |
| 29 | AsyncRead, |
| 30 | AsyncReadExt, |
| 31 | AsyncWrite, |
| 32 | ReadBuf, |
| 33 | }; |
| 34 | |
| 35 | |
| 36 | /// Held shared by every test that allocates, and alone by the memory test, so that no other |
| 37 | /// test's buffers show in its resident set. |
| 38 | static SERIAL: std::sync::RwLock<()> = std::sync::RwLock::new(()); |
| 39 | |
| 40 | const REPLY: Subject<'static> = Subject::Reply { status: 200, method: "GET" }; |
| 41 | |
| 42 | /// Relays `wire` (after `rest`) to a vector, returning what was written and what is left over. |
| 43 | async fn run( |
| 44 | wire: &[u8], |
| 45 | rest: &[u8], |
| 46 | f: Framing, |
| 47 | bound: Option<u64>, |
| 48 | trickle: bool, |
| 49 | ) |
| 50 | -> Outcome<(Vec<u8>, Vec<u8>, u64)> |
| 51 | { |
| 52 | let _shared = match SERIAL.read() { |
| 53 | Ok(g) => g, |
| 54 | Err(e) => e.into_inner(), |
| 55 | }; |
| 56 | run_unlocked(wire, rest, f, bound, trickle).await |
| 57 | } |
| 58 | |
| 59 | async fn run_unlocked( |
| 60 | wire: &[u8], |
| 61 | rest: &[u8], |
| 62 | f: Framing, |
| 63 | bound: Option<u64>, |
| 64 | trickle: bool, |
| 65 | ) |
| 66 | -> Outcome<(Vec<u8>, Vec<u8>, u64)> |
| 67 | { |
| 68 | let count = AtomicU64::new(0); |
| 69 | let mut out: Vec<u8> = Vec::new(); |
| 70 | let left = if trickle { |
| 71 | let mut r = Trickle(Cursor::new(wire.to_vec())); |
| 72 | res!(relay(&mut r, &mut out, f, &count, rest.to_vec(), RelayLimits { bound, ..RelayLimits::default() }).await) |
| 73 | } else { |
| 74 | let mut r = Cursor::new(wire.to_vec()); |
| 75 | res!(relay(&mut r, &mut out, f, &count, rest.to_vec(), RelayLimits { bound, ..RelayLimits::default() }).await) |
| 76 | }; |
| 77 | Ok((out, left, count.load(Ordering::Relaxed))) |
| 78 | } |
| 79 | |
| 80 | /// A reader that hands over one byte per read, so every line straddles reads. |
| 81 | struct Trickle(Cursor<Vec<u8>>); |
| 82 | |
| 83 | impl AsyncRead for Trickle { |
| 84 | fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) |
| 85 | -> Poll<std::io::Result<()>> |
| 86 | { |
| 87 | let mut one = [0u8; 1]; |
| 88 | let mut b = ReadBuf::new(&mut one); |
| 89 | match Pin::new(&mut self.0).poll_read(cx, &mut b) { |
| 90 | Poll::Ready(Ok(())) => { |
| 91 | buf.put_slice(b.filled()); |
| 92 | Poll::Ready(Ok(())) |
| 93 | }, |
| 94 | other => other, |
| 95 | } |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 100 | // │ FRAMING │ |
| 101 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 102 | |
| 103 | #[test] |
| 104 | fn test_framing_is_read_from_the_raw_lines() -> Outcome<()> { |
| 105 | let none: [(&str, &str); 0] = []; |
| 106 | assert_eq!(res!(framing(REPLY, [("Content-Length", " 42 ")])), Framing::Length(42)); |
| 107 | assert_eq!(res!(framing(REPLY, [("content-length", "0")])), Framing::Length(0)); |
| 108 | assert_eq!(res!(framing(REPLY, [("Transfer-Encoding", "Chunked")])), Framing::Chunked); |
| 109 | assert_eq!(res!(framing(Subject::Request, [("transfer-encoding", "chunked")])), |
| 110 | Framing::Chunked); |
| 111 | assert_eq!(res!(framing(REPLY, none)), Framing::Close); |
| 112 | assert_eq!(res!(framing(Subject::Request, none)), Framing::None); |
| 113 | let head = Subject::Reply { status: 200, method: "HEAD" }; |
| 114 | assert_eq!(res!(framing(head, [("Content-Length", "42")])), Framing::None); |
| 115 | for status in [101, 204, 304] { |
| 116 | let s = Subject::Reply { status, method: "GET" }; |
| 117 | assert_eq!(res!(framing(s, [("Content-Length", "42")])), Framing::None); |
| 118 | } |
| 119 | let connect = Subject::Reply { status: 200, method: "CONNECT" }; |
| 120 | assert_eq!(res!(framing(connect, none)), Framing::None); |
| 121 | Ok(()) |
| 122 | } |
| 123 | |
| 124 | #[test] |
| 125 | fn test_framing_refuses_every_ambiguous_form() { |
| 126 | let bad: Vec<Vec<(&str, &str)>> = vec![ |
| 127 | vec![("Content-Length", "5"), ("Transfer-Encoding", "chunked")], |
| 128 | vec![("Transfer-Encoding", "chunked"), ("Content-Length", "5")], |
| 129 | vec![("Content-Length", "5"), ("Content-Length", "5")], |
| 130 | vec![("Content-Length", "5, 5")], |
| 131 | vec![("Content-Length", "+5")], |
| 132 | vec![("Content-Length", "-1")], |
| 133 | vec![("Content-Length", "0x10")], |
| 134 | vec![("Content-Length", "")], |
| 135 | vec![("Content-Length", "99999999999999999999")], |
| 136 | vec![("Transfer-Encoding", "gzip, chunked")], |
| 137 | vec![("Transfer-Encoding", "chunked, chunked")], |
| 138 | vec![("Transfer-Encoding", "chunked"), ("Transfer-Encoding", "chunked")], |
| 139 | vec![("Transfer-Encoding", "identity")], |
| 140 | vec![("Transfer-Encoding", "")], |
| 141 | ]; |
| 142 | for f in bad { |
| 143 | assert!(framing(REPLY, f.clone()).is_err(), "framing accepted {:?}", f); |
| 144 | assert!(framing(Subject::Request, f.clone()).is_err(), "framing accepted {:?}", f); |
| 145 | // Still checked where the reply has no body. |
| 146 | let head = Subject::Reply { status: 204, method: "GET" }; |
| 147 | assert!(framing(head, f.clone()).is_err(), "framing accepted {:?} on a 204", f); |
| 148 | } |
| 149 | } |
| 150 | |
| 151 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 152 | // │ RELAY │ |
| 153 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 154 | |
| 155 | const CHUNKED: &[u8] = b"4;name=val;q=\"x y\"\r\nWiki\r\n5\r\npedia\r\n\ |
| 156 | A ; ext\r\n in chunks\r\n0;last\r\nX-Trailer: yes\r\nOther:\t2\r\n\r\n"; |
| 157 | |
| 158 | #[tokio::test] |
| 159 | async fn test_a_chunked_body_is_copied_byte_for_byte() -> Outcome<()> { |
| 160 | let next = b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"; |
| 161 | let mut wire = CHUNKED.to_vec(); |
| 162 | wire.extend_from_slice(next); |
| 163 | for trickle in [false, true] { |
| 164 | for split in [0, 3, 20, CHUNKED.len()] { |
| 165 | let (out, left, n) = res!(run(&wire[split..], &wire[..split], Framing::Chunked, |
| 166 | None, trickle).await); |
| 167 | assert_eq!(out, CHUNKED, "trickle {} split {}", trickle, split); |
| 168 | assert_eq!(n, CHUNKED.len() as u64); |
| 169 | if !trickle && split < CHUNKED.len() { |
| 170 | assert_eq!(left, next, "trickle {} split {}", trickle, split); |
| 171 | } else { |
| 172 | // Nothing past the final CRLF has been read: the reader gives one byte at a |
| 173 | // time, or the whole body was already in `rest`. |
| 174 | assert!(left.is_empty(), "trickle {} split {}", trickle, split); |
| 175 | } |
| 176 | } |
| 177 | } |
| 178 | Ok(()) |
| 179 | } |
| 180 | |
| 181 | #[tokio::test] |
| 182 | async fn test_a_length_body_leaves_the_next_response_as_rest() -> Outcome<()> { |
| 183 | let body = b"0123456789"; |
| 184 | let next = b"HTTP/1.1 204 No Content\r\n\r\n"; |
| 185 | let mut wire = body[3..].to_vec(); |
| 186 | wire.extend_from_slice(next); |
| 187 | let (out, left, n) = res!(run(&wire, &body[..3], Framing::Length(10), None, false).await); |
| 188 | assert_eq!(out, body); |
| 189 | assert_eq!(left, next); |
| 190 | assert_eq!(n, 10); |
| 191 | // All of the body, and the next message, already in `rest`. |
| 192 | let mut rest = body.to_vec(); |
| 193 | rest.extend_from_slice(next); |
| 194 | let (out, left, _) = res!(run(b"", &rest, Framing::Length(10), None, false).await); |
| 195 | assert_eq!(out, body); |
| 196 | assert_eq!(left, next); |
| 197 | // No body leaves `rest` untouched. |
| 198 | let (out, left, n) = res!(run(b"more", next, Framing::None, None, false).await); |
| 199 | assert!(out.is_empty()); |
| 200 | assert_eq!(left, next); |
| 201 | assert_eq!(n, 0); |
| 202 | // A body cut short is an error. |
| 203 | assert!(run(b"0123", b"", Framing::Length(10), None, false).await.is_err()); |
| 204 | Ok(()) |
| 205 | } |
| 206 | |
| 207 | #[tokio::test] |
| 208 | async fn test_a_close_body_runs_to_the_end_of_the_connection() -> Outcome<()> { |
| 209 | let _shared = match SERIAL.read() { |
| 210 | Ok(g) => g, |
| 211 | Err(e) => e.into_inner(), |
| 212 | }; |
| 213 | let body: Vec<u8> = (0..200_000u32).map(|i| (i % 251) as u8).collect(); |
| 214 | let (out, left, n) = res!(run_unlocked(&body[5..], &body[..5], Framing::Close, None, false).await); |
| 215 | assert_eq!(out, body); |
| 216 | assert!(left.is_empty()); |
| 217 | assert_eq!(n, body.len() as u64); |
| 218 | Ok(()) |
| 219 | } |
| 220 | |
| 221 | #[tokio::test] |
| 222 | async fn test_a_bad_chunk_line_is_an_error() { |
| 223 | let bad: [&[u8]; 16] = [ |
| 224 | b"g\r\nx\r\n0\r\n\r\n", // Not hex. |
| 225 | b"\r\n", // No size. |
| 226 | b" 4\r\nWiki\r\n0\r\n\r\n", // Leading blank. |
| 227 | b"4 \r\nWiki\r\n0\r\n\r\n", // Trailing blank and no extension. |
| 228 | b"4x\r\nWiki\r\n0\r\n\r\n", // Junk after the size. |
| 229 | b"00000000000000004\r\nWiki\r\n0\r\n\r\n", // 17 digits. |
| 230 | b"-4\r\nWiki\r\n0\r\n\r\n", |
| 231 | b"4\nWiki\r\n0\r\n\r\n", // Bare LF. |
| 232 | b"4\r;x\r\nWiki\r\n0\r\n\r\n", // Bare CR. |
| 233 | b"4;a\x01b\r\nWiki\r\n0\r\n\r\n", // Control in an extension. |
| 234 | b"4\r\nWikiXY0\r\n\r\n", // No CRLF after the data. |
| 235 | b"4\r\nWiki\r\n0\r\nnot a field\r\n\r\n", // A trailer that is not a field. |
| 236 | b"4\r\nWiki\r\n0\r\n: v\r\n\r\n", // A trailer without a name. |
| 237 | b"4\r\nWiki\r\n0\r\nX\x00: v\r\n\r\n", // A trailer name with a control. |
| 238 | b"4\r\nWi", // Cut short in the data. |
| 239 | b"4\r\nWiki\r\n0\r\nX: y\r\n", // Cut short in the trailers. |
| 240 | ]; |
| 241 | for wire in bad { |
| 242 | assert!(run(wire, b"", Framing::Chunked, None, false).await.is_err(), |
| 243 | "the relay accepted {:?}", String::from_utf8_lossy(wire)); |
| 244 | } |
| 245 | // A chunk-size line without end, kept to the line limit. |
| 246 | let long = vec![b'0'; 10_000]; |
| 247 | assert!(run(&long, b"", Framing::Chunked, None, false).await.is_err()); |
| 248 | // Sixteen digits are allowed. |
| 249 | let ok = b"0000000000000004\r\nWiki\r\n0\r\n\r\n"; |
| 250 | assert!(run(ok, b"", Framing::Chunked, None, false).await.is_ok()); |
| 251 | } |
| 252 | |
| 253 | #[tokio::test] |
| 254 | async fn test_a_body_past_the_bound_is_an_error() -> Outcome<()> { |
| 255 | assert!(run(b"0123456789", b"", Framing::Length(10), Some(9), false).await.is_err()); |
| 256 | res!(run(b"0123456789", b"", Framing::Length(10), Some(10), false).await); |
| 257 | let n = CHUNKED.len() as u64; |
| 258 | assert!(run(CHUNKED, b"", Framing::Chunked, Some(n - 1), false).await.is_err()); |
| 259 | res!(run(CHUNKED, b"", Framing::Chunked, Some(n), false).await); |
| 260 | assert!(run(&[0u8; 100], b"", Framing::Close, Some(99), false).await.is_err()); |
| 261 | Ok(()) |
| 262 | } |
| 263 | |
| 264 | /// Fault caught: a stalled write tagged only `Write`, so a caller cannot tell it from a peer gone. |
| 265 | #[tokio::test] |
| 266 | async fn test_the_idle_wait_ends_a_stalled_read_or_write_as_a_timeout() -> Outcome<()> { |
| 267 | let wait = std::time::Duration::from_millis(100); |
| 268 | let lim = RelayLimits { idle: Some(wait), ..RelayLimits::default() }; |
| 269 | let count = AtomicU64::new(0); |
| 270 | // A reader that sends nothing, its writer held open. |
| 271 | let (mut r, _w) = tokio::io::duplex(64); |
| 272 | let t0 = std::time::Instant::now(); |
| 273 | match relay(&mut r, &mut Vec::new(), Framing::Length(10), &count, Vec::new(), lim).await { |
| 274 | Err(e) => assert!(e.tags().contains(&ErrTag::Timeout) && e.tags().contains(&ErrTag::Read), "{:?}", e), |
| 275 | Ok(_) => return Err(err!("A stalled read relayed."; Test)), |
| 276 | } |
| 277 | assert!(t0.elapsed() < 3 * wait, "the read took {:?}", t0.elapsed()); |
| 278 | // A writer whose reader never reads, past its 16 byte buffer. |
| 279 | let (mut w, _r) = tokio::io::duplex(16); |
| 280 | let t0 = std::time::Instant::now(); |
| 281 | match relay(&mut Cursor::new(vec![0u8; 1_024]), &mut w, Framing::Length(1_024), &count, Vec::new(), lim).await { |
| 282 | Err(e) => assert!(e.tags().contains(&ErrTag::Timeout) && e.tags().contains(&ErrTag::Write), "{:?}", e), |
| 283 | Ok(_) => return Err(err!("A stalled write relayed."; Test)), |
| 284 | } |
| 285 | assert!(t0.elapsed() < 3 * wait, "the write took {:?}", t0.elapsed()); |
| 286 | Ok(()) |
| 287 | } |
| 288 | |
| 289 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 290 | // │ BOUNDED MEMORY │ |
| 291 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 292 | |
| 293 | /// The resident set in bytes, from `/proc/self/status`. |
| 294 | fn rss() -> u64 { |
| 295 | let s = std::fs::read_to_string("/proc/self/status").unwrap_or_default(); |
| 296 | s.lines() |
| 297 | .find(|l| l.starts_with("VmRSS:")) |
| 298 | .and_then(|l| l.split_whitespace().nth(1)) |
| 299 | .and_then(|k| k.parse::<u64>().ok()) |
| 300 | .map(|k| k * 1_024) |
| 301 | .unwrap_or(0) |
| 302 | } |
| 303 | |
| 304 | /// A writer that keeps nothing, and samples the resident set every 8 MB. |
| 305 | struct Gauge { |
| 306 | n: u64, |
| 307 | next: u64, |
| 308 | peak: u64, |
| 309 | } |
| 310 | |
| 311 | impl AsyncWrite for Gauge { |
| 312 | fn poll_write(mut self: Pin<&mut Self>, _: &mut Context<'_>, buf: &[u8]) |
| 313 | -> Poll<std::io::Result<usize>> |
| 314 | { |
| 315 | self.n += buf.len() as u64; |
| 316 | if self.n >= self.next { |
| 317 | self.next += 8 << 20; |
| 318 | let r = rss(); |
| 319 | if r > self.peak { |
| 320 | self.peak = r; |
| 321 | } |
| 322 | } |
| 323 | Poll::Ready(Ok(buf.len())) |
| 324 | } |
| 325 | fn poll_flush(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<std::io::Result<()>> { |
| 326 | Poll::Ready(Ok(())) |
| 327 | } |
| 328 | fn poll_shutdown(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<std::io::Result<()>> { |
| 329 | Poll::Ready(Ok(())) |
| 330 | } |
| 331 | } |
| 332 | |
| 333 | #[tokio::test] |
| 334 | async fn test_a_256_mb_body_streams_in_bounded_memory() -> Outcome<()> { |
| 335 | const SIZE: u64 = 256 << 20; |
| 336 | let _alone = match SERIAL.write() { |
| 337 | Ok(g) => g, |
| 338 | Err(e) => e.into_inner(), |
| 339 | }; |
| 340 | let mut r = tokio::io::repeat(7).take(SIZE).chain(Cursor::new(b"NEXT".to_vec())); |
| 341 | let start = rss(); |
| 342 | let mut g = Gauge { n: 0, next: 0, peak: start }; |
| 343 | let count = AtomicU64::new(0); |
| 344 | let left = res!(relay(&mut r, &mut g, Framing::Length(SIZE), &count, Vec::new(), RelayLimits::default()).await); |
| 345 | assert_eq!(g.n, SIZE); |
| 346 | assert_eq!(count.load(Ordering::Relaxed), SIZE); |
| 347 | assert!(left.is_empty() || left == b"NEXT"); |
| 348 | let growth = g.peak.saturating_sub(start); |
| 349 | eprintln!("256 MB relayed: peak RSS growth {} KiB (buffer {} KiB)", growth / 1_024, |
| 350 | RELAY_BUF_SIZE / 1_024); |
| 351 | assert!(growth < (1 << 20) + RELAY_BUF_SIZE as u64, |
| 352 | "peak RSS grew by {} bytes relaying 256 MB", growth); |
| 353 | Ok(()) |
| 354 | } |