oxedyne/fe2o3/fe2o3_steel/tests/upstream_limits.rs
8.1 KiB, 1 run
created by r1870400018:65947, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! What an `api_routes` proxy and a webhook forwarder do with an upstream that will not finish. |
| 2 | //! |
| 3 | //! An upstream that states no `Content-Length`, is not chunked and never closes the connection has |
| 4 | //! a body that ends nowhere. Since the outbound client learned to read such a body to the end of the |
| 5 | //! connection (for HTTP/1.0 servers such as Python's `http.server`), the forwarders, which set no |
| 6 | //! limit and no deadline, held the visitor's request open for ever and buffered whatever arrived |
| 7 | //! without bound. They are bounded now: the reply's size by the server's `http_max_*` limits (a |
| 8 | //! `502 Bad Gateway` over them), and the whole call by `upstream_timeout_ms` (a `504 Gateway |
| 9 | //! Timeout` over it). |
| 10 | //! |
| 11 | //! Each case stands up a misbehaving upstream on loopback and asks through `handle_https`, with the |
| 12 | //! whole ask under a deadline of its own so that a hang is a failed assertion and not a stuck test. |
| 13 | //! |
| 14 | //! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\ |
| 15 | //! Anthropic Claude |
| 16 | |
| 17 | mod vhost_rig; |
| 18 | |
| 19 | use vhost_rig::{ |
| 20 | Reply, |
| 21 | Rig, |
| 22 | Scratch, |
| 23 | Site, |
| 24 | }; |
| 25 | |
| 26 | use oxedyne_fe2o3_core::prelude::*; |
| 27 | use oxedyne_fe2o3_steel::srv::cfg::{ |
| 28 | ApiRoute, |
| 29 | ServerConfig, |
| 30 | WebhookRoute, |
| 31 | }; |
| 32 | |
| 33 | use std::{ |
| 34 | sync::{ |
| 35 | atomic::{ |
| 36 | AtomicUsize, |
| 37 | Ordering, |
| 38 | }, |
| 39 | Arc, |
| 40 | }, |
| 41 | time::Duration, |
| 42 | }; |
| 43 | |
| 44 | use tokio::{ |
| 45 | io::{ |
| 46 | AsyncReadExt, |
| 47 | AsyncWriteExt, |
| 48 | }, |
| 49 | net::TcpListener, |
| 50 | }; |
| 51 | |
| 52 | const HOST: &str = "limits.test"; |
| 53 | const CAP: usize = 64 * 1024; // the reply size limit the rig's config sets |
| 54 | const SPILL: usize = 64 * 1024 * 1024; // what the endless upstream would send, if let |
| 55 | |
| 56 | #[derive(Clone, Copy, Debug)] |
| 57 | enum Upstream { |
| 58 | Hangs, // answers `HTTP/1.0 200` with a few bytes, then says nothing and holds the line |
| 59 | Endless, // answers `HTTP/1.0 200`, then sends without stopping and never closes |
| 60 | } |
| 61 | |
| 62 | /// A loopback upstream; the second value counts the bytes it managed to send. |
| 63 | async fn upstream(kind: Upstream) -> Outcome<(u16, Arc<AtomicUsize>)> { |
| 64 | let listener = res!(TcpListener::bind("127.0.0.1:0").await, Network, Init); |
| 65 | let port = res!(listener.local_addr(), Network, Init).port(); |
| 66 | let sent = Arc::new(AtomicUsize::new(0)); |
| 67 | let counter = sent.clone(); |
| 68 | tokio::spawn(async move { |
| 69 | loop { |
| 70 | let (mut tcp, _) = match listener.accept().await { |
| 71 | Ok(c) => c, |
| 72 | Err(_) => return, |
| 73 | }; |
| 74 | let counter = counter.clone(); |
| 75 | tokio::spawn(async move { |
| 76 | // The request head, then whatever body it promised. |
| 77 | let mut buf = Vec::new(); |
| 78 | let mut chunk = [0u8; 4096]; |
| 79 | let split = loop { |
| 80 | if let Some(i) = buf.windows(4).position(|w| w == b"\r\n\r\n") { |
| 81 | break i; |
| 82 | } |
| 83 | match tcp.read(&mut chunk).await { |
| 84 | Ok(0) | Err(_) => return, |
| 85 | Ok(n) => buf.extend_from_slice(&chunk[..n]), |
| 86 | } |
| 87 | }; |
| 88 | let head = String::from_utf8_lossy(&buf[..split]).to_lowercase(); |
| 89 | let want = head.lines() |
| 90 | .filter_map(|l| l.strip_prefix("content-length:")) |
| 91 | .filter_map(|v| v.trim().parse::<usize>().ok()) |
| 92 | .next() |
| 93 | .unwrap_or(0); |
| 94 | while buf.len() < split + 4 + want { |
| 95 | match tcp.read(&mut chunk).await { |
| 96 | Ok(0) | Err(_) => break, |
| 97 | Ok(n) => buf.extend_from_slice(&chunk[..n]), |
| 98 | } |
| 99 | } |
| 100 | if tcp.write_all(b"HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n\r\npartial").await.is_err() { |
| 101 | return; |
| 102 | } |
| 103 | match kind { |
| 104 | Upstream::Hangs => { |
| 105 | tokio::time::sleep(Duration::from_secs(60)).await; |
| 106 | }, |
| 107 | Upstream::Endless => { |
| 108 | let block = [b'x'; 16 * 1024]; |
| 109 | while counter.load(Ordering::SeqCst) < SPILL { |
| 110 | match tcp.write_all(&block).await { |
| 111 | Ok(()) => { counter.fetch_add(block.len(), Ordering::SeqCst); }, |
| 112 | Err(_) => return, // Steel has hung up on it |
| 113 | } |
| 114 | } |
| 115 | tokio::time::sleep(Duration::from_secs(60)).await; |
| 116 | }, |
| 117 | } |
| 118 | }); |
| 119 | } |
| 120 | }); |
| 121 | Ok((port, sent)) |
| 122 | } |
| 123 | |
| 124 | fn config() -> ServerConfig { |
| 125 | let mut cfg = ServerConfig::default(); |
| 126 | cfg.http_max_body_bytes = CAP as u64; |
| 127 | cfg.upstream_timeout_ms = 500; |
| 128 | cfg |
| 129 | } |
| 130 | |
| 131 | fn api(port: u16) -> ApiRoute { |
| 132 | ApiRoute { |
| 133 | path: "/proxy".to_string(), |
| 134 | upstream_host: Some("127.0.0.1".to_string()), |
| 135 | upstream_port: Some(port), |
| 136 | upstream_path: Some("/x".to_string()), |
| 137 | upstream_tls: false, |
| 138 | headers: Vec::new(), |
| 139 | handler: None, |
| 140 | config: Vec::new(), |
| 141 | } |
| 142 | } |
| 143 | |
| 144 | fn hook(port: u16) -> WebhookRoute { |
| 145 | WebhookRoute { |
| 146 | path: "/hook".to_string(), |
| 147 | handler: None, |
| 148 | upstream_host: Some("127.0.0.1".to_string()), |
| 149 | upstream_port: Some(port), |
| 150 | upstream_path: Some("/x".to_string()), |
| 151 | upstream_tls: false, |
| 152 | config: Vec::new(), |
| 153 | } |
| 154 | } |
| 155 | |
| 156 | /// One request through a vhost whose route points at the upstream, under a deadline of its own. |
| 157 | fn ask(label: &str, kind: Upstream, method: &str, path: &str) -> Outcome<(Reply, usize)> { |
| 158 | let web = res!(Scratch::new(label)); |
| 159 | res!(web.put("index.html", "<p>home</p>")); |
| 160 | let runtime = res!(vhost_rig::runtime()); |
| 161 | runtime.block_on(async { |
| 162 | let (port, sent) = res!(upstream(kind).await); |
| 163 | let mut site = Site::new(HOST, &web.0); |
| 164 | site.api_routes = vec![api(port)]; |
| 165 | site.webhook_routes = vec![hook(port)]; |
| 166 | let rig = res!(Rig::with_config(vec![site], config())); |
| 167 | let fields: &[&str] = &[]; |
| 168 | let body = if method == "POST" { "{}" } else { "" }; |
| 169 | let reply = match tokio::time::timeout( |
| 170 | Duration::from_secs(8), |
| 171 | rig.fetch(HOST, method, path, fields, body), |
| 172 | ).await { |
| 173 | Ok(r) => res!(r), |
| 174 | Err(_) => return Err(err!( |
| 175 | "{} {} through a {:?} upstream was still unanswered after 8 s: the forwarder has \ |
| 176 | no deadline.", method, path, kind; Test, Timeout)), |
| 177 | }; |
| 178 | Ok((reply, sent.load(Ordering::SeqCst))) |
| 179 | }) |
| 180 | } |
| 181 | |
| 182 | #[test] |
| 183 | fn an_api_route_answers_504_when_its_upstream_never_finishes() -> Outcome<()> { |
| 184 | let (r, _) = res!(ask("api_hangs", Upstream::Hangs, "GET", "/proxy")); |
| 185 | assert_eq!(r.status(), 504, "a reply that never completes must be a 504: {:?}", r.head); |
| 186 | Ok(()) |
| 187 | } |
| 188 | |
| 189 | #[test] |
| 190 | fn an_api_route_answers_502_and_stops_reading_when_the_reply_has_no_end() -> Outcome<()> { |
| 191 | let (r, sent) = res!(ask("api_endless", Upstream::Endless, "GET", "/proxy")); |
| 192 | assert_eq!(r.status(), 502, "a reply over the size limit must be a 502: {:?}", r.head); |
| 193 | assert!(sent < SPILL / 2, |
| 194 | "the forwarder read {} bytes of an endless reply with a {} byte limit", sent, CAP); |
| 195 | Ok(()) |
| 196 | } |
| 197 | |
| 198 | #[test] |
| 199 | fn an_api_post_answers_504_when_its_upstream_never_finishes() -> Outcome<()> { |
| 200 | let (r, _) = res!(ask("api_post_hangs", Upstream::Hangs, "POST", "/proxy")); |
| 201 | assert_eq!(r.status(), 504, "{:?}", r.head); |
| 202 | Ok(()) |
| 203 | } |
| 204 | |
| 205 | #[test] |
| 206 | fn a_webhook_forward_answers_504_when_its_upstream_never_finishes() -> Outcome<()> { |
| 207 | let (r, _) = res!(ask("hook_hangs", Upstream::Hangs, "POST", "/hook")); |
| 208 | assert_eq!(r.status(), 504, "a reply that never completes must be a 504: {:?}", r.head); |
| 209 | Ok(()) |
| 210 | } |
| 211 | |
| 212 | #[test] |
| 213 | fn a_webhook_forward_answers_502_and_stops_reading_when_the_reply_has_no_end() -> Outcome<()> { |
| 214 | let (r, sent) = res!(ask("hook_endless", Upstream::Endless, "POST", "/hook")); |
| 215 | assert_eq!(r.status(), 502, "a reply over the size limit must be a 502: {:?}", r.head); |
| 216 | assert!(sent < SPILL / 2, |
| 217 | "the forwarder read {} bytes of an endless reply with a {} byte limit", sent, CAP); |
| 218 | Ok(()) |
| 219 | } |