Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_steel/tests/stopping_signal.rs

17.2 KiB, 18 runs

created by r1870400018:21672, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! A real signal, sent to a real Steel, and what it leaves behind.
2//!
3//! Everything else in this directory asks the code a question and reads the
4//! code's answer. This asks the operating system, because the fault it is here
5//! for cannot be seen from inside: `Server::start` ends in an accept loop with
6//! no way out, so `systemctl stop`, a reboot, and a Ctrl-C at a terminal all
7//! reach a process that hears nothing and is killed where it stands -- with a
8//! vhost's Ozone instance still open.
9//!
10//! What is checked is that Steel now hears them, and that hearing one costs
11//! nothing:
12//!
13//! 1. `SIGTERM` -- what a service manager and every reboot send -- ends the
14//! process by its own choice, with a nought, rather than felling it.
15//! 2. `SIGINT` -- Ctrl-C -- does the same.
16//! 3. The store was *shut*, not merely left readable, and it opens again after.
17//!
18//! The signal is sent by `kill`, the operating system's own tool, to a process
19//! identifier this test holds. Nothing here matches on a command line: a live
20//! Steel may be running on this machine, and anything that matched by name could
21//! take a real server with it.
22//!
23//! Unix only. Windows has no `kill` and no signals; the three console events the
24//! same listener answers there cannot be sent from here.
25//!
26//! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\
27//! Anthropic Claude
28
29#![cfg(unix)]
30
31use oxedyne_fe2o3_core::prelude::*;
32use oxedyne_fe2o3_crypto::keystore::{
33 DEFAULT_WALLET_KDF_NAME,
34 Wallet,
35};
36use oxedyne_fe2o3_iop_db::api::Database;
37use oxedyne_fe2o3_jdat::{
38 prelude::*,
39 file::JdatFile,
40 string::enc::EncoderConfig,
41};
42use oxedyne_fe2o3_steel::{
43 app::constant as app_const,
44 srv::{
45 context::new_db,
46 id,
47 },
48};
49
50use std::{
51 collections::BTreeMap,
52 net::{
53 IpAddr,
54 Ipv4Addr,
55 SocketAddr,
56 TcpListener,
57 TcpStream,
58 },
59 os::unix::process::ExitStatusExt,
60 path::{
61 Path,
62 PathBuf,
63 },
64 process::{
65 Child,
66 Command,
67 ExitStatus,
68 Stdio,
69 },
70 time::{
71 Duration,
72 Instant,
73 },
74};
75
76use secrecy::ExposeSecret;
77
78// The wallet passphrase, in the clear and on purpose: it protects a wallet that exists for a few
79// seconds in a scratch directory and holds one key to one empty database. The rig beside this one
80// says the same thing for the same reason.
81const PASS: &str = "steel-stop-test-passphrase-not-a-secret"; // allowlist secret
82
83const APP: &str = "stopsig";
84
85const START_SECS: u64 = 120;
86
87const STOP_SECS: u64 = 120;
88
89/// Where a fixture goes.
90///
91/// A name per fixture, because the tests in this file run at the same time and a
92/// shared directory is wiped by whichever reaches it last. Under `~/.cache`
93/// rather than `/tmp`: an Ozone instance is not a small thing to hold in a
94/// tmpfs, and this machine has been brought down that way before.
95fn scratch(name: &str) -> Outcome<PathBuf> {
96 let base = match std::env::var("HOME") {
97 Ok(home) => PathBuf::from(home).join(".cache"),
98 Err(_) => std::env::temp_dir(),
99 };
100 let dir = base.join(fmt!("steel_stopsignal_{}", name));
101 let _ = std::fs::remove_dir_all(&dir);
102 res!(std::fs::create_dir_all(&dir), IO, File);
103 Ok(dir)
104}
105
106/// A port nothing is listening on, asked of the operating system rather than
107/// picked.
108fn free_port() -> Outcome<u16> {
109 let probe = res!(TcpListener::bind("127.0.0.1:0"), IO, Network);
110 let port = res!(probe.local_addr(), IO, Network).port();
111 drop(probe);
112 Ok(port)
113}
114
115fn in_use(port: u16) -> bool {
116 TcpStream::connect_timeout(
117 &SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port),
118 Duration::from_millis(250),
119 ).is_ok()
120}
121
122/// Everything a Steel needs to start: a config, a wallet, and the directories
123/// the dev-mode refresh insists on.
124///
125/// The config is the test rig's, with the port substituted and the blocks a
126/// stop has nothing to do with left out. The one thing it must have is a vhost
127/// with a `db_dir_rel`, because a Steel with no store cannot demonstrate
128/// anything about closing one.
129fn lay_out(dir: &Path, port: u16) -> Outcome<Vec<u8>> {
130 for sub in ["www/public", "www/src/styles", "www/logs"] {
131 res!(std::fs::create_dir_all(dir.join(sub)), IO, File);
132 }
133 res!(std::fs::write(
134 dir.join("www").join("public").join("index.html"),
135 "<!doctype html><title>stop</title><p>here.\n",
136 ), IO, File);
137
138 let cfg = fmt!("{{
139 \"app_description\": \"Stop signal fixture\",
140 \"app_human_name\": \"Stopsig\",
141 \"app_log_level\": \"info\",
142 \"app_name\": \"{app}\",
143 \"app_root\": \"current\",
144 \"enc_name\": \"AES-256-GCM\",
145 \"kdf_name\": \"Argon2id_v0x13\",
146 \"dev_cfg\": {{
147 \"src_path_rel\": \"./www/src\",
148 \"js_bundles_rel\": {{}},
149 \"js_import_aliases_rel\": {{}},
150 \"css_source_dir_rel\": \"./www/src/styles\",
151 \"css_bundle_rel\": \"./www/public/styles.css\"
152 }},
153 \"server_cfg\": {{
154 \"log_level\": \"info\",
155 \"num_server_bots\": (u16|1),
156 \"server_address\": \"127.0.0.1\",
157 \"server_port_tcp\": (u16|{port}),
158 \"server_port_tcp_plaintext\": (u16|0),
159 \"hsts_max_age_secs\": (u32|0),
160 \"admin_local_port\": (u16|0),
161 \"session_expiry_default_secs\": (u32|604800),
162 \"ws_ping_interval_secs\": (u8|30),
163 \"server_max_errors_allowed\": (u8|30),
164 \"allow_anonymous_sessions\": (true),
165 \"http_max_header_bytes\": (u64|16384),
166 \"http_max_body_bytes\": (u64|8388608),
167 \"http_header_read_timeout_ms\": (u64|15000),
168 \"security_headers_enabled\": (true),
169 \"content_security_policy\": \"\",
170 \"addr_guard\": {{}},
171 \"auth_path_prefixes\": (vek|[\"/admin/login\"]),
172 \"auth_rps_max\": (u64|100),
173 \"tls_dir_rel\": \"./tls\",
174 \"acme\": {{
175 \"enabled\": (false),
176 \"contact_email\": \"\",
177 \"directory_url\": \"https://acme-staging-v02.api.letsencrypt.org/directory\",
178 \"cache_dir_rel\": \"./tls/acme\"
179 }},
180 \"mail\": {{}},
181 \"vhosts\": [
182 {{
183 \"hostnames\": (vek|[\"localhost\",\"localhost.\"]),
184 \"public_dir_rel\": \"./www/public\",
185 \"static_route_paths_rel\": {{}},
186 \"default_index_files\": (vek|[\"index.html\",\"index.htm\"]),
187 \"redirects\": [],
188 \"db_dir_rel\": \"./o3db\"
189 }}
190 ]
191 }}
192}}
193", app = APP, port = port);
194 res!(std::fs::write(dir.join("config.jdat"), cfg), IO, File);
195
196 // The wallet, built here rather than through the shell. Creating one at the
197 // prompt reads the passphrase in crossterm's raw mode, which needs a real
198 // terminal and is why the older rig carries a pty in Python; the library
199 // that shell calls is right here and takes bytes.
200 let mut metadata = BTreeMap::new();
201 metadata.insert(dat!("app_name"), dat!(APP));
202 let (wallet, unlocked) = res!(Wallet::create_with_first_admin(
203 metadata,
204 fmt!("operator"),
205 PASS.as_bytes(),
206 DEFAULT_WALLET_KDF_NAME,
207 ));
208 res!(wallet.save(
209 &dir.join(app_const::WALLET_NAME),
210 " ",
211 Some(EncoderConfig::<(), ()>::default()),
212 ));
213 // The wallet master key is the database encryption key, so the check that
214 // reopens the store afterwards needs it.
215 Ok(unlocked.master_key.expose_secret().clone())
216}
217
218/// Everything the server has said, wherever it said it.
219///
220/// Two files, because the two matter for different reasons: the log on disk is
221/// what an operator reads after the fact, and the console capture is what
222/// survives a failure early enough that the log was never configured.
223fn said(dir: &Path) -> String {
224 let mut all = String::new();
225 for path in [
226 dir.join("console.log"),
227 dir.join("www").join("logs").join(fmt!("{}.log", APP)),
228 ] {
229 if let Ok(text) = std::fs::read_to_string(&path) {
230 all.push_str(&text);
231 }
232 }
233 all
234}
235
236/// The last few lines of what it said, for a failure that has to show its work.
237fn tail_of(log: &str) -> String {
238 let lines: Vec<&str> = log.lines().collect();
239 let from = lines.len().saturating_sub(15);
240 lines[from..].join("\n")
241}
242
243/// Kills the child, by its own handle and never by name.
244fn put_down(mut child: Child) {
245 let _ = child.kill();
246 let _ = child.wait();
247}
248
249/// Sends one signal to one process, using the operating system's own tool.
250///
251/// `kill` rather than anything in this program: sending a signal from Rust means
252/// `libc::kill`, which is an `extern "C"` call and would need an `unsafe` block
253/// in a crate that forbids them. The external tool is also the better oracle --
254/// it is what a person at a terminal or a service manager would use, rather than
255/// Steel agreeing with itself.
256fn send(signal: &str, child: &Child) -> Outcome<()> {
257 let out = res!(Command::new("kill")
258 .arg(fmt!("-{}", signal))
259 .arg(fmt!("{}", child.id()))
260 .output(), IO, File);
261 if !out.status.success() {
262 return Err(err!(
263 "kill -{} {} answered {:?}: {}",
264 signal, child.id(), out.status,
265 String::from_utf8_lossy(&out.stderr).trim();
266 Test, System));
267 }
268 Ok(())
269}
270
271fn wait_for_end(child: &mut Child) -> Outcome<ExitStatus> {
272 let began = Instant::now();
273 while began.elapsed() < Duration::from_secs(STOP_SECS) {
274 match child.try_wait() {
275 Ok(Some(status)) => return Ok(status),
276 Ok(None) => std::thread::sleep(Duration::from_millis(200)),
277 Err(e) => return Err(err!(e,
278 "The server could not be asked how it was."; Test, IO)),
279 }
280 }
281 Err(err!(
282 "The server was still running {} seconds after it was signalled.",
283 STOP_SECS; Test, Timeout))
284}
285
286/// How a process ended, said the way a person would say it.
287///
288/// The distinction this whole file turns on: a process that *exited* chose to,
289/// and a process that was *felled by a signal* did not.
290fn how_it_ended(status: &ExitStatus) -> String {
291 match (status.code(), status.signal()) {
292 (Some(code), _) => fmt!("it exited with {}", code),
293 (None, Some(sig)) => fmt!(
294 "it was felled by signal {}, which means it caught nothing", sig),
295 (None, None) => fmt!("{:?}", status),
296 }
297}
298
299/// Starts a Steel on the fixture and waits until its database is open.
300///
301/// Waiting for the port alone would not do. A Steel starts sealed and binds
302/// before it opens anything, so a signal sent at that moment would find no store
303/// to close and the test would pass while proving nothing. `STEEL_ADMIN_PASS`
304/// unseals it at start-up, and the line waited for here is the one the opener
305/// writes when every configured Ozone is up and attached.
306fn serve_fixture(dir: &Path, port: u16) -> Outcome<Child> {
307 let exe = PathBuf::from(env!("CARGO_BIN_EXE_steel"));
308 let console = res!(std::fs::File::create(dir.join("console.log")), IO, File);
309 let errs = res!(console.try_clone(), IO, File);
310 let child = res!(Command::new(&exe)
311 .current_dir(dir)
312 .arg("server")
313 // `-d` or nothing: without it a first run refuses production mode and
314 // exits 0 in silence. Dev mode also generates the self-signed
315 // certificate this fixture would otherwise have to carry.
316 .arg("-d")
317 .env("STEEL_ADMIN_PASS", PASS)
318 // Kept open for the life of the child. The shell sits beside the
319 // listener and an EOF on stdin ends the process, which would look
320 // exactly like the clean stop this test is trying to observe.
321 .stdin(Stdio::piped())
322 .stdout(Stdio::from(console))
323 .stderr(Stdio::from(errs))
324 .spawn(), IO, File);
325
326 let here = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port);
327 let began = Instant::now();
328 let mut bound = false;
329 while began.elapsed() < Duration::from_secs(START_SECS) {
330 if !bound && TcpStream::connect_timeout(&here, Duration::from_millis(500)).is_ok() {
331 bound = true;
332 }
333 if bound && said(dir).contains("database(s) open and attached") {
334 return Ok(child);
335 }
336 std::thread::sleep(Duration::from_millis(250));
337 }
338 let log = said(dir);
339 put_down(child);
340 Err(err!(
341 "The server did not bind {} and open its database within {} seconds \
342 (bound: {}). It said:\n{}",
343 here, START_SECS, bound, tail_of(&log);
344 Test, Timeout))
345}
346
347/// One signal, one clean stop, and a store that was shut and opens again.
348///
349/// The whole claim in one function, so that `SIGTERM` and `SIGINT` are held to
350/// exactly the same standard rather than to two slightly different ones.
351fn a_signal_stops_it_cleanly(signal: &str, name: &str) -> Outcome<()> {
352 let dir = res!(scratch(name));
353 let port = res!(free_port());
354 req!(in_use(port), false,
355 "Something is already listening on port {}. Every check below would be \
356 answered by it rather than by the server this test starts.", port);
357 let key = res!(lay_out(&dir, port));
358
359 let mut child = res!(serve_fixture(&dir, port));
360
361 if let Err(e) = send(signal, &child) {
362 put_down(child);
363 return Err(e);
364 }
365
366 let status = match wait_for_end(&mut child) {
367 Ok(status) => status,
368 Err(e) => {
369 let log = said(&dir);
370 put_down(child);
371 return Err(err!(e, "It said:\n{}", tail_of(&log); Test));
372 },
373 };
374
375 let log = said(&dir);
376
377 // The teeth of this test. A process that catches nothing is felled by the
378 // signal and reports no exit code at all; a process that was asked, and did
379 // as it was asked, exits by itself with a nought. A service manager reads
380 // anything else as a unit that failed -- and a Steel felled here is a Steel
381 // killed with its Ozone open, which is the incident.
382 req!(status.success(), true,
383 "A {} did not stop the server cleanly: {}. It said:\n{}",
384 signal, how_it_ended(&status), tail_of(&log));
385
386 // The port is genuinely given back, so a restart is not refused.
387 req!(in_use(port), false, "Something is still listening on port {}.", port);
388
389 // The line at the end of `start_server` that this whole change exists to
390 // make reachable.
391 req!(log.contains("Server stopped gracefully."), true,
392 "A {} ended the server without it ever reaching the end of \
393 `start_server`. It said:\n{}", signal, tail_of(&log));
394
395 // And the store was shut, which the survey below cannot tell on its own:
396 // Ozone acknowledges each write before returning, so a killed process still
397 // leaves a readable store. Being able to reopen it is necessary and not
398 // sufficient; these two lines are the evidence that the close ran, the
399 // second of them written by Ozone's own supervisor after every bot thread
400 // had ended.
401 req!(log.contains("Closed the database for vhost 'localhost'."), true,
402 "A {} ended the server without it recording that it had closed the \
403 vhost's database. It said:\n{}", signal, tail_of(&log));
404 req!(log.contains("Shutdown: Verified."), true,
405 "The database was never verified as shut down, so whatever closed it \
406 did not finish. It said:\n{}", tail_of(&log));
407
408 // And the store opens again, and works. A stop that left it unreadable
409 // would be a worse fault than the one being fixed.
410 {
411 let db_dir = dir.join("o3db");
412 let mut db = res!(new_db(&db_dir, &key));
413 // The sequence the server itself runs, to the letter. Starting
414 // without the rest of it leaves the bots up but not answering, and
415 // every read then fails on a responder timeout that says nothing
416 // about whether the store is sound.
417 res!(db.start("db_reopen"));
418 res!(ok!(db.updated_api()).activate_gc(true));
419 std::thread::sleep(Duration::from_millis(200));
420 let (_start, msgs) = res!(db.api().ping_bots(app_const::GET_DATA_WAIT));
421 let answered = !msgs.is_empty();
422 req!(answered, true,
423 "The store left by a {} reopened but none of its bots answered.",
424 signal);
425 let uid = id::Uid::default();
426 res!(db.insert(dat!("after/the/stop"), dat!("readable"), uid, None));
427 let back = res!(db.get(&dat!("after/the/stop"), None));
428 let found = match back {
429 Some((val, _meta)) => val == dat!("readable"),
430 None => false,
431 };
432 req!(found, true,
433 "The store left by a {} would not take and return a value, so the \
434 stop cost it something.", signal);
435 res!(db.close());
436 }
437
438 let _ = std::fs::remove_dir_all(&dir);
439 Ok(())
440}
441
442/// What `systemctl stop` and every reboot send.
443///
444/// The signal from the incident: two live servers are stopped this way whenever
445/// their machines are rebooted, and until now each was killed mid-flight with
446/// its Ozone instances open.
447#[test]
448fn test_a_terminate_stops_the_server_cleanly_00() -> Outcome<()> {
449 a_signal_stops_it_cleanly("TERM", "term")
450}
451
452/// Ctrl-C, which is what a person at a terminal sends.
453#[test]
454fn test_an_interrupt_stops_the_server_cleanly_01() -> Outcome<()> {
455 a_signal_stops_it_cleanly("INT", "int")
456}