Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_steel/src/srv/admin/state.rs

18.9 KiB, 330 runs

created by r1870400018:10332, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! Runtime state shared by every dashboard request.
2//!
3//! Built once at Steel start-up and threaded through the admin
4//! handler. Holds the pieces of state that both dashboard auth and
5//! session decoding need:
6//!
7//! - A shared handle to the [`Wallet`] so login calls `unlock` against
8//! the same admin list the CLI sees, and admin management from the
9//! dashboard mutates the same file on disk.
10//! - An [`EncryptionScheme`] pre-keyed with the 32-byte dashboard
11//! session key, so session encode/decode does not re-derive on
12//! every request.
13//! - The seal: the wallet master key, when it is known.
14//!
15//! # The seal
16//!
17//! Steel starts *sealed*. The wallet file is readable without any
18//! passphrase -- it holds each admin's password-wrapped copy of the
19//! master key -- so the process can bind its listeners, serve every
20//! static vhost and renew its certificates while the master key is
21//! still unknown and the databases are still shut. Only the routes
22//! that actually need a database are refused, with a 503, until an
23//! admin unseals.
24//!
25//! This is the whole point of the arrangement: the *database* key
26//! stops being a precondition for the *websites* being up. A restart
27//! is no longer an outage that waits on a human at a terminal.
28//!
29//! The session key is therefore **not** derived from the master key.
30//! It is 32 random bytes minted at start-up, because sessions have to
31//! work while sealed -- an admin has to be able to reach the unseal
32//! page and be issued a cookie before any master key exists. A
33//! restart consequently invalidates outstanding dashboard cookies,
34//! which is the correct behaviour anyway.
35//!
36//! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\
37//! Anthropic Claude
38
39use crate::srv::{
40 admin::{
41 guard::SteelAddressGuard,
42 host_sampler::HostSampler,
43 traffic::TrafficRecorder,
44 },
45 alert::Alerter,
46 cfg::AdminKey,
47 fleet::Fleet,
48 health::{
49 F_DISK_PCT,
50 F_MAIL_DOWN,
51 F_SEALED_DBS,
52 HealthBody,
53 HealthStamp,
54 RollingCounter,
55 },
56 mail::ListenerTally,
57};
58
59use oxedyne_fe2o3_core::{
60 prelude::*,
61 rand::Rand,
62};
63use oxedyne_fe2o3_crypto::{
64 enc::EncryptionScheme,
65 keystore::Wallet,
66};
67use oxedyne_fe2o3_net::guard::nonce::NonceTracker;
68
69use std::{
70 path::PathBuf,
71 sync::{
72 Arc,
73 Mutex,
74 RwLock,
75 atomic::{
76 AtomicBool,
77 Ordering,
78 },
79 },
80 time::{
81 Duration,
82 Instant,
83 SystemTime,
84 },
85};
86
87use secrecy::ExposeSecret;
88use tokio::sync::Notify;
89
90pub const SESSION_KEY_LEN: usize = 32; // bytes
91
92/// Result of a successful passphrase unwrap against the wallet.
93#[derive(Clone, Debug)]
94pub struct Unsealed {
95 pub name: String, // admin entry whose wrap the passphrase opened
96 pub scopes: Vec<String>,
97 // True only when this call lifted the seal, rather than finding it already
98 // lifted. Only the first correct passphrase after a start unseals; a later
99 // one merely re-authenticates. Alerting turns on the distinction: an unseal
100 // is a rare, notable event worth an email, and a routine dashboard login is
101 // not.
102 pub lifted: bool,
103}
104
105/// Shared dashboard runtime state.
106///
107/// Cheaply cloneable: the wallet is behind an `Arc<RwLock<_>>` and the
108/// encryption scheme clones its key material.
109#[derive(Clone, Debug)]
110pub struct AdminState {
111 pub wallet: Arc<RwLock<Wallet>>,
112 // Held here so the dashboard's admin-management UI can call `Wallet::save`
113 // without depending on `app::constant`.
114 pub wallet_path: PathBuf,
115 // `None` while the process is sealed. Read it through
116 // `AdminState::master_key`, which fails with a `Sealed` tag rather than
117 // handing back an `Option` every caller would have to interpret for itself.
118 //
119 // Behind an `Arc<RwLock<_>>` because the unseal happens after the listeners
120 // are up: the login handler that recovers the key and the server task that
121 // opens the databases with it hold different clones of this state. The key
122 // is held in clear in process memory, in line with the wallet-v2 design --
123 // a human supplies the passphrase, and the unwrapped secret lives in RAM
124 // until the process restarts. It is never written to disk.
125 master_key: Arc<RwLock<Option<Vec<u8>>>>,
126 // An atomic alongside `master_key` so the request path can test the seal
127 // without taking a lock.
128 sealed: Arc<AtomicBool>,
129 // Signalled once, when the master key is installed. The database starter
130 // task in `Server::start` waits on this; it cannot open the Ozone instances
131 // until the key exists.
132 unseal_notify: Arc<Notify>,
133 // How many vhosts have a database configured. Zero is common: a deployment
134 // serving only static sites, redirects and proxy routes never touches
135 // Ozone. The seal is reported against this count, because telling an
136 // operator who has no database that "the databases are shut" is how a
137 // healthy server gets mistaken for a broken one.
138 db_count: usize,
139 // `None` disables alerting. It lives here because the events worth alerting
140 // on -- an unseal, a run of failed passphrase attempts -- all happen on the
141 // login path, which is the one place that already holds this state.
142 alerter: Option<Alerter>,
143 pub session_enc: EncryptionScheme, // AES-256-GCM
144 // The dashboard reads this when rendering `/admin/traffic`; the request
145 // pipeline in `srv/https.rs` writes to it on every completed response. Both
146 // sides hold the same `Arc`, so the dashboard sees live data without any
147 // per-vhost coordination.
148 pub traffic: Arc<TrafficRecorder>,
149 // A background task in `Server::start` calls `HostSampler::sample_now` on a
150 // fixed interval; the dashboard reads the same `Arc` when drawing the host
151 // resource strip.
152 pub host_sampler: Arc<HostSampler>,
153 // The TCP accept loop in `srv/server.rs` calls `check` before handing any
154 // stream to the TLS acceptor, and the dashboard's Security view reads
155 // snapshots and drives whitelist / blacklist / unblock actions against the
156 // same `Arc`.
157 pub addr_guard: Arc<SteelAddressGuard>,
158 // A tighter limiter for sensitive URL prefixes (login forms, admin login),
159 // consulted by the HTTPS handler once the request line has been parsed. A
160 // block returns 429 without reaching the application handler and without
161 // counting against, or affecting, `addr_guard`'s state for that address.
162 pub auth_guard: Arc<SteelAddressGuard>,
163 // Authorised public keys for the signed-admin-login flow, parsed from the
164 // primary vhost's `admin_keys` config block. Empty when the feature is not
165 // configured.
166 pub admin_keys: Arc<Vec<AdminKey>>,
167 pub nonce_tracker: Arc<Mutex<NonceTracker>>,
168 // Injected into every admin-served page's `<head>`, copied from the primary
169 // vhost's `head_injection_url` at start-up. `None` leaves the default head
170 // untouched.
171 pub head_injection_url: Arc<Option<String>>,
172 // When the process began serving, so the health body can report uptime.
173 pub started: Instant,
174 // Whether the address guard reported armed by its in-process self-test at
175 // start-up, surfaced as `guard_failed` in the health body so a box proves
176 // its own admission control is live without an external synthetic probe.
177 pub guard_selftest: bool,
178 // Rolling one-minute counts of `429`s emitted and connections dropped at
179 // admission, surfaced as `r429_1m` / `dropped_1m`. The accept loop and the
180 // 429 site increment these; the health route reads them.
181 pub r429: Arc<RollingCounter>,
182 pub dropped: Arc<RollingCounter>,
183 // Mail listeners asked for against those bound, surfaced as `mail_down`. The
184 // mail spawner in `Server::start` counts into it.
185 pub mail: Arc<ListenerTally>,
186 // What this host's watcher saw of each peer, drawn by `/admin/fleet`. The
187 // watcher writes the same `Arc`; an empty fleet on a host that watches nobody.
188 pub fleet: Arc<Fleet>,
189 // Job stamps whose ages the health body reports, vetted at start-up by
190 // `ServerConfig::get_health_stamps`; empty on a host that names none.
191 pub health_stamps: Arc<Vec<HealthStamp>>,
192}
193
194impl AdminState {
195 /// Builds a fresh admin state around a loaded wallet.
196 ///
197 /// The state starts **sealed**: the wallet has been read from
198 /// disk, but no passphrase has unwrapped a master key out of it
199 /// yet. Call [`AdminState::unseal`] with an admin's passphrase to
200 /// install the key. When the operator has already supplied a
201 /// passphrase before the listeners bind -- via `STEEL_ADMIN_PASS`
202 /// or the shell's `unseal` command -- the caller passes the
203 /// recovered key here and the state starts unsealed.
204 pub fn new(
205 wallet: Arc<RwLock<Wallet>>,
206 wallet_path: PathBuf,
207 master_key: Option<Vec<u8>>,
208 db_count: usize,
209 alerter: Option<Alerter>,
210 traffic: Arc<TrafficRecorder>,
211 host_sampler: Arc<HostSampler>,
212 addr_guard: Arc<SteelAddressGuard>,
213 auth_guard: Arc<SteelAddressGuard>,
214 admin_keys: Vec<AdminKey>,
215 head_injection_url: Option<String>,
216 )
217 -> Outcome<Self>
218 {
219 // The session key is random, not derived from the master key:
220 // a sealed Steel has no master key, yet it must still issue
221 // and validate the session cookie of the admin who is on
222 // their way to the unseal page.
223 let mut session_key = [0u8; SESSION_KEY_LEN];
224 Rand::fill_u8(&mut session_key);
225 let session_enc = res!(
226 EncryptionScheme::new_aes_256_gcm_with_key(&session_key));
227 // The replay window is the signed-login freshness window. The
228 // tracker holds each nonce until that window after the later
229 // of the envelope's stamp and its first showing: an envelope
230 // stamped ahead of the clock stays fresh until its stamp plus
231 // the window, so a window from the first showing alone would
232 // let it replay.
233 let tracker = NonceTracker::new(Duration::from_secs(
234 crate::srv::admin::signed_login::SIGNED_LOGIN_FRESHNESS_SECS,
235 ));
236 let sealed = master_key.is_none();
237 // Run the guard's in-process self-test once, at construction, so the
238 // health body can report armed/not without an external probe a live
239 // guard would blacklist.
240 let guard_selftest = addr_guard.self_test();
241 Ok(Self {
242 wallet,
243 wallet_path,
244 master_key: Arc::new(RwLock::new(master_key)),
245 sealed: Arc::new(AtomicBool::new(sealed)),
246 unseal_notify: Arc::new(Notify::new()),
247 db_count,
248 alerter,
249 session_enc,
250 traffic,
251 host_sampler,
252 addr_guard,
253 auth_guard,
254 admin_keys: Arc::new(admin_keys),
255 nonce_tracker: Arc::new(Mutex::new(tracker)),
256 head_injection_url: Arc::new(head_injection_url),
257 started: Instant::now(),
258 guard_selftest,
259 r429: RollingCounter::new_shared(),
260 dropped: RollingCounter::new_shared(),
261 mail: ListenerTally::new_shared(),
262 fleet: Fleet::new_shared(String::new(), None),
263 health_stamps: Arc::new(Vec::new()),
264 })
265 }
266
267 /// Hands the dashboard the rings the watcher writes, and this host's name for
268 /// the page's own row. A state built without one has an empty fleet, as a host
269 /// that watches nobody does.
270 pub fn with_fleet(mut self, fleet: Arc<Fleet>) -> Self {
271 self.fleet = fleet;
272 self
273 }
274
275 pub fn with_health_stamps(mut self, stamps: Vec<HealthStamp>) -> Self {
276 self.health_stamps = Arc::new(stamps);
277 self
278 }
279
280 /// This host's health body: what the health route serves, and what the Fleet
281 /// page draws in this host's own row.
282 ///
283 /// `assemble` makes the original fields; the ones the Fleet view added are
284 /// set here, from state this struct holds. Each is absent rather than zero
285 /// when it was never read, so no watcher mistakes a missing figure for a
286 /// good one. The stamp ages are the exception by design: each is read at
287 /// this call, and a stamp that cannot be read reports as never written.
288 pub fn health_body(&self) -> HealthBody {
289 let host = self.host_sampler.health_metrics().ok().flatten();
290 let mut b = HealthBody::assemble(
291 host,
292 self.addr_guard.live_conns(),
293 self.r429.last(60),
294 self.dropped.last(60),
295 self.guard_selftest,
296 self.started.elapsed().as_secs(),
297 self.is_sealed(),
298 );
299 let sealed_dbs = if self.seal_withholds_data() { self.db_count } else { 0 };
300 b.set(F_SEALED_DBS, sealed_dbs as i64);
301 if let Some(pct) = self.host_sampler.disk_pct().ok().flatten() {
302 b.set(F_DISK_PCT, pct);
303 }
304 if let Some(n) = self.mail.down() {
305 b.set(F_MAIL_DOWN, n as i64);
306 }
307 for r in self.host_sampler.residents().unwrap_or_default() {
308 b.set_resident(&r);
309 }
310 b.set_stamps(&self.health_stamps, SystemTime::now());
311 b
312 }
313
314 /// True while no master key is known, so the databases are shut and
315 /// DB-backed routes must refuse.
316 pub fn is_sealed(&self) -> bool {
317 self.sealed.load(Ordering::Acquire)
318 }
319
320 pub fn db_count(&self) -> usize {
321 self.db_count
322 }
323
324 pub fn alerter(&self) -> Option<&Alerter> {
325 self.alerter.as_ref()
326 }
327
328 /// Is the seal actually holding something shut -- no master key, and at
329 /// least one database that needs it?
330 ///
331 /// Distinct from `is_sealed` because a deployment of static sites, redirects
332 /// and proxy routes has no database at all, and for it the seal is
333 /// inconsequential: nothing is locked, nothing is waiting, and there is no
334 /// reason to tell an operator otherwise.
335 pub fn seal_withholds_data(&self) -> bool {
336 self.is_sealed() && self.db_count > 0
337 }
338
339 /// Fails with a `Sealed` tag while the process is sealed. Callers that merely
340 /// want the seal state should ask `is_sealed` rather than probing this for an
341 /// error.
342 pub fn master_key(&self) -> Outcome<Vec<u8>> {
343 // Always the message-carrying form of the lock macros on this
344 // field: the bare form formats the locked value into the error
345 // message with `{:?}`, which for a master key would print the
346 // secret into the log.
347 let guard = lock_read!(self.master_key, "Reading the wallet master key.");
348 match &*guard {
349 Some(k) => Ok(k.clone()),
350 None => Err(err!(
351 "Steel is sealed: no wallet master key is loaded. An admin \
352 must unseal before this operation can proceed.";
353 Sealed, Unauthorised)),
354 }
355 }
356
357 /// Unwraps the wallet master key with an admin's passphrase and installs it,
358 /// lifting the seal.
359 ///
360 /// Authenticates against the wallet **only** -- the wallet file is readable
361 /// while sealed, which is precisely what makes a web unseal page possible
362 /// without a database behind it.
363 ///
364 /// Unsealing an already-unsealed process re-verifies the passphrase and
365 /// leaves the key in place, so a second admin logging in cannot swap the key
366 /// out from under the running databases.
367 pub fn unseal(&self, passphrase: &[u8]) -> Outcome<Unsealed> {
368 let unlocked = {
369 let wallet = lock_read!(self.wallet, "Reading the wallet to unseal.");
370 res!(wallet.unlock(passphrase))
371 };
372 let name = unlocked.admin_name.clone();
373 let scopes = unlocked.admin_scopes.clone();
374
375 let mut guard = lock_write!(self.master_key,
376 "Installing the wallet master key.");
377 let lifted = guard.is_none();
378 if lifted {
379 *guard = Some(unlocked.master_key.expose_secret().clone());
380 self.sealed.store(false, Ordering::Release);
381 drop(guard);
382 // Wake the database starter. `notify_waiters` only reaches
383 // tasks already waiting, which is the case here: the
384 // starter task is spawned before the listeners accept.
385 self.unseal_notify.notify_waiters();
386 info!("Steel unsealed by admin '{}'; starting databases.", name);
387 }
388 Ok(Unsealed { name, scopes, lifted })
389 }
390
391 /// Waits until an admin installs the master key, returning immediately when
392 /// the process is already unsealed.
393 pub async fn await_master_key(&self) -> Outcome<Vec<u8>> {
394 loop {
395 if !self.is_sealed() {
396 return self.master_key();
397 }
398 // Register interest *before* re-testing the flag, so an
399 // unseal landing between the test and the wait cannot be
400 // missed.
401 let notified = self.unseal_notify.notified();
402 if !self.is_sealed() {
403 return self.master_key();
404 }
405 notified.await;
406 }
407 }
408}
409
410
411#[cfg(test)]
412mod tests {
413 use super::*;
414
415 use crate::srv::health::BUILTIN_FIELDS;
416
417 /// Every field the served body carries of its own accord is on the reserved list, so a stamp
418 /// can take none of them, and the stamps the state was handed ride beside them, a stamp that
419 /// is not there reading as never written.
420 #[test]
421 fn the_served_body_is_the_builtin_list_and_the_stamps() -> Outcome<()> {
422 let state = res!(AdminState::new(
423 Arc::new(RwLock::new(Wallet::default())),
424 PathBuf::from("./wallet.jdat"),
425 Some([0u8; 32].to_vec()),
426 1,
427 None,
428 TrafficRecorder::new_shared(0),
429 HostSampler::new_shared(),
430 res!(crate::srv::admin::guard::new_shared()),
431 res!(crate::srv::admin::guard::new_shared()),
432 Vec::new(),
433 None,
434 ));
435 let plain = state.health_body();
436 for k in plain.fields.keys() {
437 assert!(BUILTIN_FIELDS.contains(&k.as_str()),
438 "the served body carries '{}', which BUILTIN_FIELDS does not reserve", k);
439 }
440
441 let state = state.with_health_stamps(vec![HealthStamp {
442 field: fmt!("forge_state_age_s"),
443 path: PathBuf::from("/nonexistent/fe2o3_steel/stamp/state.ok"),
444 }]);
445 let body = state.health_body();
446 assert!(body.get("forge_state_age_s").unwrap_or(0) > 1_000_000_000,
447 "a missing stamp must read as never written, a very large age");
448 assert_eq!(body.fields.len(), plain.fields.len() + 1,
449 "the stamp rides beside the built-in fields and displaces none");
450 Ok(())
451 }
452}