Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_steel/src/srv/admin/fleet_view.rs

31.7 KiB, 1 run

created by r1870400018:61444, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1//! The Fleet page is gated on `dashboard.admin`, not `dashboard.view`: it shows
2//! what every watched peer says about itself, which is whether an attack on any
3//! of them is working, and that is not for a read-only login on one of them.
4//!
5//! Every field a body carries is drawn somewhere. A field no pane knows lands
6//! under *Other fields* rather than being dropped, so a peer on a newer build --
7//! one reporting the age of a stamp file, say -- is visible here before this page
8//! has been taught what the field means.
9//!
10//! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\
11//! Anthropic Claude
12
13use crate::srv::{
14 admin::{
15 AdminPrincipal,
16 assets::{
17 FLEET_JS,
18 render_layout,
19 },
20 handler::{
21 PATH_FLEET,
22 extract_principal,
23 json_escape,
24 redirect_to_login,
25 },
26 state::AdminState,
27 },
28 fleet::{
29 FleetPeer,
30 ProbeSample,
31 RowState,
32 Tone,
33 peer_state,
34 tone,
35 unix_secs,
36 },
37 health::{
38 F_CONNS,
39 F_DISK_IOPS,
40 F_DISK_PCT,
41 F_DROPPED_1M,
42 F_GUARD_FAILED,
43 F_LOAD1,
44 F_MAIL_DOWN,
45 F_MEM_PCT,
46 F_PROBE_MS,
47 F_R429_1M,
48 F_SEALED,
49 F_SEALED_DBS,
50 F_SWAP_PCT,
51 F_UPTIME_S,
52 HealthBody,
53 RES_CAP_KB,
54 RES_CAP_PCT,
55 RES_PROCS,
56 RES_RSS_KB,
57 resident_key,
58 },
59};
60
61use oxedyne_fe2o3_core::prelude::*;
62use oxedyne_fe2o3_net::http::{
63 fields::{
64 HeaderFieldValue,
65 HeaderFields,
66 HeaderName,
67 },
68 msg::HttpMessage,
69 status::HttpStatus,
70};
71
72use std::collections::{
73 BTreeMap,
74 BTreeSet,
75};
76
77/// One cell a pane always or sometimes shows.
78struct CellSpec {
79 key: &'static str,
80 label: &'static str,
81 unit: &'static str, // how the page formats the value
82 always: bool, // shown even when the body lacks the field
83}
84
85// Pane A, the box short of something. Residents follow these, one cell each.
86const PANE_A: &[CellSpec] = &[
87 CellSpec { key: F_MEM_PCT, label: "Memory", unit: "pct", always: true },
88 CellSpec { key: F_SWAP_PCT, label: "Swap", unit: "pct", always: true },
89 CellSpec { key: F_DISK_IOPS, label: "Disk ops/s", unit: "n", always: true },
90 CellSpec { key: F_LOAD1, label: "Load", unit: "load", always: true },
91 CellSpec { key: F_DISK_PCT, label: "Disk space", unit: "pct", always: true },
92];
93
94// Pane B, someone asking a lot.
95const PANE_B: &[CellSpec] = &[
96 CellSpec { key: F_CONNS, label: "Connections", unit: "n", always: true },
97 CellSpec { key: F_R429_1M, label: "429s / min", unit: "n", always: true },
98 CellSpec { key: F_DROPPED_1M, label: "Dropped / min", unit: "n", always: true },
99 CellSpec { key: F_GUARD_FAILED, label: "Guard", unit: "guard", always: true },
100 CellSpec { key: F_PROBE_MS, label: "Probe", unit: "ms", always: true },
101 CellSpec { key: F_MAIL_DOWN, label: "Mail", unit: "mail", always: false },
102 // Carries the raw `sealed` beside it, so it accounts for both fields.
103 CellSpec { key: F_SEALED_DBS, label: "Seal", unit: "seal", always: false },
104];
105
106// Read into the row head rather than a cell.
107const HEAD_KEYS: &[&str] = &[F_UPTIME_S];
108
109// The resident figures its cell draws; any other `res.` figure is shown as Other.
110const RESIDENT_FIGURES: &[&str] = &[RES_PROCS, RES_RSS_KB, RES_CAP_KB, RES_CAP_PCT];
111
112/// How the page formats a field nothing names, read from its suffix.
113fn unit_of(key: &str) -> &'static str {
114 if key.ends_with("_pct") {
115 "pct"
116 } else if key.ends_with("_ms") {
117 "ms"
118 } else if key.ends_with("_s") || key.ends_with("_secs") {
119 "secs"
120 } else if key.ends_with("_kb") {
121 "kib"
122 } else {
123 "n"
124 }
125}
126
127/// Who is asking, and whether they may see the page.
128enum Gate {
129 Admin(AdminPrincipal),
130 NotAdmin(AdminPrincipal),
131 SignedOut,
132}
133
134fn gate(state: &AdminState, headers: &HeaderFields) -> Gate {
135 match extract_principal(state, headers) {
136 None => Gate::SignedOut,
137 Some(p) if p.can_admin_dashboard() => Gate::Admin(p),
138 Some(p) => Gate::NotAdmin(p),
139 }
140}
141
142pub fn render_fleet_page(state: &AdminState, headers: &HeaderFields) -> HttpMessage {
143 let principal = match gate(state, headers) {
144 Gate::Admin(p) => p,
145 Gate::SignedOut => return redirect_to_login(),
146 Gate::NotAdmin(p) => {
147 let body = "<h1>Fleet</h1>\n\
148 <p class=\"notice error\">The Fleet view needs the \
149 <code>dashboard.admin</code> scope. It shows what every watched \
150 host reports about itself, including whether an attack on it is \
151 working, so a view-only sign-in does not reach it.</p>\n";
152 let html = render_layout("Fleet", PATH_FLEET, &p, body, "");
153 return html_with_status(HttpStatus::Forbidden, html);
154 },
155 };
156 // A `<` can only sit inside a JSON string here, where `\u003c` means the same
157 // thing and cannot end, or restart, the script element the data rides in.
158 let data = fleet_json(state).replace('<', "\\u003c");
159 let body = fmt!(
160 "<h1>Fleet</h1>\n\
161 <p class=\"meta\">What this host's watcher last read from each machine it \
162 watches, beside this host's own reading. Each cell is judged against that \
163 peer's own <code>distress</code> and <code>clear</code> thresholds, the \
164 numbers its alarm uses: red at distress, amber between the two, green at or \
165 under clear. A figure with no threshold is left uncoloured. Nothing on this \
166 page raises, silences or acknowledges an alarm.</p>\n\
167 <div id=\"fleet-notice\"></div>\n\
168 <div id=\"fleet-rows\" class=\"fleet-rows\">\
169 <p class=\"notice empty\">Drawing the fleet&hellip;</p></div>\n\
170 <p class=\"meta fleet-stamp\" id=\"fleet-stamp\"></p>\n\
171 <script id=\"fleet-data\" type=\"application/json\">{data}</script>\n\
172 <script>{js}</script>\n",
173 data = data,
174 js = FLEET_JS,
175 );
176 let html = render_layout("Fleet", PATH_FLEET, &principal, &body, "");
177 html_with_status(HttpStatus::OK, html)
178}
179
180pub fn render_fleet_json(state: &AdminState, headers: &HeaderFields) -> HttpMessage {
181 match gate(state, headers) {
182 Gate::Admin(_) => (),
183 Gate::SignedOut => return HttpMessage::respond_with_text(
184 HttpStatus::Unauthorized, "Sign in required."),
185 Gate::NotAdmin(_) => return HttpMessage::respond_with_text(
186 HttpStatus::Forbidden, "The Fleet view needs the dashboard.admin scope."),
187 }
188 HttpMessage::new_response(HttpStatus::OK)
189 .with_field(
190 HeaderName::ContentType,
191 HeaderFieldValue::Generic("application/json; charset=utf-8".to_string()),
192 )
193 .with_field(
194 HeaderName::CacheControl,
195 HeaderFieldValue::Generic("no-store".to_string()),
196 )
197 .with_body(fleet_json(state).into_bytes())
198}
199
200fn html_with_status(status: HttpStatus, html: String) -> HttpMessage {
201 HttpMessage::new_response(status)
202 .with_field(
203 HeaderName::ContentType,
204 HeaderFieldValue::Generic("text/html; charset=utf-8".to_string()),
205 )
206 .with_field(
207 HeaderName::CacheControl,
208 HeaderFieldValue::Generic("no-store".to_string()),
209 )
210 .with_body(html.into_bytes())
211}
212
213// ┌───────────────────────────────────────────────────────────────────────────┐
214// │ THE DOCUMENT │
215// └───────────────────────────────────────────────────────────────────────────┘
216
217/// The whole page's data: this host's row first, then one row per watched host,
218/// in the order the watch list first names each.
219pub fn fleet_json(state: &AdminState) -> String {
220 let fleet = &state.fleet;
221 let now = unix_secs();
222 let snap = match fleet.snapshot() {
223 Ok(s) => s,
224 Err(e) => {
225 error!(e, "dashboard: fleet snapshot failed");
226 Vec::new()
227 },
228 };
229 let whoami = fleet.whoami();
230
231 // Hosts in first-mention order, this host's own name kept for its row.
232 let mut hosts: Vec<&str> = Vec::new();
233 for (p, _) in &snap {
234 if p.host != whoami && !hosts.contains(&p.host.as_str()) {
235 hosts.push(p.host.as_str());
236 }
237 }
238
239 let mut rows = Vec::with_capacity(hosts.len() + 1);
240 let own_services: Vec<&(FleetPeer, Vec<ProbeSample>)> = snap.iter()
241 .filter(|(p, _)| !whoami.is_empty() && p.host == whoami)
242 .collect();
243 rows.push(self_row_json(state, whoami, &own_services, now));
244 for host in hosts {
245 let group: Vec<&(FleetPeer, Vec<ProbeSample>)> = snap.iter()
246 .filter(|(p, _)| p.host == host)
247 .collect();
248 rows.push(host_row_json(host, &group, now, fleet.interval_secs()));
249 }
250
251 fmt!(
252 "{{\"now\":{now},\"whoami\":{who},\"watching\":{watching},\"link_down\":{link},\
253 \"peers\":{peers},\"interval_secs\":{interval},\"fail_threshold\":{fail},\
254 \"started\":{started},\"rows\":[{rows}]}}",
255 now = now,
256 who = jstr(whoami),
257 watching = fleet.is_watching(),
258 link = fleet.is_link_down(),
259 peers = fleet.peers().len(),
260 interval = fleet.interval_secs(),
261 fail = fleet.fail_threshold(),
262 started = fleet.started_secs(),
263 rows = rows.join(","),
264 )
265}
266
267/// This host, from its own state: the reachability a self-probe would measure is
268/// meaningless from the same box, so there is no probe time, and no threshold is
269/// held here for this host -- its watchers hold those -- so nothing is coloured
270/// but the guard's own self-test.
271fn self_row_json(
272 state: &AdminState,
273 whoami: &str,
274 services: &[&(FleetPeer, Vec<ProbeSample>)],
275 now: u64,
276)
277 -> String
278{
279 let body = state.health_body();
280 let row_state = if body.get(F_SEALED_DBS).unwrap_or(0) > 0 {
281 RowState::Sealed
282 } else {
283 RowState::Up
284 };
285 let none = BTreeMap::new();
286 let panes = panes_json(Some(&body), &[], &none, &none, true);
287 let host = if whoami.is_empty() { "this host" } else { whoami };
288 let note = if row_state == RowState::Sealed {
289 "databases held shut awaiting an unseal"
290 } else {
291 ""
292 };
293 row_json(host, true, row_state, note, None, body.get(F_UPTIME_S), &panes,
294 &services_json(services, now, state.fleet.interval_secs()))
295}
296
297/// One watched host: the first entry on it that reads a health body draws the
298/// panes, and every other entry on it is a service cell with its own liveness.
299fn host_row_json(
300 host: &str,
301 group: &[&(FleetPeer, Vec<ProbeSample>)],
302 now: u64,
303 interval: u64,
304)
305 -> String
306{
307 let lead = match group.iter().position(|(p, _)| p.has_token) {
308 Some(i) => i,
309 None => 0,
310 };
311 let (peer, samples) = match group.get(lead) {
312 Some(g) => (&g.0, &g.1),
313 None => return row_json(host, false, RowState::Never, "", None, None, "", "[]"),
314 };
315 let others: Vec<&(FleetPeer, Vec<ProbeSample>)> = group.iter()
316 .enumerate()
317 .filter(|(i, _)| *i != lead)
318 .map(|(_, g)| *g)
319 .collect();
320 let services = services_json(&others, now, interval);
321 let (row_state, note) = peer_state(peer, samples, now, interval);
322
323 if !peer.has_token {
324 // Nothing on this host serves a body to this watcher: liveness is all there is.
325 let note = if note.is_empty() {
326 fmt!("liveness only: no entry for this host carries a health token")
327 } else {
328 note
329 };
330 let age = last_ok_age(samples, now);
331 return row_json(host, false, row_state, &note, age, None, "", &services);
332 }
333
334 let last_read = samples.iter().rev().find(|s| s.body.is_some());
335 let age = last_read.map(|s| now.saturating_sub(s.t_secs));
336 let (distress, clear) = host_thresholds(group, lead);
337 let (panes, uptime) = match (row_state, last_read.and_then(|s| s.body.as_ref())) {
338 (RowState::Down, _) | (RowState::Never, _) | (_, None) => (String::new(), None),
339 (st, Some(body)) => (
340 panes_json(Some(body), samples, &distress, &clear, st.is_fresh()),
341 body.get(F_UPTIME_S),
342 ),
343 };
344 row_json(host, false, row_state, &note, age, uptime, &panes, &services)
345}
346
347/// The thresholds a host's row is coloured from: the lead entry's, then, for each field the
348/// lead does not judge, the first other entry on the host that reads a body and does.
349///
350/// Two entries on one host read the same body -- jarrah's own figures and its forge copy's
351/// stamp ages, say -- and each alarm judges only its own fields, so each cell takes its colour
352/// from the entry whose alarm judges that field. A field's clear boundary always comes from the
353/// same entry as its distress value, so no dead-band is assembled from two alarms.
354fn host_thresholds(
355 group: &[&(FleetPeer, Vec<ProbeSample>)],
356 lead: usize,
357)
358 -> (BTreeMap<String, i64>, BTreeMap<String, i64>)
359{
360 let mut distress = BTreeMap::new();
361 let mut clear = BTreeMap::new();
362 let order = std::iter::once(lead).chain((0..group.len()).filter(|i| *i != lead));
363 for i in order {
364 let peer = match group.get(i) {
365 // An entry with no token never reads a body, so its thresholds judge nothing.
366 Some((p, _)) if p.has_token => p,
367 _ => continue,
368 };
369 for (field, d) in &peer.distress {
370 if distress.contains_key(field) {
371 continue;
372 }
373 distress.insert(field.clone(), *d);
374 if let Some(c) = peer.clear.get(field) {
375 clear.insert(field.clone(), *c);
376 }
377 }
378 }
379 (distress, clear)
380}
381
382fn last_ok_age(samples: &[ProbeSample], now: u64) -> Option<u64> {
383 samples.iter().rev().find(|s| s.ok).map(|s| now.saturating_sub(s.t_secs))
384}
385
386fn row_json(
387 host: &str,
388 local: bool,
389 row_state: RowState,
390 note: &str,
391 age: Option<u64>,
392 uptime: Option<i64>,
393 panes: &str,
394 services: &str,
395)
396 -> String
397{
398 let dim = matches!(row_state, RowState::Stale | RowState::Down | RowState::Never);
399 fmt!(
400 "{{\"host\":{host},\"local\":{local},\"state\":{st},\"note\":{note},\"dim\":{dim},\
401 \"age\":{age},\"uptime\":{uptime},\"panes\":[{panes}],\"services\":{services}}}",
402 host = jstr(host),
403 local = local,
404 st = jstr(row_state.word()),
405 note = jstr(note),
406 dim = dim,
407 age = jopt_u(age),
408 uptime = jopt(uptime),
409 panes = panes,
410 services = services,
411 )
412}
413
414fn services_json(
415 group: &[&(FleetPeer, Vec<ProbeSample>)],
416 now: u64,
417 interval: u64,
418)
419 -> String
420{
421 let items: Vec<String> = group.iter().map(|(p, samples)| {
422 let (st, note) = peer_state(p, samples, now, interval);
423 let probe = samples.last().map(|s| s.probe_ms);
424 fmt!(
425 "{{\"name\":{name},\"state\":{st},\"note\":{note},\"probe_ms\":{probe},\
426 \"age\":{age}}}",
427 name = jstr(&p.name),
428 st = jstr(st.word()),
429 note = jstr(&note),
430 probe = jopt_u(probe),
431 age = jopt_u(last_ok_age(samples, now)),
432 )
433 }).collect();
434 fmt!("[{}]", items.join(","))
435}
436
437/// The panes of a row. `fresh` false keeps the values and drops every colour,
438/// which is how a stale row shows its last numbers without claiming them.
439///
440/// Every field in the body, and every field a threshold names, is drawn in
441/// exactly one place: a known cell, a resident, the row head, or *Other fields*,
442/// so nothing the peer said is dropped for want of a label.
443fn panes_json(
444 body: Option<&HealthBody>,
445 samples: &[ProbeSample],
446 distress: &BTreeMap<String, i64>,
447 clear: &BTreeMap<String, i64>,
448 fresh: bool,
449)
450 -> String
451{
452 let body = match body {
453 Some(b) => b,
454 None => return String::new(),
455 };
456 let mut drawn: BTreeSet<String> = HEAD_KEYS.iter().map(|k| k.to_string()).collect();
457 let wanted = |c: &CellSpec| c.always || body.get(c.key).is_some() || distress.contains_key(c.key);
458
459 let mut a = Vec::new();
460 for c in PANE_A.iter().filter(|c| wanted(*c)) {
461 a.push(cell_json(c.key, c.label, c.unit, body, samples, distress, clear, fresh, ""));
462 drawn.insert(c.key.to_string());
463 }
464 for r in body.residents() {
465 let capped = r.get(RES_CAP_PCT).is_some();
466 let key = resident_key(&r.name, if capped { RES_CAP_PCT } else { RES_RSS_KB });
467 let extra = fmt!(
468 ",\"procs\":{procs},\"rss\":{rss},\"cap\":{cap},\"pct\":{pct}",
469 procs = jopt(r.get(RES_PROCS)),
470 rss = jopt(r.get(RES_RSS_KB)),
471 cap = jopt(r.get(RES_CAP_KB)),
472 pct = jopt(r.get(RES_CAP_PCT)),
473 );
474 a.push(cell_json(&key, &r.name, "res", body, samples, distress, clear, fresh, &extra));
475 for figure in RESIDENT_FIGURES {
476 drawn.insert(resident_key(&r.name, figure));
477 }
478 }
479
480 let mut b = Vec::new();
481 for c in PANE_B.iter().filter(|c| wanted(*c)) {
482 let extra = if c.key == F_SEALED_DBS {
483 drawn.insert(F_SEALED.to_string());
484 fmt!(",\"sealed\":{}", jopt(body.get(F_SEALED)))
485 } else {
486 String::new()
487 };
488 b.push(cell_json(c.key, c.label, c.unit, body, samples, distress, clear, fresh, &extra));
489 drawn.insert(c.key.to_string());
490 }
491
492 let mut rest: BTreeSet<&str> = body.fields.keys().map(|k| k.as_str()).collect();
493 rest.extend(distress.keys().map(|k| k.as_str()));
494 let other: Vec<String> = rest.into_iter()
495 .filter(|k| !drawn.contains(*k))
496 .map(|k| cell_json(k, k, unit_of(k), body, samples, distress, clear, fresh, ""))
497 .collect();
498
499 let mut panes = vec![
500 fmt!("{{\"id\":\"a\",\"title\":\"Resources\",\"cells\":[{}]}}", a.join(",")),
501 fmt!("{{\"id\":\"b\",\"title\":\"Traffic\",\"cells\":[{}]}}", b.join(",")),
502 ];
503 if !other.is_empty() {
504 panes.push(fmt!(
505 "{{\"id\":\"c\",\"title\":\"Other fields\",\"cells\":[{}]}}", other.join(",")));
506 }
507 panes.join(",")
508}
509
510fn cell_json(
511 key: &str,
512 label: &str,
513 unit: &str,
514 body: &HealthBody,
515 samples: &[ProbeSample],
516 distress: &BTreeMap<String, i64>,
517 clear: &BTreeMap<String, i64>,
518 fresh: bool,
519 extra: &str,
520)
521 -> String
522{
523 let v = body.get(key);
524 let d = distress.get(key).copied();
525 let c = clear.get(key).copied();
526 // Every colour, the guard's included, is an alarm threshold: a failed self-test is red
527 // where a `distress` of 1 on `guard_failed` would tell someone, and plain where nothing
528 // would (D-06 audit D2).
529 let t = match (fresh, v) {
530 (false, _) | (_, None) => Tone::Plain,
531 (true, Some(v)) => tone(v, d, c),
532 };
533 let series: Vec<String> = samples.iter()
534 .map(|s| jopt(s.body.as_ref().and_then(|b| b.get(key))))
535 .collect();
536 fmt!(
537 "{{\"k\":{k},\"label\":{label},\"unit\":{unit},\"v\":{v},\"tone\":{tone},\
538 \"d\":{d},\"c\":{c},\"s\":[{s}]{extra}}}",
539 k = jstr(key),
540 label = jstr(label),
541 unit = jstr(unit),
542 v = jopt(v),
543 tone = jstr(t.word()),
544 d = jopt(d),
545 c = jopt(c),
546 s = series.join(","),
547 extra = extra,
548 )
549}
550
551fn jstr(s: &str) -> String {
552 fmt!("\"{}\"", json_escape(s))
553}
554
555fn jopt(v: Option<i64>) -> String {
556 match v {
557 Some(n) => n.to_string(),
558 None => fmt!("null"),
559 }
560}
561
562fn jopt_u(v: Option<u64>) -> String {
563 match v {
564 Some(n) => n.to_string(),
565 None => fmt!("null"),
566 }
567}
568
569
570#[cfg(test)]
571mod tests {
572 use super::*;
573
574 use crate::srv::{
575 admin::{
576 host_sampler::HostSampler,
577 session::{
578 SESSION_COOKIE_NAME,
579 encode_session,
580 },
581 traffic::TrafficRecorder,
582 },
583 cfg::{
584 WatchConfig,
585 WatchPeer,
586 },
587 fleet::{
588 Fleet,
589 PeerHealth,
590 },
591 };
592
593 use oxedyne_fe2o3_crypto::keystore::Wallet;
594 use oxedyne_fe2o3_net::http::{
595 fields::Cookie,
596 header::HttpHeadline,
597 };
598
599 use std::{
600 path::PathBuf,
601 sync::{
602 Arc,
603 RwLock,
604 },
605 };
606
607 fn mkstate(fleet: Arc<Fleet>) -> Outcome<AdminState> {
608 let state = res!(AdminState::new(
609 Arc::new(RwLock::new(Wallet::default())),
610 PathBuf::from("./wallet.jdat"),
611 Some([0u8; 32].to_vec()),
612 1,
613 None,
614 TrafficRecorder::new_shared(0),
615 HostSampler::new_shared(),
616 res!(crate::srv::admin::guard::new_shared()),
617 res!(crate::srv::admin::guard::new_shared()),
618 Vec::new(),
619 None,
620 ));
621 Ok(state.with_fleet(fleet))
622 }
623
624 fn signed_in(state: &AdminState, scopes: &[&str]) -> Outcome<HeaderFields> {
625 let principal = AdminPrincipal {
626 name: fmt!("alice"),
627 scopes: scopes.iter().map(|s| s.to_string()).collect(),
628 expires_at: unix_secs() + 3_600,
629 };
630 let cookie = res!(encode_session(state, &principal));
631 let mut h = HeaderFields::default();
632 h.insert(
633 HeaderName::Cookie,
634 HeaderFieldValue::Cookie(vec![Cookie {
635 key: SESSION_COOKIE_NAME.to_string(),
636 val: cookie,
637 attrs: None,
638 }]),
639 None,
640 );
641 Ok(h)
642 }
643
644 fn status_of(m: &HttpMessage) -> u16 {
645 match &m.header.headline {
646 HttpHeadline::Response { status } => *status as u16,
647 _ => 0,
648 }
649 }
650
651 fn watched() -> Arc<Fleet> {
652 let mut cfg = WatchConfig::default();
653 let peer = |name: &str, host: &str, token: Option<&str>| WatchPeer {
654 name: name.to_string(),
655 host: host.to_string(),
656 url: fmt!("https://{}.test/_steel/health", name),
657 plain_ok: false,
658 distress: [(fmt!("mem_pct"), 90)].into_iter().collect(),
659 clear: [(fmt!("mem_pct"), 75)].into_iter().collect(),
660 token: token.map(|t| t.to_string()),
661 repeat_secs: None,
662 };
663 cfg.peers.push(peer("jarrah", "jarrah", Some("the-mesh-token")));
664 cfg.peers.push(peer("gateway", "jarrah", None));
665 Fleet::new_shared(fmt!("karri"), Some(&cfg))
666 }
667
668 /// Signed in is not enough: a principal holding `dashboard.view` alone is refused both the
669 /// page and its data, while `dashboard.admin` is served, and a visitor with no session is
670 /// sent to sign in.
671 #[test]
672 fn the_fleet_view_refuses_a_signed_in_non_admin() -> Outcome<()> {
673 let state = res!(mkstate(watched()));
674
675 let viewer = res!(signed_in(&state, &["dashboard.view"]));
676 assert_eq!(status_of(&render_fleet_page(&state, &viewer)), 403);
677 assert_eq!(status_of(&render_fleet_json(&state, &viewer)), 403);
678
679 let admin = res!(signed_in(&state, &["dashboard.admin"]));
680 assert_eq!(status_of(&render_fleet_page(&state, &admin)), 200);
681 assert_eq!(status_of(&render_fleet_json(&state, &admin)), 200);
682
683 let wildcard = res!(signed_in(&state, &["*"]));
684 assert_eq!(status_of(&render_fleet_json(&state, &wildcard)), 200);
685
686 let nobody = HeaderFields::default();
687 assert_eq!(status_of(&render_fleet_page(&state, &nobody)), 303, "sent to sign in");
688 assert_eq!(status_of(&render_fleet_json(&state, &nobody)), 401);
689 Ok(())
690 }
691
692 /// Every red on the page has an alarm behind it. A failed guard self-test is red only where
693 /// a `distress` threshold on `guard_failed` would tell someone, and a sealed box's word is
694 /// not drawn red at all, since nothing alarms on it (D-06 audit D2).
695 #[test]
696 fn a_red_on_the_page_always_has_an_alarm_behind_it() {
697 let mut failed = HealthBody::new();
698 failed.set(F_GUARD_FAILED, 1);
699 let mut passed = HealthBody::new();
700 passed.set(F_GUARD_FAILED, 0);
701 let none = BTreeMap::new();
702 let judged: BTreeMap<String, i64> = [(F_GUARD_FAILED.to_string(), 1)].into_iter().collect();
703 let guard = |body: &HealthBody, d: &BTreeMap<String, i64>|
704 cell_json(F_GUARD_FAILED, "Guard", "guard", body, &[], d, &none, true, "");
705
706 let cell = guard(&failed, &none);
707 assert!(cell.contains("\"tone\":\"\""), "a failed guard nothing alarms on was coloured: {}",
708 cell);
709 let cell = guard(&failed, &judged);
710 assert!(cell.contains("\"tone\":\"red\""), "{}", cell);
711 assert!(crate::srv::watch::is_over(1, 1), "the threshold that reddens it must alarm");
712 let cell = guard(&passed, &judged);
713 assert!(cell.contains("\"tone\":\"green\""), "{}", cell);
714
715 // Only the watcher's own call, `down`, draws a row's word red.
716 for rule in crate::srv::admin::assets::STYLE_CSS.split('}') {
717 if rule.contains("var(--red)") {
718 assert!(!rule.contains(".fleet-state-sealed"),
719 "a sealed row's word is drawn red, and nothing alarms on it: {}", rule);
720 }
721 }
722 }
723
724 /// The document groups a host's entries into one row, colours from that peer's
725 /// thresholds, and never carries a token.
726 #[test]
727 fn the_document_groups_by_host_and_colours_from_the_peer_thresholds() -> Outcome<()> {
728 let fleet = watched();
729 let now = unix_secs();
730 let mut body = HealthBody::new();
731 body.set(F_MEM_PCT, 80);
732 body.set(F_GUARD_FAILED, 0);
733 res!(fleet.record(0, ProbeSample {
734 t_secs: now, ok: true, probe_ms: 120, body: Some(body), health: PeerHealth::Up }));
735 res!(fleet.record(1, ProbeSample {
736 t_secs: now, ok: true, probe_ms: 45, body: None, health: PeerHealth::Up }));
737 let state = res!(mkstate(fleet));
738 let json = fleet_json(&state);
739
740 assert!(json.contains("\"host\":\"karri\",\"local\":true"), "own row first: {}", json);
741 assert_eq!(json.matches("\"host\":\"jarrah\"").count(), 1,
742 "the gateway must share jarrah's row, not make its own: {}", json);
743 assert!(json.contains("\"name\":\"gateway\",\"state\":\"up\""), "{}", json);
744 assert!(json.contains("\"k\":\"mem_pct\",\"label\":\"Memory\",\"unit\":\"pct\",\"v\":80,\
745 \"tone\":\"amber\",\"d\":90,\"c\":75"), "80 sits between clear 75 and distress 90: {}",
746 json);
747 assert!(!json.contains("the-mesh-token"), "a token must never reach the page");
748 Ok(())
749 }
750
751 /// A host watched by two entries that read one body -- jarrah's figures and its forge copy's
752 /// stamp ages -- draws both on its one row, each field coloured from the entry that judges
753 /// it, the lead's own thresholds standing where both name a field.
754 #[test]
755 fn a_hosts_row_colours_each_field_from_the_entry_that_judges_it() -> Outcome<()> {
756 let mut cfg = WatchConfig::default();
757 let entry = |name: &str, distress: &[(&str, i64)], clear: &[(&str, i64)]| WatchPeer {
758 name: name.to_string(),
759 host: fmt!("jarrah"),
760 url: fmt!("https://oxedyne.test/_steel/health"),
761 plain_ok: false,
762 distress: distress.iter().map(|(k, v)| (k.to_string(), *v)).collect(),
763 clear: clear.iter().map(|(k, v)| (k.to_string(), *v)).collect(),
764 token: Some(fmt!("the-mesh-token")),
765 repeat_secs: None,
766 };
767 cfg.peers.push(entry("jarrah", &[("mem_pct", 90)], &[("mem_pct", 75)]));
768 cfg.peers.push(entry("jarrah forge copy",
769 &[("forge_state_age_s", 10_800), ("forge_repos_age_s", 10_800), ("mem_pct", 50)],
770 &[("forge_state_age_s", 7_200), ("forge_repos_age_s", 7_200)]));
771 let fleet = Fleet::new_shared(fmt!("karri"), Some(&cfg));
772 let now = unix_secs();
773 let mut body = HealthBody::new();
774 body.set(F_MEM_PCT, 80);
775 body.set("forge_state_age_s", 12_000);
776 body.set("forge_repos_age_s", 900);
777 for i in 0..2 {
778 res!(fleet.record(i, ProbeSample { t_secs: now, ok: true, probe_ms: 90,
779 body: Some(body.clone()), health: PeerHealth::Up }));
780 }
781 let json = fleet_json(&res!(mkstate(fleet)));
782
783 assert_eq!(json.matches("\"host\":\"jarrah\"").count(), 1, "{}", json);
784 assert!(json.contains("{\"k\":\"forge_state_age_s\",\"label\":\"forge_state_age_s\",\
785 \"unit\":\"secs\",\"v\":12000,\"tone\":\"red\",\"d\":10800,\"c\":7200"),
786 "a stale stamp is red by the forge copy's own thresholds: {}", json);
787 assert!(json.contains("{\"k\":\"forge_repos_age_s\",\"label\":\"forge_repos_age_s\",\
788 \"unit\":\"secs\",\"v\":900,\"tone\":\"green\""), "{}", json);
789 assert!(json.contains("\"k\":\"mem_pct\",\"label\":\"Memory\",\"unit\":\"pct\",\"v\":80,\
790 \"tone\":\"amber\",\"d\":90,\"c\":75"),
791 "where both entries name a field, the lead's thresholds stand: {}", json);
792 assert!(json.contains("\"name\":\"jarrah forge copy\",\"state\":\"up\""), "{}", json);
793 assert!(json.contains("\"link_down\":false"), "{}", json);
794 Ok(())
795 }
796
797 /// A field no pane knows is shown under Other, formatted from its suffix and coloured from
798 /// its threshold, and a field a threshold names that the body lacks is shown empty, so a
799 /// threshold with nothing to judge is visible rather than silent.
800 #[test]
801 fn a_field_the_page_does_not_know_is_shown_not_dropped() -> Outcome<()> {
802 let mut body = HealthBody::new();
803 body.set(F_MEM_PCT, 40);
804 body.set(F_UPTIME_S, 3_600);
805 body.set(F_SEALED, 1);
806 body.set("forge_state_age_s", 7_300);
807 let mut distress = BTreeMap::new();
808 distress.insert(fmt!("forge_state_age_s"), 7_200);
809 distress.insert(fmt!("forge_repos_age_s"), 86_400);
810 let panes = panes_json(Some(&body), &[], &distress, &BTreeMap::new(), true);
811
812 assert!(panes.contains("\"id\":\"c\",\"title\":\"Other fields\""), "{}", panes);
813 assert!(panes.contains("{\"k\":\"forge_state_age_s\",\"label\":\"forge_state_age_s\",\
814 \"unit\":\"secs\",\"v\":7300,\"tone\":\"red\",\"d\":7200"), "{}", panes);
815 assert!(panes.contains("{\"k\":\"forge_repos_age_s\",\"label\":\"forge_repos_age_s\",\
816 \"unit\":\"secs\",\"v\":null"), "{}", panes);
817 // With no `sealed_dbs` to carry it, the raw seal falls to Other rather than vanishing.
818 assert!(panes.contains("{\"k\":\"sealed\",\"label\":\"sealed\""), "{}", panes);
819 assert!(!panes.contains("\"k\":\"uptime_s\""), "uptime is the row head's, not a cell's");
820
821 // Once `sealed_dbs` is there, its cell carries `sealed` and Other does not repeat it.
822 body.set(F_SEALED_DBS, 0);
823 let panes = panes_json(Some(&body), &[], &distress, &BTreeMap::new(), true);
824 assert!(panes.contains("\"k\":\"sealed_dbs\",\"label\":\"Seal\""), "{}", panes);
825 assert!(panes.contains("\"sealed\":1"), "{}", panes);
826 assert!(!panes.contains("{\"k\":\"sealed\","), "{}", panes);
827 Ok(())
828 }
829}