oxedyne/fe2o3/fe2o3_steel/src/srv/alert.rs
69.1 KiB, 303 runs
created by r1870400018:13762, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! Operator alerting, by email and by text message. |
| 2 | //! |
| 3 | //! Steel raises an alert when something happens that a human needs to know |
| 4 | //! about and would not otherwise see: it came up with data still sealed, an |
| 5 | //! admin unsealed it, or somebody is guessing at the passphrase. |
| 6 | //! |
| 7 | //! # What is deliberately not alerted |
| 8 | //! |
| 9 | //! Routine events. An alert that fires on every sign-in, every restart, every |
| 10 | //! request, is an alert the operator learns to delete unread, and the one |
| 11 | //! message that mattered goes with the rest. The set below is small on |
| 12 | //! purpose, and each member is rare in normal operation. |
| 13 | //! |
| 14 | //! # The email notifies, it never authorises |
| 15 | //! |
| 16 | //! There is no approve-by-clicking link, and there never should be. The mail |
| 17 | //! says what happened and points at `/admin`; the human authenticates there. |
| 18 | //! An authorisation that arrives by email is an authorisation anybody who can |
| 19 | //! read, spoof or replay that email holds too. |
| 20 | //! |
| 21 | //! # The machine that alerts is the machine in trouble |
| 22 | //! |
| 23 | //! Worth being honest about: Steel is reporting on itself. A Steel that is |
| 24 | //! wedged, unreachable or dead sends nothing, and silence is indistinguishable |
| 25 | //! from health. That is why the alert is addressed *off* the network -- an |
| 26 | //! external mailbox at least survives the host -- and why alerting is a |
| 27 | //! complement to external monitoring, not a substitute for it. |
| 28 | //! |
| 29 | //! # A channel that fails is reported through the other |
| 30 | //! |
| 31 | //! A channel can die as quietly as a host. From 2026-08-31 the SMS gateway |
| 32 | //! refused every text in the estate for want of credit, each refusal was |
| 33 | //! logged as sent, and it was found by an audit three weeks later. So a |
| 34 | //! refusal is a failure (see `oxedyne_fe2o3_net::sms`), and a channel that |
| 35 | //! starts failing is reported through the other one: text failures by mail, |
| 36 | //! and mail failures by text. The report comes when the failures start, again |
| 37 | //! once a day while they last, and once more when the channel delivers again |
| 38 | //! (see [`ChannelFailing`](AlertEvent::ChannelFailing)). An event that no |
| 39 | //! channel on the host carries at all, such as a notice on a host with no |
| 40 | //! mail, is logged as undelivered rather than dropped. |
| 41 | //! |
| 42 | //! [Written with AI entirely](https://need2know.ai/entirely-ai/code)\ |
| 43 | //! Anthropic Claude |
| 44 | |
| 45 | use crate::srv::cfg::{ |
| 46 | AlertConfig, |
| 47 | SmsAlertConfig, |
| 48 | }; |
| 49 | |
| 50 | use oxedyne_fe2o3_core::prelude::*; |
| 51 | use oxedyne_fe2o3_net::{ |
| 52 | dkim::DkimSigner, |
| 53 | http::{ |
| 54 | client::https_request, |
| 55 | header::HttpHeadline, |
| 56 | }, |
| 57 | imap::client::Security, |
| 58 | smtp::client::{ |
| 59 | OutboundClient, |
| 60 | SubmissionConfig, |
| 61 | }, |
| 62 | sms::{ |
| 63 | Credential, |
| 64 | Message as SmsMessage, |
| 65 | Provider as SmsProvider, |
| 66 | Receipt, |
| 67 | }, |
| 68 | }; |
| 69 | |
| 70 | use tokio_rustls::rustls::ClientConfig; |
| 71 | |
| 72 | use std::{ |
| 73 | net::SocketAddr, |
| 74 | sync::{ |
| 75 | Arc, |
| 76 | Mutex, |
| 77 | }, |
| 78 | time::{ |
| 79 | Duration, |
| 80 | Instant, |
| 81 | SystemTime, |
| 82 | UNIX_EPOCH, |
| 83 | }, |
| 84 | }; |
| 85 | |
| 86 | // How often a channel that keeps failing is reported again. Daily: a report read and forgotten |
| 87 | // comes back, and a channel dead over a long weekend is three messages, not three hundred. |
| 88 | const CHANNEL_REMIND: Duration = Duration::from_secs(86_400); |
| 89 | |
| 90 | // How long one text may take, dial to receipt. A gateway that hangs must end as a failure in the |
| 91 | // log, not as a task that waits for ever with its text neither sent nor refused. |
| 92 | const SMS_TIMEOUT: Duration = Duration::from_secs(30); |
| 93 | |
| 94 | // How long an alert's mail may take, from the lookup to the relay's or the exchange's acceptance. |
| 95 | // The SMTP client holds each step to its own deadline; this bounds the whole leg. |
| 96 | const MAIL_TIMEOUT: Duration = Duration::from_secs(120); |
| 97 | |
| 98 | /// A way this host reaches its operator. |
| 99 | #[derive(Clone, Copy, Debug, Eq, PartialEq)] |
| 100 | pub enum Channel { |
| 101 | Mail, |
| 102 | Sms, |
| 103 | } |
| 104 | |
| 105 | impl Channel { |
| 106 | /// What the alerts on this channel are called in a message about it. |
| 107 | pub fn alerts(&self) -> &'static str { |
| 108 | match self { |
| 109 | Self::Mail => "alert mail", |
| 110 | Self::Sms => "text alerts", |
| 111 | } |
| 112 | } |
| 113 | |
| 114 | /// How an alert travels on this channel: "getting through by ...". |
| 115 | pub fn by(&self) -> &'static str { |
| 116 | match self { |
| 117 | Self::Mail => "mail", |
| 118 | Self::Sms => "text message", |
| 119 | } |
| 120 | } |
| 121 | } |
| 122 | |
| 123 | /// Something worth waking an operator for. |
| 124 | #[derive(Clone, Debug)] |
| 125 | pub enum AlertEvent { |
| 126 | // Started with databases configured but no master key, so DB-backed routes |
| 127 | // answer 503 until somebody unseals. The one that matters most: the |
| 128 | // websites are up, so nothing else looks wrong, and without this the |
| 129 | // operator learns about it from a user complaint. |
| 130 | SealedStart { |
| 131 | db_count: usize, // databases waiting on the key |
| 132 | }, |
| 133 | // Rare by construction, and the audit trail an operator wants: who, when, |
| 134 | // from where. |
| 135 | Unsealed { |
| 136 | admin: String, |
| 137 | peer: SocketAddr, |
| 138 | }, |
| 139 | // Repeated failures to unwrap the wallet at the dashboard login, coalesced |
| 140 | // into one message per burst rather than one per attempt. The login form |
| 141 | // unseals, so it is worth guessing at, and an alerter that sent a message |
| 142 | // per guess would be an amplifier pointed at the operator's mailbox. |
| 143 | FailedUnseals { |
| 144 | count: u32, |
| 145 | window_secs: u64, |
| 146 | last_peer: SocketAddr, |
| 147 | }, |
| 148 | // The one event this host can raise about somebody else, and the reason |
| 149 | // crate::srv::watch exists: a dead machine sends nothing, so the alarm has |
| 150 | // to come from a live one. |
| 151 | PeerDown { |
| 152 | peer: String, // as configured |
| 153 | url: String, // what was probed |
| 154 | failures: u32, // consecutive failed probes |
| 155 | down_secs: u64, |
| 156 | // Which machine noticed. Two watchers see one outage and send two |
| 157 | // messages; without this they read as one message sent twice. |
| 158 | noticed_by: String, |
| 159 | }, |
| 160 | // Sent because an operator who was woken is owed the end of the story, and |
| 161 | // because a recovery nobody announced is one somebody drives to the office |
| 162 | // for. |
| 163 | PeerRecovered { |
| 164 | peer: String, |
| 165 | url: String, |
| 166 | away_secs: u64, |
| 167 | noticed_by: String, |
| 168 | }, |
| 169 | // A peer that is answering but reports itself unwell: a health-body class has |
| 170 | // stayed over its distress threshold. Distinct from `PeerDown` -- the box is |
| 171 | // up, so this goes by email rather than waking somebody with an SMS -- and it |
| 172 | // names the class that fired so the reader knows which resource is short. |
| 173 | PeerDistress { |
| 174 | peer: String, |
| 175 | url: String, |
| 176 | classes: String, // e.g. "mem_pct 94, swap_pct 71" |
| 177 | since_secs: u64, |
| 178 | noticed_by: String, |
| 179 | }, |
| 180 | // The end of a distress episode, owed for the same reason as a recovery from |
| 181 | // down: an operator told a box was unwell wants to know it came back under |
| 182 | // its thresholds. |
| 183 | PeerDistressCleared { |
| 184 | peer: String, |
| 185 | url: String, |
| 186 | were_secs: u64, // how long the peer was distressed |
| 187 | noticed_by: String, |
| 188 | }, |
| 189 | // Proof that the alerting path itself still works, and the point of it is |
| 190 | // that it is boring. A path used twice a year is broken when it is needed |
| 191 | // -- an expired credential, a rotated key, a changed number, a dormant |
| 192 | // account -- and it is discovered during the incident. This exercises every |
| 193 | // leg on a schedule, so the failure is found on an ordinary afternoon |
| 194 | // instead. |
| 195 | Heartbeat { |
| 196 | uptime_secs: u64, |
| 197 | peers_ok: usize, // of `peers_total`, how many are answering |
| 198 | peers_total: usize, |
| 199 | }, |
| 200 | // One of this host's own channels has stopped delivering: a text refused for want of |
| 201 | // credit, a relay that no longer takes the password. Told through the other channel, since |
| 202 | // a channel can no more report its own failure than a host can report its own death. |
| 203 | ChannelFailing { |
| 204 | channel: Channel, |
| 205 | reason: String, // the provider's own words, from the latest failure |
| 206 | missed: String, // the subject of the latest alert this channel lost |
| 207 | failures: u32, // consecutive failed deliveries |
| 208 | since_secs: u64, // since the first of them |
| 209 | }, |
| 210 | // The end of that story, owed for the same reason a recovery is: the operator told of a dead |
| 211 | // channel wants to know it works again, and how much it lost. |
| 212 | ChannelRestored { |
| 213 | channel: Channel, |
| 214 | lost: u32, // the deliveries that failed in the episode |
| 215 | were_secs: u64, // from the first of them to this success |
| 216 | }, |
| 217 | } |
| 218 | |
| 219 | /// How loudly an event should be delivered. |
| 220 | /// |
| 221 | /// The distinction is the whole of the routing rule, and it exists because the |
| 222 | /// channels differ in cost and in how much they intrude. An operator who is |
| 223 | /// texted about routine events stops reading the texts, and the one that |
| 224 | /// mattered goes with the rest -- which is the same argument the module header |
| 225 | /// makes for keeping the event set small. |
| 226 | #[derive(Clone, Copy, Debug, Eq, PartialEq)] |
| 227 | pub enum Severity { |
| 228 | Critical, // every channel, including the ones that cost money and wake somebody |
| 229 | Notice, // worth a record; mail only |
| 230 | } |
| 231 | |
| 232 | impl AlertEvent { |
| 233 | /// A peer going down is the only thing here that reaches a phone, plus the |
| 234 | /// heartbeat that proves a phone still can be reached. Everything else is |
| 235 | /// about this machine, and this machine can only report it while it is |
| 236 | /// well enough to be read about later. |
| 237 | pub fn severity(&self) -> Severity { |
| 238 | match self { |
| 239 | Self::PeerDown { .. } => Severity::Critical, |
| 240 | Self::PeerRecovered { .. } => Severity::Critical, |
| 241 | Self::Heartbeat { .. } => Severity::Critical, |
| 242 | Self::SealedStart { .. } => Severity::Notice, |
| 243 | Self::Unsealed { .. } => Severity::Notice, |
| 244 | Self::FailedUnseals { .. } => Severity::Notice, |
| 245 | // A distressed peer is up, so it is worth a record, not a phone call. |
| 246 | Self::PeerDistress { .. } => Severity::Notice, |
| 247 | Self::PeerDistressCleared { .. } => Severity::Notice, |
| 248 | // Dead mail reaches the phone, as the heartbeat does, because it is about whether |
| 249 | // the operator can be reached at all: every notice goes by mail alone. Dead texts |
| 250 | // are told by mail, the one channel left. |
| 251 | Self::ChannelFailing { channel: Channel::Mail, .. } => Severity::Critical, |
| 252 | Self::ChannelFailing { channel: Channel::Sms, .. } => Severity::Notice, |
| 253 | Self::ChannelRestored { .. } => Severity::Notice, |
| 254 | } |
| 255 | } |
| 256 | |
| 257 | /// The whole event in one line, for a channel that has no subject and no |
| 258 | /// body and charges by the segment. |
| 259 | /// |
| 260 | /// Blunt on purpose, and written separately rather than trimmed from |
| 261 | /// [`Self::body`]. It is read on a lock screen in the dark by somebody who |
| 262 | /// wants to know whether to get up, so it leads with the machine and the |
| 263 | /// verdict. How long it has been down, which machine noticed and what was |
| 264 | /// probed are all in the email, which costs nothing to make longer; here |
| 265 | /// they are noise in front of the one word that matters. |
| 266 | pub fn short(&self, host: &str) -> String { |
| 267 | match self { |
| 268 | Self::PeerDown { peer, .. } => fmt!("{} is DOWN", peer), |
| 269 | Self::PeerRecovered { peer, .. } => fmt!("{} is back", peer), |
| 270 | Self::Heartbeat { peers_ok, peers_total, .. } => fmt!( |
| 271 | "{}: alerting alive, {}/{} ok", host, peers_ok, peers_total), |
| 272 | Self::ChannelFailing { channel, .. } => fmt!( |
| 273 | "{}: {} FAILING", host, channel.alerts()), |
| 274 | Self::ChannelRestored { channel, .. } => fmt!( |
| 275 | "{}: {} working again", host, channel.alerts()), |
| 276 | other => other.subject(host), |
| 277 | } |
| 278 | } |
| 279 | |
| 280 | /// Subject line. Prefixed so the operator can filter on it. |
| 281 | pub fn subject(&self, host: &str) -> String { |
| 282 | match self { |
| 283 | Self::SealedStart { db_count } => fmt!( |
| 284 | "[steel:{}] SEALED at start -- {} database(s) shut", host, db_count), |
| 285 | Self::Unsealed { admin, .. } => fmt!( |
| 286 | "[steel:{}] unsealed by '{}'", host, admin), |
| 287 | Self::FailedUnseals { count, .. } => fmt!( |
| 288 | "[steel:{}] {} failed admin passphrase attempts", host, count), |
| 289 | Self::PeerDown { peer, down_secs, .. } => fmt!( |
| 290 | "[steel:{}] {} IS DOWN ({}m)", host, peer, down_secs / 60), |
| 291 | Self::PeerRecovered { peer, away_secs, .. } => fmt!( |
| 292 | "[steel:{}] {} recovered after {}m", host, peer, away_secs / 60), |
| 293 | Self::PeerDistress { peer, classes, .. } => fmt!( |
| 294 | "[steel:{}] {} in distress ({})", host, peer, classes), |
| 295 | Self::PeerDistressCleared { peer, were_secs, .. } => fmt!( |
| 296 | "[steel:{}] {} distress cleared after {}m", host, peer, were_secs / 60), |
| 297 | Self::Heartbeat { peers_ok, peers_total, .. } => fmt!( |
| 298 | "[steel:{}] alerting alive, {}/{} peers answering", |
| 299 | host, peers_ok, peers_total), |
| 300 | Self::ChannelFailing { channel, failures, .. } => fmt!( |
| 301 | "[steel:{}] {} FAILING ({} in a row)", host, channel.alerts(), failures), |
| 302 | Self::ChannelRestored { channel, were_secs, .. } => fmt!( |
| 303 | "[steel:{}] {} working again after {}m", host, channel.alerts(), were_secs / 60), |
| 304 | } |
| 305 | } |
| 306 | |
| 307 | /// Body text. Plain, short, and it never asks the reader to click |
| 308 | /// anything that would act on their behalf. |
| 309 | pub fn body(&self, host: &str) -> String { |
| 310 | match self { |
| 311 | Self::SealedStart { db_count } => fmt!( |
| 312 | "Steel on {host} started sealed.\n\n\ |
| 313 | The websites are serving normally -- static vhosts, redirects, \ |
| 314 | proxy routes and certificate renewal are all unaffected. But {n} \ |
| 315 | database(s) are shut because no wallet master key has been \ |
| 316 | supplied, and any route that needs one is answering 503.\n\n\ |
| 317 | Sign in at https://{host}/admin with an admin passphrase to \ |
| 318 | unseal. Nothing in this email authorises anything; you will be \ |
| 319 | asked to authenticate there.\n", |
| 320 | host = host, n = db_count), |
| 321 | Self::Unsealed { admin, peer } => fmt!( |
| 322 | "Steel on {host} was unsealed.\n\n\ |
| 323 | Admin: {admin}\n\ |
| 324 | From: {peer}\n\n\ |
| 325 | The databases are open. If this was not you, treat the wallet \ |
| 326 | passphrase for '{admin}' as compromised: rotate it with \ |
| 327 | `admin --passwd`, and review admin-audit.log.\n", |
| 328 | host = host, admin = admin, peer = peer), |
| 329 | Self::FailedUnseals { count, window_secs, last_peer } => fmt!( |
| 330 | "Steel on {host} refused {count} admin passphrase attempt(s) in \ |
| 331 | the last {mins} minute(s).\n\n\ |
| 332 | Most recent from: {peer}\n\n\ |
| 333 | The dashboard login unwraps the wallet master key, so this form \ |
| 334 | is worth guessing at. Each attempt costs the attacker an Argon2id \ |
| 335 | derivation and is rate limited per address, but a sustained \ |
| 336 | campaign is worth knowing about. Review admin-audit.log, and \ |
| 337 | consider binding the dashboard to localhost via admin_local_port \ |
| 338 | if it does not need to face the internet.\n", |
| 339 | host = host, count = count, mins = window_secs / 60, |
| 340 | peer = last_peer), |
| 341 | Self::PeerDown { peer, url, failures, down_secs, noticed_by } => fmt!( |
| 342 | "{peer} is not answering.\n\n\ |
| 343 | Probed: {url}\n\ |
| 344 | Failures: {failures} consecutive\n\ |
| 345 | Down for: {mins} minute(s)\n\ |
| 346 | Noticed by: {noticed_by}\n\n\ |
| 347 | This machine is reporting on another one, because a host that has \ |
| 348 | died cannot report its own death. Nothing has been restarted: a \ |
| 349 | watcher that repairs can flap a service in a loop and hide the \ |
| 350 | fault it was built to reveal, and a decision to restart belongs to \ |
| 351 | somebody who has read why it stopped.\n\n\ |
| 352 | If this is the Daimond gateway, its log is at \ |
| 353 | ~/usr/daimond-gateway/log/, the pane is held open after an exit, \ |
| 354 | and the last lines say what it said on the way out.\n", |
| 355 | peer = peer, url = url, failures = failures, |
| 356 | mins = down_secs / 60, noticed_by = noticed_by), |
| 357 | Self::PeerRecovered { peer, url, away_secs, noticed_by } => fmt!( |
| 358 | "{peer} is answering again after {mins} minute(s).\n\n\ |
| 359 | Probed: {url}\n\ |
| 360 | Noticed by: {noticed_by}\n\n\ |
| 361 | Nothing here did that; it came back on its own or somebody fixed \ |
| 362 | it. Worth reading the log for what stopped it, because a fault \ |
| 363 | that cleared itself is a fault that can return.\n", |
| 364 | peer = peer, url = url, mins = away_secs / 60, noticed_by = noticed_by), |
| 365 | Self::PeerDistress { peer, url, classes, since_secs, noticed_by } => fmt!( |
| 366 | "{peer} is answering but reports itself unwell.\n\n\ |
| 367 | Over threshold: {classes}\n\ |
| 368 | Probed: {url}\n\ |
| 369 | For: {mins} minute(s)\n\ |
| 370 | Noticed by: {noticed_by}\n\n\ |
| 371 | The box is up and serving; a resource it depends on is short. This \ |
| 372 | is not an outage, which is why it arrives by email and not as a \ |
| 373 | text -- but a box under sustained pressure is one on its way to an \ |
| 374 | outage, and it is cheaper to look now. The figures are read from \ |
| 375 | the box's own health body; nothing here has acted on it.\n", |
| 376 | peer = peer, url = url, classes = classes, |
| 377 | mins = since_secs / 60, noticed_by = noticed_by), |
| 378 | Self::PeerDistressCleared { peer, url, were_secs, noticed_by } => fmt!( |
| 379 | "{peer} is back under its thresholds after {mins} minute(s).\n\n\ |
| 380 | Probed: {url}\n\ |
| 381 | Noticed by: {noticed_by}\n\n\ |
| 382 | The resource that was short has recovered. Worth a glance at what \ |
| 383 | drove it, because pressure that cleared itself can build again.\n", |
| 384 | peer = peer, url = url, mins = were_secs / 60, noticed_by = noticed_by), |
| 385 | Self::Heartbeat { uptime_secs, peers_ok, peers_total } => fmt!( |
| 386 | "Alerting on {host} is alive. Nothing is wrong.\n\n\ |
| 387 | Uptime: {days} day(s)\n\ |
| 388 | Peers: {ok} of {total} answering\n\n\ |
| 389 | This message exists to prove the path still works. An alerting \ |
| 390 | route used twice a year is broken when it is needed -- an expired \ |
| 391 | credential, a rotated key, a changed number, a dormant account -- \ |
| 392 | and it is discovered during the incident. If these stop arriving, \ |
| 393 | the alerting is what has failed, not the estate.\n", |
| 394 | host = host, days = uptime_secs / 86400, |
| 395 | ok = peers_ok, total = peers_total), |
| 396 | Self::ChannelFailing { channel, reason, missed, failures, since_secs } => fmt!( |
| 397 | "Alerts from {host} are not getting through by {by}.\n\n\ |
| 398 | Failed: {failures} in a row, over {mins} minute(s)\n\ |
| 399 | Last lost: {missed}\n\ |
| 400 | Why: {reason}\n\n\ |
| 401 | {advice}\n\n\ |
| 402 | This report comes by another channel, because a channel cannot report \ |
| 403 | its own failure. It is repeated once a day while the failures go on, \ |
| 404 | and the first alert that gets through again ends it with a message of \ |
| 405 | its own.\n", |
| 406 | host = host, by = channel.by(), failures = failures, |
| 407 | mins = since_secs / 60, missed = missed, reason = reason, |
| 408 | advice = match channel { |
| 409 | Channel::Sms => "A text needs a funded account and a working credential \ |
| 410 | with the gateway, and the gateway's own words are above. Until it is \ |
| 411 | fixed, a peer going down reaches you by mail alone, and mail needs the \ |
| 412 | data connection a phone does not always have.", |
| 413 | Channel::Mail => "Mail needs a relay or a recipient's server that accepts \ |
| 414 | it, and a working credential where it goes through a relay; the \ |
| 415 | server's own words are above. Until it is fixed, a sealed start, an \ |
| 416 | unseal, a burst of failed passphrases and a peer in distress reach \ |
| 417 | nobody, because they go by mail alone.", |
| 418 | }), |
| 419 | Self::ChannelRestored { channel, lost, were_secs } => fmt!( |
| 420 | "Alerts from {host} are getting through by {by} again.\n\n\ |
| 421 | Lost: {lost} alert(s) in a row failed on this channel\n\ |
| 422 | Over: {mins} minute(s)\n\n\ |
| 423 | Those alerts were not sent again. Each is in the log on {host}, under \ |
| 424 | \"ALERT NOT DELIVERED\".\n", |
| 425 | host = host, by = channel.by(), lost = lost, mins = were_secs / 60), |
| 426 | } |
| 427 | } |
| 428 | } |
| 429 | |
| 430 | |
| 431 | /// Coalescing state for failed passphrase attempts. |
| 432 | #[derive(Debug)] |
| 433 | struct FailureWindow { |
| 434 | count: u32, // failures since the window opened |
| 435 | opened: Instant, // when the first uncounted failure arrived |
| 436 | last: Option<SocketAddr>, |
| 437 | sent: Option<Instant>, // so a persistent attacker is not a stream of email |
| 438 | } |
| 439 | |
| 440 | /// A run of failed deliveries on one channel. |
| 441 | #[derive(Clone, Debug)] |
| 442 | struct Failing { |
| 443 | since: Instant, // the first failure of the run |
| 444 | failures: u32, // consecutive failed deliveries |
| 445 | told: Instant, // when the run was last reported |
| 446 | } |
| 447 | |
| 448 | /// What this alerter has seen of its own channels, so a channel that stops delivering is |
| 449 | /// reported rather than only logged. |
| 450 | #[derive(Debug, Default)] |
| 451 | struct ChannelBook { |
| 452 | mail: Option<Failing>, |
| 453 | sms: Option<Failing>, |
| 454 | } |
| 455 | |
| 456 | impl ChannelBook { |
| 457 | /// Fold one delivery's outcome into the book, returning the report it is owed, if any. |
| 458 | /// |
| 459 | /// `failure` is the provider's words for a failed delivery and `None` for a success; |
| 460 | /// `missed` is the subject of the alert concerned. The first failure of a run is reported, |
| 461 | /// then one each [`CHANNEL_REMIND`] while the run lasts, and the first success after it |
| 462 | /// ends the run with a report of its own. Each of those happens once, which is what stops a |
| 463 | /// report whose own delivery fails from setting off another. |
| 464 | fn note( |
| 465 | &mut self, |
| 466 | ch: Channel, |
| 467 | failure: Option<&str>, |
| 468 | missed: &str, |
| 469 | now: Instant, |
| 470 | ) |
| 471 | -> Option<AlertEvent> |
| 472 | { |
| 473 | let slot = match ch { |
| 474 | Channel::Mail => &mut self.mail, |
| 475 | Channel::Sms => &mut self.sms, |
| 476 | }; |
| 477 | let why = match failure { |
| 478 | Some(w) => w, |
| 479 | None => return match slot.take() { |
| 480 | Some(f) => Some(AlertEvent::ChannelRestored { |
| 481 | channel: ch, |
| 482 | lost: f.failures, |
| 483 | were_secs: now.saturating_duration_since(f.since).as_secs(), |
| 484 | }), |
| 485 | None => None, |
| 486 | }, |
| 487 | }; |
| 488 | match slot { |
| 489 | None => { |
| 490 | *slot = Some(Failing { since: now, failures: 1, told: now }); |
| 491 | Some(AlertEvent::ChannelFailing { |
| 492 | channel: ch, |
| 493 | reason: why.to_string(), |
| 494 | missed: missed.to_string(), |
| 495 | failures: 1, |
| 496 | since_secs: 0, |
| 497 | }) |
| 498 | }, |
| 499 | Some(f) => { |
| 500 | f.failures = f.failures.saturating_add(1); |
| 501 | if now.saturating_duration_since(f.told) < CHANNEL_REMIND { |
| 502 | return None; |
| 503 | } |
| 504 | f.told = now; |
| 505 | Some(AlertEvent::ChannelFailing { |
| 506 | channel: ch, |
| 507 | reason: why.to_string(), |
| 508 | missed: missed.to_string(), |
| 509 | failures: f.failures, |
| 510 | since_secs: now.saturating_duration_since(f.since).as_secs(), |
| 511 | }) |
| 512 | }, |
| 513 | } |
| 514 | } |
| 515 | } |
| 516 | |
| 517 | /// The channels one event goes by on this host. |
| 518 | #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] |
| 519 | struct Route { |
| 520 | mail: bool, |
| 521 | sms: bool, |
| 522 | } |
| 523 | |
| 524 | impl Route { |
| 525 | fn is_empty(&self) -> bool { |
| 526 | !self.mail && !self.sms |
| 527 | } |
| 528 | } |
| 529 | |
| 530 | /// Sends [`AlertEvent`]s by email and, for the critical ones, by text message. |
| 531 | /// |
| 532 | /// Cheap to clone: the configuration, the SMTP client and the channel book are shared. |
| 533 | #[derive(Clone)] |
| 534 | pub struct Alerter { |
| 535 | cfg: Arc<AlertConfig>, |
| 536 | client: Arc<OutboundClient>, |
| 537 | // Where to post, when posting through a provider rather than delivering |
| 538 | // straight to the recipient's MX. Built once, at start-up. |
| 539 | submission: Option<Arc<SubmissionConfig>>, |
| 540 | // DKIM identities to sign the alert with, when the host is configured to |
| 541 | // sign its mail at all. An unsigned message from a domain that signs |
| 542 | // everything else is exactly what a spam filter is entitled to distrust -- |
| 543 | // and the alert is the one message that has to arrive. The alerter posts |
| 544 | // straight through the SMTP client rather than through the mail handler, so |
| 545 | // it has to sign for itself. |
| 546 | dkim: Vec<Arc<DkimSigner>>, |
| 547 | host: Arc<String>, // public, used in subject lines and the `/admin` link |
| 548 | failures: Arc<Mutex<FailureWindow>>, |
| 549 | channels: Arc<Mutex<ChannelBook>>, // shared by every clone, so every delivery counts |
| 550 | // Outbound TLS, for the SMS gateway. Absent on a host with no outbound |
| 551 | // client, in which case the mail leg still works and the text leg says so |
| 552 | // rather than failing silently. |
| 553 | tls: Option<Arc<ClientConfig>>, |
| 554 | } |
| 555 | |
| 556 | impl std::fmt::Debug for Alerter { |
| 557 | /// Written by hand because `OutboundClient` is not `Debug`, and because |
| 558 | /// the recipient list is the only part of the configuration worth seeing |
| 559 | /// in a log line. |
| 560 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { |
| 561 | f.debug_struct("Alerter") |
| 562 | .field("host", &self.host) |
| 563 | .field("from", &self.cfg.from) |
| 564 | .field("to", &self.cfg.to) |
| 565 | .finish() |
| 566 | } |
| 567 | } |
| 568 | |
| 569 | impl Alerter { |
| 570 | |
| 571 | /// Build an alerter, or `None` when alerting is not configured. |
| 572 | /// |
| 573 | /// A misconfigured alerter is a start-up error rather than a silent |
| 574 | /// no-op: an operator who has written an `alerts` block believes they |
| 575 | /// will be told when something goes wrong, and the failure mode of |
| 576 | /// discovering otherwise is that they find out from an outage. |
| 577 | pub fn new( |
| 578 | cfg: AlertConfig, |
| 579 | host: String, |
| 580 | dkim: Vec<Arc<DkimSigner>>, |
| 581 | tls: Option<Arc<ClientConfig>>, |
| 582 | ) |
| 583 | -> Outcome<Option<Self>> |
| 584 | { |
| 585 | if !cfg.enabled { |
| 586 | return Ok(None); |
| 587 | } |
| 588 | // Mail is the usual channel and not the only one. A host that cannot |
| 589 | // send mail -- its provider blocks outbound port 25 and it has no relay |
| 590 | // to submit through -- still has a phone to reach, and refusing to |
| 591 | // alert at all because one channel is unavailable would leave it silent |
| 592 | // for the reason it most needs to speak. |
| 593 | let texts = cfg.sms.as_ref().map(|s| s.enabled && !s.to.is_empty()).unwrap_or(false); |
| 594 | let mails = !cfg.to.is_empty() && !cfg.from.is_empty(); |
| 595 | if !mails && !texts { |
| 596 | return Err(err!( |
| 597 | "Alerting is enabled but has no way to reach anybody: set \ |
| 598 | 'alerts.from' and 'alerts.to' for mail, or 'alerts.sms' for \ |
| 599 | text messages, or disable alerting. An alerter with nobody to \ |
| 600 | tell is worse than none, because it looks like cover."; |
| 601 | Configuration, Invalid, Missing)); |
| 602 | } |
| 603 | if !mails && texts { |
| 604 | warn!("Alerting has no mail recipient, so only a peer going down or coming back, \ |
| 605 | and the heartbeat, reach anybody: they go by text message. A sealed start, an \ |
| 606 | unseal, a burst of failed passphrases, a peer in distress and a failing text \ |
| 607 | channel go by mail alone, so on this host they reach nobody and are logged as \ |
| 608 | undelivered. Give 'alerts' a 'from' and a 'to', with a 'submission' relay \ |
| 609 | where this host cannot send direct."); |
| 610 | } |
| 611 | let ehlo = if cfg.ehlo_hostname.is_empty() { |
| 612 | host.clone() |
| 613 | } else { |
| 614 | cfg.ehlo_hostname.clone() |
| 615 | }; |
| 616 | let client = res!(OutboundClient::with_system_roots(ehlo)); |
| 617 | let submission = match &cfg.submission { |
| 618 | Some(s) => { |
| 619 | let security = match s.security.as_str() { |
| 620 | "implicit" => Security::ImplicitTls, |
| 621 | "plain" => Security::Plain, |
| 622 | _ => Security::StartTls, |
| 623 | }; |
| 624 | Some(Arc::new(SubmissionConfig::new( |
| 625 | s.host.clone(), |
| 626 | s.port, |
| 627 | security, |
| 628 | s.user.clone(), |
| 629 | s.password.clone(), |
| 630 | ))) |
| 631 | } |
| 632 | None => None, |
| 633 | }; |
| 634 | Ok(Some(Self { |
| 635 | cfg: Arc::new(cfg), |
| 636 | client: Arc::new(client), |
| 637 | submission, |
| 638 | tls, |
| 639 | dkim, |
| 640 | host: Arc::new(host), |
| 641 | failures: Arc::new(Mutex::new(FailureWindow { |
| 642 | count: 0, |
| 643 | opened: Instant::now(), |
| 644 | last: None, |
| 645 | sent: None, |
| 646 | })), |
| 647 | channels: Arc::new(Mutex::new(ChannelBook::default())), |
| 648 | })) |
| 649 | } |
| 650 | |
| 651 | /// Does this alerter have a mail recipient? Without one, a `Notice` event -- distress among |
| 652 | /// them -- has no channel at all. |
| 653 | pub fn sends_mail(&self) -> bool { |
| 654 | !self.cfg.to.is_empty() && !self.cfg.from.is_empty() |
| 655 | } |
| 656 | |
| 657 | /// Does this alerter have a text-message gateway and a number to text? |
| 658 | pub fn sends_sms(&self) -> bool { |
| 659 | self.cfg.sms.as_ref().map(|s| s.enabled && !s.to.is_empty()).unwrap_or(false) |
| 660 | } |
| 661 | |
| 662 | /// The channels an event goes by on this host: mail when there is a recipient, a text when |
| 663 | /// the event is critical and there is a gateway, and never the channel whose failure the |
| 664 | /// event reports. |
| 665 | fn route(&self, event: &AlertEvent) -> Route { |
| 666 | let failing = match event { |
| 667 | AlertEvent::ChannelFailing { channel, .. } => Some(*channel), |
| 668 | _ => None, |
| 669 | }; |
| 670 | Route { |
| 671 | mail: self.sends_mail() && failing != Some(Channel::Mail), |
| 672 | sms: self.sends_sms() |
| 673 | && event.severity() == Severity::Critical |
| 674 | && failing != Some(Channel::Sms), |
| 675 | } |
| 676 | } |
| 677 | |
| 678 | /// Send an alert, without blocking the caller. |
| 679 | /// |
| 680 | /// Delivery runs on its own task. The request path must never wait on an |
| 681 | /// MX lookup and an SMTP round trip, and must never fail because a |
| 682 | /// mail server did not answer -- an alerter that can take the site down |
| 683 | /// is a liability, not a safeguard. |
| 684 | /// |
| 685 | /// An event that no channel on this host carries, such as a notice on a host with no mail |
| 686 | /// recipient, is logged as an error rather than dropped: it is an alert nobody will get. |
| 687 | pub fn raise(&self, event: AlertEvent) { |
| 688 | let route = self.route(&event); |
| 689 | if route.is_empty() { |
| 690 | fault!("ALERT NOT DELIVERED: no channel on this host carries it. {} goes by {}, \ |
| 691 | and this host has {}. The event still happened: {}", |
| 692 | match event.severity() { |
| 693 | Severity::Critical => "A critical alert", |
| 694 | Severity::Notice => "A notice", |
| 695 | }, |
| 696 | match event.severity() { |
| 697 | Severity::Critical => "mail and text message", |
| 698 | Severity::Notice => "mail alone", |
| 699 | }, |
| 700 | match (self.sends_mail(), self.sends_sms()) { |
| 701 | (true, true) => "both, one of which is the channel this reports on", |
| 702 | (true, false) => "mail alone, the channel this reports on", |
| 703 | (false, true) => "no mail recipient", |
| 704 | (false, false) => "neither", |
| 705 | }, |
| 706 | event.subject(&self.host)); |
| 707 | return; |
| 708 | } |
| 709 | let this = self.clone(); |
| 710 | tokio::spawn(async move { |
| 711 | this.deliver(event, route).await; |
| 712 | }); |
| 713 | } |
| 714 | |
| 715 | /// Deliver one event by each channel its route names, and account for each. |
| 716 | /// |
| 717 | /// Both legs are attempted, and neither is allowed to prevent the other. They fail for |
| 718 | /// different reasons -- mail needs a working MX and a mailbox somebody reads, a text needs a |
| 719 | /// funded account and a carrier -- and an alerter that abandoned the second because the |
| 720 | /// first threw would have exactly one channel on the night both were needed. |
| 721 | /// |
| 722 | /// Nor may either wait on the other, so the legs run side by side, each bounded and each |
| 723 | /// settled the moment it ends. Until 2026-09-24 the text went after the mail, and a relay |
| 724 | /// that took the connection and never spoke held it back for ever (D-06 audit A2): a wedged |
| 725 | /// karri would have stopped conifer texting that karri was down. |
| 726 | async fn deliver(&self, event: AlertEvent, route: Route) { |
| 727 | let mail = async { |
| 728 | if !route.mail { |
| 729 | return; |
| 730 | } |
| 731 | let sent = match tokio::time::timeout(MAIL_TIMEOUT, self.send(&event)).await { |
| 732 | Ok(r) => r, |
| 733 | Err(_) => Err(err!( |
| 734 | "The alert mail was not accepted within {}s, so it is not known to have \ |
| 735 | been sent.", MAIL_TIMEOUT.as_secs(); |
| 736 | Network, Timeout)), |
| 737 | }; |
| 738 | match sent { |
| 739 | Ok(()) => self.settle(Channel::Mail, None, &event), |
| 740 | Err(e) => { |
| 741 | self.settle(Channel::Mail, Some(e.plain()), &event); |
| 742 | // Log loudly. This is the case where the operator believes they are |
| 743 | // covered and are not. |
| 744 | error!(e, "ALERT NOT DELIVERED BY MAIL. The event still happened: {}", |
| 745 | event.subject(&self.host)); |
| 746 | }, |
| 747 | } |
| 748 | }; |
| 749 | // Bounded per number by SMS_TIMEOUT, since every number is tried. |
| 750 | let sms = async { |
| 751 | if !route.sms { |
| 752 | return; |
| 753 | } |
| 754 | match self.send_sms(&event).await { |
| 755 | Ok(()) => self.settle(Channel::Sms, None, &event), |
| 756 | Err(e) => { |
| 757 | self.settle(Channel::Sms, Some(e.plain()), &event); |
| 758 | error!(e, "ALERT NOT DELIVERED BY SMS. The event still happened: {}", |
| 759 | event.subject(&self.host)); |
| 760 | }, |
| 761 | } |
| 762 | }; |
| 763 | tokio::join!(mail, sms); |
| 764 | } |
| 765 | |
| 766 | /// Record how a channel did, and raise the report that is owed: a channel that has started |
| 767 | /// failing, is still failing a day on, or has just recovered is told through the others. |
| 768 | fn settle(&self, ch: Channel, failure: Option<String>, event: &AlertEvent) { |
| 769 | let report = { |
| 770 | let mut book = lock_mutex_or_recover!(self.channels, |
| 771 | "The alerter's channel book was poisoned; carrying on with what it held."); |
| 772 | book.note(ch, failure.as_deref(), &event.subject(&self.host), Instant::now()) |
| 773 | }; |
| 774 | if let Some(r) = report { |
| 775 | self.raise(r); |
| 776 | } |
| 777 | } |
| 778 | |
| 779 | /// Send the one-line form of an event as a text message, to every configured number. |
| 780 | /// |
| 781 | /// Every number is tried, and the alert fails if the gateway refused any of them. A text |
| 782 | /// is logged as sent only on the gateway's receipt for it. |
| 783 | /// |
| 784 | /// **The credential is read from the environment, never from the |
| 785 | /// configuration file.** A configuration file is copied between machines, |
| 786 | /// pasted into a chat window to ask why a server will not start, and |
| 787 | /// committed by accident; an environment variable is none of those things |
| 788 | /// by default. The two names are configurable so a host can carry more than |
| 789 | /// one account without a second Steel. |
| 790 | async fn send_sms(&self, event: &AlertEvent) -> Outcome<()> { |
| 791 | let sms = match &self.cfg.sms { |
| 792 | Some(s) if s.enabled => s, |
| 793 | _ => return Err(err!( |
| 794 | "An SMS alert cannot be sent: no SMS gateway is configured."; Configuration, Missing)), |
| 795 | }; |
| 796 | let tls = match &self.tls { |
| 797 | Some(t) => t.clone(), |
| 798 | None => return Err(err!( |
| 799 | "An SMS alert cannot be sent: this Steel has no outbound TLS client, so it \ |
| 800 | cannot reach {}.", sms.provider.id(); |
| 801 | Init, Missing)), |
| 802 | }; |
| 803 | let user = res!(std::env::var(&sms.user_env).map_err(|_| err!( |
| 804 | "The SMS gateway account is read from ${}, which is not set. The alert was not \ |
| 805 | sent as a text.", sms.user_env; |
| 806 | Configuration, Missing))); |
| 807 | let secret = res!(std::env::var(&sms.secret_env).map_err(|_| err!( |
| 808 | "The SMS gateway secret is read from ${}, which is not set. The alert was not \ |
| 809 | sent as a text.", sms.secret_env; |
| 810 | Configuration, Missing))); |
| 811 | |
| 812 | let cred = Credential { user: &user, secret: &secret }; |
| 813 | let text = event.short(&self.host); |
| 814 | let mut outcomes = Vec::with_capacity(sms.to.len()); |
| 815 | for to in &sms.to { |
| 816 | let got = text_one(sms, &cred, to, &text, tls.clone()).await; |
| 817 | outcomes.push((to.clone(), got)); |
| 818 | } |
| 819 | tally_texts(sms.provider, outcomes, &event.subject(&self.host)) |
| 820 | } |
| 821 | |
| 822 | /// Record a failed passphrase attempt, raising a coalesced alert once |
| 823 | /// the burst crosses the configured threshold. |
| 824 | /// |
| 825 | /// One message per burst, then a cooldown. A brute-force attempt must |
| 826 | /// not turn the alerter into a mail flood pointed at the operator. |
| 827 | pub fn note_failed_unseal(&self, peer: SocketAddr) { |
| 828 | if let Some(event) = self.record_failure(peer) { |
| 829 | self.raise(event); |
| 830 | } |
| 831 | } |
| 832 | |
| 833 | /// Compose and send one alert. |
| 834 | /// |
| 835 | /// Posts through the configured provider when there is one, and otherwise |
| 836 | /// delivers straight to the recipient's MX. The provider is the better |
| 837 | /// road: a message that arrives unannounced and unauthenticated from a |
| 838 | /// host with no PTR record is one a strict receiver may bin, and the alert |
| 839 | /// saying something is wrong is precisely the one that must not land in a |
| 840 | /// spam folder. |
| 841 | async fn send(&self, event: &AlertEvent) -> Outcome<()> { |
| 842 | let mut msg = self.compose(event).into_bytes(); |
| 843 | |
| 844 | // Sign with every configured key, as the mail path does. A key that |
| 845 | // will not sign is skipped rather than fatal: an alert that goes out |
| 846 | // unsigned still reaches the operator, and an alert that does not go |
| 847 | // out at all reaches nobody. |
| 848 | let now = match SystemTime::now().duration_since(UNIX_EPOCH) { |
| 849 | Ok(d) => d.as_secs(), |
| 850 | Err(_) => 0, |
| 851 | }; |
| 852 | for signer in &self.dkim { |
| 853 | match signer.sign(&msg, &[], now) { |
| 854 | Ok(b) => msg = b, |
| 855 | Err(e) => warn!("Signing an alert with the {} key for selector \ |
| 856 | '{}' failed; sending it unsigned: {}", |
| 857 | signer.algorithm(), signer.selector(), e), |
| 858 | } |
| 859 | } |
| 860 | let msg = String::from_utf8_lossy(&msg).into_owned(); |
| 861 | let queue_id = match &self.submission { |
| 862 | Some(cfg) => res!(self.client.submit( |
| 863 | cfg, |
| 864 | &self.cfg.from, |
| 865 | &self.cfg.to, |
| 866 | msg.as_bytes(), |
| 867 | ).await), |
| 868 | None => res!(self.client.deliver( |
| 869 | &self.cfg.from, |
| 870 | &self.cfg.to, |
| 871 | msg.as_bytes(), |
| 872 | ).await), |
| 873 | }; |
| 874 | info!("Alert sent ({}): {}", queue_id, event.subject(&self.host)); |
| 875 | Ok(()) |
| 876 | } |
| 877 | |
| 878 | /// Decide whether a failed attempt should raise an alert, and update the |
| 879 | /// window. Separated from [`Self::note_failed_unseal`] so the coalescing |
| 880 | /// rule can be tested without an SMTP server. |
| 881 | fn record_failure(&self, peer: SocketAddr) -> Option<AlertEvent> { |
| 882 | let threshold = self.cfg.failed_threshold; |
| 883 | let cooldown = Duration::from_secs(self.cfg.failed_cooldown_secs); |
| 884 | let window = Duration::from_secs(self.cfg.failed_window_secs); |
| 885 | |
| 886 | let mut f = match self.failures.lock() { |
| 887 | Ok(g) => g, |
| 888 | Err(_) => { |
| 889 | fault!("The alerter's failure-window lock is poisoned; a failed \ |
| 890 | passphrase attempt from {} was not counted.", peer); |
| 891 | return None; |
| 892 | } |
| 893 | }; |
| 894 | // A burst is only a burst if it is recent. An attempt long after the |
| 895 | // last one starts a fresh window rather than topping up a stale count, |
| 896 | // so a slow trickle over weeks does not eventually trip the threshold |
| 897 | // and read as an attack. |
| 898 | if f.opened.elapsed() > window { |
| 899 | f.count = 0; |
| 900 | f.opened = Instant::now(); |
| 901 | } |
| 902 | f.count = f.count.saturating_add(1); |
| 903 | f.last = Some(peer); |
| 904 | |
| 905 | let cooled = match f.sent { |
| 906 | Some(t) => t.elapsed() >= cooldown, |
| 907 | None => true, |
| 908 | }; |
| 909 | if f.count >= threshold && cooled { |
| 910 | let event = AlertEvent::FailedUnseals { |
| 911 | count: f.count, |
| 912 | window_secs: f.opened.elapsed().as_secs(), |
| 913 | last_peer: peer, |
| 914 | }; |
| 915 | f.sent = Some(Instant::now()); |
| 916 | f.count = 0; |
| 917 | f.opened = Instant::now(); |
| 918 | return Some(event); |
| 919 | } |
| 920 | None |
| 921 | } |
| 922 | |
| 923 | /// Build an RFC 5322 message. |
| 924 | fn compose(&self, event: &AlertEvent) -> String { |
| 925 | let date = match SystemTime::now().duration_since(UNIX_EPOCH) { |
| 926 | Ok(d) => fmt!("{}", d.as_secs()), |
| 927 | Err(_) => fmt!("0"), |
| 928 | }; |
| 929 | fmt!( |
| 930 | "From: {from}\r\n\ |
| 931 | To: {to}\r\n\ |
| 932 | Subject: {subject}\r\n\ |
| 933 | X-Steel-Host: {host}\r\n\ |
| 934 | X-Steel-Unix-Time: {date}\r\n\ |
| 935 | Content-Type: text/plain; charset=utf-8\r\n\ |
| 936 | \r\n\ |
| 937 | {body}", |
| 938 | from = self.cfg.from, |
| 939 | to = self.cfg.to.join(", "), |
| 940 | subject = event.subject(&self.host), |
| 941 | host = self.host, |
| 942 | date = date, |
| 943 | body = event.body(&self.host).replace('\n', "\r\n"), |
| 944 | ) |
| 945 | } |
| 946 | } |
| 947 | |
| 948 | |
| 949 | /// Dial the gateway for one number and read its answer, which is a receipt only when the |
| 950 | /// gateway took the message. |
| 951 | async fn text_one( |
| 952 | sms: &SmsAlertConfig, |
| 953 | cred: &Credential<'_>, |
| 954 | to: &str, |
| 955 | text: &str, |
| 956 | tls: Arc<ClientConfig>, |
| 957 | ) |
| 958 | -> Outcome<Receipt> |
| 959 | { |
| 960 | let m = SmsMessage { to, from: &sms.from, body: text }; |
| 961 | let call = res!(sms.provider.request(cred, &m)); |
| 962 | let headers: Vec<(&str, &str)> = call.headers.iter() |
| 963 | .map(|(n, v)| (n.as_str(), v.as_str())) |
| 964 | .collect(); |
| 965 | let dial = https_request( |
| 966 | &call.host, call.port, call.method, &call.path, &headers, &call.body, tls); |
| 967 | let reply = match tokio::time::timeout(SMS_TIMEOUT, dial).await { |
| 968 | Ok(r) => res!(r), |
| 969 | Err(_) => return Err(err!( |
| 970 | "{} did not answer within {}s, so the text is not known to have been sent.", |
| 971 | sms.provider.id(), SMS_TIMEOUT.as_secs(); |
| 972 | Network, Timeout)), |
| 973 | }; |
| 974 | let status = match &reply.header.headline { |
| 975 | HttpHeadline::Response { status } => *status as u16, |
| 976 | // Not a response at all, which is no receipt. |
| 977 | _ => 0, |
| 978 | }; |
| 979 | // The whole answer is read, status and body. Two of these gateways answer a rejected |
| 980 | // credential with 200 and an object explaining themselves, and one answers an unfunded |
| 981 | // account with 200 and a refusal inside the message record, so a sender that trusted the |
| 982 | // status would log every alert as delivered while none of them was. |
| 983 | sms.provider.parse(status, &reply.body) |
| 984 | } |
| 985 | |
| 986 | /// Account for one alert's texts: each accepted text is logged as sent, and the alert fails |
| 987 | /// when any number's was refused or not sent, naming each number and the provider's words. |
| 988 | /// |
| 989 | /// One refusal among several numbers fails the alert. More than one number is configured |
| 990 | /// because any of them may be the one that is read, and a refusal is a person not told. |
| 991 | fn tally_texts( |
| 992 | provider: SmsProvider, |
| 993 | outcomes: Vec<(String, Outcome<Receipt>)>, |
| 994 | subject: &str, |
| 995 | ) |
| 996 | -> Outcome<()> |
| 997 | { |
| 998 | let total = outcomes.len(); |
| 999 | if total == 0 { |
| 1000 | return Err(err!( |
| 1001 | "SMS alerting is enabled with no numbers to send to."; Configuration, Missing)); |
| 1002 | } |
| 1003 | let mut refused = Vec::new(); |
| 1004 | for (to, got) in outcomes { |
| 1005 | match got { |
| 1006 | Ok(r) => info!("Alert texted to {} ({} {}, id {}): {}", |
| 1007 | to, provider.id(), r.status, r.id, subject), |
| 1008 | Err(e) => refused.push(fmt!("{}: {}", to, e.plain())), |
| 1009 | } |
| 1010 | } |
| 1011 | if refused.is_empty() { |
| 1012 | return Ok(()); |
| 1013 | } |
| 1014 | Err(err!("{} of {} text(s) were not sent. {}", refused.len(), total, refused.join("; "); |
| 1015 | Network)) |
| 1016 | } |
| 1017 | |
| 1018 | |
| 1019 | // ┌───────────────────────────────────────────────────────────────────────────┐ |
| 1020 | // │ TESTS │ |
| 1021 | // └───────────────────────────────────────────────────────────────────────────┘ |
| 1022 | |
| 1023 | #[cfg(test)] |
| 1024 | mod tests { |
| 1025 | use super::*; |
| 1026 | |
| 1027 | fn mkalerter(threshold: u32, cooldown_secs: u64) -> Alerter { |
| 1028 | let cfg = AlertConfig { |
| 1029 | enabled: true, |
| 1030 | from: "steel@example.com".to_string(), |
| 1031 | submission: None, |
| 1032 | to: vec!["operator@example.com".to_string()], |
| 1033 | ehlo_hostname: "example.com".to_string(), |
| 1034 | failed_threshold: threshold, |
| 1035 | failed_window_secs: 900, |
| 1036 | failed_cooldown_secs: cooldown_secs, |
| 1037 | sms: None, |
| 1038 | }; |
| 1039 | match Alerter::new(cfg, "example.com".to_string(), Vec::new(), None) { |
| 1040 | Ok(Some(a)) => a, |
| 1041 | _ => panic!("alerter"), |
| 1042 | } |
| 1043 | } |
| 1044 | |
| 1045 | fn peer() -> SocketAddr { |
| 1046 | match "203.0.113.7:44321".parse() { |
| 1047 | Ok(p) => p, |
| 1048 | Err(_) => panic!("peer"), |
| 1049 | } |
| 1050 | } |
| 1051 | |
| 1052 | /// Below the threshold, nothing is raised. One wrong passphrase is a |
| 1053 | /// typo, not an attack, and an operator emailed about typos stops |
| 1054 | /// reading the emails. |
| 1055 | #[test] |
| 1056 | fn test_failures_below_the_threshold_are_silent_00() { |
| 1057 | let a = mkalerter(5, 3600); |
| 1058 | for _ in 0..4 { |
| 1059 | assert!(a.record_failure(peer()).is_none()); |
| 1060 | } |
| 1061 | // The fifth crosses it. |
| 1062 | match a.record_failure(peer()) { |
| 1063 | Some(AlertEvent::FailedUnseals { count, .. }) => assert_eq!(count, 5), |
| 1064 | other => panic!("expected a coalesced alert, got {:?}", other), |
| 1065 | } |
| 1066 | } |
| 1067 | |
| 1068 | /// A brute-force run must produce one message per burst, not one per |
| 1069 | /// guess. An alerter that relays every attempt is an amplifier aimed at |
| 1070 | /// the operator's mailbox, and does the attacker's work for them. |
| 1071 | #[test] |
| 1072 | fn test_a_burst_coalesces_into_one_alert_00() { |
| 1073 | let a = mkalerter(5, 3600); |
| 1074 | let mut raised = 0; |
| 1075 | for _ in 0..100 { |
| 1076 | if a.record_failure(peer()).is_some() { |
| 1077 | raised += 1; |
| 1078 | } |
| 1079 | } |
| 1080 | assert_eq!(raised, 1, |
| 1081 | "100 guesses must yield one alert, not {}", raised); |
| 1082 | } |
| 1083 | |
| 1084 | /// Once the cooldown lapses, a continuing campaign alerts again -- |
| 1085 | /// otherwise a single message would cover an attack running for days. |
| 1086 | #[test] |
| 1087 | fn test_a_lapsed_cooldown_alerts_again_00() { |
| 1088 | let a = mkalerter(2, 0); // zero cooldown: every burst reports |
| 1089 | let mut raised = 0; |
| 1090 | for _ in 0..10 { |
| 1091 | if a.record_failure(peer()).is_some() { |
| 1092 | raised += 1; |
| 1093 | } |
| 1094 | } |
| 1095 | assert_eq!(raised, 5, "ten failures at a threshold of two, no cooldown"); |
| 1096 | } |
| 1097 | |
| 1098 | /// An alerter with nobody to tell is a start-up error, not a quiet |
| 1099 | /// no-op: it looks like cover, and the operator finds out it was not |
| 1100 | /// when something goes wrong and no message arrives. |
| 1101 | #[test] |
| 1102 | fn test_alerting_without_a_recipient_is_refused_00() { |
| 1103 | let cfg = AlertConfig { |
| 1104 | enabled: true, |
| 1105 | from: "steel@example.com".to_string(), |
| 1106 | to: Vec::new(), |
| 1107 | ..Default::default() |
| 1108 | }; |
| 1109 | assert!(Alerter::new(cfg, "example.com".to_string(), Vec::new(), None).is_err()); |
| 1110 | } |
| 1111 | |
| 1112 | /// A stand-in submission server on loopback. For each of `conns` connections in turn it |
| 1113 | /// greets, advertises AUTH, accepts the credential, takes the message and records every |
| 1114 | /// line it was sent. |
| 1115 | fn stand_in_provider(conns: usize) |
| 1116 | -> (SocketAddr, Arc<Mutex<Vec<String>>>, std::thread::JoinHandle<()>) |
| 1117 | { |
| 1118 | use std::io::{BufRead, BufReader, Write}; |
| 1119 | use std::net::TcpListener; |
| 1120 | |
| 1121 | let listener = match TcpListener::bind("127.0.0.1:0") { |
| 1122 | Ok(l) => l, |
| 1123 | Err(e) => panic!("bind: {}", e), |
| 1124 | }; |
| 1125 | let addr = match listener.local_addr() { |
| 1126 | Ok(a) => a, |
| 1127 | Err(e) => panic!("addr: {}", e), |
| 1128 | }; |
| 1129 | let seen = Arc::new(Mutex::new(Vec::<String>::new())); |
| 1130 | let log = seen.clone(); |
| 1131 | let jh = std::thread::spawn(move || { |
| 1132 | for _ in 0..conns { |
| 1133 | let (sock, _) = match listener.accept() { |
| 1134 | Ok(x) => x, |
| 1135 | Err(_) => return, |
| 1136 | }; |
| 1137 | let mut w = match sock.try_clone() { |
| 1138 | Ok(s) => s, |
| 1139 | Err(_) => return, |
| 1140 | }; |
| 1141 | let mut lines = BufReader::new(sock).lines(); |
| 1142 | let _ = w.write_all(b"220 provider.example.com ESMTP\r\n"); |
| 1143 | let mut in_data = false; |
| 1144 | while let Some(Ok(line)) = lines.next() { |
| 1145 | if let Ok(mut g) = log.lock() { |
| 1146 | g.push(line.clone()); |
| 1147 | } |
| 1148 | if in_data { |
| 1149 | if line == "." { |
| 1150 | in_data = false; |
| 1151 | let _ = w.write_all(b"250 2.0.0 Ok: queued as TEST1\r\n"); |
| 1152 | } |
| 1153 | continue; |
| 1154 | } |
| 1155 | let upper = line.to_uppercase(); |
| 1156 | if upper.starts_with("EHLO") { |
| 1157 | let _ = w.write_all( |
| 1158 | b"250-provider.example.com\r\n250-AUTH PLAIN\r\n250 8BITMIME\r\n"); |
| 1159 | } else if upper.starts_with("AUTH PLAIN") { |
| 1160 | let _ = w.write_all(b"235 2.7.0 Accepted\r\n"); |
| 1161 | } else if upper.starts_with("DATA") { |
| 1162 | in_data = true; |
| 1163 | let _ = w.write_all(b"354 End data\r\n"); |
| 1164 | } else if upper.starts_with("QUIT") { |
| 1165 | let _ = w.write_all(b"221 2.0.0 Bye\r\n"); |
| 1166 | break; |
| 1167 | } else { |
| 1168 | let _ = w.write_all(b"250 2.0.0 Ok\r\n"); |
| 1169 | } |
| 1170 | } |
| 1171 | } |
| 1172 | }); |
| 1173 | (addr, seen, jh) |
| 1174 | } |
| 1175 | |
| 1176 | /// An alerter that submits its mail to a stand-in at `addr`, with the given text leg. |
| 1177 | fn alerter_via(addr: SocketAddr, sms: Option<SmsAlertConfig>) -> Alerter { |
| 1178 | let cfg = AlertConfig { |
| 1179 | enabled: true, |
| 1180 | from: "steel@example.com".to_string(), |
| 1181 | submission: Some(crate::srv::cfg::AlertSubmission { |
| 1182 | host: "provider.example.com".to_string(), |
| 1183 | port: addr.port(), |
| 1184 | security: "plain".to_string(), |
| 1185 | user: "steel@example.com".to_string(), |
| 1186 | password: "app-password".to_string(), |
| 1187 | }), |
| 1188 | to: vec!["operator@elsewhere.example".to_string()], |
| 1189 | ehlo_hostname: "example.com".to_string(), |
| 1190 | failed_threshold: 5, |
| 1191 | failed_window_secs: 900, |
| 1192 | failed_cooldown_secs: 3600, |
| 1193 | sms, |
| 1194 | }; |
| 1195 | let mut a = match Alerter::new(cfg, "example.com".to_string(), Vec::new(), None) { |
| 1196 | Ok(Some(a)) => a, |
| 1197 | other => panic!("alerter: {:?}", other.is_err()), |
| 1198 | }; |
| 1199 | // Pin the dialled address: the certificate name stays |
| 1200 | // provider.example.com, and no resolver is involved. |
| 1201 | a.submission = a.submission.map(|s| { |
| 1202 | Arc::new((*s).clone().with_addr(addr)) |
| 1203 | }); |
| 1204 | a |
| 1205 | } |
| 1206 | |
| 1207 | /// The whole of what a stand-in was sent, one line per line. |
| 1208 | fn transcript(seen: &Arc<Mutex<Vec<String>>>) -> String { |
| 1209 | match seen.lock() { |
| 1210 | Ok(g) => g.join("\n"), |
| 1211 | Err(_) => panic!("lock"), |
| 1212 | } |
| 1213 | } |
| 1214 | |
| 1215 | fn runtime() -> tokio::runtime::Runtime { |
| 1216 | match tokio::runtime::Builder::new_current_thread().enable_all().build() { |
| 1217 | Ok(r) => r, |
| 1218 | Err(e) => panic!("runtime: {}", e), |
| 1219 | } |
| 1220 | } |
| 1221 | |
| 1222 | /// End to end, through a stand-in provider: the alert must actually be |
| 1223 | /// composed, authenticated and submitted, and arrive with the event in |
| 1224 | /// it. Alerting's failure mode is *looking like cover* -- an operator who |
| 1225 | /// believes they will be told, and is not -- so it is worth proving the |
| 1226 | /// message leaves the building rather than only that the code was called. |
| 1227 | #[test] |
| 1228 | fn test_an_alert_is_submitted_through_a_provider_00() { |
| 1229 | let (addr, seen, jh) = stand_in_provider(1); |
| 1230 | let a = alerter_via(addr, None); |
| 1231 | let ev = AlertEvent::SealedStart { db_count: 3 }; |
| 1232 | match runtime().block_on(a.send(&ev)) { |
| 1233 | Ok(()) => (), |
| 1234 | Err(e) => panic!("the alert was not submitted: {}", e), |
| 1235 | } |
| 1236 | let _ = jh.join(); |
| 1237 | |
| 1238 | let transcript = transcript(&seen); |
| 1239 | assert!(transcript.contains("AUTH PLAIN"), |
| 1240 | "the alerter must authenticate to the provider:\n{}", transcript); |
| 1241 | assert!(transcript.contains("MAIL FROM:<steel@example.com>"), |
| 1242 | "envelope sender missing:\n{}", transcript); |
| 1243 | assert!(transcript.contains("RCPT TO:<operator@elsewhere.example>"), |
| 1244 | "envelope recipient missing:\n{}", transcript); |
| 1245 | assert!(transcript.contains("SEALED at start -- 3 database(s) shut"), |
| 1246 | "the event did not reach the message:\n{}", transcript); |
| 1247 | } |
| 1248 | |
| 1249 | fn texting(to: &str) -> SmsAlertConfig { |
| 1250 | SmsAlertConfig { |
| 1251 | enabled: true, |
| 1252 | to: vec![to.to_string()], |
| 1253 | ..Default::default() |
| 1254 | } |
| 1255 | } |
| 1256 | |
| 1257 | /// An alerter with or without each channel. Nothing is dialled by one of these. |
| 1258 | fn alerter_with(mail: bool, sms: bool) -> Alerter { |
| 1259 | let cfg = AlertConfig { |
| 1260 | enabled: true, |
| 1261 | from: if mail { fmt!("steel@example.com") } else { String::new() }, |
| 1262 | to: if mail { vec![fmt!("operator@example.com")] } else { Vec::new() }, |
| 1263 | sms: if sms { Some(texting("+61400000000")) } else { None }, |
| 1264 | ..Default::default() |
| 1265 | }; |
| 1266 | match Alerter::new(cfg, "karri".to_string(), Vec::new(), None) { |
| 1267 | Ok(Some(a)) => a, |
| 1268 | other => panic!("alerter: {:?}", other.is_err()), |
| 1269 | } |
| 1270 | } |
| 1271 | |
| 1272 | fn jarrah_down() -> AlertEvent { |
| 1273 | AlertEvent::PeerDown { |
| 1274 | peer: fmt!("jarrah"), |
| 1275 | url: fmt!("https://daimond.oxedyne.com/api/health"), |
| 1276 | failures: 3, |
| 1277 | down_secs: 180, |
| 1278 | noticed_by: fmt!("karri"), |
| 1279 | } |
| 1280 | } |
| 1281 | |
| 1282 | fn failing(channel: Channel) -> AlertEvent { |
| 1283 | AlertEvent::ChannelFailing { |
| 1284 | channel, |
| 1285 | reason: fmt!("refused"), |
| 1286 | missed: fmt!("[steel:karri] jarrah IS DOWN (3m)"), |
| 1287 | failures: 1, |
| 1288 | since_secs: 0, |
| 1289 | } |
| 1290 | } |
| 1291 | |
| 1292 | // A text refused for want of credit, in the shape ClickSend answered with from 2026-08-31: |
| 1293 | // success for the call, a refusal for the message. |
| 1294 | const UNFUNDED: &[u8] = br#"{"http_code":200,"response_code":"SUCCESS", |
| 1295 | "response_msg":"Messages queued for delivery.","data":{"total_price":0,"total_count":1, |
| 1296 | "queued_count":0,"messages":[{"direction":"out","to":"+61400000000", |
| 1297 | "body":"jarrah is DOWN","from":"","message_id":"4C1F2D3E","message_parts":1, |
| 1298 | "message_price":"0.0000","status":"INSUFFICIENT_CREDIT"}]}}"#; |
| 1299 | const ACCEPTED: &[u8] = br#"{"http_code":200,"response_code":"SUCCESS","data":{"messages":[ |
| 1300 | {"message_id":"ABC-123","status":"SUCCESS","message_parts":1,"message_price":"0.0790"}]}}"#; |
| 1301 | |
| 1302 | /// The failure path from the gateway's own answer. An unfunded account's refusal, read by |
| 1303 | /// the parser the text leg uses, fails the alert and names the number and the vendor's |
| 1304 | /// status, and the report it is owed carries those words by mail. |
| 1305 | #[test] |
| 1306 | fn test_a_refused_text_fails_the_alert_and_is_reported_by_mail_00() { |
| 1307 | let subject = "[steel:karri] jarrah IS DOWN (3m)"; |
| 1308 | let refused = SmsProvider::ClickSend.parse(200, UNFUNDED); |
| 1309 | assert!(refused.is_err(), "an unfunded account's answer read as a receipt"); |
| 1310 | let words = match tally_texts( |
| 1311 | SmsProvider::ClickSend, vec![(fmt!("+61400000000"), refused)], subject) |
| 1312 | { |
| 1313 | Ok(()) => panic!("a refused text was counted as sent"), |
| 1314 | Err(e) => e.plain(), |
| 1315 | }; |
| 1316 | assert!(words.contains("1 of 1") && words.contains("+61400000000") |
| 1317 | && words.contains("INSUFFICIENT_CREDIT"), |
| 1318 | "the failure must name the number and the vendor's status: {}", words); |
| 1319 | |
| 1320 | // One refusal among two numbers still fails the alert: that number's person was not told. |
| 1321 | let outcomes = vec![ |
| 1322 | (fmt!("+61400000001"), SmsProvider::ClickSend.parse(200, ACCEPTED)), |
| 1323 | (fmt!("+61400000000"), SmsProvider::ClickSend.parse(200, UNFUNDED)), |
| 1324 | ]; |
| 1325 | match tally_texts(SmsProvider::ClickSend, outcomes, subject) { |
| 1326 | Ok(()) => panic!("a refusal among several numbers was counted as sent"), |
| 1327 | Err(e) => assert!(e.plain().contains("1 of 2"), "{}", e.plain()), |
| 1328 | } |
| 1329 | let taken = vec![(fmt!("+61400000001"), SmsProvider::ClickSend.parse(200, ACCEPTED))]; |
| 1330 | assert!(tally_texts(SmsProvider::ClickSend, taken, subject).is_ok()); |
| 1331 | |
| 1332 | // The report goes by mail, never by the channel that refused, and says why. |
| 1333 | let both = alerter_with(true, true); |
| 1334 | let mut book = ChannelBook::default(); |
| 1335 | let report = match book.note(Channel::Sms, Some(&words), subject, Instant::now()) { |
| 1336 | Some(r) => r, |
| 1337 | None => panic!("the first refusal of a run owes a report"), |
| 1338 | }; |
| 1339 | assert_eq!(both.route(&report), Route { mail: true, sms: false }); |
| 1340 | let body = report.body("karri"); |
| 1341 | assert!(body.contains("INSUFFICIENT_CREDIT") && body.contains("jarrah IS DOWN"), |
| 1342 | "the report must carry the vendor's words and the alert that was lost:\n{}", body); |
| 1343 | } |
| 1344 | |
| 1345 | /// A channel's run of failures is told when it starts, once a day while it lasts, and when |
| 1346 | /// it ends -- not once per failure, and not for the other channel. |
| 1347 | #[test] |
| 1348 | fn test_a_channel_run_is_told_once_reminded_daily_and_closed_00() { |
| 1349 | let mut book = ChannelBook::default(); |
| 1350 | let t0 = Instant::now(); |
| 1351 | let at = |s: u64| t0 + Duration::from_secs(s); |
| 1352 | let why = "clicksend refused the message: INSUFFICIENT_CREDIT"; |
| 1353 | |
| 1354 | assert!(book.note(Channel::Sms, None, "x", t0).is_none(), "a working channel owes nothing"); |
| 1355 | match book.note(Channel::Sms, Some(why), "[steel:karri] jarrah IS DOWN (3m)", at(60)) { |
| 1356 | Some(AlertEvent::ChannelFailing { |
| 1357 | channel: Channel::Sms, reason, missed, failures: 1, since_secs: 0, |
| 1358 | }) => { |
| 1359 | assert_eq!(reason, why); |
| 1360 | assert_eq!(missed, "[steel:karri] jarrah IS DOWN (3m)"); |
| 1361 | }, |
| 1362 | other => panic!("the first failure must be reported, got {:?}", other), |
| 1363 | } |
| 1364 | for i in 2..10u64 { |
| 1365 | assert!(book.note(Channel::Sms, Some(why), "x", at(60 + i * 900)).is_none(), |
| 1366 | "failure {} of a run already told was reported again", i); |
| 1367 | } |
| 1368 | assert!(book.note(Channel::Mail, None, "x", at(9_000)).is_none(), |
| 1369 | "mail working says nothing about the texts"); |
| 1370 | match book.note(Channel::Sms, Some(why), "y", at(60 + 86_400)) { |
| 1371 | Some(AlertEvent::ChannelFailing { failures: 10, since_secs: 86_400, missed, .. }) => |
| 1372 | assert_eq!(missed, "y", "a reminder names the latest alert lost"), |
| 1373 | other => panic!("a day on, a failure must remind, got {:?}", other), |
| 1374 | } |
| 1375 | assert!(book.note(Channel::Sms, Some(why), "z", at(60 + 86_401)).is_none()); |
| 1376 | match book.note(Channel::Sms, None, "w", at(60 + 90_000)) { |
| 1377 | Some(AlertEvent::ChannelRestored { channel: Channel::Sms, lost: 11, were_secs: 90_000 }) => (), |
| 1378 | other => panic!("the first success after a run must close it, got {:?}", other), |
| 1379 | } |
| 1380 | assert!(book.note(Channel::Sms, None, "v", at(60 + 90_060)).is_none(), "the run is over"); |
| 1381 | } |
| 1382 | |
| 1383 | /// Mail when there is a recipient, a text for what is critical, and never the channel whose |
| 1384 | /// failure is the news. With no mail, a notice has nowhere to go. |
| 1385 | #[test] |
| 1386 | fn test_each_event_goes_by_the_channels_it_should_00() { |
| 1387 | let both = alerter_with(true, true); |
| 1388 | let texts_only = alerter_with(false, true); |
| 1389 | let distress = AlertEvent::PeerDistress { |
| 1390 | peer: fmt!("jarrah"), |
| 1391 | url: fmt!("https://oxedyne.com/_steel/health"), |
| 1392 | classes: fmt!("swap_pct 31"), |
| 1393 | since_secs: 180, |
| 1394 | noticed_by: fmt!("conifer"), |
| 1395 | }; |
| 1396 | let restored = AlertEvent::ChannelRestored { channel: Channel::Sms, lost: 3, were_secs: 600 }; |
| 1397 | |
| 1398 | assert_eq!(both.route(&jarrah_down()), Route { mail: true, sms: true }); |
| 1399 | assert_eq!(both.route(&distress), Route { mail: true, sms: false }); |
| 1400 | assert_eq!(both.route(&failing(Channel::Sms)), Route { mail: true, sms: false }, |
| 1401 | "failing texts are told by mail"); |
| 1402 | assert_eq!(both.route(&failing(Channel::Mail)), Route { mail: false, sms: true }, |
| 1403 | "failing mail is told by text"); |
| 1404 | assert_eq!(both.route(&restored), Route { mail: true, sms: false }); |
| 1405 | |
| 1406 | assert_eq!(texts_only.route(&jarrah_down()), Route { mail: false, sms: true }); |
| 1407 | assert!(texts_only.route(&distress).is_empty()); |
| 1408 | assert!(texts_only.route(&AlertEvent::SealedStart { db_count: 1 }).is_empty()); |
| 1409 | assert!(texts_only.route(&failing(Channel::Sms)).is_empty()); |
| 1410 | } |
| 1411 | |
| 1412 | /// An event with no channel is logged as undelivered and goes no further. There is no |
| 1413 | /// runtime here, so a delivery task spawned for it would panic. |
| 1414 | #[test] |
| 1415 | fn test_an_event_with_no_channel_is_not_sent_anywhere_00() { |
| 1416 | let texts_only = alerter_with(false, true); |
| 1417 | texts_only.raise(AlertEvent::SealedStart { db_count: 2 }); |
| 1418 | } |
| 1419 | |
| 1420 | /// THE PATH THAT WAS MISSING: a text that fails is reported by mail. The text leg here |
| 1421 | /// fails for want of an outbound TLS client, a failure met before any gateway is asked. The |
| 1422 | /// alert still goes by mail, and a second message follows it saying the texts are failing, |
| 1423 | /// why, and which alert was lost. |
| 1424 | #[test] |
| 1425 | fn test_a_failed_text_is_reported_by_mail_00() { |
| 1426 | let (addr, seen, jh) = stand_in_provider(2); |
| 1427 | let a = alerter_via(addr, Some(texting("+61400000000"))); |
| 1428 | let subjects = |seen: &Arc<Mutex<Vec<String>>>| -> usize { |
| 1429 | match seen.lock() { |
| 1430 | Ok(g) => g.iter().filter(|l| l.starts_with("Subject:")).count(), |
| 1431 | Err(_) => 0, |
| 1432 | } |
| 1433 | }; |
| 1434 | runtime().block_on(async { |
| 1435 | a.raise(AlertEvent::PeerDown { |
| 1436 | peer: fmt!("birch"), |
| 1437 | url: fmt!("https://oregami.oxegen.io/health"), |
| 1438 | failures: 3, |
| 1439 | down_secs: 180, |
| 1440 | noticed_by: fmt!("conifer"), |
| 1441 | }); |
| 1442 | let deadline = Instant::now() + Duration::from_secs(20); |
| 1443 | while subjects(&seen) < 2 && Instant::now() < deadline { |
| 1444 | tokio::time::sleep(Duration::from_millis(50)).await; |
| 1445 | } |
| 1446 | }); |
| 1447 | let transcript = transcript(&seen); |
| 1448 | assert!(transcript.contains("Subject: [steel:example.com] birch IS DOWN (3m)"), |
| 1449 | "the alert itself must still go by mail:\n{}", transcript); |
| 1450 | assert!(transcript.contains("Subject: [steel:example.com] text alerts FAILING (1 in a row)"), |
| 1451 | "the failed text must be reported by mail:\n{}", transcript); |
| 1452 | assert!(transcript.contains("no outbound TLS client"), |
| 1453 | "the report must say why:\n{}", transcript); |
| 1454 | assert!(transcript.contains("Last lost: [steel:example.com] birch IS DOWN (3m)"), |
| 1455 | "the report must name the alert that was lost:\n{}", transcript); |
| 1456 | let _ = jh.join(); |
| 1457 | } |
| 1458 | |
| 1459 | /// THE STALL THAT WAS D-06 A2: the relay takes the connection and never speaks, as a wedged |
| 1460 | /// karri would while its kernel still completes handshakes. The text leg must settle while |
| 1461 | /// the mail leg is still waiting, not after it; the mail leg must then end as a failure; and |
| 1462 | /// that failure must be told by text. The text leg fails here for want of an outbound TLS |
| 1463 | /// client, a failure met before any gateway is asked, which is enough to show it ran. |
| 1464 | #[test] |
| 1465 | fn test_a_stuck_mail_leg_does_not_delay_the_text_leg_00() { |
| 1466 | let rt = runtime(); |
| 1467 | let a = rt.block_on(async { |
| 1468 | let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await { |
| 1469 | Ok(l) => l, |
| 1470 | Err(e) => panic!("bind: {}", e), |
| 1471 | }; |
| 1472 | let addr = match listener.local_addr() { |
| 1473 | Ok(a) => a, |
| 1474 | Err(e) => panic!("addr: {}", e), |
| 1475 | }; |
| 1476 | // Every connection is held open, and nothing is ever written to it. |
| 1477 | tokio::spawn(async move { |
| 1478 | let mut held = Vec::new(); |
| 1479 | while let Ok((sock, _)) = listener.accept().await { |
| 1480 | held.push(sock); |
| 1481 | } |
| 1482 | }); |
| 1483 | let mut a = alerter_via(addr, Some(texting("+61400000000"))); |
| 1484 | // Each SMTP step waits this long, so the mail leg is still stuck when the text leg |
| 1485 | // settles unless the text waited behind it. |
| 1486 | let stuck = Duration::from_secs(3); |
| 1487 | a.submission = a.submission.map(|s| Arc::new((*s).clone().with_timeout(stuck))); |
| 1488 | a |
| 1489 | }); |
| 1490 | let book = |a: &Alerter| -> (Option<Failing>, Option<Failing>) { |
| 1491 | match a.channels.lock() { |
| 1492 | Ok(g) => (g.mail.clone(), g.sms.clone()), |
| 1493 | Err(_) => panic!("the channel book was poisoned"), |
| 1494 | } |
| 1495 | }; |
| 1496 | rt.block_on(async { |
| 1497 | let start = Instant::now(); |
| 1498 | a.raise(AlertEvent::PeerDown { |
| 1499 | peer: fmt!("karri"), |
| 1500 | url: fmt!("https://mail.oxegen.io/health"), |
| 1501 | failures: 3, |
| 1502 | down_secs: 180, |
| 1503 | noticed_by: fmt!("conifer"), |
| 1504 | }); |
| 1505 | let deadline = start + Duration::from_secs(20); |
| 1506 | let (mail, sms) = loop { |
| 1507 | let (mail, sms) = book(&a); |
| 1508 | if sms.is_some() || Instant::now() > deadline { |
| 1509 | break (mail, sms); |
| 1510 | } |
| 1511 | tokio::time::sleep(Duration::from_millis(10)).await; |
| 1512 | }; |
| 1513 | assert!(sms.is_some(), "the text leg never settled while the mail leg was stuck"); |
| 1514 | assert!(mail.is_none(), |
| 1515 | "the text leg settled only after the mail leg had ended, {:?} in: it waited \ |
| 1516 | behind a relay that never spoke", start.elapsed()); |
| 1517 | |
| 1518 | // The mail leg ends, as a failure, and the failure is told by the other channel. |
| 1519 | loop { |
| 1520 | let (mail, sms) = book(&a); |
| 1521 | let told = sms.map(|f| f.failures >= 2).unwrap_or(false); |
| 1522 | if (mail.is_some() && told) || Instant::now() > deadline { |
| 1523 | assert!(mail.is_some(), "the stuck mail leg never ended"); |
| 1524 | assert!(told, "the failing mail was not told by text"); |
| 1525 | break; |
| 1526 | } |
| 1527 | tokio::time::sleep(Duration::from_millis(10)).await; |
| 1528 | } |
| 1529 | }); |
| 1530 | } |
| 1531 | |
| 1532 | /// The message must never carry an action link. An authorisation that |
| 1533 | /// arrives by email is one that anybody able to read, spoof or replay |
| 1534 | /// the email holds too. |
| 1535 | #[test] |
| 1536 | fn test_the_email_notifies_but_never_authorises_00() { |
| 1537 | let ev = AlertEvent::SealedStart { db_count: 2 }; |
| 1538 | let body = ev.body("example.com"); |
| 1539 | assert!(body.contains("https://example.com/admin"), |
| 1540 | "the mail should point the operator at the dashboard"); |
| 1541 | for bait in ["token=", "approve", "confirm=", "unseal?key", "click here"] { |
| 1542 | assert!(!body.to_lowercase().contains(bait), |
| 1543 | "the alert must not carry an authorising link ({})", bait); |
| 1544 | } |
| 1545 | } |
| 1546 | } |