oxedyne/fe2o3/fe2o3_sys/src/supervise.rs
27.9 KiB, 5 runs
created by r1870400018:75883, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! Supervision of a child process: a memory cap it cannot pass, a death that follows its parent's, |
| 2 | //! and a budget on how often it may be restarted. |
| 3 | //! |
| 4 | //! **The cap.** Given a cgroup v2 root delegated to the caller with the `memory` controller (a |
| 5 | //! systemd unit or scope with `Delegate=yes`), [`Confined`] makes a child cgroup under it, sets |
| 6 | //! its `memory.max`, a `memory.swap.max` of 0 so that the cap is not met by swapping, and |
| 7 | //! `memory.oom.group`, so that an out-of-memory kill takes every process in it. The child enters |
| 8 | //! the cgroup before the program runs: it starts as `/bin/sh`, which writes itself into the |
| 9 | //! cgroup's `cgroup.procs`, sets any address-space limit, and then replaces itself with the |
| 10 | //! program, keeping its process number. When it ends, [`Supervised::wait`] reads the cgroup's |
| 11 | //! `memory.events` to report whether the kernel killed it for memory, kills anything it left |
| 12 | //! behind, and removes the cgroup. The root is a parameter: [`CgroupRoot::open`] takes it as |
| 13 | //! given, and [`CgroupRoot::from_own_cgroup`] makes one of the caller's own delegated cgroup. |
| 14 | //! |
| 15 | //! Where there is no delegated root, the cap falls back to `RLIMIT_AS`, the address-space limit, |
| 16 | //! which an allocation meets as a failure rather than a kill. |
| 17 | //! |
| 18 | //! **The parent's death.** A child that is itself a program of ours calls [`arm_parent_death`] |
| 19 | //! first thing, to have the kernel send it a signal when its parent ends. The kernel sends it when |
| 20 | //! the *thread* that started the child ends, so a supervisor starts its children from a thread |
| 21 | //! that lives as long as it does. The parent's number travels in [`SUPERVISOR_ENV`], so a child |
| 22 | //! whose parent ended before it was armed finds out and is told so. |
| 23 | //! |
| 24 | //! **The restart budget.** [`RestartBudget`] allows at most `n` restarts in a sliding window, on a |
| 25 | //! clock the caller passes in. |
| 26 | //! |
| 27 | //! The system calls come from `nix`, which owns the `unsafe` this crate forbids. |
| 28 | |
| 29 | use crate::signal::Signal; |
| 30 | |
| 31 | use oxedyne_fe2o3_core::prelude::*; |
| 32 | |
| 33 | use nix::sys::prctl::set_pdeathsig; |
| 34 | use nix::unistd::getppid; |
| 35 | |
| 36 | use std::{ |
| 37 | collections::VecDeque, |
| 38 | env, |
| 39 | ffi::OsStr, |
| 40 | fs, |
| 41 | io::ErrorKind, |
| 42 | path::{ |
| 43 | Path, |
| 44 | PathBuf, |
| 45 | }, |
| 46 | process::{ |
| 47 | Child, |
| 48 | Command, |
| 49 | ExitStatus, |
| 50 | }, |
| 51 | thread, |
| 52 | time::{ |
| 53 | Duration, |
| 54 | Instant, |
| 55 | }, |
| 56 | }; |
| 57 | |
| 58 | /// Where the cgroup v2 hierarchy is mounted. |
| 59 | pub const CGROUP_FS: &str = "/sys/fs/cgroup"; |
| 60 | |
| 61 | /// The variable through which a child learns its supervisor's process number. |
| 62 | pub const SUPERVISOR_ENV: &str = "FE2O3_SUPERVISOR_PID"; |
| 63 | |
| 64 | // The shell that enters the cgroup, sets the address-space limit and becomes the program. Its |
| 65 | // arguments are the cgroup directory or nothing, the limit in KiB or nothing, then the program and |
| 66 | // its arguments. |
| 67 | const TRAMPOLINE: &str = "cg=$1; kb=$2; shift 2 |
| 68 | if [ -n \"$cg\" ]; then echo $$ > \"$cg/cgroup.procs\" || exit 126; fi |
| 69 | if [ -n \"$kb\" ]; then ulimit -v \"$kb\" || exit 126; fi |
| 70 | exec \"$@\" |
| 71 | "; |
| 72 | |
| 73 | // How long a finished child's cgroup is given to empty before it is removed. |
| 74 | const EMPTYING: Duration = Duration::from_secs(5); |
| 75 | const NAP: Duration = Duration::from_millis(10); |
| 76 | |
| 77 | fn io<T>(r: std::io::Result<T>, what: &str, path: &Path) -> Outcome<T> { |
| 78 | match r { |
| 79 | Ok(v) => Ok(v), |
| 80 | Err(e) => Err(err!("Cannot {} {}: {}.", what, path.display(), e; IO, File)), |
| 81 | } |
| 82 | } |
| 83 | |
| 84 | // A cgroup name is one path component the caller chose, never a way up or across. |
| 85 | fn check_name(name: &str) -> Outcome<()> { |
| 86 | if name.is_empty() || name == "." || name == ".." || name.contains('/') || name.contains('\0') { |
| 87 | return Err(err!( |
| 88 | "The cgroup name {:?} is not one path component.", name; |
| 89 | Invalid, Input, Name)); |
| 90 | } |
| 91 | Ok(()) |
| 92 | } |
| 93 | |
| 94 | /// The caller's own cgroup v2 directory, read from `/proc/self/cgroup`. |
| 95 | pub fn own_cgroup() -> Outcome<PathBuf> { |
| 96 | let src = Path::new("/proc/self/cgroup"); |
| 97 | let text = res!(io(fs::read_to_string(src), "read", src)); |
| 98 | match text.lines().find_map(|l| l.strip_prefix("0::")) { |
| 99 | Some(rel) => Ok(Path::new(CGROUP_FS).join(rel.trim().trim_start_matches('/'))), |
| 100 | None => Err(err!( |
| 101 | "The caller is not in a cgroup v2 hierarchy: {:?}.", text; |
| 102 | Missing, System)), |
| 103 | } |
| 104 | } |
| 105 | |
| 106 | /// A cgroup v2 directory delegated to the caller, under which each child gets a cgroup of its own |
| 107 | /// with the `memory` controller. |
| 108 | #[derive(Clone, Debug, Eq, PartialEq)] |
| 109 | pub struct CgroupRoot { |
| 110 | path: PathBuf, |
| 111 | } |
| 112 | |
| 113 | impl CgroupRoot { |
| 114 | /// Takes `path` as the root, enabling the `memory` controller for its children where it is not |
| 115 | /// already. It refuses a root whose parent does not offer `memory`, and one that still holds |
| 116 | /// processes of its own, which cgroup v2 forbids from enabling it. |
| 117 | pub fn open<P: AsRef<Path>>(path: P) -> Outcome<Self> { |
| 118 | let path = path.as_ref().to_path_buf(); |
| 119 | let controllers = path.join("cgroup.controllers"); |
| 120 | let offered = res!(io(fs::read_to_string(&controllers), "read", &controllers)); |
| 121 | if !offered.split_whitespace().any(|c| c == "memory") { |
| 122 | return Err(err!( |
| 123 | "The cgroup {} is not given the memory controller ({:?}); it must be delegated, \ |
| 124 | as by a systemd unit or scope with Delegate=yes.", path.display(), offered.trim(); |
| 125 | Missing, System)); |
| 126 | } |
| 127 | let subtree = path.join("cgroup.subtree_control"); |
| 128 | let enabled = res!(io(fs::read_to_string(&subtree), "read", &subtree)); |
| 129 | if !enabled.split_whitespace().any(|c| c == "memory") { |
| 130 | if let Err(e) = fs::write(&subtree, "+memory") { |
| 131 | return Err(err!( |
| 132 | "Cannot enable the memory controller below {}: {}. A cgroup that holds \ |
| 133 | processes of its own cannot; CgroupRoot::from_own_cgroup moves them to a leaf.", |
| 134 | path.display(), e; |
| 135 | IO, System)); |
| 136 | } |
| 137 | } |
| 138 | Ok(Self { path }) |
| 139 | } |
| 140 | |
| 141 | /// Makes a root of the caller's own cgroup, which must be delegated to it: every process in it, |
| 142 | /// the caller among them, moves to the child cgroup `leaf`, and the cgroup is then opened as the |
| 143 | /// root. A caller already in a cgroup named `leaf` opens that cgroup's parent, so a second call, |
| 144 | /// or a restart in place, finds the same root. |
| 145 | pub fn from_own_cgroup(leaf: &str) -> Outcome<Self> { |
| 146 | res!(check_name(leaf)); |
| 147 | let own = res!(own_cgroup()); |
| 148 | if own.file_name() == Some(OsStr::new(leaf)) { |
| 149 | return match own.parent() { |
| 150 | Some(p) => Self::open(p), |
| 151 | None => Err(err!("The cgroup {} has no parent.", own.display(); Missing, Path)), |
| 152 | }; |
| 153 | } |
| 154 | let leaf_dir = own.join(leaf); |
| 155 | if let Err(e) = fs::create_dir(&leaf_dir) { |
| 156 | if e.kind() != ErrorKind::AlreadyExists { |
| 157 | return Err(err!("Cannot make the cgroup {}: {}.", leaf_dir.display(), e; IO, Create)); |
| 158 | } |
| 159 | } |
| 160 | let procs = own.join("cgroup.procs"); |
| 161 | let into = leaf_dir.join("cgroup.procs"); |
| 162 | // A process forked while the others move lands in the old cgroup, so read until it is empty. |
| 163 | for _ in 0..8 { |
| 164 | let text = res!(io(fs::read_to_string(&procs), "read", &procs)); |
| 165 | let pids: Vec<&str> = text.split_whitespace().collect(); |
| 166 | if pids.is_empty() { |
| 167 | return Self::open(own); |
| 168 | } |
| 169 | for pid in pids { |
| 170 | if let Err(e) = fs::write(&into, pid) { |
| 171 | // A process that ended meanwhile is no longer there to move. |
| 172 | if e.raw_os_error() != Some(3) { |
| 173 | return Err(err!( |
| 174 | "Cannot move process {} into {}: {}.", pid, leaf_dir.display(), e; |
| 175 | IO, System)); |
| 176 | } |
| 177 | } |
| 178 | } |
| 179 | } |
| 180 | Err(err!("The cgroup {} kept gaining processes as they were moved out.", own.display(); |
| 181 | Excessive, System)) |
| 182 | } |
| 183 | |
| 184 | pub fn path(&self) -> &Path { |
| 185 | &self.path |
| 186 | } |
| 187 | } |
| 188 | |
| 189 | /// The memory cap a child is held to. |
| 190 | #[derive(Clone, Debug)] |
| 191 | pub struct Limit<'a> { |
| 192 | /// The cap in bytes: the cgroup's `memory.max`, or without a cgroup the address-space limit. |
| 193 | pub memory_max: u64, |
| 194 | /// The root and the name of the child's cgroup below it, if there is one. |
| 195 | pub cgroup: Option<(&'a CgroupRoot, String)>, |
| 196 | /// An address-space limit in bytes to set as well as the cgroup's cap. Without a cgroup the |
| 197 | /// limit is this, or failing it `memory_max`. |
| 198 | pub rlimit_as: Option<u64>, |
| 199 | } |
| 200 | |
| 201 | impl<'a> Limit<'a> { |
| 202 | /// A cap of `memory_max` bytes in the cgroup `name` below `root`. |
| 203 | pub fn cgroup(root: &'a CgroupRoot, name: &str, memory_max: u64) -> Self { |
| 204 | Self { memory_max, cgroup: Some((root, name.to_string())), rlimit_as: None } |
| 205 | } |
| 206 | |
| 207 | /// A cap of `memory_max` bytes of address space, the fallback where no root is delegated. |
| 208 | pub fn rlimit(memory_max: u64) -> Self { |
| 209 | Self { memory_max, cgroup: None, rlimit_as: None } |
| 210 | } |
| 211 | |
| 212 | /// Sets an address-space limit as well, in bytes. |
| 213 | pub fn with_rlimit_as(mut self, bytes: u64) -> Self { |
| 214 | self.rlimit_as = Some(bytes); |
| 215 | self |
| 216 | } |
| 217 | } |
| 218 | |
| 219 | /// A child ready to start under its cap. Its [`Command`] takes further arguments, environment, |
| 220 | /// a working directory and its standard streams before [`Confined::spawn`] starts it. A child |
| 221 | /// never started has its cgroup removed when this is dropped. |
| 222 | #[derive(Debug)] |
| 223 | pub struct Confined { |
| 224 | cmd: Command, |
| 225 | cgroup: Option<PathBuf>, |
| 226 | } |
| 227 | |
| 228 | impl Confined { |
| 229 | /// Makes the child's cgroup, if the limit names one, and the command that starts `program` |
| 230 | /// inside it. |
| 231 | pub fn new<P: AsRef<OsStr>>(program: P, limit: &Limit) -> Outcome<Self> { |
| 232 | let cgroup = match &limit.cgroup { |
| 233 | Some((root, name)) => Some(res!(Self::make_cgroup(root, name, limit.memory_max))), |
| 234 | None => None, |
| 235 | }; |
| 236 | let rlimit = match (&cgroup, limit.rlimit_as) { |
| 237 | (_, Some(b)) => Some(b), |
| 238 | (None, None) => Some(limit.memory_max), |
| 239 | (Some(_), None) => None, |
| 240 | }; |
| 241 | let mut cmd = Command::new("/bin/sh"); |
| 242 | cmd.arg("-c").arg(TRAMPOLINE).arg("sh"); |
| 243 | match &cgroup { |
| 244 | Some(p) => cmd.arg(p), |
| 245 | None => cmd.arg(""), |
| 246 | }; |
| 247 | match rlimit { |
| 248 | // ulimit counts in KiB; a cap below one is no cap the program could run in. |
| 249 | Some(b) => cmd.arg(format!("{}", std::cmp::max(b / 1024, 1))), |
| 250 | None => cmd.arg(""), |
| 251 | }; |
| 252 | cmd.arg(program).env(SUPERVISOR_ENV, format!("{}", std::process::id())); |
| 253 | Ok(Self { cmd, cgroup }) |
| 254 | } |
| 255 | |
| 256 | fn make_cgroup(root: &CgroupRoot, name: &str, memory_max: u64) -> Outcome<PathBuf> { |
| 257 | res!(check_name(name)); |
| 258 | let dir = root.path().join(name); |
| 259 | if dir.exists() { |
| 260 | // One left by a supervisor that died is removed if it is empty, and refused if not. |
| 261 | if let Err(e) = fs::remove_dir(&dir) { |
| 262 | return Err(err!( |
| 263 | "The cgroup {} already exists and cannot be removed: {}.", dir.display(), e; |
| 264 | Exists, System)); |
| 265 | } |
| 266 | } |
| 267 | res!(io(fs::create_dir(&dir), "make the cgroup", &dir)); |
| 268 | let set = |file: &str, value: String| -> Outcome<()> { |
| 269 | let path = dir.join(file); |
| 270 | io(fs::write(&path, value), "write", &path) |
| 271 | }; |
| 272 | let made = (|| -> Outcome<()> { |
| 273 | res!(set("memory.max", format!("{}", memory_max))); |
| 274 | // Absent where the kernel was built without swap accounting, and then there is no swap |
| 275 | // to meet the cap with. |
| 276 | if dir.join("memory.swap.max").exists() { |
| 277 | res!(set("memory.swap.max", "0".to_string())); |
| 278 | } |
| 279 | res!(set("memory.oom.group", "1".to_string())); |
| 280 | Ok(()) |
| 281 | })(); |
| 282 | match made { |
| 283 | Ok(()) => Ok(dir), |
| 284 | Err(e) => { |
| 285 | let _ = fs::remove_dir(&dir); |
| 286 | Err(e) |
| 287 | } |
| 288 | } |
| 289 | } |
| 290 | |
| 291 | /// The command that starts the child, for its further arguments, environment and streams. |
| 292 | pub fn command(&mut self) -> &mut Command { |
| 293 | &mut self.cmd |
| 294 | } |
| 295 | |
| 296 | /// The child's cgroup, if it has one. |
| 297 | pub fn cgroup(&self) -> Option<&Path> { |
| 298 | self.cgroup.as_deref() |
| 299 | } |
| 300 | |
| 301 | /// Starts the child. |
| 302 | pub fn spawn(mut self) -> Outcome<Supervised> { |
| 303 | let cgroup = self.cgroup.take(); |
| 304 | match self.cmd.spawn() { |
| 305 | Ok(child) => Ok(Supervised { child, cgroup, ended: false }), |
| 306 | Err(e) => { |
| 307 | if let Some(dir) = &cgroup { |
| 308 | let _ = fs::remove_dir(dir); |
| 309 | } |
| 310 | Err(err!("Cannot start the child: {}.", e; IO, System)) |
| 311 | } |
| 312 | } |
| 313 | } |
| 314 | } |
| 315 | |
| 316 | impl Drop for Confined { |
| 317 | fn drop(&mut self) { |
| 318 | if let Some(dir) = self.cgroup.take() { |
| 319 | let _ = fs::remove_dir(&dir); |
| 320 | } |
| 321 | } |
| 322 | } |
| 323 | |
| 324 | /// How a supervised child ended. |
| 325 | #[derive(Clone, Copy, Debug, Eq, PartialEq)] |
| 326 | pub struct Ended { |
| 327 | pub status: ExitStatus, |
| 328 | /// How many processes the kernel killed in the child's cgroup for want of memory. |
| 329 | pub oom_kills: u64, |
| 330 | } |
| 331 | |
| 332 | impl Ended { |
| 333 | /// Did the kernel kill the child, or a process it started, for passing its cap? |
| 334 | pub fn oom_killed(&self) -> bool { |
| 335 | self.oom_kills > 0 |
| 336 | } |
| 337 | } |
| 338 | |
| 339 | /// A child started under its cap. Dropping one that has not been waited on kills it and everything |
| 340 | /// in its cgroup, and removes the cgroup. |
| 341 | #[derive(Debug)] |
| 342 | pub struct Supervised { |
| 343 | child: Child, |
| 344 | cgroup: Option<PathBuf>, |
| 345 | ended: bool, |
| 346 | } |
| 347 | |
| 348 | impl Supervised { |
| 349 | pub fn pid(&self) -> u32 { |
| 350 | self.child.id() |
| 351 | } |
| 352 | |
| 353 | /// The child's cgroup, if it has one. |
| 354 | pub fn cgroup(&self) -> Option<&Path> { |
| 355 | self.cgroup.as_deref() |
| 356 | } |
| 357 | |
| 358 | /// The child, for its standard streams. |
| 359 | pub fn child(&mut self) -> &mut Child { |
| 360 | &mut self.child |
| 361 | } |
| 362 | |
| 363 | /// Kills the child, and with a cgroup everything in it. |
| 364 | pub fn kill(&mut self) -> Outcome<()> { |
| 365 | if let Some(dir) = &self.cgroup { |
| 366 | if fs::write(dir.join("cgroup.kill"), "1").is_ok() { |
| 367 | return Ok(()); |
| 368 | } |
| 369 | } |
| 370 | match self.child.kill() { |
| 371 | Ok(()) => Ok(()), |
| 372 | Err(e) => Err(err!("Cannot kill the child {}: {}.", self.child.id(), e; IO, System)), |
| 373 | } |
| 374 | } |
| 375 | |
| 376 | /// Waits for the child to end, then reports how, and removes its cgroup. |
| 377 | pub fn wait(mut self) -> Outcome<Ended> { |
| 378 | let status = match self.child.wait() { |
| 379 | Ok(s) => s, |
| 380 | Err(e) => return Err(err!("Cannot wait on the child {}: {}.", self.child.id(), e; |
| 381 | IO, System)), |
| 382 | }; |
| 383 | self.finish(status) |
| 384 | } |
| 385 | |
| 386 | /// Reports how the child ended, if it has, and then removes its cgroup. |
| 387 | pub fn try_wait(&mut self) -> Outcome<Option<Ended>> { |
| 388 | match self.child.try_wait() { |
| 389 | Ok(Some(s)) => match self.finish(s) { |
| 390 | Ok(e) => Ok(Some(e)), |
| 391 | Err(e) => Err(e), |
| 392 | }, |
| 393 | Ok(None) => Ok(None), |
| 394 | Err(e) => Err(err!("Cannot ask whether the child {} has ended: {}.", |
| 395 | self.child.id(), e; IO, System)), |
| 396 | } |
| 397 | } |
| 398 | |
| 399 | fn finish(&mut self, status: ExitStatus) -> Outcome<Ended> { |
| 400 | self.ended = true; |
| 401 | let dir = match self.cgroup.take() { |
| 402 | Some(d) => d, |
| 403 | None => return Ok(Ended { status, oom_kills: 0 }), |
| 404 | }; |
| 405 | let oom_kills = res!(oom_kills(&dir.join("memory.events"))); |
| 406 | res!(remove_cgroup(&dir)); |
| 407 | Ok(Ended { status, oom_kills }) |
| 408 | } |
| 409 | } |
| 410 | |
| 411 | impl Drop for Supervised { |
| 412 | fn drop(&mut self) { |
| 413 | if self.ended { |
| 414 | return; |
| 415 | } |
| 416 | let _ = self.kill(); |
| 417 | if let Ok(status) = self.child.wait() { |
| 418 | let _ = self.finish(status); |
| 419 | } |
| 420 | } |
| 421 | } |
| 422 | |
| 423 | // Kills whatever a finished child left in its cgroup, and removes the cgroup once it is empty. |
| 424 | fn remove_cgroup(dir: &Path) -> Outcome<()> { |
| 425 | let _ = fs::write(dir.join("cgroup.kill"), "1"); |
| 426 | let begun = Instant::now(); |
| 427 | loop { |
| 428 | match fs::remove_dir(dir) { |
| 429 | Ok(()) => return Ok(()), |
| 430 | Err(e) => if begun.elapsed() > EMPTYING { |
| 431 | return Err(err!( |
| 432 | "The cgroup {} did not empty in {} s: {}.", dir.display(), EMPTYING.as_secs(), e; |
| 433 | IO, Timeout)); |
| 434 | }, |
| 435 | } |
| 436 | thread::sleep(NAP); |
| 437 | } |
| 438 | } |
| 439 | |
| 440 | /// Has the kernel send `sig` to the caller when its parent ends, for a child to call before |
| 441 | /// anything else. It answers an error, and the caller should then end, where the parent has already |
| 442 | /// ended: the supervisor named in [`SUPERVISOR_ENV`], or where that is not set, any parent but |
| 443 | /// init. |
| 444 | pub fn arm_parent_death(sig: Signal) -> Outcome<()> { |
| 445 | if let Err(e) = set_pdeathsig(sig.nix()) { |
| 446 | return Err(err!("Cannot arm the parent's death signal: {}.", e; IO, System)); |
| 447 | } |
| 448 | let parent = getppid().as_raw(); |
| 449 | let expected: Option<i32> = match env::var(SUPERVISOR_ENV) { |
| 450 | Ok(v) => match v.trim().parse() { |
| 451 | Ok(p) => Some(p), |
| 452 | Err(_) => return Err(err!( |
| 453 | "{} holds {:?}, which is not a process number.", SUPERVISOR_ENV, v; |
| 454 | Invalid, Input)), |
| 455 | }, |
| 456 | Err(_) => None, |
| 457 | }; |
| 458 | let orphaned = match expected { |
| 459 | Some(p) => parent != p, |
| 460 | None => parent == 1, |
| 461 | }; |
| 462 | if orphaned { |
| 463 | return Err(err!( |
| 464 | "The parent ended before its death signal was armed (parent now {}, expected {:?}).", |
| 465 | parent, expected; |
| 466 | Missing, System)); |
| 467 | } |
| 468 | Ok(()) |
| 469 | } |
| 470 | |
| 471 | /// At most `max` restarts in any `window`, on a clock the caller passes in. A refused restart is |
| 472 | /// not counted, so the budget frees as the oldest restart ages out of the window. |
| 473 | #[derive(Clone, Debug)] |
| 474 | pub struct RestartBudget { |
| 475 | max: usize, |
| 476 | window: Duration, |
| 477 | starts: VecDeque<Instant>, |
| 478 | } |
| 479 | |
| 480 | impl RestartBudget { |
| 481 | pub fn new(max: usize, window: Duration) -> Self { |
| 482 | Self { max, window, starts: VecDeque::new() } |
| 483 | } |
| 484 | |
| 485 | fn age_out(&mut self, now: Instant) { |
| 486 | while let Some(first) = self.starts.front() { |
| 487 | match now.checked_duration_since(*first) { |
| 488 | Some(age) if age >= self.window => { self.starts.pop_front(); } |
| 489 | _ => break, |
| 490 | } |
| 491 | } |
| 492 | } |
| 493 | |
| 494 | /// Records a restart at `now` and answers true, or answers false, recording nothing, where |
| 495 | /// `max` already fall in the window ending at `now`. |
| 496 | pub fn try_restart(&mut self, now: Instant) -> bool { |
| 497 | self.age_out(now); |
| 498 | if self.starts.len() >= self.max { |
| 499 | return false; |
| 500 | } |
| 501 | self.starts.push_back(now); |
| 502 | true |
| 503 | } |
| 504 | |
| 505 | /// How many restarts the window ending at `now` still allows. |
| 506 | pub fn remaining(&self, now: Instant) -> usize { |
| 507 | let used = self.starts.iter() |
| 508 | .filter(|s| match now.checked_duration_since(**s) { |
| 509 | Some(age) => age < self.window, |
| 510 | None => true, |
| 511 | }) |
| 512 | .count(); |
| 513 | self.max.saturating_sub(used) |
| 514 | } |
| 515 | } |
| 516 | |
| 517 | // ── A transient scope of the user manager ─────────────────────────────────────────────────────── |
| 518 | |
| 519 | /// The byte a child of ours writes first, before it reads anything, so that its supervisor knows |
| 520 | /// the program itself started and did not fail on the way in. |
| 521 | pub const HELLO: u8 = b'\x06'; // fixed: ASCII ACK, one byte that no frame begins with |
| 522 | |
| 523 | /// Writes [`HELLO`] on the standard output, the first thing a [`Scoped`] child of ours does. |
| 524 | pub fn say_hello() -> Outcome<()> { |
| 525 | use std::io::Write; |
| 526 | let mut out = std::io::stdout().lock(); |
| 527 | res!(out.write_all(&[HELLO]), IO, Write); |
| 528 | res!(out.flush(), IO, Write); |
| 529 | Ok(()) |
| 530 | } |
| 531 | |
| 532 | /// How many processes the kernel has killed for want of memory in the cgroup whose `memory.events` |
| 533 | /// is at `events`, counted over the cgroup and everything below it. |
| 534 | pub fn oom_kills(events: &Path) -> Outcome<u64> { |
| 535 | let text = res!(io(fs::read_to_string(events), "read", events)); |
| 536 | Ok(text.lines() |
| 537 | .find_map(|l| l.strip_prefix("oom_kill ")) |
| 538 | .and_then(|v| v.trim().parse().ok()) |
| 539 | .unwrap_or(0)) |
| 540 | } |
| 541 | |
| 542 | fn systemctl_user(args: &[&str]) -> Outcome<String> { |
| 543 | let out = res!(Command::new("systemctl").arg("--user").args(args).output(), IO, System); |
| 544 | match out.status.success() { |
| 545 | true => Ok(String::from_utf8_lossy(&out.stdout).trim().to_string()), |
| 546 | false => Err(err!("`systemctl --user {}` failed: {}", args.join(" "), |
| 547 | String::from_utf8_lossy(&out.stderr).trim(); IO, System)), |
| 548 | } |
| 549 | } |
| 550 | |
| 551 | /// Does the user manager have the `memory` controller, without which a scope's `MemoryMax` is |
| 552 | /// silently ignored? Fails where there is no user manager to ask. |
| 553 | pub fn manager_has_memory() -> Outcome<bool> { |
| 554 | let cg = res!(systemctl_user(&["show", "-p", "ControlGroup", "--value"])); |
| 555 | if cg.is_empty() { |
| 556 | return Ok(false); |
| 557 | } |
| 558 | let at = Path::new(CGROUP_FS).join(cg.trim_start_matches('/')).join("cgroup.controllers"); |
| 559 | let text = res!(io(fs::read_to_string(&at), "read", &at)); |
| 560 | Ok(text.split_whitespace().any(|c| c == "memory")) |
| 561 | } |
| 562 | |
| 563 | /// Starts the user manager's slice `slice` (a full name, `x.slice`) and answers the path of its |
| 564 | /// `memory.events`. |
| 565 | pub fn start_slice(slice: &str) -> Outcome<PathBuf> { |
| 566 | res!(systemctl_user(&["start", slice])); |
| 567 | let cg = res!(systemctl_user(&["show", "-p", "ControlGroup", "--value", slice])); |
| 568 | if cg.is_empty() { |
| 569 | return Err(err!("The slice {} was started and has no cgroup.", slice; Missing, System)); |
| 570 | } |
| 571 | let at = Path::new(CGROUP_FS).join(cg.trim_start_matches('/')).join("memory.events"); |
| 572 | match at.is_file() { |
| 573 | true => Ok(at), |
| 574 | false => Err(err!("The slice {} has no {}, so its memory is not accounted.", slice, |
| 575 | at.display(); Missing, System)), |
| 576 | } |
| 577 | } |
| 578 | |
| 579 | /// A child to be run in a transient scope of the user manager, `systemd-run --user --scope`, in a |
| 580 | /// slice whose `memory.events` counts the kills of every scope in it. The scope is capped at |
| 581 | /// `memory_max` with no swap, and its name is `<prefix>-<pid>-<n>-<random>`, so that neither a |
| 582 | /// failed scope left loaded nor a reused process number can collide with it. The scope execs in |
| 583 | /// place, so the child keeps the caller's standard streams and environment, and lives in the |
| 584 | /// manager's cgroup, not the caller's: a child outlives its caller's death and stop. |
| 585 | /// |
| 586 | /// The child is to write [`HELLO`] first; [`ScopedChild::hello`] reads it, which tells a scope that |
| 587 | /// never started (no bus, a name taken, a property refused) from the program's own failure. |
| 588 | #[derive(Debug)] |
| 589 | pub struct Scoped { |
| 590 | cmd: Command, |
| 591 | unit: String, |
| 592 | events: PathBuf, |
| 593 | } |
| 594 | |
| 595 | impl Scoped { |
| 596 | /// The command that runs `program` in a scope named after `prefix`, in `slice` (a full name), |
| 597 | /// whose `memory.events` is at `events`. Each call within a process takes the next `n`. |
| 598 | pub fn new<P: AsRef<OsStr>>(program: P, prefix: &str, slice: &str, events: &Path, memory_max: u64) |
| 599 | -> Outcome<Self> |
| 600 | { |
| 601 | use std::sync::atomic::{ |
| 602 | AtomicU64, |
| 603 | Ordering, |
| 604 | }; |
| 605 | static N: AtomicU64 = AtomicU64::new(0); |
| 606 | res!(check_name(prefix)); |
| 607 | let n = N.fetch_add(1, Ordering::SeqCst); |
| 608 | let random = oxedyne_fe2o3_core::rand::Rand::rand_u32(); |
| 609 | let unit = format!("{}-{}-{}-{:08x}", prefix, std::process::id(), n, random); |
| 610 | let mut cmd = Command::new("systemd-run"); |
| 611 | cmd.args(["--user", "--scope", "--quiet"]) |
| 612 | .arg(format!("--slice={}", slice)) |
| 613 | .arg(format!("--unit={}", unit)) |
| 614 | .arg("-p").arg(format!("MemoryMax={}", memory_max)) |
| 615 | .arg("-p").arg("MemorySwapMax=0") |
| 616 | .arg("--") |
| 617 | .arg(program) |
| 618 | .env(SUPERVISOR_ENV, format!("{}", std::process::id())); |
| 619 | Ok(Self { cmd, unit, events: events.to_path_buf() }) |
| 620 | } |
| 621 | |
| 622 | /// The command, for the program's further arguments, environment and streams. Its arguments |
| 623 | /// so far are `systemd-run`'s; those added are the program's. |
| 624 | pub fn command(&mut self) -> &mut Command { |
| 625 | &mut self.cmd |
| 626 | } |
| 627 | |
| 628 | /// The scope's name, without `.scope`. |
| 629 | pub fn unit(&self) -> &str { |
| 630 | &self.unit |
| 631 | } |
| 632 | |
| 633 | /// Reads the slice's count, then starts the child. |
| 634 | pub fn spawn(mut self) -> Outcome<ScopedChild> { |
| 635 | let before = res!(oom_kills(&self.events)); |
| 636 | match self.cmd.spawn() { |
| 637 | Ok(child) => Ok(ScopedChild { child, unit: self.unit, events: self.events, before }), |
| 638 | Err(e) => Err(err!("`systemd-run` could not be started for the scope {}: {}.", |
| 639 | self.unit, e; IO, System)), |
| 640 | } |
| 641 | } |
| 642 | } |
| 643 | |
| 644 | /// A child started in its scope. |
| 645 | #[derive(Debug)] |
| 646 | pub struct ScopedChild { |
| 647 | child: Child, |
| 648 | unit: String, |
| 649 | events: PathBuf, |
| 650 | before: u64, |
| 651 | } |
| 652 | |
| 653 | /// How a scoped child ended, with what its slice's counter says of the kills while it ran. |
| 654 | #[derive(Clone, Copy, Debug, Eq, PartialEq)] |
| 655 | pub struct ScopeEnded { |
| 656 | pub status: ExitStatus, |
| 657 | pub kills: SliceKills, |
| 658 | } |
| 659 | |
| 660 | /// What the slice's `oom_kill` count did while one child ran. |
| 661 | #[derive(Clone, Copy, Debug, Eq, PartialEq)] |
| 662 | pub enum SliceKills { |
| 663 | None, // the count held |
| 664 | Rose(u64), // the kernel killed this many processes in the slice for memory |
| 665 | Unknown, // the count fell, so the slice was started again, or it could not be read |
| 666 | } |
| 667 | |
| 668 | impl ScopeEnded { |
| 669 | /// Was the child itself killed for memory: a kill in the slice, and the child ended by a |
| 670 | /// signal? A kill with the child ending cleanly was of another scope in the slice. |
| 671 | pub fn oom_killed(&self) -> bool { |
| 672 | use std::os::unix::process::ExitStatusExt; |
| 673 | matches!(self.kills, SliceKills::Rose(_)) && self.status.signal().is_some() |
| 674 | } |
| 675 | } |
| 676 | |
| 677 | impl ScopedChild { |
| 678 | pub fn pid(&self) -> u32 { |
| 679 | self.child.id() |
| 680 | } |
| 681 | |
| 682 | pub fn unit(&self) -> &str { |
| 683 | &self.unit |
| 684 | } |
| 685 | |
| 686 | /// The child, for its standard streams. |
| 687 | pub fn child(&mut self) -> &mut Child { |
| 688 | &mut self.child |
| 689 | } |
| 690 | |
| 691 | /// Reads one byte from the child's piped standard output, and answers whether it was |
| 692 | /// [`HELLO`]. `false` means the program never started, and the child's end is |
| 693 | /// `systemd-run`'s own; its reason is on the standard error. |
| 694 | pub fn hello(&mut self) -> Outcome<bool> { |
| 695 | use std::io::Read; |
| 696 | let out = match self.child.stdout.as_mut() { |
| 697 | Some(o) => o, |
| 698 | None => return Err(err!("The scope {}'s standard output is not piped, so its hello \ |
| 699 | cannot be read.", self.unit; Invalid, Input)), |
| 700 | }; |
| 701 | let mut byte = [0u8; 1]; |
| 702 | loop { |
| 703 | match out.read(&mut byte) { |
| 704 | Ok(0) => return Ok(false), |
| 705 | Ok(_) => return Ok(byte[0] == HELLO), |
| 706 | Err(e) if e.kind() == ErrorKind::Interrupted => continue, |
| 707 | Err(e) => return Err(err!("The scope {}'s hello could not be read: {}.", |
| 708 | self.unit, e; IO, Read)), |
| 709 | } |
| 710 | } |
| 711 | } |
| 712 | |
| 713 | /// Waits for the child to end and reads the slice's count again. The kernel completes the |
| 714 | /// count before the killed child's wait returns, so the read cannot race. A scope that failed |
| 715 | /// is reset, best effort, since the manager keeps it loaded until it is. |
| 716 | pub fn wait(mut self) -> Outcome<ScopeEnded> { |
| 717 | let status = match self.child.wait() { |
| 718 | Ok(s) => s, |
| 719 | Err(e) => return Err(err!("Cannot wait on the scope {}: {}.", self.unit, e; |
| 720 | IO, System)), |
| 721 | }; |
| 722 | let kills = match oom_kills(&self.events) { |
| 723 | Ok(after) if after == self.before => SliceKills::None, |
| 724 | Ok(after) if after > self.before => SliceKills::Rose(after - self.before), |
| 725 | _ => SliceKills::Unknown, |
| 726 | }; |
| 727 | if !status.success() { |
| 728 | let _ = systemctl_user(&["reset-failed", &format!("{}.scope", self.unit)]); |
| 729 | } |
| 730 | Ok(ScopeEnded { status, kills }) |
| 731 | } |
| 732 | } |