From 987a7b4708d13d232726bbcd27b8ce9fe7d55f37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iwo=20Strzebo=C5=84ski?= Date: Thu, 29 Oct 2026 10:47:26 +0000 Subject: [PATCH] Stop a go2rtc that outlived a hard-killed proxy SIGTERM already stops go2rtc (the signal handler), but SIGKILL and the OOM killer give the proxy no chance to, and the orphan keeps :1984/:8555 bound and a printer's access code in memory. Two layers, neither needing unsafe or a new dependency: - The watchdog records go2rtc's pid next to its config and, on startup, stops a leftover. It acts only if /proc confirms the pid is still *our* go2rtc - the configured binary on our own `-config` path - so a reused pid or another proxy's go2rtc is never touched; without /proc it does nothing. The pid file is removed when the supervisor goes away. - deploy/continuum-proxy.service, whose KillMode=control-group (written out, with the reason) makes systemd kill the whole cgroup when the proxy dies. Syntax-checked with systemd-analyze only; not run under a real systemd. Tests: the identity check, and a real stand-in process - a leftover is terminated while an unrelated `sleep` survives. Making the identity check always say yes fails both, so they do guard the behaviour. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 2 + deploy/continuum-proxy.service | 38 ++++++++++ src/go2rtc.rs | 135 +++++++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+) create mode 100644 deploy/continuum-proxy.service diff --git a/README.md b/README.md index a574d00..eccbf25 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,8 @@ src/ uplink/ WebSocket client to continuum-backend: connect, heartbeat, reconnect on drop camera.rs Writes go2rtc's config from the camera URLs the printers report go2rtc.rs Restarts the go2rtc camera-restreaming process if it dies, or when the config changes + and stops one a hard-killed proxy left behind (pid file next to its config) + deploy/ systemd unit template; its KillMode=control-group is what stops go2rtc outliving a SIGKILL'd proxy printer/ mod.rs GenericPrinter trait + PrinterBase + PrinterHandle enum - the trait+enum "inheritance" pattern this project uses instead of class inheritance diff --git a/deploy/continuum-proxy.service b/deploy/continuum-proxy.service new file mode 100644 index 0000000..b9fd3fc --- /dev/null +++ b/deploy/continuum-proxy.service @@ -0,0 +1,38 @@ +# Continuum proxy as a systemd service - the template for a Proxy Box or a +# Linux PC install. Adjust the paths; create the `continuum` user and put the +# settings it reads (continuum.toml, or the signed bundle on a Box) under +# /var/lib/continuum. +# +# Not exercised under a real systemd yet: this container's init is not +# systemd, so only `systemd-analyze verify` has been run against it. +[Unit] +Description=Continuum proxy (printer gateway) +Wants=network-online.target +After=network-online.target + +[Service] +Type=simple +User=continuum +# The proxy's relative defaults (./go2rtc.yaml, ./printers.toml, the go2rtc +# pid file) resolve here, and StateDirectory creates it owned by the user. +StateDirectory=continuum +WorkingDirectory=/var/lib/continuum +ExecStart=/usr/local/bin/continuum-proxy + +# Always, including a clean exit: installing a new config from a pendrive +# deliberately ends the process and relies on this to start it on the new +# settings (src/main.rs, config_import). +Restart=always +RestartSec=2 + +# Do not change this to `process` or `none`. go2rtc is the proxy's child, so +# it lives in this unit's cgroup, and this is what makes systemd kill it +# whenever the proxy goes - including SIGKILL and the OOM killer, which give +# the proxy no chance to stop it itself. It is systemd's default; it is +# written out so nobody weakens it by accident. An orphaned go2rtc keeps +# :1984/:8555 bound and a printer's access code in memory. +KillMode=control-group +TimeoutStopSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/src/go2rtc.rs b/src/go2rtc.rs index 8a6925d..606dc6a 100644 --- a/src/go2rtc.rs +++ b/src/go2rtc.rs @@ -1,3 +1,4 @@ +use std::path::PathBuf; use std::process::Stdio; use std::sync::Arc; use std::time::Duration; @@ -6,6 +7,76 @@ use tokio::process::Command; use tokio::sync::Notify; use tracing::{error, info, warn}; +/// Where the running go2rtc's pid is recorded: next to its config, because +/// that is the one path this process already owns and the one a restarted +/// proxy is certain to compute again. +fn pid_path(config_path: &str) -> PathBuf { + PathBuf::from(format!("{config_path}.pid")) +} + +/// Removes the pid file when the supervisor goes away, however it does: +/// the watchdog is dropped, not returned from, on a normal shutdown. +struct PidFile(PathBuf); + +impl Drop for PidFile { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} + +/// Whether a `/proc//cmdline` is the go2rtc this proxy started: the +/// configured binary as the program (or as the script an interpreter was +/// given, which is what a wrapper looks like) *and* our own config path after +/// `-config`. Both, because a pid alone proves nothing once the number has +/// been reused, and "some go2rtc" is not necessarily ours - another proxy on +/// the machine would run one against its own config. +fn is_our_go2rtc(cmdline: &[u8], bin: &str, config_path: &str) -> bool { + let args: Vec<&[u8]> = cmdline.split(|b| *b == 0).filter(|a| !a.is_empty()).collect(); + let launched = args.iter().take(2).any(|a| *a == bin.as_bytes()); + let configured = args.windows(2).any(|w| w[0] == b"-config" && w[1] == config_path.as_bytes()); + launched && configured +} + +/// Stops a go2rtc left behind by a proxy that was killed without a chance to +/// stop it (SIGKILL, the OOM killer, a power cut that kept the container). +/// +/// Only acts on a pid this proxy recorded, and only after `/proc` confirms +/// that pid is still our go2rtc (`is_our_go2rtc`). Anywhere without `/proc` +/// it cannot confirm that, so it leaves the process alone: a leftover +/// go2rtc is a nuisance, killing someone else's process is not. The orphan +/// matters because it keeps :1984 and :8555 bound - so the new one cannot +/// start - and holds a printer's access code in memory. +async fn reclaim_orphan(bin: &str, config_path: &str) { + let pid_file = pid_path(config_path); + let Ok(text) = std::fs::read_to_string(&pid_file) else { return }; + let _ = std::fs::remove_file(&pid_file); + let Ok(pid) = text.trim().parse::() else { return }; + + let ours = || { + std::fs::read(format!("/proc/{pid}/cmdline")) + .map(|cmdline| is_our_go2rtc(&cmdline, bin, config_path)) + .unwrap_or(false) + }; + if !ours() { + return; + } + + warn!(pid, "a go2rtc left by an earlier proxy is still running; stopping it"); + // The `kill` program rather than a libc call: signalling is the only + // reason this would need `unsafe` and a new dependency. + for (signal, patience) in [("-TERM", 30), ("-KILL", 30)] { + let _ = Command::new("kill").arg(signal).arg(pid.to_string()).status().await; + for _ in 0..patience { + if !ours() { + info!(pid, "the old go2rtc is gone"); + return; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + error!(pid, "could not stop the old go2rtc; the new one may fail to bind its ports"); +} + /// Supervises the go2rtc subprocess used for camera restreaming /// (RTSP -> WebRTC). Restarts it with a fixed backoff whenever it exits, and /// immediately when `restart` fires. @@ -16,6 +87,11 @@ use tracing::{error, info, warn}; /// Before that existed this watchdog supervised a process whose config file /// nothing wrote - it restarted go2rtc forever over a file that did not exist. pub async fn watchdog(bin: String, config_path: String, restart: Arc) { + // Before the first spawn: whatever the last proxy left behind owns the + // ports this one is about to ask for. + reclaim_orphan(&bin, &config_path).await; + let _pid_file = PidFile(pid_path(&config_path)); + loop { info!(%bin, %config_path, "starting go2rtc"); @@ -29,6 +105,9 @@ pub async fn watchdog(bin: String, config_path: String, restart: Arc) { match spawned { Ok(mut child) => { + if let Some(Err(err)) = child.id().map(|pid| std::fs::write(pid_path(&config_path), pid.to_string())) { + warn!(?err, "could not record go2rtc's pid; a hard kill of the proxy would orphan it unnoticed"); + } tokio::select! { // Whichever happens first. `kill_on_drop` means the // child dies when `child` goes out of scope at the end @@ -61,3 +140,59 @@ pub async fn watchdog(bin: String, config_path: String, restart: Arc) { tokio::time::sleep(Duration::from_secs(5)).await; } } + +#[cfg(test)] +mod tests { + use super::*; + + fn cmdline(args: &[&str]) -> Vec { + args.iter().flat_map(|a| a.bytes().chain([0])).collect() + } + + #[test] + fn only_our_binary_on_our_config_counts() { + let ours = cmdline(&["go2rtc", "-config", "./go2rtc.yaml"]); + assert!(is_our_go2rtc(&ours, "go2rtc", "./go2rtc.yaml")); + // A wrapper script puts the interpreter first. + assert!(is_our_go2rtc(&cmdline(&["/bin/sh", "go2rtc", "-config", "./go2rtc.yaml"]), "go2rtc", "./go2rtc.yaml")); + // Another proxy's go2rtc, an unrelated process, and a reused pid. + assert!(!is_our_go2rtc(&cmdline(&["go2rtc", "-config", "/other.yaml"]), "go2rtc", "./go2rtc.yaml")); + assert!(!is_our_go2rtc(&cmdline(&["sleep", "-config", "./go2rtc.yaml"]), "go2rtc", "./go2rtc.yaml")); + assert!(!is_our_go2rtc(&cmdline(&["vim", "go2rtc.yaml"]), "go2rtc", "./go2rtc.yaml")); + assert!(!is_our_go2rtc(b"", "go2rtc", "./go2rtc.yaml"), "a zombie has an empty cmdline"); + } + + /// A stand-in go2rtc that runs until killed, started the way the real + /// one is, whose pid file is then found by a "restarted proxy". + #[cfg(target_os = "linux")] + #[tokio::test] + async fn a_leftover_go2rtc_is_stopped_and_a_stranger_is_not() { + let dir = std::env::temp_dir().join(format!("continuum-go2rtc-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let script = dir.join("go2rtc"); + // Stays alive until signalled, and (run as `sh