From 84025d6bc9e4f7f33aab0f0cd2ca4da8e7046f9c Mon Sep 17 00:00:00 2001 From: hydrangeas20 Date: Sat, 24 Oct 2026 21:00:08 -0400 Subject: [PATCH 1/2] fix: harden cross-platform server shutdown and add CI Root cause (macOS). _alive() read /proc//stat and treated a missing file as "dead". macOS has no /proc, so stop_server() for a server started by an earlier kernel stopped waiting immediately after SIGTERM and checked /health while uvicorn was still shutting down, failing test_stop_server_started_by_another_kernel. Replacing it with plain kill(pid, 0) would not work either: that succeeds on zombies, so an exited but unreaped server would look alive until the deadline. Further defects found while fixing it: - The pid file held only a pid, and stop_server() sent killpg() to it unverified: a stale file whose pid had been reused by another session leader would signal an unrelated process group (reproduced in a test). - The port was reported free whenever /health stopped answering, even if something still listened on it. - A failed or timed-out start left the pid file, _PROC and the log handle behind; the log handle was never closed. benchmarks/colab_helpers.py: - Liveness: kill(pid, 0) first (ESRCH = gone, EPERM = exists), then the state from /proc on Linux or `ps -o stat=` elsewhere; zombies count as exited; an unreadable state counts as alive, so shutdown waits rather than declaring a live process dead. - Ownership: the pid file is JSON {pid, port, start}; start is the process start time (/proc starttime, or `ps -o lstart=`), fixed at fork. A pid is signalled only if its start time and command line (-m echorelay --port N) still match; otherwise the file is stale and is removed without signalling. Legacy pid-only files are accepted when the command line matches. If ownership cannot be verified at all, stop_server() raises and keeps the file. - Shutdown: SIGTERM to the group, bounded wait (monotonic deadlines, backoff polling), identity re-checked, then SIGKILL; raises if the process outlives SIGKILL. A child of this kernel is reaped; an already exited child is never signalled (its pid may be reused). - Port: after stopping, waits for nothing to accept TCP connections on 127.0.0.1:PORT and raises otherwise, naming whether the listener answers /health. - start_server(): cleans up the process, pid file and log on a crash or a health timeout before raising. Public signatures are unchanged; timeouts are module constants because start_server(**settings) forwards keywords to the server. tests/test_colab_helpers.py: 16 new tests (21 total): another kernel on the /proc and ps (macOS) paths, graceful SIGTERM only, repeated stop, already-exited servers, stale legacy/mismatched/garbage pid files, pid reuse by start time, unverifiable ownership, SIGKILL escalation, an unkillable process, ps-based and unknown-state liveness, Linux zombies, a non-HTTP listener holding the port, and cleanup after failed starts. Spawned processes are killed in teardown even when a test fails. .github/workflows/ci.yml: offline suite on Linux for Python 3.11 and 3.13 x both scheduling flags (8 jobs), on macOS for Python 3.13 (default flags), and the benchmark evidence verifier. Real-model tests excluded; read-only permissions; job timeouts; no secrets or GPU. README.md: CI badge. No changes to inference, scheduling, the streaming protocol, benchmark results or dependencies. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017DfLG9q5h7eiawGNwaBswq --- .github/workflows/ci.yml | 81 +++++++++ README.md | 2 + benchmarks/colab_helpers.py | 325 +++++++++++++++++++++++++++++++----- tests/test_colab_helpers.py | 273 ++++++++++++++++++++++++++++-- 4 files changed, 622 insertions(+), 59 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..feb2912 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,81 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + test-linux: + # The offline suite (fake and gated backends) under every combination of + # the two scheduling flags. Real-model tests (`-m whisper`) need Whisper + # weights and are excluded; nothing here needs a GPU, secrets or network + # access beyond installing packages. + name: Linux · py${{ matrix.python }} · final_priority=${{ matrix.final_priority }} · cancel=${{ matrix.cancel }} + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + python: ["3.11", "3.13"] + final_priority: ["false", "true"] + cancel: ["false", "true"] + env: + ECHORELAY_FINAL_PRIORITY: ${{ matrix.final_priority }} + ECHORELAY_CANCEL_QUEUED_PARTIAL_ON_END: ${{ matrix.cancel }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + cache: pip + cache-dependency-path: pyproject.toml + - name: Install + run: python -m pip install -e ".[dev]" + - name: Test (offline; real-model tests excluded) + run: python -m pytest -q -m "not whisper" + + test-macos: + # macOS has no /proc: covers the server-shutdown path that failed there. + # Default scheduling flags only. + name: macOS · py3.13 + runs-on: macos-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install + run: python -m pip install -e ".[dev]" + - name: Test (offline; real-model tests excluded) + run: python -m pytest -q -m "not whisper" + + evidence: + # Every committed benchmark file is present, hash-matched and tracked. + name: Benchmark evidence + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: Verify evidence manifests + run: python -m benchmarks.verify_evidence diff --git a/README.md b/README.md index 04fe49a..24ed124 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ Python · FastAPI · WebSockets · faster-whisper / CTranslate2 · NVIDIA T4 +[![CI](https://github.com/hydrangeas20/echorelay/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/hydrangeas20/echorelay/actions/workflows/ci.yml) + [Architecture](#architecture) · [Results](#experimental-findings) · [Quick Start](#quick-start-local-cpu) · [Reproducibility](#reproducibility) diff --git a/benchmarks/colab_helpers.py b/benchmarks/colab_helpers.py index 3cc5769..daae50f 100644 --- a/benchmarks/colab_helpers.py +++ b/benchmarks/colab_helpers.py @@ -9,7 +9,8 @@ ``sha256sum`` on the .wav files would include the header and never match). * ``start_server(log, **settings)`` / ``stop_server()``: run EchoRelay in its own process group with ``ECHORELAY_`` settings, and stop it - reliably. Avoids ``pkill -f``, which can match the shell running it. + reliably, including one started by an earlier notebook kernel. Works on + Linux and macOS. Avoids ``pkill -f``, which can match the shell running it. * ``verify_server(expected)``: assert model and config via ``/metrics``. * ``save_provenance(out_dir, name)``: commit, packages, GPU, server metrics. """ @@ -19,18 +20,30 @@ import json import os import signal +import socket import subprocess import sys import time import urllib.error import urllib.request from pathlib import Path +from typing import IO REPO = Path(__file__).resolve().parents[1] PORT = int(os.environ.get("ECHORELAY_BENCH_PORT", "8000")) PIDFILE = REPO.parent / f"echorelay_server_{PORT}.pid" +# Bounded waits (seconds). Module constants rather than parameters because +# start_server(**settings) forwards every keyword to the server. +START_TIMEOUT_S = 300.0 # the first start downloads the model weights +STOP_GRACE_S = 20.0 # SIGTERM -> exit (uvicorn waits for open streams) +STOP_KILL_S = 10.0 # SIGKILL -> exit +PORT_RELEASE_S = 5.0 # process gone -> nothing listening on PORT + +_HAVE_PROC = os.path.isdir("/proc/self") # Linux; absent on macOS + _PROC: subprocess.Popen | None = None # server started by this kernel +_LOG: IO[str] | None = None # its log file, held open by this kernel __all__ = ["REPO", "save_provenance", "start_server", "stop_server", "verify_commit", "verify_eval_audio", "verify_server"] @@ -72,73 +85,293 @@ def verify_eval_audio(audio_dir: str | Path, manifest_result: str | Path) -> Non print(f"evaluation audio verified: {len(have)} files identical to {Path(manifest_result).name}") +# ------------------------------------------------------------ processes +# A server started by an earlier notebook kernel is known only by the pid +# file, and its pid may since have been reused by an unrelated process. The +# pid file therefore records the process start time, and the server is +# signalled only if its start time and command line still match. + + +def _ps(pid: int, field: str) -> str | None: + """One ``ps`` field for pid (Linux procps and macOS), or None.""" + try: + r = subprocess.run(["ps", "-ww", "-o", f"{field}=", "-p", str(pid)], check=False, + capture_output=True, text=True, timeout=5, + env={**os.environ, "LC_ALL": "C"}) # returncode checked below + except FileNotFoundError: # no ps installed + return None + out = r.stdout.strip() + return out if r.returncode == 0 and out else None + + +def _proc_state(pid: int) -> str | None: + """Scheduler state of pid ('R', 'S', 'Z', ...), '?' if the process exists + but its state cannot be read, or None if there is no such process. + + ``kill(pid, 0)`` alone is not enough: it succeeds on a zombie, which has + exited but not been reaped. Linux reads the state from /proc; macOS has + no /proc, so ``ps`` reads it there. + """ + try: + os.kill(pid, 0) + except ProcessLookupError: + return None + except PermissionError: + pass # exists, owned by another user + if _HAVE_PROC: + try: + with open(f"/proc/{pid}/stat") as f: + return f.read().rsplit(")", 1)[1].split()[0] + except FileNotFoundError: + return None # exited between the two checks + state = _ps(pid, "stat") + if state: + return state[0] + try: + os.kill(pid, 0) # ps found nothing: gone, or ps unavailable + except ProcessLookupError: + return None + except PermissionError: + pass + return "?" # exists, state unknown: callers treat it as alive + + def _alive(pid: int) -> bool: - """True if pid exists and is not a zombie (kill(pid, 0) succeeds on zombies).""" + """True if pid exists and has not exited (a zombie counts as exited).""" + return _proc_state(pid) not in (None, "Z", "X") + + +def _identity(pid: int) -> dict | None: + """Start time and command line of pid, or None if they cannot be read.""" + if _HAVE_PROC: + try: + with open(f"/proc/{pid}/stat") as f: + start = f.read().rsplit(")", 1)[1].split()[19] # field 22: starttime + with open(f"/proc/{pid}/cmdline", "rb") as f: + command = f.read().replace(b"\0", b" ").decode(errors="replace").strip() + except FileNotFoundError: + return None + return {"start": f"proc:{start}", "command": command} + start, command = _ps(pid, "lstart"), _ps(pid, "command") + if start is None or command is None: + return None + return {"start": f"ps:{start}", "command": command} + + +def _server_marker() -> tuple[str, str]: + return ("-m echorelay", f"--port {PORT}") + + +def _ownership(rec: dict) -> str: + """'owned', 'foreign' (pid reused or not our server), 'gone', or 'unknown'.""" + pid = rec["pid"] + if not _alive(pid): + return "gone" # includes a zombie, whose command line is no longer readable + ident = _identity(pid) + if ident is None: + return "gone" if _proc_state(pid) is None else "unknown" + if not all(m in ident["command"] for m in _server_marker()): + return "foreign" + if rec.get("start") is not None and rec["start"] != ident["start"]: + return "foreign" # same pid, different process: the pid was reused + return "owned" # a legacy pid file has no start time: the command line must match + + +def _signal_group(pid: int, sig: int) -> bool: + """Send sig to pid's process group (the server leads its own session). + Returns False if the process is already gone. Only called for this + kernel's child or a process whose ownership was verified, both started + with start_new_session=True, so their group id equals their pid.""" try: - with open(f"/proc/{pid}/stat") as f: - return f.read().rsplit(")", 1)[1].split()[0] != "Z" - except (FileNotFoundError, IndexError): + pgid = os.getpgid(pid) + except ProcessLookupError: return False + except PermissionError: # some platforms restrict cross-session queries + pgid = pid + try: + if pgid == pid: + os.killpg(pgid, sig) + else: # not a group leader: never signal someone else's group + os.kill(pid, sig) + except ProcessLookupError: + return False + except PermissionError as exc: + raise RuntimeError(f"not permitted to signal pid {pid}: {exc}") from exc + return True +def _wait_exit(pid: int, timeout: float, proc: subprocess.Popen | None = None) -> bool: + """Wait at most timeout seconds for pid to exit. A child of this kernel is + also reaped; any other process is polled with a growing interval.""" + if proc is not None: + try: + proc.wait(timeout=timeout) + return True + except subprocess.TimeoutExpired: + return False + deadline = time.monotonic() + timeout + delay = 0.02 + while _alive(pid): + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + time.sleep(min(delay, remaining)) + delay = min(delay * 2, 0.25) + return True + + +def _stop_pid(pid: int, proc: subprocess.Popen | None = None, still_ours=None) -> None: + """SIGTERM pid's group; SIGKILL it if it outlives STOP_GRACE_S.""" + if proc is not None: + if proc.poll() is not None: + return # exited and reaped: its pid may already belong to someone else + elif not _alive(pid): + return + if not _signal_group(pid, signal.SIGTERM): + return + if _wait_exit(pid, STOP_GRACE_S, proc): + return + if still_ours is not None and not still_ours(): + return # it exited and the pid was reused while we waited + print(f"pid {pid} still running {STOP_GRACE_S:g} s after SIGTERM; sending SIGKILL") + _signal_group(pid, signal.SIGKILL) + if not _wait_exit(pid, STOP_KILL_S, proc): + raise RuntimeError(f"pid {pid} did not exit {STOP_KILL_S:g} s after SIGKILL") + + +# --------------------------------------------------------------- pid file +def _write_pidfile(proc: subprocess.Popen) -> None: + ident = _identity(proc.pid) # start time is fixed at fork, so it is valid before exec + rec = {"pid": proc.pid, "port": PORT, "start": ident["start"] if ident else None} + tmp = PIDFILE.with_name(PIDFILE.name + ".tmp") + tmp.write_text(json.dumps(rec) + "\n") + os.replace(tmp, PIDFILE) + + +def _read_pidfile() -> dict | None: + """The pid file's record ({"pid", "port", "start"}), or None if absent or + unreadable. A legacy file holds only the pid.""" + try: + text = PIDFILE.read_text() + except FileNotFoundError: + return None + try: + rec = json.loads(text) + except json.JSONDecodeError: + rec = None + if isinstance(rec, int) and not isinstance(rec, bool): + rec = {"pid": rec} + if not (isinstance(rec, dict) and isinstance(rec.get("pid"), int) and rec["pid"] > 0): + print(f"ignoring unreadable pid file {PIDFILE}") + return None + return rec + + +# ------------------------------------------------------------------ port +def _port_listening() -> bool: + """True if something accepts TCP connections on 127.0.0.1:PORT.""" + with socket.socket() as s: + s.settimeout(0.5) + return s.connect_ex(("127.0.0.1", PORT)) == 0 + + +def _wait_port_free(timeout: float) -> bool: + deadline = time.monotonic() + timeout + while _port_listening(): + if time.monotonic() >= deadline: + return False + time.sleep(0.05) + return True + + +def _close_log() -> None: + global _LOG + if _LOG is not None: + _LOG.close() + _LOG = None + + +# ---------------------------------------------------------------- public def stop_server() -> None: - """Stop the server started by start_server (its whole process group).""" + """Stop the server started by start_server (its whole process group). + + Also stops a server started by an earlier kernel, found through the pid + file, but only after verifying it is that server. Safe to call repeatedly. + Raises if a process cannot be stopped, if ownership cannot be verified, or + if anything still listens on PORT afterwards. + """ global _PROC - if PIDFILE.exists(): - pid = int(PIDFILE.read_text()) - try: - os.killpg(pid, signal.SIGTERM) - except ProcessLookupError: - pass - if _PROC is not None and _PROC.pid == pid: - try: - _PROC.wait(timeout=20) # also reaps it - except subprocess.TimeoutExpired: - os.killpg(pid, signal.SIGKILL) - _PROC.wait(timeout=10) - else: # started by an earlier kernel: cannot reap, poll instead - deadline = time.monotonic() + 20 - while _alive(pid) and time.monotonic() < deadline: - time.sleep(0.2) - if _alive(pid): - try: - os.killpg(pid, signal.SIGKILL) - except ProcessLookupError: - pass - _PROC = None - PIDFILE.unlink() try: - _get("/health", timeout=1) - except (urllib.error.URLError, OSError): - print(f"server stopped; port {PORT} free") - return - raise RuntimeError(f"something is still serving on :{PORT}; restart the runtime") + own = _PROC + if own is not None: + _stop_pid(own.pid, proc=own) + _PROC = None + rec = _read_pidfile() + if rec is not None and (own is None or rec["pid"] != own.pid): + status = _ownership(rec) + if status == "unknown": + raise RuntimeError( + f"cannot verify that pid {rec['pid']} from {PIDFILE} is the EchoRelay server " + f"(no /proc and no ps); stop it yourself, then delete the pid file") + if status == "owned": + _stop_pid(rec["pid"], still_ours=lambda: _ownership(rec) == "owned") + elif status == "foreign": + print(f"stale pid file: pid {rec['pid']} is not the EchoRelay server on :{PORT}; " + "not signalling it") + else: # "gone" + print(f"stale pid file: pid {rec['pid']} has already exited") + PIDFILE.unlink(missing_ok=True) + finally: + _close_log() + if not _wait_port_free(PORT_RELEASE_S): + try: + _get("/health", timeout=1) + what = "an HTTP server answering /health (not one these helpers can stop)" + except (urllib.error.URLError, OSError): + what = "a process that is not an EchoRelay server" + raise RuntimeError(f"port {PORT} is still in use by {what}; stop it or restart the runtime") + print(f"server stopped; port {PORT} free") def start_server(log_path: str | Path, **settings) -> subprocess.Popen: - """Start EchoRelay with ECHORELAY_=value settings and wait until healthy.""" - global _PROC + """Start EchoRelay with ECHORELAY_=value settings and wait until healthy. + + Stops any previous server first. If the new server exits or never becomes + healthy, it is stopped and the pid file removed before the error is raised. + """ + global _PROC, _LOG stop_server() env = dict(os.environ) env.update({f"ECHORELAY_{k.upper()}": str(v) for k, v in settings.items()}) Path(log_path).parent.mkdir(parents=True, exist_ok=True) log = open(log_path, "w") - proc = subprocess.Popen( - [sys.executable, "-m", "echorelay", "--host", "127.0.0.1", "--port", str(PORT)], - cwd=REPO, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True, - ) - PIDFILE.write_text(str(proc.pid)) - _PROC = proc - for _ in range(300): # the first start downloads the model weights + try: + proc = subprocess.Popen( + [sys.executable, "-m", "echorelay", "--host", "127.0.0.1", "--port", str(PORT)], + cwd=REPO, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True, + ) + except BaseException: + log.close() + raise + _PROC, _LOG = proc, log + _write_pidfile(proc) + deadline = time.monotonic() + START_TIMEOUT_S + while True: if proc.poll() is not None: - raise RuntimeError(f"server exited with code {proc.returncode}; see {log_path}") + failure = f"server exited with code {proc.returncode}; see {log_path}" + break try: _get("/health", timeout=1) print(f"server up (pid {proc.pid}, port {PORT}); log: {log_path}") return proc except (urllib.error.URLError, OSError): - time.sleep(1) - raise RuntimeError(f"server not healthy after 300 s; see {log_path}") + pass + if time.monotonic() >= deadline: + failure = f"server not healthy after {START_TIMEOUT_S:g} s; see {log_path}" + break + time.sleep(0.25) + stop_server() # clean up the failed server, its pid file and log handle + raise RuntimeError(failure) def verify_server(expected: dict) -> dict: diff --git a/tests/test_colab_helpers.py b/tests/test_colab_helpers.py index 47d8ec5..532ba09 100644 --- a/tests/test_colab_helpers.py +++ b/tests/test_colab_helpers.py @@ -1,17 +1,27 @@ """Notebook helpers: real server process start/verify/restart/stop, commit and -audio verification. Uses the fake backend on a free port.""" +audio verification, and shutdown reliability (another kernel, stale pid +files, pid reuse, escalation, macOS-style liveness, port checks). Server tests +use the fake backend on a free port.""" from __future__ import annotations import importlib import json +import os +import shutil +import signal import socket import subprocess +import sys +import time import wave import numpy as np import pytest +HAS_PS = shutil.which("ps") is not None +HAS_PROC = os.path.isdir("/proc/self") + @pytest.fixture def helpers(monkeypatch, tmp_path): @@ -25,11 +35,78 @@ def helpers(monkeypatch, tmp_path): monkeypatch.setattr(h, "PIDFILE", tmp_path / "server.pid") yield h h.stop_server() + assert not h.PIDFILE.exists() + + +@pytest.fixture +def spawned(): + """Start helper processes; every one is killed and reaped at teardown, + even if the test fails.""" + procs: list[subprocess.Popen] = [] + + def spawn(code: str, *args: str) -> subprocess.Popen: + p = subprocess.Popen([sys.executable, "-c", code, *args], stdout=subprocess.PIPE, + text=True, start_new_session=True) + procs.append(p) + return p + + yield spawn + for p in procs: + if p.poll() is None: + try: + os.killpg(p.pid, signal.SIGKILL) + except ProcessLookupError: + pass + p.wait(timeout=10) + if p.stdout: + p.stdout.close() + + +def _until(predicate, timeout: float = 10.0, what: str = "condition") -> None: + deadline = time.monotonic() + timeout + while not predicate(): + if time.monotonic() > deadline: + raise AssertionError(f"timed out waiting for {what}") + time.sleep(0.01) + + +# Waits for SIGTERM (or ignores it), printing "ready" once its handler is set. +SLEEPER = ("import signal, sys, time\n" + "if 'ignore-term' in sys.argv: signal.signal(signal.SIGTERM, signal.SIG_IGN)\n" + "print('ready', flush=True)\n" + "time.sleep(120)\n") + + +def _ready(p: subprocess.Popen) -> subprocess.Popen: + assert p.stdout.readline().strip() == "ready" + return p + + +def _record_signals(monkeypatch, h) -> list[tuple[int, int]]: + """Record every real signal (not kill(pid, 0) probes) the helpers send.""" + sent: list[tuple[int, int]] = [] + real_kill, real_killpg = os.kill, os.killpg + + def kill(pid, sig): + if sig: + sent.append((pid, sig)) + return real_kill(pid, sig) + + def killpg(pgid, sig): + sent.append((pgid, sig)) + return real_killpg(pgid, sig) + + monkeypatch.setattr(h.os, "kill", kill) + monkeypatch.setattr(h.os, "killpg", killpg) + return sent +# ----------------------------------------------------------- lifecycle def test_start_verify_restart_stop(helpers, tmp_path): h = helpers first = h.start_server(tmp_path / "a.log", backend="fake", event_timing="true") + rec = json.loads(h.PIDFILE.read_text()) + assert rec["pid"] == first.pid and rec["port"] == h.PORT and rec["start"] h.verify_server({"backend": "fake", "event_timing": True, "max_concurrent_inference": 1, "partial_interval_s": 1.0, "final_beam_size": 5, "active_streams": 0}) with pytest.raises(AssertionError, match="device"): @@ -45,16 +122,198 @@ def test_start_verify_restart_stop(helpers, tmp_path): assert prov["server_metrics"]["config"]["max_concurrent_inference"] == 2 assert len(prov["git_commit"]) == 40 + log = h._LOG h.stop_server() assert second.wait(timeout=10) is not None + assert not h.PIDFILE.exists() and h._PROC is None and h._LOG is None and log.closed + assert not h._port_listening() h.stop_server() # idempotent def test_start_server_reports_a_crashing_server(helpers, tmp_path): with pytest.raises(RuntimeError, match="exited"): helpers.start_server(tmp_path / "bad.log", backend="fake", max_concurrent_streams=0) + # nothing left behind for the next start + assert not helpers.PIDFILE.exists() and helpers._PROC is None and helpers._LOG is None + + +def test_start_server_cleans_up_when_never_healthy(helpers, tmp_path, monkeypatch): + monkeypatch.setattr(helpers, "START_TIMEOUT_S", 0.0) + with pytest.raises(RuntimeError, match="not healthy"): + helpers.start_server(tmp_path / "slow.log", backend="fake") + assert not helpers.PIDFILE.exists() and helpers._PROC is None and helpers._LOG is None + assert not helpers._port_listening() + + +def test_repeated_stop_without_a_server(helpers): + for _ in range(3): + helpers.stop_server() + assert not helpers.PIDFILE.exists() + + +@pytest.mark.parametrize("liveness", [ + pytest.param("proc", marks=pytest.mark.skipif(not HAS_PROC, reason="needs /proc")), + pytest.param("ps", marks=pytest.mark.skipif(not HAS_PS, reason="needs ps")), +]) +def test_stop_server_started_by_another_kernel(helpers, tmp_path, monkeypatch, liveness): + """After a notebook kernel restart only the pid file remains. With + liveness="ps" the helpers run as on macOS, which has no /proc.""" + monkeypatch.setattr(helpers, "_HAVE_PROC", liveness == "proc") + proc = helpers.start_server(tmp_path / "c.log", backend="fake") + helpers._PROC = None # simulate a fresh kernel + sent = _record_signals(monkeypatch, helpers) + t0 = time.monotonic() + helpers.stop_server() + assert time.monotonic() - t0 < 15 + assert sent == [(proc.pid, signal.SIGTERM)] # graceful: no SIGKILL needed + assert proc.wait(timeout=5) is not None + assert not helpers._port_listening() + + +def test_graceful_shutdown_sends_only_sigterm(helpers, tmp_path, monkeypatch): + proc = helpers.start_server(tmp_path / "g.log", backend="fake") + sent = _record_signals(monkeypatch, helpers) + helpers.stop_server() + assert sent == [(proc.pid, signal.SIGTERM)] + + +def test_already_exited_server_is_not_signalled(helpers, tmp_path, monkeypatch): + proc = helpers.start_server(tmp_path / "x.log", backend="fake") + os.killpg(proc.pid, signal.SIGKILL) + proc.wait(timeout=10) # exited and reaped: its pid may be reused + sent = _record_signals(monkeypatch, helpers) + helpers.stop_server() # same kernel + assert sent == [] + helpers.PIDFILE.write_text(json.dumps({"pid": proc.pid, "port": helpers.PORT, "start": "x"})) + helpers.stop_server() # another kernel, stale pid file + assert sent == [] and not helpers.PIDFILE.exists() +# ------------------------------------------------- stale pid files, reuse +@pytest.mark.parametrize("content", ["legacy", "wrong-start", "garbage"]) +def test_stale_pid_file_never_signals_an_unrelated_process(helpers, spawned, monkeypatch, content): + """The pid in the file now belongs to someone else's session leader, the + dangerous case for killpg.""" + other = _ready(spawned(SLEEPER)) + text = {"legacy": str(other.pid), + "wrong-start": json.dumps({"pid": other.pid, "port": helpers.PORT, "start": "proc:1"}), + "garbage": "not a pid"}[content] + helpers.PIDFILE.write_text(text) + sent = _record_signals(monkeypatch, helpers) + helpers.stop_server() + assert sent == [] and other.poll() is None + assert not helpers.PIDFILE.exists() + + +def test_pid_reuse_detected_by_start_time(helpers, spawned, monkeypatch): + """Same pid, a command line that looks like the server, but a different + start time: a reused pid must not be signalled.""" + lookalike = _ready(spawned(SLEEPER, "-m", "echorelay", "--port", str(helpers.PORT))) + assert helpers._ownership({"pid": lookalike.pid, "start": helpers._identity(lookalike.pid)["start"]}) == "owned" + helpers.PIDFILE.write_text(json.dumps({"pid": lookalike.pid, "port": helpers.PORT, "start": "proc:0"})) + sent = _record_signals(monkeypatch, helpers) + helpers.stop_server() + assert sent == [] and lookalike.poll() is None + + +def test_unverifiable_ownership_fails_safe(helpers, spawned, monkeypatch): + other = _ready(spawned(SLEEPER, "-m", "echorelay", "--port", str(helpers.PORT))) + helpers.PIDFILE.write_text(json.dumps({"pid": other.pid, "port": helpers.PORT, "start": None})) + monkeypatch.setattr(helpers, "_identity", lambda pid: None) # no /proc and no ps + sent = _record_signals(monkeypatch, helpers) + with pytest.raises(RuntimeError, match="cannot verify"): + helpers.stop_server() + assert sent == [] and other.poll() is None + assert helpers.PIDFILE.exists() # kept so the user can act on it + helpers.PIDFILE.unlink() + + +# ------------------------------------------------------------ escalation +def test_sigterm_ignored_escalates_to_sigkill(helpers, spawned, monkeypatch): + stubborn = _ready(spawned(SLEEPER, "ignore-term", "-m", "echorelay", "--port", str(helpers.PORT))) + ident = helpers._identity(stubborn.pid) + helpers.PIDFILE.write_text(json.dumps({"pid": stubborn.pid, "port": helpers.PORT, + "start": ident["start"]})) + monkeypatch.setattr(helpers, "STOP_GRACE_S", 0.3) + sent = _record_signals(monkeypatch, helpers) + t0 = time.monotonic() + helpers.stop_server() + assert time.monotonic() - t0 < 5 + assert sent == [(stubborn.pid, signal.SIGTERM), (stubborn.pid, signal.SIGKILL)] + assert stubborn.wait(timeout=5) == -signal.SIGKILL + + +def test_unkillable_process_raises_and_keeps_pid_file(helpers, spawned, monkeypatch): + stubborn = _ready(spawned(SLEEPER, "ignore-term", "-m", "echorelay", "--port", str(helpers.PORT))) + helpers.PIDFILE.write_text(json.dumps({"pid": stubborn.pid, "port": helpers.PORT, + "start": helpers._identity(stubborn.pid)["start"]})) + monkeypatch.setattr(helpers, "STOP_GRACE_S", 0.1) + monkeypatch.setattr(helpers, "STOP_KILL_S", 0.2) + monkeypatch.setattr(helpers, "_signal_group", lambda pid, sig: True) # signals never arrive + with pytest.raises(RuntimeError, match="did not exit"): + helpers.stop_server() + assert helpers.PIDFILE.exists() and stubborn.poll() is None + helpers.PIDFILE.unlink() + + +# -------------------------------------------------------------- liveness +def _zombie(spawned): + p = spawned("pass") + return p # exits at once; stays a zombie until p.wait() + + +@pytest.mark.skipif(not HAS_PS, reason="needs ps") +def test_liveness_without_proc_uses_ps(helpers, spawned, monkeypatch): + """macOS has no /proc: live, zombie and reaped processes are told apart + with ps.""" + monkeypatch.setattr(helpers, "_HAVE_PROC", False) + live = _ready(spawned(SLEEPER)) + assert helpers._alive(live.pid) + z = _zombie(spawned) + _until(lambda: helpers._proc_state(z.pid) == "Z", what="zombie state") + assert not helpers._alive(z.pid) # kill(pid, 0) alone would call it alive + z.wait(timeout=5) + assert helpers._proc_state(z.pid) is None and not helpers._alive(z.pid) + ident = helpers._identity(live.pid) + assert ident["start"].startswith("ps:") and "time.sleep" in ident["command"] + + +def test_unknown_state_is_treated_as_alive(helpers, spawned, monkeypatch): + """With neither /proc nor ps, a process that exists is assumed alive, so + stop_server waits for it instead of declaring it dead.""" + monkeypatch.setattr(helpers, "_HAVE_PROC", False) + monkeypatch.setattr(helpers, "_ps", lambda pid, field: None) + live = _ready(spawned(SLEEPER)) + assert helpers._proc_state(live.pid) == "?" and helpers._alive(live.pid) + live.kill() + live.wait(timeout=5) + assert helpers._proc_state(live.pid) is None + + +@pytest.mark.skipif(not HAS_PROC, reason="Linux /proc") +def test_linux_zombie_detection(helpers, spawned): + z = _zombie(spawned) + _until(lambda: helpers._proc_state(z.pid) == "Z", what="zombie state") + assert not helpers._alive(z.pid) + assert helpers._identity(z.pid)["start"].startswith("proc:") + + +# ------------------------------------------------------------------ port +def test_port_held_by_a_non_http_listener_is_not_reported_free(helpers, tmp_path, monkeypatch): + """/health failing does not mean the port is free.""" + monkeypatch.setattr(helpers, "PORT_RELEASE_S", 0.2) + with socket.socket() as listener: + listener.bind(("127.0.0.1", helpers.PORT)) + listener.listen() + with pytest.raises(RuntimeError, match="still in use by a process that is not"): + helpers.stop_server() + with pytest.raises(RuntimeError, match="still in use"): + helpers.start_server(tmp_path / "busy.log", backend="fake") + assert not helpers.PIDFILE.exists() and helpers._PROC is None + helpers.stop_server() + + +# --------------------------------------------------- commit, audio checks def test_verify_commit(helpers): head = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=helpers.REPO, text=True).strip() dirty = subprocess.check_output(["git", "status", "--porcelain", "--untracked-files=no"], @@ -87,15 +346,3 @@ def test_verify_eval_audio_hashes_pcm_not_file(helpers, tmp_path): _wav(audio / "b.wav", np.ones(800)) # same length, different samples with pytest.raises(AssertionError, match="differs"): helpers.verify_eval_audio(audio, manifest) - - -def test_stop_server_started_by_another_kernel(helpers, tmp_path): - """After a notebook kernel restart only the pid file remains.""" - import time - - proc = helpers.start_server(tmp_path / "c.log", backend="fake") - helpers._PROC = None # simulate a fresh kernel - t0 = time.monotonic() - helpers.stop_server() - assert time.monotonic() - t0 < 15 - assert proc.wait(timeout=5) is not None From 770e29c9377e6bd9a1e11ae56782e144a3a1dc28 Mon Sep 17 00:00:00 2001 From: hydrangeas20 Date: Sat, 24 Oct 2026 21:27:18 -0400 Subject: [PATCH 2/2] fix: never signal a pid-file process whose identity is unconfirmed A pid file without a recorded start time (written by an older version, or when the start time was unreadable at start-up) was accepted as the server on a command-line match alone. Anything started with the same arguments, including another EchoRelay server, would pass. stop_server now refuses to signal it, keeps the pid file, and raises with manual-stop instructions. Stale entries whose process has exited, or whose command line does not match, are still removed without signalling. On Linux, the recorded process is pinned with a pidfd before verification: an identity read after the pinned process exited is not trusted, nothing is signalled once it has exited, and exit is awaited on the pidfd, which a reused pid cannot fool. Falls back to the previous path where pidfd is unavailable (macOS, old kernels, seccomp). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017DfLG9q5h7eiawGNwaBswq --- benchmarks/colab_helpers.py | 119 ++++++++++++++++++++++++++++-------- tests/test_colab_helpers.py | 110 ++++++++++++++++++++++++++++++++- 2 files changed, 202 insertions(+), 27 deletions(-) diff --git a/benchmarks/colab_helpers.py b/benchmarks/colab_helpers.py index daae50f..0d09b5e 100644 --- a/benchmarks/colab_helpers.py +++ b/benchmarks/colab_helpers.py @@ -19,6 +19,7 @@ import json import os +import select import signal import socket import subprocess @@ -162,8 +163,42 @@ def _server_marker() -> tuple[str, str]: return ("-m echorelay", f"--port {PORT}") -def _ownership(rec: dict) -> str: - """'owned', 'foreign' (pid reused or not our server), 'gone', or 'unknown'.""" +def _pidfd(pid: int) -> int | None: + """A Linux pidfd pinning this exact process, or None where unsupported + (macOS, old kernels, sandboxes). Raises ProcessLookupError if pid is gone. + + A pidfd keeps referring to the process it was opened for even if that + process exits and its pid is reused, which closes the window between + verifying a process and signalling or waiting for it. + """ + if not hasattr(os, "pidfd_open"): + return None + try: + return os.pidfd_open(pid) + except ProcessLookupError: + raise + except OSError: # ENOSYS, EPERM under seccomp, ... + return None + + +def _pinned_exited(pidfd: int, timeout: float = 0.0) -> bool: + """True once the pinned process has exited (its pidfd becomes readable), + waiting at most timeout seconds. poll, unlike select, takes any fd number.""" + poller = select.poll() + poller.register(pidfd, select.POLLIN) + return bool(poller.poll(max(timeout, 0.0) * 1000)) + + +def _ownership(rec: dict, pidfd: int | None = None) -> str: + """'owned', 'foreign' (pid reused or not our server), 'gone', 'unknown' + (identity unreadable), or 'unconfirmed' (no recorded start time). + + A process is 'owned' only if its start time matches the one recorded at + start-up AND its command line is the server's. A matching command line + alone is never enough: another EchoRelay server, or anything started with + the same arguments, would pass. With a pidfd, the identity read is + confirmed to belong to the pinned process. + """ pid = rec["pid"] if not _alive(pid): return "gone" # includes a zombie, whose command line is no longer readable @@ -172,16 +207,23 @@ def _ownership(rec: dict) -> str: return "gone" if _proc_state(pid) is None else "unknown" if not all(m in ident["command"] for m in _server_marker()): return "foreign" - if rec.get("start") is not None and rec["start"] != ident["start"]: + if rec.get("start") is None: + return "unconfirmed" # legacy pid file, or start time unreadable at start-up + if rec["start"] != ident["start"]: return "foreign" # same pid, different process: the pid was reused - return "owned" # a legacy pid file has no start time: the command line must match + if pidfd is not None and _pinned_exited(pidfd): + return "gone" # pinned process gone: the identity read may be a successor's + return "owned" -def _signal_group(pid: int, sig: int) -> bool: +def _signal_group(pid: int, sig: int, pidfd: int | None = None) -> bool: """Send sig to pid's process group (the server leads its own session). Returns False if the process is already gone. Only called for this kernel's child or a process whose ownership was verified, both started - with start_new_session=True, so their group id equals their pid.""" + with start_new_session=True, so their group id equals their pid. With a + pidfd, nothing is sent once the pinned process has exited.""" + if pidfd is not None and _pinned_exited(pidfd): + return False try: pgid = os.getpgid(pid) except ProcessLookupError: @@ -200,15 +242,19 @@ def _signal_group(pid: int, sig: int) -> bool: return True -def _wait_exit(pid: int, timeout: float, proc: subprocess.Popen | None = None) -> bool: +def _wait_exit(pid: int, timeout: float, proc: subprocess.Popen | None = None, + pidfd: int | None = None) -> bool: """Wait at most timeout seconds for pid to exit. A child of this kernel is - also reaped; any other process is polled with a growing interval.""" + also reaped; a pinned process is waited for on its pidfd, which a reused + pid cannot fool; anything else is polled with a growing interval.""" if proc is not None: try: proc.wait(timeout=timeout) return True except subprocess.TimeoutExpired: return False + if pidfd is not None: + return _pinned_exited(pidfd, timeout) deadline = time.monotonic() + timeout delay = 0.02 while _alive(pid): @@ -220,22 +266,23 @@ def _wait_exit(pid: int, timeout: float, proc: subprocess.Popen | None = None) - return True -def _stop_pid(pid: int, proc: subprocess.Popen | None = None, still_ours=None) -> None: +def _stop_pid(pid: int, proc: subprocess.Popen | None = None, still_ours=None, + pidfd: int | None = None) -> None: """SIGTERM pid's group; SIGKILL it if it outlives STOP_GRACE_S.""" if proc is not None: if proc.poll() is not None: return # exited and reaped: its pid may already belong to someone else elif not _alive(pid): return - if not _signal_group(pid, signal.SIGTERM): + if not _signal_group(pid, signal.SIGTERM, pidfd): return - if _wait_exit(pid, STOP_GRACE_S, proc): + if _wait_exit(pid, STOP_GRACE_S, proc, pidfd): return if still_ours is not None and not still_ours(): return # it exited and the pid was reused while we waited print(f"pid {pid} still running {STOP_GRACE_S:g} s after SIGTERM; sending SIGKILL") - _signal_group(pid, signal.SIGKILL) - if not _wait_exit(pid, STOP_KILL_S, proc): + _signal_group(pid, signal.SIGKILL, pidfd) + if not _wait_exit(pid, STOP_KILL_S, proc, pidfd): raise RuntimeError(f"pid {pid} did not exit {STOP_KILL_S:g} s after SIGKILL") @@ -291,6 +338,39 @@ def _close_log() -> None: _LOG = None +def _stop_recorded(rec: dict) -> None: + """Stop the server named by the pid file, only if it is provably that + server. Raises, keeping the pid file, when ownership cannot be confirmed.""" + pid = rec["pid"] + try: + pidfd = _pidfd(pid) # pin first, so the identity checked is the one signalled + except ProcessLookupError: + print(f"stale pid file: pid {pid} has already exited") + return + try: + status = _ownership(rec, pidfd) + if status == "owned": + _stop_pid(pid, still_ours=lambda: _ownership(rec, pidfd) == "owned", pidfd=pidfd) + elif status == "foreign": + print(f"stale pid file: pid {pid} is not the EchoRelay server on :{PORT}; " + "not signalling it") + elif status == "gone": + print(f"stale pid file: pid {pid} has already exited") + elif status == "unconfirmed": + raise RuntimeError( + f"{PIDFILE} records no start time (written by an older version, or the start " + f"time was unreadable), so pid {pid} cannot be confirmed as the server even " + f"though its command line matches; not signalling it. If it is the server, " + f"stop it with `kill -TERM -- -{pid}`, then delete the pid file") + else: # "unknown" + raise RuntimeError( + f"cannot verify that pid {pid} from {PIDFILE} is the EchoRelay server " + f"(no /proc and no ps); stop it yourself, then delete the pid file") + finally: + if pidfd is not None: + os.close(pidfd) + + # ---------------------------------------------------------------- public def stop_server() -> None: """Stop the server started by start_server (its whole process group). @@ -308,18 +388,7 @@ def stop_server() -> None: _PROC = None rec = _read_pidfile() if rec is not None and (own is None or rec["pid"] != own.pid): - status = _ownership(rec) - if status == "unknown": - raise RuntimeError( - f"cannot verify that pid {rec['pid']} from {PIDFILE} is the EchoRelay server " - f"(no /proc and no ps); stop it yourself, then delete the pid file") - if status == "owned": - _stop_pid(rec["pid"], still_ours=lambda: _ownership(rec) == "owned") - elif status == "foreign": - print(f"stale pid file: pid {rec['pid']} is not the EchoRelay server on :{PORT}; " - "not signalling it") - else: # "gone" - print(f"stale pid file: pid {rec['pid']} has already exited") + _stop_recorded(rec) PIDFILE.unlink(missing_ok=True) finally: _close_log() diff --git a/tests/test_colab_helpers.py b/tests/test_colab_helpers.py index 532ba09..91183c2 100644 --- a/tests/test_colab_helpers.py +++ b/tests/test_colab_helpers.py @@ -21,6 +21,7 @@ HAS_PS = shutil.which("ps") is not None HAS_PROC = os.path.isdir("/proc/self") +HAS_PIDFD = hasattr(os, "pidfd_open") @pytest.fixture @@ -229,7 +230,13 @@ def test_unverifiable_ownership_fails_safe(helpers, spawned, monkeypatch): # ------------------------------------------------------------ escalation -def test_sigterm_ignored_escalates_to_sigkill(helpers, spawned, monkeypatch): +@pytest.mark.parametrize("pidfd", [ + pytest.param(True, marks=pytest.mark.skipif(not HAS_PIDFD, reason="Linux pidfd")), + False, +]) +def test_sigterm_ignored_escalates_to_sigkill(helpers, spawned, monkeypatch, pidfd): + if not pidfd: + monkeypatch.setattr(helpers, "_pidfd", lambda pid: None) # as on macOS stubborn = _ready(spawned(SLEEPER, "ignore-term", "-m", "echorelay", "--port", str(helpers.PORT))) ident = helpers._identity(stubborn.pid) helpers.PIDFILE.write_text(json.dumps({"pid": stubborn.pid, "port": helpers.PORT, @@ -249,13 +256,112 @@ def test_unkillable_process_raises_and_keeps_pid_file(helpers, spawned, monkeypa "start": helpers._identity(stubborn.pid)["start"]})) monkeypatch.setattr(helpers, "STOP_GRACE_S", 0.1) monkeypatch.setattr(helpers, "STOP_KILL_S", 0.2) - monkeypatch.setattr(helpers, "_signal_group", lambda pid, sig: True) # signals never arrive + monkeypatch.setattr(helpers, "_signal_group", lambda *a: True) # signals never arrive with pytest.raises(RuntimeError, match="did not exit"): helpers.stop_server() assert helpers.PIDFILE.exists() and stubborn.poll() is None helpers.PIDFILE.unlink() +# A pid file without a recorded start time names a process whose command line +# may match but whose identity cannot be confirmed: never signal it. +@pytest.mark.parametrize("liveness", [ + pytest.param("proc", marks=pytest.mark.skipif(not HAS_PROC, reason="needs /proc")), + pytest.param("ps", marks=pytest.mark.skipif(not HAS_PS, reason="needs ps")), +]) +@pytest.mark.parametrize("content", ["legacy-int", "start-null", "start-missing"]) +def test_pid_file_without_start_time_fails_safe(helpers, spawned, monkeypatch, content, liveness): + monkeypatch.setattr(helpers, "_HAVE_PROC", liveness == "proc") + lookalike = _ready(spawned(SLEEPER, "-m", "echorelay", "--port", str(helpers.PORT))) + assert all(m in helpers._identity(lookalike.pid)["command"] for m in helpers._server_marker()) + text = {"legacy-int": str(lookalike.pid), + "start-null": json.dumps({"pid": lookalike.pid, "port": helpers.PORT, "start": None}), + "start-missing": json.dumps({"pid": lookalike.pid, "port": helpers.PORT})}[content] + helpers.PIDFILE.write_text(text) + sent = _record_signals(monkeypatch, helpers) + for _ in range(2): # stays safe when retried + with pytest.raises(RuntimeError, match="no start time"): + helpers.stop_server() + assert sent == [] and lookalike.poll() is None + assert helpers.PIDFILE.read_text() == text # kept, unchanged, for the user to act on + assert helpers._LOG is None + helpers.PIDFILE.unlink() + + +def test_pid_file_without_start_time_for_an_exited_process_is_removed(helpers, spawned, monkeypatch): + gone = spawned("pass") + gone.wait(timeout=10) + helpers.PIDFILE.write_text(str(gone.pid)) + sent = _record_signals(monkeypatch, helpers) + helpers.stop_server() + assert sent == [] and not helpers.PIDFILE.exists() + + +# ------------------------------------------------- pidfd (Linux) pinning +@pytest.mark.skipif(not HAS_PIDFD, reason="Linux pidfd") +def test_pinned_process_that_exited_is_never_signalled(helpers, spawned, monkeypatch): + """Exit between verification and signalling: the pid may already be + someone else's, so nothing is sent.""" + p = _ready(spawned(SLEEPER)) + fd = os.pidfd_open(p.pid) + try: + os.killpg(p.pid, signal.SIGKILL) + p.wait(timeout=10) # exited and reaped: the pid number is free for reuse + sent = _record_signals(monkeypatch, helpers) + assert helpers._signal_group(p.pid, signal.SIGTERM, fd) is False + assert helpers._wait_exit(p.pid, 5, pidfd=fd) is True + assert sent == [] + finally: + os.close(fd) + + +@pytest.mark.skipif(not HAS_PIDFD, reason="Linux pidfd") +def test_identity_read_after_pinned_exit_is_not_trusted(helpers, spawned, monkeypatch): + """The identity is read by pid number; if the pinned process exited first, + that read may describe a successor with the same pid, so it is 'gone'.""" + p = _ready(spawned(SLEEPER, "-m", "echorelay", "--port", str(helpers.PORT))) + rec = {"pid": p.pid, "start": helpers._identity(p.pid)["start"]} + fd = os.pidfd_open(p.pid) + try: + assert helpers._ownership(rec, fd) == "owned" + ident = helpers._identity(p.pid) + os.killpg(p.pid, signal.SIGKILL) + p.wait(timeout=10) + # pretend a successor with an identical identity took the pid + monkeypatch.setattr(helpers, "_alive", lambda pid: True) + monkeypatch.setattr(helpers, "_identity", lambda pid: ident) + assert helpers._ownership(rec) == "owned" # without the pin it would be fooled + assert helpers._ownership(rec, fd) == "gone" + finally: + os.close(fd) + + +@pytest.mark.skipif(not HAS_PIDFD, reason="Linux pidfd") +def test_pidfd_wait_is_bounded_and_wakes_on_exit(helpers, spawned): + p = _ready(spawned(SLEEPER)) + fd = os.pidfd_open(p.pid) + try: + t0 = time.monotonic() + assert helpers._wait_exit(p.pid, 0.2, pidfd=fd) is False + assert 0.15 < time.monotonic() - t0 < 2 + os.killpg(p.pid, signal.SIGTERM) + t0 = time.monotonic() + assert helpers._wait_exit(p.pid, 10, pidfd=fd) is True + assert time.monotonic() - t0 < 2 + finally: + os.close(fd) + + +def test_pidfd_unavailable_falls_back(helpers, monkeypatch): + """Old kernels and seccomp sandboxes refuse pidfd_open: fall back to the + unpinned path instead of failing.""" + if HAS_PIDFD: + def refuse(pid): + raise PermissionError(1, "Operation not permitted") + monkeypatch.setattr(helpers.os, "pidfd_open", refuse) + assert helpers._pidfd(os.getpid()) is None + + # -------------------------------------------------------------- liveness def _zombie(spawned): p = spawned("pass")