diff --git a/.github/workflows/fresh-host-acceptance.yml b/.github/workflows/fresh-host-acceptance.yml index 19bc16d1..c77f8f7b 100644 --- a/.github/workflows/fresh-host-acceptance.yml +++ b/.github/workflows/fresh-host-acceptance.yml @@ -44,6 +44,7 @@ on: options: - none - orbstack-stop-during-sidecars + - orbstack-stop-during-sidecars-health concurrency: group: >- diff --git a/platform/docs/OBSERVABILITY.md b/platform/docs/OBSERVABILITY.md index 4a8163ae..e4784d90 100644 --- a/platform/docs/OBSERVABILITY.md +++ b/platform/docs/OBSERVABILITY.md @@ -40,11 +40,14 @@ Those signals reuse the shared ClawOps telemetry path instead of adding a separa - `clawops.ops.sidecars.wait.start` - `clawops.ops.sidecars.wait.ready` - `clawops.ops.sidecars.wait.timeout` +- `clawops.ops.sidecars.wait.backend_lost` - `clawops.ops.sidecars.ready` - `clawops.ops.sidecars.status` Wait-timeout events include `service`, `target`, `observed`, and `timeout_seconds` so failed dependencies can be diagnosed without replaying the command interactively. +A wait whose `docker compose ps` fails probes the Docker backend. When the backend socket is gone, or `docker info` fails three probes in a row, the wait stops early with `wait.backend_lost` (`service`, `target`, `detail`, `elapsed_ms`) instead of polling out its timeout. + ## Host Service Activation Events `clawops services install --activate` now emits: diff --git a/src/clawops/assets/platform/docs/OBSERVABILITY.md b/src/clawops/assets/platform/docs/OBSERVABILITY.md index 4a8163ae..e4784d90 100644 --- a/src/clawops/assets/platform/docs/OBSERVABILITY.md +++ b/src/clawops/assets/platform/docs/OBSERVABILITY.md @@ -40,11 +40,14 @@ Those signals reuse the shared ClawOps telemetry path instead of adding a separa - `clawops.ops.sidecars.wait.start` - `clawops.ops.sidecars.wait.ready` - `clawops.ops.sidecars.wait.timeout` +- `clawops.ops.sidecars.wait.backend_lost` - `clawops.ops.sidecars.ready` - `clawops.ops.sidecars.status` Wait-timeout events include `service`, `target`, `observed`, and `timeout_seconds` so failed dependencies can be diagnosed without replaying the command interactively. +A wait whose `docker compose ps` fails probes the Docker backend. When the backend socket is gone, or `docker info` fails three probes in a row, the wait stops early with `wait.backend_lost` (`service`, `target`, `detail`, `elapsed_ms`) instead of polling out its timeout. + ## Host Service Activation Events `clawops services install --activate` now emits: diff --git a/src/clawops/strongclaw_ops.py b/src/clawops/strongclaw_ops.py index 21c616dd..cdf53d83 100644 --- a/src/clawops/strongclaw_ops.py +++ b/src/clawops/strongclaw_ops.py @@ -30,8 +30,11 @@ from clawops.strongclaw_runtime import ( VARLOCK_ENV_MODE_MANAGED, CommandError, + DockerBackendLostError, + DockerBackendWatch, ensure_docker_backend_ready, load_env_assignments, + probe_docker_backend, rendered_openclaw_hypermemory_config_path, rendered_openclaw_uses_hypermemory, rendered_openclaw_uses_qmd, @@ -346,7 +349,11 @@ def _wait_for_compose_service( health: str | None = None, timeout_seconds: int, ) -> None: - """Wait for one compose service to reach the requested state.""" + """Wait for one compose service to reach the requested state. + + A failed status inspection probes the Docker backend; once it is confirmed lost the wait + raises `DockerBackendLostError` instead of polling out the rest of *timeout_seconds*. + """ started_at = time.monotonic() target = state if health is None else f"{state}/{health}" emit_structured_log( @@ -360,12 +367,31 @@ def _wait_for_compose_service( deadline = time.monotonic() + timeout_seconds last_status: _ComposeServiceStatus | None = None last_error: CommandError | None = None + backend = DockerBackendWatch(lambda: probe_docker_backend(env=execution.env, cwd=execution.cwd)) while True: try: last_status = _compose_service_statuses(execution).get(service_name) last_error = None + backend.reset() except CommandError as exc: last_error = exc + lost_detail = backend.confirm_lost() + if lost_detail is not None: + emit_structured_log( + "clawops.ops.sidecars.wait.backend_lost", + { + "service": service_name, + "target": target, + "detail": lost_detail, + "elapsed_ms": int((time.monotonic() - started_at) * 1000), + "statusInspectionError": str(exc), + }, + ) + raise DockerBackendLostError( + f"Docker backend became unreachable while waiting for compose service " + f"'{service_name}' to reach {target}: {lost_detail}; last status inspection " + f"error: {exc}. Restart the Docker runtime, then rerun the operation." + ) from exc if time.monotonic() >= deadline: break time.sleep(COMPOSE_POLL_INTERVAL_SECONDS) @@ -763,6 +789,19 @@ def _record_step(step: str, exit_code: int) -> int: health="healthy", timeout_seconds=POSTGRES_HEALTH_TIMEOUT_SECONDS, ) + except DockerBackendLostError as exc: + # Recreating Postgres cannot help once the backend itself is gone. + if not suppress_wait_failures: + raise + return _sidecars_report_payload( + execution, + profile_flags=profile_flags, + targets=targets, + command_steps=command_steps, + command_exit_code=1, + error=str(exc), + failed_service=POSTGRES_SERVICE_NAME, + ) except CommandError as exc: postgres_wait_error = exc if postgres_wait_error is not None: diff --git a/src/clawops/strongclaw_runtime.py b/src/clawops/strongclaw_runtime.py index 5e249bb4..f078bbe4 100644 --- a/src/clawops/strongclaw_runtime.py +++ b/src/clawops/strongclaw_runtime.py @@ -14,7 +14,7 @@ import subprocess import sys import time -from collections.abc import Generator, Iterable, Mapping, Sequence +from collections.abc import Callable, Generator, Iterable, Mapping, Sequence from contextlib import contextmanager from typing import Any, Final, Literal, cast @@ -1201,6 +1201,95 @@ def ensure_docker_backend_ready() -> None: ) +DOCKER_BACKEND_PROBE_TIMEOUT_SECONDS: Final[int] = 15 +DOCKER_BACKEND_LOSS_PROBE_LIMIT: Final[int] = 3 + + +class DockerBackendLostError(CommandError): + """Raised when the Docker backend disappears while a command is waiting on it.""" + + +@dataclasses.dataclass(frozen=True, slots=True) +class DockerBackendProbe: + """One live Docker backend probe; a `definitive` failure confirms loss without repeats.""" + + reachable: bool + detail: str = "" + definitive: bool = False + + +def _docker_host_socket(docker_host: str) -> pathlib.Path | None: + """Return the Unix socket a `DOCKER_HOST` value points at, when it is one.""" + if not docker_host.startswith("unix://"): + return None + return pathlib.Path(docker_host.removeprefix("unix://")) + + +def probe_docker_backend( + *, + env: Mapping[str, str] | None = None, + cwd: pathlib.Path | None = None, + timeout_seconds: int = DOCKER_BACKEND_PROBE_TIMEOUT_SECONDS, +) -> DockerBackendProbe: + """Probe the Docker backend *env* targets: its Unix socket first, then `docker info`.""" + docker_host = (os.environ if env is None else env).get("DOCKER_HOST", "").strip() + socket_path = _docker_host_socket(docker_host) + if socket_path is not None and not socket_path.is_socket(): + return DockerBackendProbe( + reachable=False, detail=f"Docker socket {socket_path} is missing", definitive=True + ) + result = run_command(["docker", "info"], cwd=cwd, env=env, timeout_seconds=timeout_seconds) + if result.ok: + return DockerBackendProbe(reachable=True) + if result.timed_out: + return DockerBackendProbe( + reachable=False, detail=f"`docker info` timed out after {timeout_seconds}s" + ) + # `docker info` prints client details to stdout even when the daemon is gone. + error = " ".join(result.stderr.split()) + return DockerBackendProbe( + reachable=False, + detail=f"`docker info` failed: {error}" if error else "`docker info` failed", + ) + + +class DockerBackendWatch: + """Tell a lost Docker backend from a slow one across a status-polling loop. + + Call `confirm_lost` after a status inspection fails and `reset` after one succeeds. A + missing socket confirms loss at once; otherwise *probe_limit* failed probes in a row do. + """ + + def __init__( + self, + probe: Callable[[], DockerBackendProbe], + *, + probe_limit: int = DOCKER_BACKEND_LOSS_PROBE_LIMIT, + ) -> None: + if probe_limit < 1: + raise ValueError("probe_limit must be positive") + self._probe = probe + self._probe_limit = probe_limit + self._failures = 0 + + def reset(self) -> None: + """Record that the backend answered, clearing the failure streak.""" + self._failures = 0 + + def confirm_lost(self) -> str | None: + """Probe once; return why the backend is lost, or `None` while it may still answer.""" + probe = self._probe() + if probe.reachable: + self._failures = 0 + return None + self._failures += 1 + if probe.definitive: + return probe.detail + if self._failures >= self._probe_limit: + return f"{probe.detail} ({self._failures} failed probes in a row)" + return None + + def repair_linux_runtime_user_docker_access(runtime_user: str) -> None: """Ensure the runtime user can talk to Docker on Linux.""" group_result = run_command(["getent", "group", "docker"], timeout_seconds=10) diff --git a/tests/suites/contracts/repo/test_ci_workflow_surfaces.py b/tests/suites/contracts/repo/test_ci_workflow_surfaces.py index 60391929..717fba30 100644 --- a/tests/suites/contracts/repo/test_ci_workflow_surfaces.py +++ b/tests/suites/contracts/repo/test_ci_workflow_surfaces.py @@ -1016,13 +1016,14 @@ def _job_step(workflow_name: str, job_id: str, step_name: str) -> dict[str, obje def test_fault_injection_cannot_be_enabled_on_push_to_main() -> None: - """The fault drill reaches the scenario only from a manual dispatch off main.""" - leaked_push_env = { - FAULT_INJECTION_ENV: next(iter(KNOWN_FAULTS)), - "GITHUB_EVENT_NAME": "push", - "GITHUB_REF": "refs/heads/main", - } - assert requested_fault(leaked_push_env) is None + """Every fault drill reaches the scenario only from a manual dispatch off main.""" + for fault in sorted(KNOWN_FAULTS): + leaked_push_env = { + FAULT_INJECTION_ENV: fault, + "GITHUB_EVENT_NAME": "push", + "GITHUB_REF": "refs/heads/main", + } + assert requested_fault(leaked_push_env) is None, fault core_input = _as_str_object_dict( _workflow_trigger_inputs("fresh-host-core.yml", "workflow_call").get("fault_injection") diff --git a/tests/suites/unit/ci/test_fresh_host.py b/tests/suites/unit/ci/test_fresh_host.py index 0d201a44..b2a3497c 100644 --- a/tests/suites/unit/ci/test_fresh_host.py +++ b/tests/suites/unit/ci/test_fresh_host.py @@ -13,7 +13,7 @@ import pytest from clawops.strongclaw_compose import compose_project_name -from clawops.strongclaw_runtime import resolve_repo_local_compose_state_dir +from clawops.strongclaw_runtime import DockerBackendProbe, resolve_repo_local_compose_state_dir from tests.plugins.infrastructure.context import TestContext from tests.scripts import fresh_host as fresh_host_script from tests.utils.helpers import fresh_host @@ -22,6 +22,7 @@ from tests.utils.helpers._fresh_host import reporting as fresh_host_reporting from tests.utils.helpers._fresh_host import scenario as fresh_host_scenario from tests.utils.helpers._fresh_host import shell as fresh_host_shell +from tests.utils.helpers.clock import FakeClock _parse_args = cast( Callable[[list[str] | None], argparse.Namespace], @@ -2209,3 +2210,143 @@ def test_write_summary_marks_missing_child_durations_as_na( summary_text = summary_path.read_text(encoding="utf-8") assert "| Hosted runtime install | n/a |" in summary_text assert "| Hosted image ensure | n/a |" in summary_text + + +_DAEMON_DOWN_STDERR = "Cannot connect to the Docker daemon at unix:///tmp/docker.sock." + + +class _BackendProbes: + """Answer every backend probe with one fixed result.""" + + def __init__(self, probe: DockerBackendProbe) -> None: + self.probe = probe + self.calls = 0 + + def __call__(self, **kwargs: object) -> DockerBackendProbe: + del kwargs + self.calls += 1 + return self.probe + + +class _ComposePs: + """Scripted `docker compose ps` runs that record each per-poll timeout.""" + + def __init__(self, *outcomes: str | BaseException) -> None: + self._outcomes = list(outcomes) + self.timeouts: list[float | None] = [] + + def __call__( + self, args: list[str], *, timeout: float | None = None, **kwargs: object + ) -> subprocess.CompletedProcess[str]: + del kwargs + self.timeouts.append(timeout) + outcome = self._outcomes.pop(0) if len(self._outcomes) > 1 else self._outcomes[0] + if isinstance(outcome, BaseException): + raise outcome + if outcome == _DAEMON_DOWN_STDERR: + return subprocess.CompletedProcess(args, 1, stdout="", stderr=outcome) + return subprocess.CompletedProcess(args, 0, stdout=outcome, stderr="") + + +def _sidecars_payload(litellm_health: str) -> str: + return json.dumps( + [ + {"Service": "postgres", "State": "running", "Health": "healthy"}, + {"Service": "litellm", "State": "running", "Health": litellm_health}, + {"Service": "otel-collector", "State": "running"}, + {"Service": "qdrant", "State": "running", "Health": "healthy"}, + ] + ) + + +def _install_health_wait( + test_context: TestContext, compose_ps: _ComposePs, probe: DockerBackendProbe +) -> tuple[FakeClock, _BackendProbes]: + clock = FakeClock() + probes = _BackendProbes(probe) + test_context.patch.patch_object(fresh_host_shell.subprocess, "run", new=compose_ps) + test_context.patch.patch_object(fresh_host_shell, "time", new=clock) + test_context.patch.patch_object(fresh_host_shell, "probe_docker_backend", new=probes) + return clock, probes + + +def _verify_sidecars(tmp_path: Path, *, timeout_seconds: int = 300) -> None: + fresh_host_shell.verify_sidecar_services_running( + tmp_path / "compose.yaml", + cwd=tmp_path, + env={"PATH": "/usr/bin"}, + timeout_seconds=timeout_seconds, + ) + + +def test_sidecar_health_wait_fails_fast_when_the_docker_socket_disappears( + tmp_path: Path, test_context: TestContext +) -> None: + missing = DockerBackendProbe( + reachable=False, detail="Docker socket /tmp/docker.sock is missing", definitive=True + ) + clock, probes = _install_health_wait(test_context, _ComposePs(_DAEMON_DOWN_STDERR), missing) + + with pytest.raises(fresh_host.FreshHostError) as exc_info: + _verify_sidecars(tmp_path) + + assert str(exc_info.value) == ( + "Docker backend became unreachable while waiting for docker compose services: " + f"Docker socket /tmp/docker.sock is missing; last status error: {_DAEMON_DOWN_STDERR}" + ) + assert clock.sleeps == [] + assert probes.calls == 1 + + +def test_sidecar_health_wait_fails_fast_after_consecutive_failed_docker_probes( + tmp_path: Path, test_context: TestContext +) -> None: + down = DockerBackendProbe(reachable=False, detail="`docker info` failed: EOF") + clock, probes = _install_health_wait(test_context, _ComposePs(_DAEMON_DOWN_STDERR), down) + + with pytest.raises(fresh_host.FreshHostError, match=r"\(3 failed probes in a row\)"): + _verify_sidecars(tmp_path) + + assert clock.sleeps == [2.0, 2.0] + assert probes.calls == 3 + + +def test_sidecar_health_wait_treats_a_hung_compose_ps_as_a_failed_poll( + tmp_path: Path, test_context: TestContext +) -> None: + hung = subprocess.TimeoutExpired(cmd=["docker", "compose", "ps"], timeout=30) + missing = DockerBackendProbe(reachable=False, detail="socket gone", definitive=True) + compose_ps = _ComposePs(hung) + _install_health_wait(test_context, compose_ps, missing) + + with pytest.raises(fresh_host.FreshHostError, match="docker compose ps timed out after 30s"): + _verify_sidecars(tmp_path, timeout_seconds=300) + + assert compose_ps.timeouts == [30] + + +def test_sidecar_health_wait_keeps_waiting_for_a_slow_container_on_a_healthy_backend( + tmp_path: Path, test_context: TestContext +) -> None: + down = DockerBackendProbe(reachable=False, detail="never probed") + compose_ps = _ComposePs(*[_sidecars_payload("starting")] * 60, _sidecars_payload("healthy")) + clock, probes = _install_health_wait(test_context, compose_ps, down) + + _verify_sidecars(tmp_path, timeout_seconds=300) + + assert clock.sleeps == [2.0] * 60 + assert probes.calls == 0 + assert set(compose_ps.timeouts) == {30} + + +def test_sidecar_health_wait_keeps_polling_failed_compose_ps_while_the_backend_answers( + tmp_path: Path, test_context: TestContext +) -> None: + up = DockerBackendProbe(reachable=True) + compose_ps = _ComposePs(_DAEMON_DOWN_STDERR, _DAEMON_DOWN_STDERR, _sidecars_payload("healthy")) + clock, probes = _install_health_wait(test_context, compose_ps, up) + + _verify_sidecars(tmp_path) + + assert clock.sleeps == [2.0, 2.0] + assert probes.calls == 2 diff --git a/tests/suites/unit/ci/test_fresh_host_faults.py b/tests/suites/unit/ci/test_fresh_host_faults.py index 9e6e88ee..e23f7203 100644 --- a/tests/suites/unit/ci/test_fresh_host_faults.py +++ b/tests/suites/unit/ci/test_fresh_host_faults.py @@ -3,7 +3,10 @@ from __future__ import annotations import json +import threading +from collections.abc import Callable from pathlib import Path +from typing import Any, cast import pytest @@ -108,3 +111,196 @@ def test_sidecars_runtime_loss_is_inert_on_push_to_main( assert fired is False assert commands == [] assert not (Path(context.diagnostics_dir) / "fault-injection").exists() + + +_HEALTH_FAULT = faults.ORBSTACK_STOP_DURING_SIDECARS_HEALTH +_COMPOSE_FILE = Path("docker-compose.aux-stack.ci-hosted-macos.yaml") + + +@pytest.mark.parametrize("fault", sorted(faults.KNOWN_FAULTS)) +def test_every_fault_is_inert_on_push_to_main(fault: str) -> None: + assert faults.requested_fault(_environ(fault, event="push", ref="refs/heads/main")) is None + + +def _entries(**health: str) -> dict[str, dict[str, object]]: + """Return a running sidecars stack with per-service health overrides.""" + services: dict[str, dict[str, object]] = { + "postgres": {"Service": "postgres", "State": "running", "Health": "healthy"}, + "litellm": {"Service": "litellm", "State": "running", "Health": "healthy"}, + "otel-collector": {"Service": "otel-collector", "State": "running"}, + "qdrant": {"Service": "qdrant", "State": "running", "Health": "healthy"}, + } + for name, value in health.items(): + services[name]["Health"] = value + return services + + +_WAITING = _entries(litellm="starting", qdrant="starting") +_PARTIAL = {"postgres": _entries()["postgres"], "litellm": _entries(litellm="starting")["litellm"]} + + +@pytest.mark.parametrize( + ("services", "expected"), + [ + (_WAITING, "litellm=starting, qdrant=starting"), + (_entries(), None), + (_PARTIAL, None), + ({}, None), + ], +) +def test_pending_sidecar_health_sees_only_a_started_stack_waiting_on_health( + services: dict[str, dict[str, object]], expected: str | None +) -> None: + assert faults.pending_sidecar_health(services) == expected + + +class _InstantEvent(threading.Event): + """An event whose waits return at once, so polling loops run without real sleeps.""" + + def wait(self, timeout: float | None = None) -> bool: + del timeout + return self.is_set() + + +def _script_compose_ps( + test_context: TestContext, + *outcomes: dict[str, dict[str, object]] | FreshHostError, + on_poll: Callable[[], None] | None = None, +) -> list[int]: + polls: list[int] = [] + remaining = list(outcomes) + + def fake_entries(compose_file: Path, **kwargs: object) -> dict[str, dict[str, object]]: + del compose_file, kwargs + polls.append(len(polls) + 1) + if on_poll is not None: + on_poll() + outcome = remaining.pop(0) if len(remaining) > 1 else remaining[0] + if isinstance(outcome, FreshHostError): + raise outcome + return outcome + + test_context.patch.patch_object(faults, "compose_service_entries", new=fake_entries) + return polls + + +def test_health_wait_watcher_stops_orbstack_on_the_second_sighting( + tmp_path: Path, test_context: TestContext +) -> None: + context = _macos_context(tmp_path, test_context) + commands = _record_commands(test_context) + polls = _script_compose_ps( + test_context, _PARTIAL, _WAITING, FreshHostError("blip"), _WAITING, _WAITING + ) + + cast(Any, faults)._stop_orbstack_in_health_wait( + context, fault=_HEALTH_FAULT, compose_file=_COMPOSE_FILE, env={}, stop=_InstantEvent() + ) + + assert polls == [1, 2, 3, 4, 5] + assert commands == [["orb", "stop"]] + marker = json.loads( + (Path(context.diagnostics_dir) / "fault-injection" / f"{_HEALTH_FAULT}.json").read_text( + encoding="utf-8" + ) + ) + assert marker["when"] == "inside the sidecars health wait: litellm=starting, qdrant=starting" + + +def test_health_wait_watcher_never_stops_orbstack_after_it_is_told_to_stop( + tmp_path: Path, test_context: TestContext +) -> None: + context = _macos_context(tmp_path, test_context) + commands = _record_commands(test_context) + stop = _InstantEvent() + polls = _script_compose_ps(test_context, _WAITING, on_poll=stop.set) + + cast(Any, faults)._stop_orbstack_in_health_wait( + context, fault=_HEALTH_FAULT, compose_file=_COMPOSE_FILE, env={}, stop=stop + ) + + assert polls == [1] + assert commands == [] + + +def test_health_wait_fault_is_inert_unless_requested( + tmp_path: Path, test_context: TestContext +) -> None: + context = _macos_context(tmp_path, test_context) + commands = _record_commands(test_context) + + def unexpected_probe_env(*args: object, **kwargs: object) -> dict[str, str]: + raise AssertionError((args, kwargs)) + + test_context.patch.patch_object(faults, "compose_probe_env", new=unexpected_probe_env) + + for environ in ( + _environ(_HEALTH_FAULT, event="push", ref="refs/heads/main"), + _environ(_FAULT, event="workflow_dispatch", ref=_BRANCH_REF), + {}, + ): + with faults.orbstack_stop_during_sidecars_health( + context, compose_file=_COMPOSE_FILE, environ=environ + ): + pass + + assert commands == [] + assert not (Path(context.diagnostics_dir) / "fault-injection").exists() + + +def _arm_health_fault(test_context: TestContext, *, poll_seconds: float) -> threading.Event: + """Stub the watcher's env and poll pace; return an event set when `orb stop` runs.""" + fired = threading.Event() + + def fake_run_command(command: list[str], **kwargs: object) -> None: + del kwargs + assert command == ["orb", "stop"] + fired.set() + + def fake_probe_env(*args: object, **kwargs: object) -> dict[str, str]: + del args, kwargs + return {} + + test_context.patch.patch_object(faults, "run_command", new=fake_run_command) + test_context.patch.patch_object(faults, "compose_probe_env", new=fake_probe_env) + test_context.patch.patch_object(faults, "_HEALTH_WAIT_POLL_SECONDS", new=poll_seconds) + return fired + + +def test_health_wait_fault_fires_once_while_sidecars_up_waits( + tmp_path: Path, test_context: TestContext +) -> None: + context = _macos_context(tmp_path, test_context) + fired = _arm_health_fault(test_context, poll_seconds=0.0) + _script_compose_ps(test_context, _WAITING) + environ = _environ(_HEALTH_FAULT, event="workflow_dispatch", ref=_BRANCH_REF) + + with faults.orbstack_stop_during_sidecars_health( + context, compose_file=_COMPOSE_FILE, environ=environ + ): + assert fired.wait(timeout=10) + fired.clear() + with faults.orbstack_stop_during_sidecars_health( + context, compose_file=_COMPOSE_FILE, environ=environ + ): + pass + + assert not fired.is_set() + assert (Path(context.diagnostics_dir) / "fault-injection" / f"{_HEALTH_FAULT}.json").is_file() + + +def test_health_wait_fault_reports_when_no_health_wait_was_seen( + tmp_path: Path, test_context: TestContext, capsys: pytest.CaptureFixture[str] +) -> None: + context = _macos_context(tmp_path, test_context) + fired = _arm_health_fault(test_context, poll_seconds=0.01) + _script_compose_ps(test_context, _entries()) + environ = _environ(_HEALTH_FAULT, event="workflow_dispatch", ref=_BRANCH_REF) + + with faults.orbstack_stop_during_sidecars_health( + context, compose_file=_COMPOSE_FILE, environ=environ + ): + pass + + assert not fired.is_set() + assert f"Fault injection {_HEALTH_FAULT} did not fire" in capsys.readouterr().out diff --git a/tests/suites/unit/ci/test_fresh_host_runtime_loss.py b/tests/suites/unit/ci/test_fresh_host_runtime_loss.py index 75e06f4d..1466e1a3 100644 --- a/tests/suites/unit/ci/test_fresh_host_runtime_loss.py +++ b/tests/suites/unit/ci/test_fresh_host_runtime_loss.py @@ -9,11 +9,13 @@ import pytest +from clawops.strongclaw_runtime import DockerBackendProbe from tests.plugins.infrastructure.context import TestContext from tests.utils.helpers import fresh_host from tests.utils.helpers._fresh_host import faults as fresh_host_faults from tests.utils.helpers._fresh_host import macos as fresh_host_macos from tests.utils.helpers._fresh_host import runtime_loss +from tests.utils.helpers._fresh_host import shell as fresh_host_shell from tests.utils.helpers._fresh_host.budget import ( SCENARIO_RECOVERY_CAP_SECONDS, SCENARIO_RETRY_RESERVE_SECONDS, @@ -515,3 +517,57 @@ def fake_verify(compose_file: Path, **kwargs: object) -> None: Path(context.diagnostics_dir) / "fault-injection" / "orbstack-stop-during-sidecars.json" ) assert marker.is_file() + + +def test_backend_lost_inside_a_health_wait_fails_fast_into_recovery_and_one_retry( + tmp_path: Path, test_context: TestContext +) -> None: + clock = FakeClock() + context = _context(tmp_path, test_context) + orbstack = _FakeOrbStack(clock, up=False) + _install(test_context, orbstack, tmp_path) + test_context.patch.patch_object(fresh_host_shell, "time", new=clock) + + def fake_compose_service_entries( + compose_file: Path, **kwargs: object + ) -> dict[str, dict[str, object]]: + del compose_file, kwargs + if not orbstack.up: + raise fresh_host_shell.ComposeStatusCommandError(_EOF_STDERR) + return { + name: {"Service": name, "State": "running", "Health": "healthy"} + for name in fresh_host_shell.SIDECAR_EXPECTED_SERVICES + } + + def fake_probe_docker_backend(**kwargs: object) -> DockerBackendProbe: + del kwargs + return DockerBackendProbe(reachable=orbstack.up, detail="`docker info` failed: EOF") + + test_context.patch.patch_object( + fresh_host_shell, "compose_service_entries", new=fake_compose_service_entries + ) + test_context.patch.patch_object( + fresh_host_shell, "probe_docker_backend", new=fake_probe_docker_backend + ) + wait_seconds: list[float] = [] + + def health_wait() -> list[str]: + started = clock.monotonic() + try: + fresh_host_shell.verify_sidecar_services_running( + tmp_path / "compose.yaml", cwd=tmp_path, env={}, timeout_seconds=300 + ) + finally: + wait_seconds.append(clock.monotonic() - started) + return ["verify", "ok"] + + result = runtime_loss.run_with_runtime_recovery( + context, step=_STEP, window=_window(context, clock), action=health_wait + ) + + assert result == ["verify", "ok"] + assert wait_seconds[0] == pytest.approx(4.0) + assert len(wait_seconds) == 2 + report = _loss_report(context) + assert report["recovered"] is True + assert report["trigger_reason"] == "docker_socket_eof" diff --git a/tests/suites/unit/clawops/test_strongclaw_ops.py b/tests/suites/unit/clawops/test_strongclaw_ops.py index 820c7a0a..9736dcd4 100644 --- a/tests/suites/unit/clawops/test_strongclaw_ops.py +++ b/tests/suites/unit/clawops/test_strongclaw_ops.py @@ -13,8 +13,9 @@ from clawops import strongclaw_ops from clawops.qdrant_storage import StorageDecision -from clawops.strongclaw_runtime import CommandError +from clawops.strongclaw_runtime import CommandError, DockerBackendLostError, DockerBackendProbe from tests.plugins.infrastructure.context import TestContext +from tests.utils.helpers.clock import FakeClock from tests.utils.helpers.repo import REPO_ROOT @@ -1153,6 +1154,7 @@ def _sleep_noop(_seconds: float) -> None: return None monkeypatch.setattr(strongclaw_ops.time, "sleep", _sleep_noop) + probes = _install_backend_probes(monkeypatch, _BACKEND_UP) cast(Any, strongclaw_ops)._wait_for_compose_service( execution, @@ -1163,6 +1165,259 @@ def _sleep_noop(_seconds: float) -> None: ) assert attempts["count"] == 2 + assert probes.calls == 1 + + +_BACKEND_UP = DockerBackendProbe(reachable=True) +_BACKEND_DOWN = DockerBackendProbe( + reachable=False, detail="`docker info` failed: Cannot connect to the Docker daemon" +) +_SOCKET_MISSING = DockerBackendProbe( + reachable=False, detail="Docker socket /tmp/orbstack/docker.sock is missing", definitive=True +) + + +class _BackendProbes: + """Answer every backend probe with one fixed result, recording the probed env.""" + + def __init__(self, probe: DockerBackendProbe) -> None: + self.probe = probe + self.calls = 0 + self.envs: list[Mapping[str, str] | None] = [] + + def __call__( + self, + *, + env: Mapping[str, str] | None = None, + cwd: pathlib.Path | None = None, + timeout_seconds: int = 15, + ) -> DockerBackendProbe: + del cwd, timeout_seconds + self.calls += 1 + self.envs.append(env) + return self.probe + + +def _install_backend_probes( + monkeypatch: pytest.MonkeyPatch, probe: DockerBackendProbe +) -> _BackendProbes: + probes = _BackendProbes(probe) + monkeypatch.setattr(strongclaw_ops, "probe_docker_backend", probes) + return probes + + +def _wait_execution() -> object: + return cast(Any, strongclaw_ops)._ComposeExecution( + repo_root=REPO_ROOT, + compose_path=pathlib.Path("/tmp/compose.yaml"), + cwd=pathlib.Path("/tmp"), + env={"DOCKER_HOST": "unix:///tmp/orbstack/docker.sock"}, + ) + + +def _failing_compose_statuses(_execution: object) -> dict[str, object]: + raise CommandError("failed to inspect sidecar services: Cannot connect to the Docker daemon") + + +def _wait_for_litellm(timeout_seconds: int = 300) -> None: + cast(Any, strongclaw_ops)._wait_for_compose_service( + _wait_execution(), + service_name="litellm", + state="running", + health="healthy", + timeout_seconds=timeout_seconds, + ) + + +def _record_events(monkeypatch: pytest.MonkeyPatch) -> list[str]: + events: list[str] = [] + + def _emit_structured_log(event: str, payload: object) -> None: + del payload + events.append(event) + + monkeypatch.setattr(strongclaw_ops, "emit_structured_log", _emit_structured_log) + return events + + +def test_wait_for_compose_service_fails_fast_when_the_docker_socket_disappears( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = FakeClock() + monkeypatch.setattr(strongclaw_ops, "time", clock) + monkeypatch.setattr(strongclaw_ops, "_compose_service_statuses", _failing_compose_statuses) + probes = _install_backend_probes(monkeypatch, _SOCKET_MISSING) + events = _record_events(monkeypatch) + + with pytest.raises(DockerBackendLostError) as exc_info: + _wait_for_litellm() + + message = str(exc_info.value) + assert message.startswith( + "Docker backend became unreachable while waiting for compose service 'litellm' to " + "reach running/healthy: Docker socket /tmp/orbstack/docker.sock is missing; " + ) + assert "Cannot connect to the Docker daemon" in message + assert clock.sleeps == [] + assert probes.envs == [{"DOCKER_HOST": "unix:///tmp/orbstack/docker.sock"}] + assert events == ["clawops.ops.sidecars.wait.start", "clawops.ops.sidecars.wait.backend_lost"] + + +def test_wait_for_compose_service_fails_fast_after_consecutive_failed_docker_probes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = FakeClock() + started = clock.monotonic() + monkeypatch.setattr(strongclaw_ops, "time", clock) + monkeypatch.setattr(strongclaw_ops, "_compose_service_statuses", _failing_compose_statuses) + probes = _install_backend_probes(monkeypatch, _BACKEND_DOWN) + + with pytest.raises(DockerBackendLostError, match=r"\(3 failed probes in a row\)"): + _wait_for_litellm(timeout_seconds=300) + + assert probes.calls == 3 + assert clock.sleeps == [2.0, 2.0] + assert clock.monotonic() - started == pytest.approx(4.0) + + +def test_wait_for_compose_service_keeps_waiting_for_a_slow_container_on_a_healthy_backend( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = FakeClock() + monkeypatch.setattr(strongclaw_ops, "time", clock) + probes = _install_backend_probes(monkeypatch, _BACKEND_DOWN) + polls = {"count": 0} + + def _slow_litellm(_execution: object) -> dict[str, object]: + polls["count"] += 1 + health = "healthy" if polls["count"] > 100 else "starting" + return { + "litellm": cast(Any, strongclaw_ops)._ComposeServiceStatus( + name="litellm", state="running", health=health + ) + } + + monkeypatch.setattr(strongclaw_ops, "_compose_service_statuses", _slow_litellm) + + _wait_for_litellm(timeout_seconds=300) + + assert polls["count"] == 101 + assert clock.sleeps == [2.0] * 100 + assert probes.calls == 0 + + +def test_wait_for_compose_service_still_times_out_while_the_backend_answers( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = FakeClock() + started = clock.monotonic() + monkeypatch.setattr(strongclaw_ops, "time", clock) + monkeypatch.setattr(strongclaw_ops, "_compose_service_statuses", _failing_compose_statuses) + probes = _install_backend_probes(monkeypatch, _BACKEND_UP) + events = _record_events(monkeypatch) + + with pytest.raises(CommandError) as exc_info: + _wait_for_litellm(timeout_seconds=10) + + assert not isinstance(exc_info.value, DockerBackendLostError) + assert str(exc_info.value).startswith( + "timed out waiting for compose service 'litellm' to reach running/healthy; " + "last observed service not listed in compose status; last status inspection error: " + ) + assert clock.monotonic() - started == pytest.approx(10.0) + assert probes.calls == 6 + assert events[-1] == "clawops.ops.sidecars.wait.timeout" + + +def _stub_sidecars_up_with_postgres_wait( + monkeypatch: pytest.MonkeyPatch, + tmp_path: pathlib.Path, + postgres_error: CommandError, +) -> tuple[pathlib.Path, list[tuple[str, ...]]]: + """Stub sidecars orchestration so the Postgres health wait raises *postgres_error*.""" + compose_path = tmp_path / "compose.yaml" + compose_path.write_text("services: {}\n", encoding="utf-8") + inherited_calls: list[tuple[str, ...]] = [] + + def _compose_path_override(_repo_root: pathlib.Path, _compose_name: str) -> pathlib.Path: + return compose_path + + def _profile_flags(_path: pathlib.Path) -> dict[str, bool | str]: + return {"usesQmd": True, "usesHypermemory": True, "source": "test"} + + def _wait_for_compose_service( + _execution: object, + *, + service_name: str, + state: str, + health: str | None = None, + timeout_seconds: int, + ) -> None: + del state, health, timeout_seconds + if service_name == "postgres": + raise postgres_error + + def _empty_compose_statuses(_execution: object) -> dict[str, object]: + return {} + + def fake_run_command_inherited( + command: list[str], + *, + cwd: pathlib.Path | None = None, + env: dict[str, str] | None = None, + timeout_seconds: int = 1800, + ) -> int: + del cwd, env, timeout_seconds + inherited_calls.append(tuple(command)) + return 0 + + monkeypatch.setattr(strongclaw_ops, "ensure_docker_backend_ready", lambda: None) + monkeypatch.setattr(strongclaw_ops, "_compose_path", _compose_path_override) + monkeypatch.setattr(strongclaw_ops, "_compose_env", _fixed_compose_env) + monkeypatch.setattr(strongclaw_ops, "_resolve_profile_dependency_flags", _profile_flags) + monkeypatch.setattr(strongclaw_ops, "_wait_for_compose_service", _wait_for_compose_service) + monkeypatch.setattr(strongclaw_ops, "_compose_service_statuses", _empty_compose_statuses) + monkeypatch.setattr(strongclaw_ops, "run_command_inherited", fake_run_command_inherited) + _stub_qdrant_storage(monkeypatch) + return compose_path, inherited_calls + + +_POSTGRES_BACKEND_LOST = DockerBackendLostError( + "Docker backend became unreachable while waiting for compose service 'postgres'" +) + + +def test_sidecars_up_does_not_recreate_postgres_once_the_backend_is_lost( + monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path +) -> None: + compose_path, inherited_calls = _stub_sidecars_up_with_postgres_wait( + monkeypatch, tmp_path, _POSTGRES_BACKEND_LOST + ) + + with pytest.raises(DockerBackendLostError, match="Docker backend became unreachable"): + strongclaw_ops.sidecars_up(REPO_ROOT, repo_local_state=False) + + assert inherited_calls == [ + ("docker", "compose", "-f", str(compose_path), "up", "-d", "postgres") + ] + + +def test_sidecars_up_json_reports_backend_loss_during_the_postgres_wait( + monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path +) -> None: + _, inherited_calls = _stub_sidecars_up_with_postgres_wait( + monkeypatch, tmp_path, _POSTGRES_BACKEND_LOST + ) + + payload = cast(Any, strongclaw_ops)._sidecars_up_report( + REPO_ROOT, repo_local_state=False, suppress_wait_failures=True + ) + + assert payload["ok"] is False + assert payload["failedService"] == "postgres" + assert payload["error"] == str(_POSTGRES_BACKEND_LOST) + assert payload["command"]["exitCode"] == 1 + assert len(inherited_calls) == 1 def test_gateway_start_uses_unbounded_subprocess_timeout( diff --git a/tests/suites/unit/clawops/test_strongclaw_runtime.py b/tests/suites/unit/clawops/test_strongclaw_runtime.py index 19f9c2bf..eb0a00a4 100644 --- a/tests/suites/unit/clawops/test_strongclaw_runtime.py +++ b/tests/suites/unit/clawops/test_strongclaw_runtime.py @@ -589,3 +589,120 @@ def test_profile_requirement_helpers_include_non_managed_runtime_profiles() -> N assert runtime.profile_requires_qmd("acp") is True assert runtime.profile_requires_lossless_claw("acp") is False assert runtime.profile_requires_hypermemory_backend("acp") is False + + +def _docker_info_result( + returncode: int | None, *, stderr: str = "", timed_out: bool = False +) -> ExecResult: + return ExecResult(("docker", "info"), returncode, "Client: ...", stderr, 3, timed_out=timed_out) + + +def test_probe_docker_backend_reports_a_missing_socket_without_running_docker( + tmp_path: pathlib.Path, test_context: TestContext +) -> None: + def _unexpected_run_command(*args: object, **kwargs: object) -> ExecResult: + raise AssertionError((args, kwargs)) + + test_context.patch.patch_object(runtime, "run_command", new=_unexpected_run_command) + socket_path = tmp_path / "docker.sock" + + probe = runtime.probe_docker_backend(env={"DOCKER_HOST": f"unix://{socket_path}"}) + + assert probe == runtime.DockerBackendProbe( + reachable=False, detail=f"Docker socket {socket_path} is missing", definitive=True + ) + + +@pytest.mark.parametrize( + ("result", "expected"), + [ + (_docker_info_result(0), runtime.DockerBackendProbe(reachable=True)), + ( + _docker_info_result(1, stderr="Cannot connect to the Docker daemon.\n Is it up?"), + runtime.DockerBackendProbe( + reachable=False, + detail="`docker info` failed: Cannot connect to the Docker daemon. Is it up?", + ), + ), + ( + _docker_info_result(None, timed_out=True), + runtime.DockerBackendProbe(reachable=False, detail="`docker info` timed out after 15s"), + ), + ( + _docker_info_result(1), + runtime.DockerBackendProbe(reachable=False, detail="`docker info` failed"), + ), + ], +) +def test_probe_docker_backend_classifies_docker_info( + result: ExecResult, expected: runtime.DockerBackendProbe, test_context: TestContext +) -> None: + calls: list[tuple[list[str], dict[str, object]]] = [] + + def _fake_run_command(argv: list[str], **kwargs: object) -> ExecResult: + calls.append((list(argv), kwargs)) + return result + + test_context.patch.patch_object(runtime, "run_command", new=_fake_run_command) + env = {"DOCKER_HOST": "tcp://127.0.0.1:2375"} + + assert runtime.probe_docker_backend(env=env, cwd=pathlib.Path("/tmp")) == expected + assert calls == [ + (["docker", "info"], {"cwd": pathlib.Path("/tmp"), "env": env, "timeout_seconds": 15}) + ] + + +class _ScriptedProbe: + """Return scripted probe results in order, counting calls.""" + + def __init__(self, *probes: runtime.DockerBackendProbe) -> None: + self._probes = list(probes) + self.calls = 0 + + def __call__(self) -> runtime.DockerBackendProbe: + self.calls += 1 + return self._probes.pop(0) + + +_BACKEND_DOWN = runtime.DockerBackendProbe(reachable=False, detail="`docker info` failed: EOF") +_BACKEND_UP = runtime.DockerBackendProbe(reachable=True) + + +def test_docker_backend_watch_confirms_loss_after_consecutive_failures() -> None: + watch = runtime.DockerBackendWatch(_ScriptedProbe(*[_BACKEND_DOWN] * 3)) + + assert [watch.confirm_lost() for _ in range(3)] == [ + None, + None, + "`docker info` failed: EOF (3 failed probes in a row)", + ] + + +def test_docker_backend_watch_restarts_the_streak_when_the_backend_answers() -> None: + probe = _ScriptedProbe(_BACKEND_DOWN, _BACKEND_DOWN, _BACKEND_UP, _BACKEND_DOWN, _BACKEND_DOWN) + watch = runtime.DockerBackendWatch(probe, probe_limit=3) + + assert [watch.confirm_lost() for _ in range(5)] == [None] * 5 + + +def test_docker_backend_watch_reset_clears_the_failure_streak() -> None: + watch = runtime.DockerBackendWatch(_ScriptedProbe(*[_BACKEND_DOWN] * 3), probe_limit=2) + + assert watch.confirm_lost() is None + watch.reset() + assert watch.confirm_lost() is None + assert watch.confirm_lost() == "`docker info` failed: EOF (2 failed probes in a row)" + + +def test_docker_backend_watch_confirms_a_definitive_failure_at_once() -> None: + missing = runtime.DockerBackendProbe( + reachable=False, detail="Docker socket /tmp/docker.sock is missing", definitive=True + ) + watch = runtime.DockerBackendWatch(_ScriptedProbe(missing)) + + assert watch.confirm_lost() == "Docker socket /tmp/docker.sock is missing" + + +def test_docker_backend_watch_rejects_a_non_positive_probe_limit() -> None: + with pytest.raises(ValueError, match="probe_limit must be positive"): + runtime.DockerBackendWatch(_ScriptedProbe(), probe_limit=0) diff --git a/tests/utils/helpers/_fresh_host/faults.py b/tests/utils/helpers/_fresh_host/faults.py index 5134a7f3..7f954c36 100644 --- a/tests/utils/helpers/_fresh_host/faults.py +++ b/tests/utils/helpers/_fresh_host/faults.py @@ -11,22 +11,44 @@ import json import os -from collections.abc import Mapping +import threading +from collections.abc import Generator, Mapping +from contextlib import contextmanager from pathlib import Path from typing import Final from tests.utils.helpers._fresh_host.models import FreshHostContext, FreshHostError -from tests.utils.helpers._fresh_host.shell import run_command +from tests.utils.helpers._fresh_host.shell import ( + SIDECAR_EXPECTED_SERVICES, + SIDECAR_HEALTHY_SERVICES, + compose_probe_env, + compose_service_entries, + compose_service_health, + compose_service_state, + run_command, +) from tests.utils.helpers._fresh_host.storage import log, now_iso FAULT_INJECTION_ENV: Final[str] = "FRESH_HOST_FAULT_INJECTION" FAULT_INJECTION_OFF: Final[str] = "none" # Stops the OrbStack VM right before the macOS sidecars stack starts. ORBSTACK_STOP_DURING_SIDECARS: Final[str] = "orbstack-stop-during-sidecars" -KNOWN_FAULTS: Final[frozenset[str]] = frozenset({ORBSTACK_STOP_DURING_SIDECARS}) +# Stops the OrbStack VM once `sidecars up` has started the stack and waits on its health. +ORBSTACK_STOP_DURING_SIDECARS_HEALTH: Final[str] = "orbstack-stop-during-sidecars-health" +KNOWN_FAULTS: Final[frozenset[str]] = frozenset( + {ORBSTACK_STOP_DURING_SIDECARS, ORBSTACK_STOP_DURING_SIDECARS_HEALTH} +) FAULT_INJECTION_EVENT: Final[str] = "workflow_dispatch" PROTECTED_REF: Final[str] = "refs/heads/main" _ORB_STOP_TIMEOUT_SECONDS: Final[int] = 120 +_HEALTH_WAIT_POLL_SECONDS: Final[float] = 1.0 +# `up -d` returns right after its containers start; a second sighting puts the stop inside +# the health wait that follows instead of racing the tail of `up`. +_HEALTH_WAIT_SIGHTINGS: Final[int] = 2 +_HEALTH_WAIT_PROBE_TIMEOUT_SECONDS: Final[int] = 30 +_WATCHER_JOIN_SECONDS: Final[float] = float( + _ORB_STOP_TIMEOUT_SECONDS + _HEALTH_WAIT_PROBE_TIMEOUT_SECONDS +) def requested_fault(environ: Mapping[str, str]) -> str | None: @@ -54,22 +76,22 @@ def _marker_path(context: FreshHostContext, fault: str) -> Path: return Path(context.diagnostics_dir).resolve() / "fault-injection" / f"{fault}.json" -def inject_sidecars_runtime_loss( - context: FreshHostContext, *, environ: Mapping[str, str] | None = None -) -> bool: - """Run `orb stop` once when the sidecars runtime-loss fault is requested; return if it fired.""" - fault = ORBSTACK_STOP_DURING_SIDECARS +def _armed(context: FreshHostContext, fault: str, environ: Mapping[str, str] | None) -> bool: + """Return whether *fault* is requested, allowed here, and has not fired yet.""" if requested_fault(os.environ if environ is None else environ) != fault: return False + return not _marker_path(context, fault).exists() + + +def _stop_orbstack(context: FreshHostContext, fault: str, *, when: str) -> None: + """Record the one-shot marker for *fault*, then run `orb stop`.""" marker = _marker_path(context, fault) - if marker.exists(): - return False marker.parent.mkdir(parents=True, exist_ok=True) marker.write_text( - json.dumps({"fault": fault, "fired_at": now_iso()}, sort_keys=True) + "\n", + json.dumps({"fault": fault, "fired_at": now_iso(), "when": when}, sort_keys=True) + "\n", encoding="utf-8", ) - log(f"Fault injection {fault}: stopping OrbStack.") + log(f"Fault injection {fault}: stopping OrbStack ({when}).") # The runner env keeps the real HOME, which `orb` resolves its control socket from. run_command( ["orb", "stop"], @@ -77,4 +99,104 @@ def inject_sidecars_runtime_loss( env=dict(os.environ), timeout_seconds=_ORB_STOP_TIMEOUT_SECONDS, ) + + +def inject_sidecars_runtime_loss( + context: FreshHostContext, *, environ: Mapping[str, str] | None = None +) -> bool: + """Run `orb stop` once when the sidecars runtime-loss fault is requested; return if it fired.""" + fault = ORBSTACK_STOP_DURING_SIDECARS + if not _armed(context, fault, environ): + return False + _stop_orbstack(context, fault, when="before the sidecars stack starts") return True + + +def pending_sidecar_health(services: Mapping[str, dict[str, object]]) -> str | None: + """Describe the sidecars still unhealthy once the whole stack runs; `None` otherwise.""" + if any( + compose_service_state(services.get(name, {})) != "running" + for name in SIDECAR_EXPECTED_SERVICES + ): + return None + pending = [ + f"{name}={compose_service_health(services[name]) or 'unknown'}" + for name in SIDECAR_HEALTHY_SERVICES + if compose_service_health(services[name]) != "healthy" + ] + return ", ".join(pending) or None + + +def _stop_orbstack_in_health_wait( + context: FreshHostContext, + *, + fault: str, + compose_file: Path, + env: dict[str, str], + stop: threading.Event, +) -> None: + """Poll the sidecars stack and stop OrbStack once it is up and waiting on health.""" + cwd = Path(context.repo_root).resolve() / "platform" / "compose" + sightings = 0 + while not stop.wait(_HEALTH_WAIT_POLL_SECONDS): + try: + services = compose_service_entries( + compose_file, + cwd=cwd, + env=env, + timeout_seconds=_HEALTH_WAIT_PROBE_TIMEOUT_SECONDS, + ) + except FreshHostError: + sightings = 0 + continue + if stop.is_set(): + return + pending = pending_sidecar_health(services) + sightings = sightings + 1 if pending is not None else 0 + if sightings < _HEALTH_WAIT_SIGHTINGS: + continue + try: + _stop_orbstack(context, fault, when=f"inside the sidecars health wait: {pending}") + except FreshHostError as exc: + log(f"Fault injection {fault}: `orb stop` failed: {exc}") + return + + +@contextmanager +def orbstack_stop_during_sidecars_health( + context: FreshHostContext, + *, + compose_file: Path, + environ: Mapping[str, str] | None = None, +) -> Generator[None]: + """While the body runs `sidecars up`, stop OrbStack once inside its health wait. + + Inert unless the health-wait fault is requested and allowed. On exit the watcher is + joined, so a recovery that follows never races an `orb stop` still in flight. + """ + fault = ORBSTACK_STOP_DURING_SIDECARS_HEALTH + if not _armed(context, fault, environ): + yield + return + stop = threading.Event() + watcher = threading.Thread( + target=_stop_orbstack_in_health_wait, + kwargs={ + "context": context, + "fault": fault, + "compose_file": compose_file, + "env": compose_probe_env(context, compose_file=compose_file, repo_local_state=True), + "stop": stop, + }, + name=f"fault-injection-{fault}", + daemon=True, + ) + log(f"Fault injection {fault}: armed; watching for the sidecars health wait.") + watcher.start() + try: + yield + finally: + stop.set() + watcher.join(timeout=_WATCHER_JOIN_SECONDS) + if not _marker_path(context, fault).exists(): + log(f"Fault injection {fault} did not fire: no sidecars health wait was observed.") diff --git a/tests/utils/helpers/_fresh_host/macos.py b/tests/utils/helpers/_fresh_host/macos.py index 1791a206..382f97cb 100644 --- a/tests/utils/helpers/_fresh_host/macos.py +++ b/tests/utils/helpers/_fresh_host/macos.py @@ -4,10 +4,14 @@ import os import subprocess +from contextlib import AbstractContextManager, nullcontext from dataclasses import dataclass from pathlib import Path -from tests.utils.helpers._fresh_host.faults import inject_sidecars_runtime_loss +from tests.utils.helpers._fresh_host.faults import ( + inject_sidecars_runtime_loss, + orbstack_stop_during_sidecars_health, +) from tests.utils.helpers._fresh_host.linux import run_clawops_bootstrap from tests.utils.helpers._fresh_host.models import FreshHostContext, FreshHostError from tests.utils.helpers._fresh_host.runtime_loss import ( @@ -295,16 +299,22 @@ def _run_repo_local_cycle( "down", "--repo-local-state", ) + health_wait_fault: AbstractContextManager[None] = ( + orbstack_stop_during_sidecars_health(context, compose_file=compose_file) + if component == "sidecars" + else nullcontext() + ) # Hosted OrbStack can transiently lose its task-service connection while the # repo-local stack starts. Reconcile once after cleanup and a fresh backend probe. - run_command( - up_command, - cwd=repo_root, - env=env, - retries=1, - retry_on_exit_codes=RETRYABLE_REPO_LOCAL_UP_EXIT_CODES, - on_retry=lambda: _recover_repo_local_stack(down_command, cwd=repo_root, env=env), - ) + with health_wait_fault: + run_command( + up_command, + cwd=repo_root, + env=env, + retries=1, + retry_on_exit_codes=RETRYABLE_REPO_LOCAL_UP_EXIT_CODES, + on_retry=lambda: _recover_repo_local_stack(down_command, cwd=repo_root, env=env), + ) if component == "sidecars": verify_sidecar_services_running( compose_file, diff --git a/tests/utils/helpers/_fresh_host/shell.py b/tests/utils/helpers/_fresh_host/shell.py index ed3cd5e3..604ab2be 100644 --- a/tests/utils/helpers/_fresh_host/shell.py +++ b/tests/utils/helpers/_fresh_host/shell.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import math import os import stat import subprocess @@ -17,8 +18,10 @@ from clawops.strongclaw_runtime import ( VARLOCK_ENV_MODE_ENV, VARLOCK_ENV_MODE_LEGACY, + DockerBackendWatch, expand_user_path, load_env_assignments, + probe_docker_backend, resolve_openclaw_config_path, resolve_openclaw_state_dir, resolve_repo_local_compose_state_dir, @@ -35,6 +38,8 @@ "qdrant", ) SIDECAR_HEALTHY_SERVICES: tuple[str, ...] = ("postgres", "litellm", "qdrant") +COMPOSE_PS_TIMEOUT_SECONDS = 30 +COMPOSE_POLL_SECONDS = 2.0 # Exit codes that mean the OS killed the process (128 + SIGKILL / 128 + SIGTERM). # On the memory-constrained hosted macOS Docker runner these almost always signal @@ -362,6 +367,87 @@ def compose_probe_env( ) +class ComposeStatusCommandError(FreshHostError): + """Raised when `docker compose ps` itself fails or times out, rather than reporting state.""" + + +def compose_service_entries( + compose_file: Path, + *, + cwd: Path, + env: dict[str, str], + timeout_seconds: int = COMPOSE_PS_TIMEOUT_SECONDS, +) -> dict[str, dict[str, object]]: + """Return `docker compose ps` entries keyed by service name. + + Raises `ComposeStatusCommandError` when the command fails or times out, and + `FreshHostError` when it answers without usable service data. + """ + try: + completed = subprocess.run( + ["docker", "compose", "-f", str(compose_file), "ps", "--format", "json"], + cwd=cwd, + env=env, + check=False, + capture_output=True, + text=True, + timeout=timeout_seconds, + ) + except subprocess.TimeoutExpired as exc: + raise ComposeStatusCommandError( + f"docker compose ps timed out after {timeout_seconds}s." + ) from exc + if completed.returncode != 0: + raise ComposeStatusCommandError( + completed.stderr.strip() or completed.stdout.strip() or "docker compose ps failed" + ) + services: dict[str, dict[str, object]] = {} + for entry in _coerce_compose_ps_entries(completed.stdout): + service_name = _string_field(entry, "Service", "service") + if service_name is not None: + services[service_name] = entry + return services + + +def compose_service_state(entry: dict[str, object]) -> str: + """Return one compose entry's lower-cased state, empty when unreported.""" + return (_string_field(entry, "State", "state") or "").lower() + + +def compose_service_health(entry: dict[str, object]) -> str: + """Return one compose entry's lower-cased health, empty when unreported.""" + return (_string_field(entry, "Health", "health") or "").lower() + + +def _first_unready_service( + services: dict[str, dict[str, object]], + *, + expected_services: tuple[str, ...], + healthy_services: Collection[str], +) -> FreshHostError | None: + """Return why the expected services are not ready yet, or `None` once they are.""" + missing_services = sorted(name for name in expected_services if name not in services) + if missing_services: + return FreshHostError( + "docker compose ps is missing expected services: " + ", ".join(missing_services) + ) + for service_name in expected_services: + state = compose_service_state(services[service_name]) + if state != "running": + return FreshHostError( + f"docker compose ps reports service '{service_name}' in state " + f"'{state or 'unknown'}'." + ) + if service_name in healthy_services: + health = compose_service_health(services[service_name]) + if health != "healthy": + return FreshHostError( + f"docker compose ps reports service '{service_name}' health " + f"'{health or 'unknown'}'." + ) + return None + + def verify_compose_services_running( compose_file: Path, *, @@ -373,7 +459,11 @@ def verify_compose_services_running( repo_root_path: Path | None = None, repo_local_state: bool = False, ) -> None: - """Assert that the expected compose services become running within the timeout window.""" + """Assert that the expected compose services become running within the timeout window. + + A failed `docker compose ps` probes the Docker backend; once it is confirmed lost the + wait fails at once, so runtime-loss recovery does not wait out the timeout first. + """ deadline = time.monotonic() + timeout_seconds last_error: FreshHostError | None = None probe_env = ( @@ -386,75 +476,44 @@ def verify_compose_services_running( if repo_root_path is not None else env ) + backend = DockerBackendWatch(lambda: probe_docker_backend(env=probe_env, cwd=cwd)) while True: - completed = subprocess.run( - ["docker", "compose", "-f", str(compose_file), "ps", "--format", "json"], - cwd=cwd, - env=probe_env, - check=False, - capture_output=True, - text=True, - timeout=timeout_seconds, + # One hung poll must not outlast the wait it belongs to. + poll_timeout = max( + 1, min(COMPOSE_PS_TIMEOUT_SECONDS, math.ceil(deadline - time.monotonic())) ) - if completed.returncode == 0: - try: - entries = _coerce_compose_ps_entries(completed.stdout) - except FreshHostError as exc: - last_error = exc - else: - services: dict[str, dict[str, object]] = {} - for entry in entries: - service_name = _string_field(entry, "Service", "service") - if service_name is not None: - services[service_name] = entry - - missing_services = sorted( - service_name - for service_name in expected_services - if service_name not in services - ) - if not missing_services: - for service_name in expected_services: - state = ( - _string_field(services[service_name], "State", "state") or "" - ).lower() - if state != "running": - last_error = FreshHostError( - "docker compose ps reports service " - f"'{service_name}' in state '{state or 'unknown'}'." - ) - break - if service_name in healthy_services: - health = ( - _string_field(services[service_name], "Health", "health") or "" - ).lower() - if health != "healthy": - last_error = FreshHostError( - "docker compose ps reports service " - f"'{service_name}' health '{health or 'unknown'}'." - ) - break - else: - return - else: - last_error = FreshHostError( - "docker compose ps is missing expected services: " - + ", ".join(missing_services) - ) + try: + services = compose_service_entries( + compose_file, cwd=cwd, env=probe_env, timeout_seconds=poll_timeout + ) + except ComposeStatusCommandError as exc: + last_error = exc + lost_detail = backend.confirm_lost() + if lost_detail is not None: + raise FreshHostError( + "Docker backend became unreachable while waiting for docker compose " + f"services: {lost_detail}; last status error: {exc}" + ) from exc + except FreshHostError as exc: + last_error = exc + backend.reset() else: - detail = ( - completed.stderr.strip() or completed.stdout.strip() or "docker compose ps failed" + backend.reset() + last_error = _first_unready_service( + services, + expected_services=expected_services, + healthy_services=healthy_services, ) - last_error = FreshHostError(detail) + if last_error is None: + return if time.monotonic() >= deadline: break - time.sleep(2.0) + time.sleep(COMPOSE_POLL_SECONDS) assert last_error is not None raise last_error - raise FreshHostError("Timed out waiting for docker compose services to start.") def verify_sidecar_services_running(