diff --git a/.github/workflows/fresh-host-acceptance.yml b/.github/workflows/fresh-host-acceptance.yml index bb014514..19bc16d1 100644 --- a/.github/workflows/fresh-host-acceptance.yml +++ b/.github/workflows/fresh-host-acceptance.yml @@ -35,6 +35,15 @@ on: required: true type: boolean default: true + # Dispatch-only: callers through workflow_call cannot request a fault. + fault_injection: + description: CI-only fault drill for the macOS sidecars scenario; ignored on main. + required: true + type: choice + default: none + options: + - none + - orbstack-stop-during-sidecars concurrency: group: >- @@ -57,3 +66,4 @@ jobs: docker_pull_parallelism: ${{ inputs.docker_pull_parallelism }} docker_pull_max_attempts: ${{ inputs.docker_pull_max_attempts }} enable_package_cache: ${{ inputs.enable_package_cache }} + fault_injection: ${{ github.event_name == 'workflow_dispatch' && inputs.fault_injection || '' }} diff --git a/.github/workflows/fresh-host-core.yml b/.github/workflows/fresh-host-core.yml index ddc0843c..a42654e5 100644 --- a/.github/workflows/fresh-host-core.yml +++ b/.github/workflows/fresh-host-core.yml @@ -16,6 +16,11 @@ on: required: false type: boolean default: true + # CI-only fault drill (see tests/utils/helpers/_fresh_host/faults.py); empty is off. + fault_injection: + required: false + type: string + default: "" permissions: contents: read @@ -203,8 +208,12 @@ jobs: - name: Ensure hosted macOS Docker images timeout-minutes: 32 run: ./tests/scripts/hosted_docker.py ensure-images --context "${FRESH_HOST_CONTEXT}" + # The scenario recovers a Docker runtime lost mid-run inside its own budget. The fault + # drill reaches it only from a manual dispatch off main; the helper enforces that too. - name: Run macOS fresh-host scenario timeout-minutes: 35 + env: + FRESH_HOST_FAULT_INJECTION: ${{ github.event_name == 'workflow_dispatch' && github.ref != 'refs/heads/main' && inputs.fault_injection || '' }} run: ./tests/scripts/fresh_host.py run-scenario --context "${FRESH_HOST_CONTEXT}" - name: Collect hosted macOS Docker diagnostics if: ${{ always() && env.FRESH_HOST_CONTEXT != '' }} diff --git a/tests/suites/contracts/repo/test_ci_workflow_surfaces.py b/tests/suites/contracts/repo/test_ci_workflow_surfaces.py index 0befa038..60391929 100644 --- a/tests/suites/contracts/repo/test_ci_workflow_surfaces.py +++ b/tests/suites/contracts/repo/test_ci_workflow_surfaces.py @@ -19,6 +19,12 @@ RUNTIME_PHASE_CAP_SECONDS, RUNTIME_PHASE_MIN_SECONDS, ) +from tests.utils.helpers._fresh_host.faults import ( + FAULT_INJECTION_ENV, + FAULT_INJECTION_OFF, + KNOWN_FAULTS, + requested_fault, +) from tests.utils.helpers._hosted_docker.diagnostics import DIAGNOSTICS_COLLECTION_CAP_SECONDS from tests.utils.helpers.repo import REPO_ROOT @@ -971,3 +977,83 @@ def test_codeql_config_ignores_unmaintained_and_packaged_vendor_code() -> None: assert "src/clawops/assets" in text assert "platform/plugins/memory-lancedb-pro" in text assert "platform/plugins/strongclaw-hypermemory" not in text + + +_CORE_FAULT_GATE = ( + "${{ github.event_name == 'workflow_dispatch' && github.ref != 'refs/heads/main' " + "&& inputs.fault_injection || '' }}" +) +_ACCEPTANCE_FAULT_FORWARD = ( + "${{ github.event_name == 'workflow_dispatch' && inputs.fault_injection || '' }}" +) + + +def _workflow_trigger_inputs(workflow_name: str, trigger: str) -> dict[str, object]: + """Return one trigger's inputs; YAML 1.1 loads the `on:` key as `True`.""" + loaded: object = yaml.safe_load(_workflow_text(workflow_name)) + assert isinstance(loaded, dict), workflow_name + raw = cast(dict[object, object], loaded) + triggers = _as_str_object_dict(raw.get("on", raw.get(True))) + assert triggers is not None, workflow_name + trigger_config = _as_str_object_dict(triggers.get(trigger)) + assert trigger_config is not None, (workflow_name, trigger) + inputs = _as_str_object_dict(trigger_config.get("inputs")) + assert inputs is not None, (workflow_name, trigger) + return inputs + + +def _job_step(workflow_name: str, job_id: str, step_name: str) -> dict[str, object]: + """Return one named step of one job.""" + job = _as_str_object_dict(_workflow_jobs(workflow_name)[job_id]) + assert job is not None, job_id + steps = job.get("steps") + assert isinstance(steps, list), job_id + for raw_step in cast(list[object], steps): + step = _as_str_object_dict(raw_step) + if step is not None and step.get("name") == step_name: + return step + raise AssertionError(f"{workflow_name}:{job_id} has no step {step_name!r}") + + +def test_fault_injection_cannot_be_enabled_on_push_to_main() -> None: + """The fault drill reaches the scenario only from a manual dispatch off main.""" + leaked_push_env = { + FAULT_INJECTION_ENV: next(iter(KNOWN_FAULTS)), + "GITHUB_EVENT_NAME": "push", + "GITHUB_REF": "refs/heads/main", + } + assert requested_fault(leaked_push_env) is None + + core_input = _as_str_object_dict( + _workflow_trigger_inputs("fresh-host-core.yml", "workflow_call").get("fault_injection") + ) + assert core_input is not None + assert core_input["default"] == "" + assert _workflow_text("fresh-host-core.yml").count("inputs.fault_injection") == 1 + scenario_step = _job_step( + "fresh-host-core.yml", "macos-fresh-host", "Run macOS fresh-host scenario" + ) + assert scenario_step.get("env") == {FAULT_INJECTION_ENV: _CORE_FAULT_GATE} + + assert "fault_injection" not in _workflow_trigger_inputs( + "fresh-host-acceptance.yml", "workflow_call" + ) + dispatch_input = _as_str_object_dict( + _workflow_trigger_inputs("fresh-host-acceptance.yml", "workflow_dispatch").get( + "fault_injection" + ) + ) + assert dispatch_input is not None + assert dispatch_input["type"] == "choice" + assert dispatch_input["default"] == FAULT_INJECTION_OFF + assert dispatch_input["options"] == [FAULT_INJECTION_OFF, *sorted(KNOWN_FAULTS)] + acceptance_job = _as_str_object_dict( + _workflow_jobs("fresh-host-acceptance.yml")["fresh-host-acceptance"] + ) + assert acceptance_job is not None + acceptance_with = _as_str_object_dict(acceptance_job.get("with")) + assert acceptance_with is not None + assert acceptance_with["fault_injection"] == _ACCEPTANCE_FAULT_FORWARD + + for caller in ("ci-gate.yml", "nightly.yml", "release.yml", "e2e-acceptance.yml"): + assert "fault_injection" not in _workflow_text(caller), caller diff --git a/tests/suites/unit/ci/test_fresh_host.py b/tests/suites/unit/ci/test_fresh_host.py index 73854e57..0d201a44 100644 --- a/tests/suites/unit/ci/test_fresh_host.py +++ b/tests/suites/unit/ci/test_fresh_host.py @@ -374,9 +374,12 @@ def test_run_scenario_records_successful_phase_sequence( def fake_run_named_phase( loaded_context: fresh_host.FreshHostContext, phase_name: str, + *, + recovery_window: object = None, ) -> list[str]: calls.append(phase_name) assert loaded_context.scenario_id == "linux" + assert recovery_window is None, "Linux scenarios never arm runtime-loss recovery" return ["echo", phase_name] test_context.patch.patch_object( diff --git a/tests/suites/unit/ci/test_fresh_host_budget.py b/tests/suites/unit/ci/test_fresh_host_budget.py index 139da8b7..8a4e956e 100644 --- a/tests/suites/unit/ci/test_fresh_host_budget.py +++ b/tests/suites/unit/ci/test_fresh_host_budget.py @@ -25,6 +25,7 @@ phase_deadline, record_job_budget, runtime_phase_deadline, + scenario_phase_deadline, ) from tests.utils.helpers._fresh_host.models import FreshHostError from tests.utils.helpers.clock import FakeClock @@ -300,3 +301,36 @@ def test_initialize_budget_cli_writes_env_once(tmp_path: Path, test_context: Tes ) assert JobBudget.from_env(exported) is not None assert fresh_host_script.main([*argv[:2], "5", *argv[3:]]) == 1 + + +def test_scenario_phase_deadline_follows_the_scenario_step_timeout_when_the_job_has_room() -> None: + clock = FakeClock() + budget = _macos_budget(clock) + clock.advance(600) + + deadline = scenario_phase_deadline(budget, clock=clock) + + assert deadline.remaining(clock) == budget.scenario_reserve_seconds + assert deadline.limited_by == "phase-cap" + + +def test_scenario_phase_deadline_keeps_the_observer_reserve_when_the_scenario_starts_late() -> None: + clock = FakeClock() + budget = _macos_budget(clock) + observer_limit = budget.deadline_epoch - budget.observer_reserve_seconds + clock.advance(observer_limit - clock.time() - 300) + + deadline = scenario_phase_deadline(budget, clock=clock) + + assert deadline.wall_deadline_epoch == observer_limit + assert deadline.remaining(clock) == 300 + assert deadline.limited_by == "job-reserve" + + +def test_scenario_phase_deadline_fails_fast_inside_the_observer_reserve() -> None: + clock = FakeClock() + budget = _macos_budget(clock) + clock.advance(budget.deadline_epoch - budget.observer_reserve_seconds - clock.time()) + + with pytest.raises(BudgetExhaustedError, match="no scenario time remains"): + scenario_phase_deadline(budget, clock=clock) diff --git a/tests/suites/unit/ci/test_fresh_host_faults.py b/tests/suites/unit/ci/test_fresh_host_faults.py new file mode 100644 index 00000000..9e6e88ee --- /dev/null +++ b/tests/suites/unit/ci/test_fresh_host_faults.py @@ -0,0 +1,110 @@ +"""Unit coverage for the CI-only fresh-host fault-injection switch.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from tests.plugins.infrastructure.context import TestContext +from tests.utils.helpers import fresh_host +from tests.utils.helpers._fresh_host import faults +from tests.utils.helpers._fresh_host.models import FreshHostContext, FreshHostError + +_FAULT = faults.ORBSTACK_STOP_DURING_SIDECARS +_BRANCH_REF = "refs/heads/fix/runtime-loss" + + +def _environ(fault: str, *, event: str, ref: str) -> dict[str, str]: + return {faults.FAULT_INJECTION_ENV: fault, "GITHUB_EVENT_NAME": event, "GITHUB_REF": ref} + + +@pytest.mark.parametrize( + ("event", "ref"), + [ + ("push", "refs/heads/main"), + ("push", _BRANCH_REF), + ("pull_request", "refs/pull/252/merge"), + ("schedule", "refs/heads/main"), + ("workflow_dispatch", "refs/heads/main"), + ("", ""), + ], +) +def test_fault_requests_are_ignored_outside_manual_dispatch_off_main(event: str, ref: str) -> None: + assert faults.requested_fault(_environ(_FAULT, event=event, ref=ref)) is None + + +def test_fault_requests_are_honored_on_manual_dispatch_off_main() -> None: + environ = _environ(_FAULT, event="workflow_dispatch", ref=_BRANCH_REF) + + assert faults.requested_fault(environ) == _FAULT + + +@pytest.mark.parametrize("value", ["", " ", faults.FAULT_INJECTION_OFF]) +def test_unset_or_off_fault_requests_inject_nothing(value: str) -> None: + environ = _environ(value, event="workflow_dispatch", ref=_BRANCH_REF) + + assert faults.requested_fault(environ) is None + + +def test_unknown_fault_requests_fail_loudly() -> None: + environ = _environ("kill-everything", event="workflow_dispatch", ref=_BRANCH_REF) + + with pytest.raises(FreshHostError, match="is not one of"): + faults.requested_fault(environ) + + +def _macos_context(tmp_path: Path, test_context: TestContext) -> FreshHostContext: + workspace = tmp_path / "workspace" + workspace.mkdir() + test_context.apply_profiles("fresh_host_macos_orbstack") + return fresh_host.prepare_context( + scenario_id="macos-sidecars", + repo_root=workspace, + runner_temp=tmp_path / "runner-temp", + workspace=workspace, + github_env_file=tmp_path / "github.env", + ) + + +def _record_commands(test_context: TestContext) -> list[list[str]]: + commands: list[list[str]] = [] + + def fake_run_command(command: list[str], **kwargs: object) -> None: + del kwargs + commands.append(list(command)) + + test_context.patch.patch_object(faults, "run_command", new=fake_run_command) + return commands + + +def test_sidecars_runtime_loss_fires_orb_stop_exactly_once( + tmp_path: Path, test_context: TestContext +) -> None: + context = _macos_context(tmp_path, test_context) + commands = _record_commands(test_context) + environ = _environ(_FAULT, event="workflow_dispatch", ref=_BRANCH_REF) + + first = faults.inject_sidecars_runtime_loss(context, environ=environ) + second = faults.inject_sidecars_runtime_loss(context, environ=environ) + + assert (first, second) == (True, False) + assert commands == [["orb", "stop"]] + marker = Path(context.diagnostics_dir) / "fault-injection" / f"{_FAULT}.json" + assert json.loads(marker.read_text(encoding="utf-8"))["fault"] == _FAULT + + +def test_sidecars_runtime_loss_is_inert_on_push_to_main( + tmp_path: Path, test_context: TestContext +) -> None: + context = _macos_context(tmp_path, test_context) + commands = _record_commands(test_context) + + fired = faults.inject_sidecars_runtime_loss( + context, environ=_environ(_FAULT, event="push", ref="refs/heads/main") + ) + + assert fired is False + assert commands == [] + assert not (Path(context.diagnostics_dir) / "fault-injection").exists() diff --git a/tests/suites/unit/ci/test_fresh_host_runtime_loss.py b/tests/suites/unit/ci/test_fresh_host_runtime_loss.py new file mode 100644 index 00000000..75e06f4d --- /dev/null +++ b/tests/suites/unit/ci/test_fresh_host_runtime_loss.py @@ -0,0 +1,517 @@ +"""Unit coverage for hosted macOS runtime-loss recovery inside fresh-host scenarios.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import cast + +import pytest + +from tests.plugins.infrastructure.context import TestContext +from tests.utils.helpers import fresh_host +from tests.utils.helpers._fresh_host import faults as fresh_host_faults +from tests.utils.helpers._fresh_host import macos as fresh_host_macos +from tests.utils.helpers._fresh_host import runtime_loss +from tests.utils.helpers._fresh_host.budget import ( + SCENARIO_RECOVERY_CAP_SECONDS, + SCENARIO_RETRY_RESERVE_SECONDS, + BudgetExhaustedError, + Clock, + JobBudget, + PhaseDeadline, + initialize_job_budget, +) +from tests.utils.helpers._fresh_host.models import FreshHostContext, FreshHostError +from tests.utils.helpers._hosted_docker import diagnostics as hosted_docker_diagnostics +from tests.utils.helpers._hosted_docker import runtime as hosted_docker_runtime +from tests.utils.helpers.clock import FakeClock + +_EOF_STDERR = 'error during connect: Get "http://docker.sock/_ping": EOF' +_STEP = "exercise-sidecars" + + +class _FakeOrbStack: + """Scripted OrbStack runtime: once down, it stays down until a recovery command runs.""" + + def __init__( + self, + clock: FakeClock, + *, + up: bool, + recovers: bool = True, + command_seconds: float = 0.0, + docker_info_error: BaseException | None = None, + ) -> None: + self.clock = clock + self.up = up + self.recovers = recovers + self.command_seconds = command_seconds + self.docker_info_error = docker_info_error + self.commands: list[list[str]] = [] + + def run_command( + self, + command: list[str], + *, + cwd: Path, + env: dict[str, str], + timeout_seconds: float = 3600, + capture_output: bool = False, + ) -> subprocess.CompletedProcess[str]: + del cwd, env, timeout_seconds, capture_output + self.commands.append(list(command)) + self.clock.advance(self.command_seconds) + if command == ["docker", "info"]: + if self.docker_info_error is not None: + raise self.docker_info_error + if not self.up: + return subprocess.CompletedProcess(command, 1, stdout="", stderr=_EOF_STDERR) + if command in (["orb", "start"], ["orb", "restart", "docker"]): + self.up = self.recovers + if command == ["orb", "stop"]: + self.up = False + return subprocess.CompletedProcess(command, 0, stdout="ok", stderr="") + + def run_checked( + self, + command: list[str], + *, + cwd: Path, + env: dict[str, str], + timeout_seconds: float = 3600, + capture_output: bool = False, + ) -> subprocess.CompletedProcess[str]: + completed = self.run_command( + command, + cwd=cwd, + env=env, + timeout_seconds=timeout_seconds, + capture_output=capture_output, + ) + if completed.returncode != 0: + raise FreshHostError(f"{' '.join(command)} failed: {completed.stderr}") + return completed + + def wait_for_docker_ready( + self, *, cwd: Path, env: dict[str, str], deadline: PhaseDeadline, clock: Clock + ) -> None: + del cwd, env, deadline, clock + if not self.up: + raise FreshHostError(f"docker did not become ready; last: {_EOF_STDERR}") + + @property + def orb_commands(self) -> list[list[str]]: + return [command for command in self.commands if command[0] == "orb"] + + +def _install(test_context: TestContext, orbstack: _FakeOrbStack, tmp_path: Path) -> None: + """Route every runtime, recovery, and diagnostics command through *orbstack*.""" + test_context.patch.patch_object(hosted_docker_runtime, "run_command", new=orbstack.run_command) + test_context.patch.patch_object(hosted_docker_runtime, "run_checked", new=orbstack.run_checked) + test_context.patch.patch_object( + hosted_docker_runtime, "wait_for_docker_ready", new=orbstack.wait_for_docker_ready + ) + test_context.patch.patch_object( + hosted_docker_diagnostics, "run_command", new=orbstack.run_command + ) + test_context.patch.patch_object(hosted_docker_diagnostics, "sysctl_int", new=_fake_sysctl_int) + test_context.patch.patch_object( + hosted_docker_diagnostics, "_orbstack_log_dir", new=lambda: tmp_path / "no-orbstack-logs" + ) + + +def _fake_sysctl_int(name: str, *, timeout_seconds: float = 30) -> int | None: + del name, timeout_seconds + return 4 + + +def _no_op(**kwargs: object) -> None: + del kwargs + + +def _context( + tmp_path: Path, test_context: TestContext, *, scenario_id: str = "macos-sidecars" +) -> FreshHostContext: + workspace = tmp_path / "workspace" + workspace.mkdir() + test_context.apply_profiles( + "fresh_host_push" if scenario_id == "linux" else "fresh_host_macos_orbstack" + ) + # A TCP host skips the socket check, so only the scripted `docker info` decides health. + test_context.env.set("DOCKER_HOST", "tcp://127.0.0.1:2375") + return fresh_host.prepare_context( + scenario_id=scenario_id, + repo_root=workspace, + runner_temp=tmp_path / "runner-temp", + workspace=workspace, + github_env_file=tmp_path / "github.env", + ) + + +def _budget(clock: FakeClock) -> JobBudget: + return initialize_job_budget( + job_timeout_minutes=90, + scenario_reserve_minutes=35, + observer_reserve_minutes=15, + environ={}, + clock=clock, + ) + + +def _window(context: FreshHostContext, clock: FakeClock) -> runtime_loss.RecoveryWindow: + window = runtime_loss.arm_runtime_recovery( + context, environ=_budget(clock).to_env(), clock=clock + ) + assert window is not None + return window + + +class _Step: + """A scenario step that fails a scripted number of times, then returns its command.""" + + def __init__(self, *, failures: int, error: str = "Command failed with exit 1") -> None: + self.failures = failures + self.error = error + self.calls = 0 + + def __call__(self) -> list[str]: + self.calls += 1 + if self.calls <= self.failures: + raise FreshHostError(self.error) + return ["step", "ok"] + + +def _loss_report(context: FreshHostContext) -> dict[str, object]: + path = Path(context.diagnostics_dir) / "runtime-loss" / _STEP / "report.json" + payload: object = json.loads(path.read_text(encoding="utf-8")) + assert isinstance(payload, dict) + return {str(key): value for key, value in cast(dict[object, object], payload).items()} + + +def _attempts(report: dict[str, object]) -> list[dict[str, object]]: + attempts = report["recovery_attempts"] + assert isinstance(attempts, list) + return cast(list[dict[str, object]], attempts) + + +@pytest.mark.parametrize( + ("scenario_id", "with_budget", "armed"), + [ + ("macos-sidecars", True, True), + ("macos-sidecars", False, False), + ("linux", True, False), + ], +) +def test_scenario_recovery_deadline_arms_only_budgeted_hosted_macos_runs( + tmp_path: Path, + test_context: TestContext, + scenario_id: str, + with_budget: bool, + armed: bool, +) -> None: + clock = FakeClock() + context = _context(tmp_path, test_context, scenario_id=scenario_id) + environ = _budget(clock).to_env() if with_budget else {} + + window = runtime_loss.arm_runtime_recovery(context, environ=environ, clock=clock) + + assert (window is not None) is armed + if window is not None: + assert window.clock is clock + assert window.deadline.remaining(clock) == 35 * 60 + + +def test_scenario_recovery_deadline_disarms_inside_the_observer_reserve( + tmp_path: Path, test_context: TestContext +) -> None: + clock = FakeClock() + context = _context(tmp_path, test_context) + budget = _budget(clock) + clock.advance(budget.deadline_epoch - budget.observer_reserve_seconds - clock.time()) + + assert runtime_loss.arm_runtime_recovery(context, environ=budget.to_env(), clock=clock) is None + + +def test_failures_pass_through_untouched_without_a_recovery_deadline( + tmp_path: Path, test_context: TestContext +) -> None: + clock = FakeClock() + context = _context(tmp_path, test_context) + orbstack = _FakeOrbStack(clock, up=False) + _install(test_context, orbstack, tmp_path) + step = _Step(failures=1) + + with pytest.raises(FreshHostError, match="Command failed with exit 1") as raised: + runtime_loss.run_with_runtime_recovery(context, step=_STEP, window=None, action=step) + + assert raised.value.__cause__ is None + assert step.calls == 1 + assert orbstack.commands == [] + + +def test_linux_scenarios_fail_fast_even_with_a_recorded_budget( + tmp_path: Path, test_context: TestContext +) -> None: + clock = FakeClock() + context = _context(tmp_path, test_context, scenario_id="linux") + orbstack = _FakeOrbStack(clock, up=False) + _install(test_context, orbstack, tmp_path) + window = runtime_loss.arm_runtime_recovery( + context, environ=_budget(clock).to_env(), clock=clock + ) + step = _Step(failures=1) + + assert window is None + with pytest.raises(FreshHostError, match="Command failed with exit 1"): + runtime_loss.run_with_runtime_recovery(context, step=_STEP, window=window, action=step) + + assert step.calls == 1 + assert orbstack.commands == [] + + +def test_step_failures_with_a_reachable_backend_are_not_runtime_loss( + tmp_path: Path, test_context: TestContext +) -> None: + clock = FakeClock() + context = _context(tmp_path, test_context) + orbstack = _FakeOrbStack(clock, up=True) + _install(test_context, orbstack, tmp_path) + # Timeout wording in a step failure must not be mistaken for a dead runtime. + step = _Step(failures=1, error="timed out waiting for compose service 'litellm'") + + with pytest.raises(FreshHostError, match="timed out waiting for compose service"): + runtime_loss.run_with_runtime_recovery( + context, + step=_STEP, + window=_window(context, clock), + action=step, + ) + + assert step.calls == 1 + assert orbstack.commands == [["docker", "info"]] + assert not (Path(context.diagnostics_dir) / "runtime-loss").exists() + + +def test_socket_eof_runtime_loss_restarts_orbstack_and_retries_the_step_once( + tmp_path: Path, test_context: TestContext +) -> None: + clock = FakeClock() + context = _context(tmp_path, test_context) + orbstack = _FakeOrbStack(clock, up=False) + _install(test_context, orbstack, tmp_path) + step = _Step(failures=1) + + result = runtime_loss.run_with_runtime_recovery( + context, + step=_STEP, + window=_window(context, clock), + action=step, + ) + + assert result == ["step", "ok"] + assert step.calls == 2 + report = _loss_report(context) + attempts = _attempts(report) + assert len(attempts) == 1 + assert attempts[0]["recovery_commands"] == [["orb", "restart", "docker"]] + assert report["step"] == _STEP + assert report["trigger_reason"] == "docker_socket_eof" + assert report["recovered"] is True + assert report["exhausted"] is False + assert report["deadline_limited_by"] == runtime_loss.RUNTIME_LOSS_CAP + attempt_root = Path(context.diagnostics_dir) / "runtime-loss" / _STEP / "attempt-01" + assert (attempt_root / "before").is_dir() + assert (attempt_root / "after").is_dir() + assert (attempt_root / "recovery-command.txt").is_file() + + +def test_missing_orbstack_socket_starts_orbstack_instead_of_restarting_docker( + tmp_path: Path, test_context: TestContext +) -> None: + clock = FakeClock() + context = _context(tmp_path, test_context) + test_context.env.set("DOCKER_HOST", f"unix://{tmp_path / 'missing.sock'}") + orbstack = _FakeOrbStack(clock, up=False) + _install(test_context, orbstack, tmp_path) + step = _Step(failures=1) + + runtime_loss.run_with_runtime_recovery( + context, + step=_STEP, + window=_window(context, clock), + action=step, + ) + + assert ["orb", "start"] in orbstack.orb_commands + assert ["orb", "restart", "docker"] not in orbstack.orb_commands + assert _loss_report(context)["trigger_reason"] == "orbstack_socket_missing" + + +def test_retry_after_recovery_runs_the_retry_action( + tmp_path: Path, test_context: TestContext +) -> None: + clock = FakeClock() + context = _context(tmp_path, test_context) + _install(test_context, _FakeOrbStack(clock, up=False), tmp_path) + step = _Step(failures=1) + retried: list[str] = [] + + def retry() -> list[str]: + retried.append("retry") + return ["retry", "ok"] + + result = runtime_loss.run_with_runtime_recovery( + context, + step=_STEP, + window=_window(context, clock), + action=step, + retry_action=retry, + ) + + assert result == ["retry", "ok"] + assert step.calls == 1 + assert retried == ["retry"] + + +def test_recovery_refuses_to_start_once_only_the_retry_reserve_is_left( + tmp_path: Path, test_context: TestContext +) -> None: + clock = FakeClock() + context = _context(tmp_path, test_context) + orbstack = _FakeOrbStack(clock, up=False) + _install(test_context, orbstack, tmp_path) + window = _window(context, clock) + clock.advance(window.deadline.remaining(clock) - SCENARIO_RETRY_RESERVE_SECONDS) + step = _Step(failures=1) + + with pytest.raises(BudgetExhaustedError, match="runtime_loss recovery for") as raised: + runtime_loss.run_with_runtime_recovery(context, step=_STEP, window=window, action=step) + + assert str(raised.value).startswith("budget_exhausted: ") + assert isinstance(raised.value.__cause__, FreshHostError) + assert step.calls == 1 + assert orbstack.commands == [] + + +def test_recovery_that_outlives_its_slice_ends_with_budget_exhaustion( + tmp_path: Path, test_context: TestContext +) -> None: + clock = FakeClock() + context = _context(tmp_path, test_context) + # Every command burns most of an attempt's share, so the slice runs out mid-recovery. + orbstack = _FakeOrbStack(clock, up=False, command_seconds=SCENARIO_RECOVERY_CAP_SECONDS / 3) + _install(test_context, orbstack, tmp_path) + window = _window(context, clock) + started = clock.monotonic() + step = _Step(failures=1) + + with pytest.raises(BudgetExhaustedError, match="recovery ran out of time"): + runtime_loss.run_with_runtime_recovery(context, step=_STEP, window=window, action=step) + + assert step.calls == 1 + report = _loss_report(context) + assert report["recovered"] is False + assert report["exhausted"] is True + # The slice bounds recovery; one in-flight command may finish past it, never two. + assert clock.monotonic() - started <= SCENARIO_RECOVERY_CAP_SECONDS + orbstack.command_seconds + + +def test_orbstack_that_never_recovers_fails_the_step_with_attempt_evidence( + tmp_path: Path, test_context: TestContext +) -> None: + clock = FakeClock() + context = _context(tmp_path, test_context) + orbstack = _FakeOrbStack(clock, up=False, recovers=False) + _install(test_context, orbstack, tmp_path) + step = _Step(failures=1) + + with pytest.raises( + FreshHostError, + match=r"orbstack_recovery_failed: runtime_loss during exercise-sidecars " + r"\(docker_socket_eof\)", + ): + runtime_loss.run_with_runtime_recovery( + context, + step=_STEP, + window=_window(context, clock), + action=step, + ) + + assert step.calls == 1 + report = _loss_report(context) + assert report["recovered"] is False + assert report["exhausted"] is False + assert len(_attempts(report)) == 2 + + +def test_unrecoverable_runtime_loss_is_reported_without_restarting_orbstack( + tmp_path: Path, test_context: TestContext +) -> None: + clock = FakeClock() + context = _context(tmp_path, test_context) + orbstack = _FakeOrbStack(clock, up=False, docker_info_error=FileNotFoundError("docker")) + _install(test_context, orbstack, tmp_path) + step = _Step(failures=1) + + with pytest.raises(FreshHostError, match=r"unrecoverable runtime_loss .*docker_binary_missing"): + runtime_loss.run_with_runtime_recovery( + context, + step=_STEP, + window=_window(context, clock), + action=step, + ) + + assert step.calls == 1 + assert orbstack.orb_commands == [] + + +def test_macos_sidecars_phase_recovers_from_an_injected_orbstack_stop( + tmp_path: Path, test_context: TestContext +) -> None: + clock = FakeClock() + context = _context(tmp_path, test_context) + test_context.env.set(fresh_host_faults.FAULT_INJECTION_ENV, "orbstack-stop-during-sidecars") + test_context.env.set("GITHUB_EVENT_NAME", "workflow_dispatch") + test_context.env.set("GITHUB_REF", "refs/heads/fix/runtime-loss") + orbstack = _FakeOrbStack(clock, up=True) + _install(test_context, orbstack, tmp_path) + calls: list[str] = [] + + def fake_fault_run_command(command: list[str], **kwargs: object) -> None: + del kwargs + calls.append(" ".join(command)) + orbstack.run_command(command, cwd=tmp_path, env={}) + + def fake_run_command(command: list[str], **kwargs: object) -> None: + del kwargs + calls.append(command[-2] if "--repo-local-state" in command else " ".join(command)) + if not orbstack.up: + raise FreshHostError("Docker backend is not reachable from this shell.") + + def fake_verify(compose_file: Path, **kwargs: object) -> None: + del kwargs + calls.append(f"verify:{compose_file.name}") + + test_context.patch.patch_object(fresh_host_faults, "run_command", new=fake_fault_run_command) + test_context.patch.patch_object(fresh_host_macos, "run_command", new=fake_run_command) + test_context.patch.patch_object(fresh_host_macos, "wait_for_docker_backend", new=_no_op) + test_context.patch.patch_object( + fresh_host_macos, "verify_sidecar_services_running", new=fake_verify + ) + + fresh_host_macos.exercise_macos_sidecars(context, recovery_window=_window(context, clock)) + + assert calls == [ + "orb stop", + "up", + "up", + "verify:docker-compose.aux-stack.ci-hosted-macos.yaml", + ] + assert ["orb", "restart", "docker"] in orbstack.orb_commands + assert _loss_report(context)["recovered"] is True + marker = ( + Path(context.diagnostics_dir) / "fault-injection" / "orbstack-stop-during-sidecars.json" + ) + assert marker.is_file() diff --git a/tests/suites/unit/ci/test_hosted_docker.py b/tests/suites/unit/ci/test_hosted_docker.py index 0f33c334..91c94ffe 100644 --- a/tests/suites/unit/ci/test_hosted_docker.py +++ b/tests/suites/unit/ci/test_hosted_docker.py @@ -65,6 +65,14 @@ def no_preflight(**kwargs: object) -> None: test_context.patch.patch_object(hosted_docker_images, "preflight_linux_disk", new=no_preflight) +@pytest.fixture(autouse=True) +def _isolate_orbstack_logs(tmp_path: Path, test_context: TestContext) -> None: + """Keep diagnostics tests off the host's real OrbStack logs.""" + test_context.patch.patch_object( + hosted_docker_diagnostics, "_orbstack_log_dir", new=lambda: tmp_path / "orbstack-log" + ) + + def _no_jitter(cap_seconds: float) -> float: return cap_seconds @@ -1793,3 +1801,37 @@ def test_run_command_kills_the_whole_process_group_on_timeout( while not _process_is_gone(grandchild) and time.monotonic() < give_up: time.sleep(0.05) assert _process_is_gone(grandchild) + + +def test_runtime_diagnostics_keep_the_tail_of_each_orbstack_log( + tmp_path: Path, test_context: TestContext +) -> None: + """OrbStack's own logs explain a stopped VM; diagnostics keep a bounded tail of each.""" + context = _macos_context(tmp_path, test_context) + log_dir = tmp_path / "orbstack-log" + log_dir.mkdir() + (log_dir / "vmgr.log").write_bytes(b"old-line\n" + b"vm stopped: guest panic\n") + (log_dir / "gui.log").write_bytes(b"gui ok\n") + (log_dir / "notes.txt").write_bytes(b"not a log\n") + test_context.patch.patch_object(hosted_docker_diagnostics, "ORBSTACK_LOG_TAIL_BYTES", new=24) + + def unexpected_wait( + *, cwd: Path, env: dict[str, str], deadline: PhaseDeadline, clock: Clock + ) -> None: + raise AssertionError("diagnostics never wait for Docker") + + commands = _patch_runtime_commands(test_context, wait=unexpected_wait) + diagnostics_dir = tmp_path / "diagnostics" + clock = FakeClock() + + hosted_docker_diagnostics.collect_runtime_diagnostics_for_context( + context, diagnostics_dir, deadline=PhaseDeadline.after(30, clock=clock), clock=clock + ) + + copied = diagnostics_dir / "orbstack-logs" + assert sorted(path.name for path in copied.iterdir()) == ["gui.log", "vmgr.log"] + assert (copied / "vmgr.log").read_bytes() == b"vm stopped: guest panic\n" + unified_log = next(command for command, _ in commands if command[0] == "/usr/bin/log") + predicate = unified_log[unified_log.index("--predicate") + 1] + assert 'subsystem BEGINSWITH "com.apple.Virtualization"' in predicate + assert "memorystatus" in predicate diff --git a/tests/utils/helpers/_fresh_host/budget.py b/tests/utils/helpers/_fresh_host/budget.py index 9d516b59..51fd166b 100644 --- a/tests/utils/helpers/_fresh_host/budget.py +++ b/tests/utils/helpers/_fresh_host/budget.py @@ -45,6 +45,11 @@ RUNTIME_PHASE_MIN_SECONDS: Final[int] = ( RUNTIME_INITIAL_MIN_SECONDS + RUNTIME_RECOVERY_RESERVE_SECONDS ) +# A runtime lost mid-scenario gets the same OrbStack recovery slice as the runtime phase. +SCENARIO_RECOVERY_CAP_SECONDS: Final[int] = RUNTIME_RECOVERY_RESERVE_SECONDS +# Left after scenario recovery for the retried step and the phases after it. Hosted macOS +# exercise-sidecars took up to 411s plus 41s of later phases (Nov 2026 CI); ~30% margin. +SCENARIO_RETRY_RESERVE_SECONDS: Final[int] = 600 # Never start a blocking command with less time than this left before its deadline. MIN_COMMAND_SECONDS: Final[float] = 1.0 BUDGET_EXHAUSTED: Final[str] = "budget_exhausted" @@ -275,20 +280,37 @@ def phase_deadline( The scenario and observer reserves are always later phases; *later_reserve_seconds* adds the minimum slices of any phase that runs between this one and the scenario. """ + reserve_limit_epoch = ( + None + if budget is None + else budget.deadline_epoch + - budget.scenario_reserve_seconds + - budget.observer_reserve_seconds + - later_reserve_seconds + ) + return _bounded_deadline( + phase=phase, + phase_cap_seconds=phase_cap_seconds, + reserve_limit_epoch=reserve_limit_epoch, + clock=clock, + ) + + +def _bounded_deadline( + *, + phase: str, + phase_cap_seconds: int, + reserve_limit_epoch: float | None, + clock: Clock, +) -> PhaseDeadline: + """Return min(now + phase cap, reserve limit); raise when that is already past.""" now_wall = clock.time() now_monotonic = clock.monotonic() deadline = now_wall + phase_cap_seconds limited_by = PHASE_CAP - if budget is not None: - reserve_limit = ( - budget.deadline_epoch - - budget.scenario_reserve_seconds - - budget.observer_reserve_seconds - - later_reserve_seconds - ) - if reserve_limit < deadline: - deadline = reserve_limit - limited_by = JOB_RESERVE + if reserve_limit_epoch is not None and reserve_limit_epoch < deadline: + deadline = reserve_limit_epoch + limited_by = JOB_RESERVE if deadline <= now_wall: raise BudgetExhaustedError( f"no {phase} time remains before the later-phase reserves " @@ -322,3 +344,17 @@ def image_phase_deadline(budget: JobBudget | None, *, clock: Clock = SYSTEM_CLOC phase_cap_seconds=IMAGE_PHASE_CAP_SECONDS, clock=clock, ) + + +def scenario_phase_deadline(budget: JobBudget, *, clock: Clock = SYSTEM_CLOCK) -> PhaseDeadline: + """Return min(now + scenario reserve, job deadline - observer reserve). + + Call it when the scenario starts: the scenario step's timeout equals its reserve, and the + observers that upload evidence keep their reserve however late the scenario began. + """ + return _bounded_deadline( + phase="scenario", + phase_cap_seconds=budget.scenario_reserve_seconds, + reserve_limit_epoch=budget.deadline_epoch - budget.observer_reserve_seconds, + clock=clock, + ) diff --git a/tests/utils/helpers/_fresh_host/faults.py b/tests/utils/helpers/_fresh_host/faults.py new file mode 100644 index 00000000..5134a7f3 --- /dev/null +++ b/tests/utils/helpers/_fresh_host/faults.py @@ -0,0 +1,80 @@ +"""CI-only fault injection for proving fresh-host recovery paths on real runners. + +A fault is requested through `FRESH_HOST_FAULT_INJECTION` and honored only on a manual +`workflow_dispatch` run off `main`; anywhere else the request is logged and ignored, so the +switch is inert on push, pull-request, schedule, and local runs. Each fault fires at most +once per scenario: a marker under the diagnostics directory records the shot, so a retried +step runs clean and the artifacts show when the fault fired. +""" + +from __future__ import annotations + +import json +import os +from collections.abc import Mapping +from pathlib import Path +from typing import Final + +from tests.utils.helpers._fresh_host.models import FreshHostContext, FreshHostError +from tests.utils.helpers._fresh_host.shell import run_command +from tests.utils.helpers._fresh_host.storage import log, now_iso + +FAULT_INJECTION_ENV: Final[str] = "FRESH_HOST_FAULT_INJECTION" +FAULT_INJECTION_OFF: Final[str] = "none" +# Stops the OrbStack VM right before the macOS sidecars stack starts. +ORBSTACK_STOP_DURING_SIDECARS: Final[str] = "orbstack-stop-during-sidecars" +KNOWN_FAULTS: Final[frozenset[str]] = frozenset({ORBSTACK_STOP_DURING_SIDECARS}) +FAULT_INJECTION_EVENT: Final[str] = "workflow_dispatch" +PROTECTED_REF: Final[str] = "refs/heads/main" +_ORB_STOP_TIMEOUT_SECONDS: Final[int] = 120 + + +def requested_fault(environ: Mapping[str, str]) -> str | None: + """Return the fault this run may inject, or `None` when injection is off or not allowed.""" + fault = environ.get(FAULT_INJECTION_ENV, "").strip() + if not fault or fault == FAULT_INJECTION_OFF: + return None + if fault not in KNOWN_FAULTS: + raise FreshHostError( + f"{FAULT_INJECTION_ENV}={fault!r} is not one of {sorted(KNOWN_FAULTS)}" + ) + event = environ.get("GITHUB_EVENT_NAME", "") + ref = environ.get("GITHUB_REF", "") + if event != FAULT_INJECTION_EVENT or ref == PROTECTED_REF: + log( + f"Ignoring {FAULT_INJECTION_ENV}={fault}: fault injection runs only on " + f"{FAULT_INJECTION_EVENT} off {PROTECTED_REF} (event={event!r}, ref={ref!r})." + ) + return None + return fault + + +def _marker_path(context: FreshHostContext, fault: str) -> Path: + """Return the one-shot marker that records when *fault* fired.""" + return Path(context.diagnostics_dir).resolve() / "fault-injection" / f"{fault}.json" + + +def inject_sidecars_runtime_loss( + context: FreshHostContext, *, environ: Mapping[str, str] | None = None +) -> bool: + """Run `orb stop` once when the sidecars runtime-loss fault is requested; return if it fired.""" + fault = ORBSTACK_STOP_DURING_SIDECARS + if requested_fault(os.environ if environ is None else environ) != fault: + return False + marker = _marker_path(context, fault) + if marker.exists(): + return False + marker.parent.mkdir(parents=True, exist_ok=True) + marker.write_text( + json.dumps({"fault": fault, "fired_at": now_iso()}, sort_keys=True) + "\n", + encoding="utf-8", + ) + log(f"Fault injection {fault}: stopping OrbStack.") + # The runner env keeps the real HOME, which `orb` resolves its control socket from. + run_command( + ["orb", "stop"], + cwd=Path(context.repo_root).resolve(), + env=dict(os.environ), + timeout_seconds=_ORB_STOP_TIMEOUT_SECONDS, + ) + return True diff --git a/tests/utils/helpers/_fresh_host/macos.py b/tests/utils/helpers/_fresh_host/macos.py index 1bd4d46d..1791a206 100644 --- a/tests/utils/helpers/_fresh_host/macos.py +++ b/tests/utils/helpers/_fresh_host/macos.py @@ -7,8 +7,13 @@ from dataclasses import dataclass from pathlib import Path +from tests.utils.helpers._fresh_host.faults import inject_sidecars_runtime_loss from tests.utils.helpers._fresh_host.linux import run_clawops_bootstrap from tests.utils.helpers._fresh_host.models import FreshHostContext, FreshHostError +from tests.utils.helpers._fresh_host.runtime_loss import ( + RecoveryWindow, + run_with_runtime_recovery, +) from tests.utils.helpers._fresh_host.shell import ( TRANSIENT_KILL_EXIT_CODES, best_effort, @@ -276,6 +281,8 @@ def _run_repo_local_cycle( context, "browser-lab" if component == "browser-lab" else "sidecars" ) wait_for_docker_backend(cwd=repo_root, env=env) + if component == "sidecars": + inject_sidecars_runtime_loss(context) up_command = venv_clawops_command( context, "ops", "--asset-root", ".", component, "up", "--repo-local-state" ) @@ -328,35 +335,65 @@ def _venv_python(context: FreshHostContext) -> str: return venv_clawops_command(context)[0] -def exercise_macos_sidecars(context: FreshHostContext) -> list[str]: +def exercise_macos_sidecars( + context: FreshHostContext, *, recovery_window: RecoveryWindow | None = None +) -> list[str]: """Exercise macOS repo-local sidecars. When *exercise-channels-runtime* is also scheduled the sidecars stack is left running after verification so the next phase can reuse it without a redundant - up/down cycle. + up/down cycle. A *recovery_window* lets a lost Docker runtime be recovered once. """ bring_down = "exercise-channels-runtime" not in context.phase_names - return _run_repo_local_cycle(context, "sidecars", bring_down=bring_down) + return run_with_runtime_recovery( + context, + step="exercise-sidecars", + window=recovery_window, + action=lambda: _run_repo_local_cycle(context, "sidecars", bring_down=bring_down), + ) -def exercise_macos_browser_lab(context: FreshHostContext) -> list[str]: - """Exercise macOS repo-local browser-lab.""" - return _run_repo_local_cycle(context, "browser-lab") +def exercise_macos_browser_lab( + context: FreshHostContext, *, recovery_window: RecoveryWindow | None = None +) -> list[str]: + """Exercise macOS repo-local browser-lab; a lost Docker runtime is recovered once.""" + return run_with_runtime_recovery( + context, + step="exercise-browser-lab", + window=recovery_window, + action=lambda: _run_repo_local_cycle(context, "browser-lab"), + ) -def exercise_macos_channels_runtime(context: FreshHostContext) -> list[str]: +def exercise_macos_channels_runtime( + context: FreshHostContext, *, recovery_window: RecoveryWindow | None = None +) -> list[str]: """Exercise channels acceptance while sidecars are running repo-locally. When *exercise-sidecars* is also scheduled the sidecars stack is already running from that phase, so this function skips the redundant up/verify cycle and proceeds - directly to channels exercises before tearing the stack down. + directly to channels exercises before tearing the stack down. A retry after a + recovered runtime loss starts the stack again, since the lost runtime took it down. """ + sidecars_already_up = "exercise-sidecars" in context.phase_names + return run_with_runtime_recovery( + context, + step="exercise-channels-runtime", + window=recovery_window, + action=lambda: _exercise_channels_runtime(context, sidecars_already_up=sidecars_already_up), + retry_action=lambda: _exercise_channels_runtime(context, sidecars_already_up=False), + ) + + +def _exercise_channels_runtime( + context: FreshHostContext, *, sidecars_already_up: bool +) -> list[str]: + """Run the channels exercises, starting the sidecars stack unless it is already up.""" repo_root, _ = repo_paths(context) env = phase_env(context) env.setdefault("STRONGCLAW_CHANNELS_RUNTIME_TELEGRAM_BOT_TOKEN", "fresh-host-smoke-token") wait_for_docker_backend(cwd=repo_root, env=env) - sidecars_already_up = "exercise-sidecars" in context.phase_names if not sidecars_already_up: compose_file = compose_file_for_component(context, "sidecars") up_command = venv_clawops_command( diff --git a/tests/utils/helpers/_fresh_host/runtime_loss.py b/tests/utils/helpers/_fresh_host/runtime_loss.py new file mode 100644 index 00000000..fb7b1b2a --- /dev/null +++ b/tests/utils/helpers/_fresh_host/runtime_loss.py @@ -0,0 +1,148 @@ +"""Runtime-loss recovery for hosted macOS fresh-host scenario steps. + +The runtime and image phases recover OrbStack before the scenario starts. A VM that dies +once the scenario runs would otherwise fail the job, so Docker-backed steps run through +`run_with_runtime_recovery`: when a step fails and a live probe finds the backend gone, the +runtime phase's OrbStack recovery runs inside a slice of the scenario's recovery window and +the step is retried once. Without a recorded job budget (local runs) or off hosted macOS OrbStack, +failures pass through untouched. +""" + +from __future__ import annotations + +import os +from collections.abc import Callable, Mapping +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Final, TypeVar + +from tests.utils.helpers._fresh_host.budget import ( + SCENARIO_RECOVERY_CAP_SECONDS, + SCENARIO_RETRY_RESERVE_SECONDS, + SYSTEM_CLOCK, + BudgetExhaustedError, + Clock, + JobBudget, + PhaseDeadline, + scenario_phase_deadline, +) +from tests.utils.helpers._fresh_host.models import FreshHostContext, FreshHostError +from tests.utils.helpers._fresh_host.storage import log +from tests.utils.helpers._hosted_docker.io import write_json +from tests.utils.helpers._hosted_docker.runtime import recover_lost_runtime + +RUNTIME_LOSS: Final[str] = "runtime_loss" +RUNTIME_LOSS_CAP: Final[str] = "runtime-loss-cap" + +_T = TypeVar("_T") + + +@dataclass(frozen=True, slots=True) +class RecoveryWindow: + """The scenario deadline runtime-loss recovery runs inside, with the clock it is pinned to.""" + + deadline: PhaseDeadline + clock: Clock + + +def arm_runtime_recovery( + context: FreshHostContext, + *, + environ: Mapping[str, str] | None = None, + clock: Clock = SYSTEM_CLOCK, +) -> RecoveryWindow | None: + """Return the window runtime-loss recovery runs inside; `None` leaves recovery off. + + Recovery is enabled only on hosted macOS OrbStack runs that recorded a job budget. Call + it when the scenario starts, so the window follows the scenario step's own timeout. + """ + if context.platform != "macos" or (context.runtime_provider or "orbstack") != "orbstack": + return None + budget = JobBudget.from_env(os.environ if environ is None else environ) + if budget is None: + return None + try: + deadline = scenario_phase_deadline(budget, clock=clock) + except BudgetExhaustedError as exc: + log(f"Runtime-loss recovery disabled: {exc}.") + return None + log( + f"Runtime-loss recovery armed: {deadline.remaining(clock):.0f}s scenario window, " + f"limited by {deadline.limited_by}." + ) + return RecoveryWindow(deadline=deadline, clock=clock) + + +def _recovery_slice(deadline: PhaseDeadline, clock: Clock) -> PhaseDeadline: + """Return the recovery slice; it always leaves the retried step its reserve.""" + return deadline.slice( + clock, + cap_seconds=SCENARIO_RECOVERY_CAP_SECONDS, + reserve_seconds=SCENARIO_RETRY_RESERVE_SECONDS, + limited_by=RUNTIME_LOSS_CAP, + ) + + +def _recover( + context: FreshHostContext, + *, + step: str, + failure: FreshHostError, + window: RecoveryWindow, +) -> bool: + """Recover a lost runtime; `False` means the failure was not runtime loss.""" + clock = window.clock + try: + report = recover_lost_runtime( + context, step=step, deadline=_recovery_slice(window.deadline, clock), clock=clock + ) + except BudgetExhaustedError as exc: + raise BudgetExhaustedError( + f"{RUNTIME_LOSS} recovery for {step} has no time left ({exc}); step failure: " + f"{failure}" + ) from failure + if report is None: + return False + write_json( + asdict(report), + Path(context.diagnostics_dir).resolve() / "runtime-loss" / step / "report.json", + ) + if report.recovered: + return True + detail = ( + f"{RUNTIME_LOSS} during {step} ({report.trigger_reason}); " + f"last reason: {report.last_failure_reason}; step failure: {failure}" + ) + if report.exhausted: + raise BudgetExhaustedError( + f"{RUNTIME_LOSS} recovery ran out of time: {detail}" + ) from failure + if not report.recovery_attempts: + raise FreshHostError(f"unrecoverable {detail}") from failure + raise FreshHostError(f"orbstack_recovery_failed: {detail}") from failure + + +def run_with_runtime_recovery( + context: FreshHostContext, + *, + step: str, + window: RecoveryWindow | None, + action: Callable[[], _T], + retry_action: Callable[[], _T] | None = None, +) -> _T: + """Run one Docker-backed step; after a recovered runtime loss, retry it once. + + *retry_action* replaces *action* for the retry when the step must rebuild state the + lost runtime took with it. A `None` *window* leaves recovery off. + """ + try: + return action() + except FreshHostError as failure: + if window is None: + raise + log(f"{step} failed ({failure}); probing the Docker runtime for loss.") + if not _recover(context, step=step, failure=failure, window=window): + log(f"Docker runtime is reachable; {step} failed for another reason.") + raise + log(f"Docker runtime recovered; retrying {step} once.") + return (retry_action or action)() diff --git a/tests/utils/helpers/_fresh_host/scenario.py b/tests/utils/helpers/_fresh_host/scenario.py index 5ee27179..93d68333 100644 --- a/tests/utils/helpers/_fresh_host/scenario.py +++ b/tests/utils/helpers/_fresh_host/scenario.py @@ -13,6 +13,7 @@ FreshHostReport, PhaseResult, ) +from tests.utils.helpers._fresh_host.runtime_loss import RecoveryWindow, arm_runtime_recovery from tests.utils.helpers._fresh_host.storage import ( context_path, load_context, @@ -28,8 +29,16 @@ def scenario_phase_names(context: FreshHostContext) -> list[str]: return list(context.phase_names) -def run_named_phase(context: FreshHostContext, phase_name: str) -> list[str] | None: - """Execute one named scenario phase.""" +def run_named_phase( + context: FreshHostContext, + phase_name: str, + *, + recovery_window: RecoveryWindow | None = None, +) -> list[str] | None: + """Execute one named scenario phase. + + *recovery_window* arms hosted macOS runtime-loss recovery for Docker-backed phases. + """ if phase_name == "bootstrap": return ( linux.linux_bootstrap(context) @@ -57,13 +66,13 @@ def run_named_phase(context: FreshHostContext, phase_name: str) -> list[str] | N return ( linux.exercise_linux_sidecars(context) if context.platform == "linux" - else macos.exercise_macos_sidecars(context) + else macos.exercise_macos_sidecars(context, recovery_window=recovery_window) ) if phase_name == "exercise-channels-runtime": return ( linux.exercise_linux_channels_runtime(context) if context.platform == "linux" - else macos.exercise_macos_channels_runtime(context) + else macos.exercise_macos_channels_runtime(context, recovery_window=recovery_window) ) if phase_name == "exercise-recovery-smoke": return ( @@ -75,7 +84,7 @@ def run_named_phase(context: FreshHostContext, phase_name: str) -> list[str] | N return ( linux.exercise_linux_browser_lab(context) if context.platform == "linux" - else macos.exercise_macos_browser_lab(context) + else macos.exercise_macos_browser_lab(context, recovery_window=recovery_window) ) if phase_name == "normalize-machine-name": return macos.normalize_macos_machine_name(context) @@ -132,11 +141,12 @@ def run_scenario(context_file: Path) -> FreshHostReport: report = load_report(report_file) report.status = "running" write_report(report, report_file) + recovery_window = arm_runtime_recovery(context) for phase_name in scenario_phase_names(context): log(f"Starting phase={phase_name}.") def _run_phase(*, selected_phase_name: str = phase_name) -> list[str] | None: - return run_named_phase(context, selected_phase_name) + return run_named_phase(context, selected_phase_name, recovery_window=recovery_window) record_phase( report=report, diff --git a/tests/utils/helpers/_hosted_docker/diagnostics.py b/tests/utils/helpers/_hosted_docker/diagnostics.py index dc840d18..44ded918 100644 --- a/tests/utils/helpers/_hosted_docker/diagnostics.py +++ b/tests/utils/helpers/_hosted_docker/diagnostics.py @@ -27,7 +27,16 @@ SYSCTL_TIMEOUT_SECONDS: Final[int] = 10 # Wall budget for the standalone `collect-diagnostics` observer step. DIAGNOSTICS_COLLECTION_CAP_SECONDS: Final[int] = 120 +# Tail kept from each OrbStack log; the VM manager log holds why a VM stopped. +ORBSTACK_LOG_TAIL_BYTES: Final[int] = 1024 * 1024 _ORB_START_LOG: Final[Path] = Path("/tmp/orb-start.log") +# Host-side OrbStack processes, the Virtualization framework that runs its VM, and kernel +# memory-pressure kills, so a stopped VM can be told apart from a killed one. +_UNIFIED_LOG_PREDICATE: Final[str] = ( + 'process CONTAINS[c] "OrbStack" OR subsystem CONTAINS[c] "orbstack" ' + 'OR subsystem BEGINSWITH "com.apple.Virtualization" ' + 'OR (process == "kernel" AND eventMessage CONTAINS[c] "memorystatus")' +) def _socket_state(env: dict[str, str]) -> str: @@ -101,11 +110,39 @@ def _diagnostic_commands(context: FreshHostContext, root: Path) -> dict[Path, li "--last", "30m", "--predicate", - 'process CONTAINS[c] "OrbStack" OR subsystem CONTAINS[c] "orbstack"', + _UNIFIED_LOG_PREDICATE, ] return commands +def _tail_bytes(path: Path, limit: int) -> bytes: + """Return at most the last *limit* bytes of one file.""" + with path.open("rb") as stream: + stream.seek(0, os.SEEK_END) + stream.seek(max(0, stream.tell() - limit)) + return stream.read() + + +def _orbstack_log_dir() -> Path: + """Return the runner's OrbStack log directory.""" + return Path.home() / ".orbstack" / "log" + + +def _copy_orbstack_logs(diagnostics_dir: Path) -> None: + """Copy the tail of each OrbStack log; they hold the VM manager's account of a stopped VM.""" + log_dir = _orbstack_log_dir() + if not log_dir.is_dir(): + return + output_dir = diagnostics_dir / "orbstack-logs" + output_dir.mkdir(parents=True, exist_ok=True) + for log_path in sorted(log_dir.glob("*.log")): + try: + content = _tail_bytes(log_path, ORBSTACK_LOG_TAIL_BYTES) + except OSError as exc: + content = f"failed to read {log_path}: {exc}\n".encode() + (output_dir / log_path.name).write_bytes(content) + + def _capture( command: list[str], output_path: Path, @@ -173,6 +210,7 @@ def collect_runtime_diagnostics_for_context( orb_start_content, encoding="utf-8", ) + _copy_orbstack_logs(resolved_diagnostics_dir) for output_path, sysctl_name in ( (resolved_diagnostics_dir / "host-cpu-count.txt", "hw.ncpu"), (resolved_diagnostics_dir / "host-memory-bytes.txt", "hw.memsize"), diff --git a/tests/utils/helpers/_hosted_docker/models.py b/tests/utils/helpers/_hosted_docker/models.py index 86b672da..a4bfff5f 100644 --- a/tests/utils/helpers/_hosted_docker/models.py +++ b/tests/utils/helpers/_hosted_docker/models.py @@ -108,6 +108,22 @@ class RuntimeInstallReport: deadline_limited_by: str | None = None +@dataclass(slots=True) +class RuntimeLossReport: + """Structured report describing one runtime-loss recovery inside a scenario step.""" + + step: str + trigger_reason: str + recovered: bool + exhausted: bool + last_failure_reason: str | None + recovery_attempts: list[RuntimeRecoveryAttemptReport] + deadline_epoch: float + deadline_limited_by: str + started_at: str + finished_at: str + + @dataclass(slots=True) class ImageEnsureReport: """Structured report describing image ensure status.""" diff --git a/tests/utils/helpers/_hosted_docker/runtime.py b/tests/utils/helpers/_hosted_docker/runtime.py index 3c92b329..340c98b4 100644 --- a/tests/utils/helpers/_hosted_docker/runtime.py +++ b/tests/utils/helpers/_hosted_docker/runtime.py @@ -4,6 +4,7 @@ OrbStack recovery slice reserved up front so a slow gate can never starve it. Every probe, validation command, recovery command, backoff, and diagnostics collection is clipped to the time left through one injected clock, so the phase ends at its deadline with a report. +`recover_lost_runtime` reuses the same recovery for a runtime lost mid-scenario. """ from __future__ import annotations @@ -35,6 +36,7 @@ from tests.utils.helpers._hosted_docker.io import log, now_iso, write_github_env, write_json from tests.utils.helpers._hosted_docker.models import ( RuntimeInstallReport, + RuntimeLossReport, RuntimeRecoveryAttemptReport, ) from tests.utils.helpers._hosted_docker.shell import ( @@ -226,6 +228,26 @@ def _classify_runtime_failure( if exception_reason is not None and exception_reason != "docker_not_ready": return exception_reason + try: + probe_reason = _probe_backend_failure(cwd=cwd, env=env, deadline=deadline, clock=clock) + except BudgetExhaustedError: + return exception_reason or "docker_not_ready" + if probe_reason is None: + return exception_reason or "runtime_tool_validation_failed" + return probe_reason + + +def _probe_backend_failure( + *, + cwd: Path, + env: dict[str, str], + deadline: PhaseDeadline, + clock: Clock, +) -> str | None: + """Return why a live probe cannot reach the Docker backend now; `None` when it answers. + + Raises `BudgetExhaustedError` when *deadline* leaves no time to probe. + """ docker_host = env.get("DOCKER_HOST") or os.environ.get("DOCKER_HOST", "") socket_path = _docker_socket_path(docker_host) if socket_path is not None and (not socket_path.exists() or not socket_path.is_socket()): @@ -239,8 +261,6 @@ def _classify_runtime_failure( timeout_seconds=deadline.command_timeout(_READINESS_PROBE_TIMEOUT_SECONDS, clock), capture_output=True, ) - except BudgetExhaustedError: - return exception_reason or "docker_not_ready" except subprocess.TimeoutExpired: return "docker_probe_timeout" except FileNotFoundError: @@ -249,11 +269,13 @@ def _classify_runtime_failure( return _classify_failure_text(str(probe_exc)) or "docker_not_ready" if completed.returncode == 0: - return exception_reason or "runtime_tool_validation_failed" + return None probe_text = "\n".join( - chunk for chunk in (completed.stdout.strip(), completed.stderr.strip()) if chunk + chunk + for chunk in ((completed.stdout or "").strip(), (completed.stderr or "").strip()) + if chunk ) - return _classify_failure_text(probe_text) or exception_reason or "docker_not_ready" + return _classify_failure_text(probe_text) or "docker_not_ready" def _validate_runtime_ready( @@ -369,17 +391,17 @@ def _recover_orbstack_runtime( trigger_reason: str, deadline: PhaseDeadline, clock: Clock, + recovery_root: Path, ) -> _RecoveryResult: """Split the recovery slice across bounded OrbStack recovery attempts. Each attempt gets an equal share of what is left; inside it, before and after diagnostics each get a fixed slice, and recovery commands, backoff, and the readiness - gate share the rest. + gate share the rest. Attempt evidence lands under *recovery_root*. """ attempts: list[RuntimeRecoveryAttemptReport] = [] last_readiness_reason: str | None = trigger_reason exhausted = False - recovery_root = Path(context.diagnostics_dir).resolve() / "runtime-recovery" attempt_total = len(_RECOVERY_BACKOFF_SECONDS) for attempt_number, backoff_seconds in enumerate(_RECOVERY_BACKOFF_SECONDS, start=1): @@ -528,6 +550,7 @@ def _bring_up_runtime( trigger_reason=initial_reason, deadline=deadline, clock=clock, + recovery_root=Path(context.diagnostics_dir).resolve() / "runtime-recovery", ) if recovery.recovered: return _RuntimeOutcome(None, None, None, recovery.attempts) @@ -542,6 +565,72 @@ def _bring_up_runtime( ) +def _runtime_command_env(runtime_provider: str) -> dict[str, str]: + """Return the runner env for Docker probes and OrbStack commands. + + It keeps the runner's own HOME: `orb` and the Docker CLI resolve their sockets and config + from it, unlike the scenario env, which points HOME at the isolated app home. + """ + env = macos_env() + env["DOCKER_CONFIG"] = os.environ.get("DOCKER_CONFIG", str((Path.home() / ".docker").resolve())) + docker_host = os.environ.get("DOCKER_HOST", "") + # When OrbStack is started in the background its socket appears asynchronously. + # Fall back to OrbStack's canonical socket path if DOCKER_HOST is not yet set. + if not docker_host and runtime_provider == "orbstack": + docker_host = f"unix://{Path.home() / '.orbstack' / 'run' / 'docker.sock'}" + if docker_host: + env["DOCKER_HOST"] = docker_host + return env + + +def recover_lost_runtime( + context: FreshHostContext, + *, + step: str, + deadline: PhaseDeadline, + clock: Clock = SYSTEM_CLOCK, +) -> RuntimeLossReport | None: + """Recover a Docker runtime lost after the runtime phase, inside *deadline*. + + A live probe decides whether a failed scenario step lost its runtime: `None` means the + backend answers, so the step failed for another reason. A recoverable loss on OrbStack + runs the runtime phase's recovery attempts, with their evidence under + `diagnostics/runtime-loss/`. Raises `BudgetExhaustedError` when *deadline* leaves + no time to probe. + """ + started_at = now_iso() + runtime_provider = context.runtime_provider or "orbstack" + repo_root = Path(context.repo_root).resolve() + env = _runtime_command_env(runtime_provider) + reason = _probe_backend_failure(cwd=repo_root, env=env, deadline=deadline, clock=clock) + if reason is None: + return None + log(f"Docker runtime lost during {step}: {reason}.") + recovery = _RecoveryResult(recovered=False, attempts=[], last_reason=reason, exhausted=False) + if runtime_provider == "orbstack" and reason in _RECOVERABLE_RUNTIME_REASONS: + recovery = _recover_orbstack_runtime( + context=context, + repo_root=repo_root, + env=env, + trigger_reason=reason, + deadline=deadline, + clock=clock, + recovery_root=Path(context.diagnostics_dir).resolve() / "runtime-loss" / step, + ) + return RuntimeLossReport( + step=step, + trigger_reason=reason, + recovered=recovery.recovered, + exhausted=recovery.exhausted, + last_failure_reason=recovery.last_reason, + recovery_attempts=recovery.attempts, + deadline_epoch=deadline.wall_deadline_epoch, + deadline_limited_by=deadline.limited_by, + started_at=started_at, + finished_at=now_iso(), + ) + + def wait_runtime_ready( context_path: Path, *, @@ -571,23 +660,13 @@ def wait_runtime_ready( host_memory_gib = ( max(1, host_memory_bytes // 1073741824) if host_memory_bytes is not None else None ) - docker_config = os.environ.get("DOCKER_CONFIG", str((Path.home() / ".docker").resolve())) - docker_host = os.environ.get("DOCKER_HOST", "") - env = macos_env() - env["DOCKER_CONFIG"] = docker_config - if docker_host: - env["DOCKER_HOST"] = docker_host + env = _runtime_command_env(runtime_provider) + docker_config = env["DOCKER_CONFIG"] + docker_host = env.get("DOCKER_HOST", "") started_at = now_iso() started = clock.monotonic() - # When OrbStack is started in the background its socket appears asynchronously. - # Fall back to OrbStack's canonical socket path if DOCKER_HOST is not yet set. - if not docker_host and runtime_provider == "orbstack": - orbstack_socket = Path.home() / ".orbstack" / "run" / "docker.sock" - docker_host = f"unix://{orbstack_socket}" - env["DOCKER_HOST"] = docker_host - log(f"wait_runtime_ready: DOCKER_HOST={docker_host!r} DOCKER_CONFIG={docker_config!r}") phase_error: BudgetExhaustedError | None = None