Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions .github/workflows/fresh-host-acceptance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,15 @@ on:
required: true
type: boolean
default: true
# Dispatch-only: callers through workflow_call cannot request a fault.
fault_injection:
description: CI-only fault drill for the macOS sidecars scenario; ignored on main.
required: true
type: choice
default: none
options:
- none
- orbstack-stop-during-sidecars

concurrency:
group: >-
Expand All @@ -57,3 +66,4 @@ jobs:
docker_pull_parallelism: ${{ inputs.docker_pull_parallelism }}
docker_pull_max_attempts: ${{ inputs.docker_pull_max_attempts }}
enable_package_cache: ${{ inputs.enable_package_cache }}
fault_injection: ${{ github.event_name == 'workflow_dispatch' && inputs.fault_injection || '' }}
9 changes: 9 additions & 0 deletions .github/workflows/fresh-host-core.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ on:
required: false
type: boolean
default: true
# CI-only fault drill (see tests/utils/helpers/_fresh_host/faults.py); empty is off.
fault_injection:
required: false
type: string
default: ""

permissions:
contents: read
Expand Down Expand Up @@ -203,8 +208,12 @@ jobs:
- name: Ensure hosted macOS Docker images
timeout-minutes: 32
run: ./tests/scripts/hosted_docker.py ensure-images --context "${FRESH_HOST_CONTEXT}"
# The scenario recovers a Docker runtime lost mid-run inside its own budget. The fault
# drill reaches it only from a manual dispatch off main; the helper enforces that too.
- name: Run macOS fresh-host scenario
timeout-minutes: 35
env:
FRESH_HOST_FAULT_INJECTION: ${{ github.event_name == 'workflow_dispatch' && github.ref != 'refs/heads/main' && inputs.fault_injection || '' }}
run: ./tests/scripts/fresh_host.py run-scenario --context "${FRESH_HOST_CONTEXT}"
- name: Collect hosted macOS Docker diagnostics
if: ${{ always() && env.FRESH_HOST_CONTEXT != '' }}
Expand Down
86 changes: 86 additions & 0 deletions tests/suites/contracts/repo/test_ci_workflow_surfaces.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,12 @@
RUNTIME_PHASE_CAP_SECONDS,
RUNTIME_PHASE_MIN_SECONDS,
)
from tests.utils.helpers._fresh_host.faults import (
FAULT_INJECTION_ENV,
FAULT_INJECTION_OFF,
KNOWN_FAULTS,
requested_fault,
)
from tests.utils.helpers._hosted_docker.diagnostics import DIAGNOSTICS_COLLECTION_CAP_SECONDS
from tests.utils.helpers.repo import REPO_ROOT

Expand Down Expand Up @@ -971,3 +977,83 @@ def test_codeql_config_ignores_unmaintained_and_packaged_vendor_code() -> None:
assert "src/clawops/assets" in text
assert "platform/plugins/memory-lancedb-pro" in text
assert "platform/plugins/strongclaw-hypermemory" not in text


_CORE_FAULT_GATE = (
"${{ github.event_name == 'workflow_dispatch' && github.ref != 'refs/heads/main' "
"&& inputs.fault_injection || '' }}"
)
_ACCEPTANCE_FAULT_FORWARD = (
"${{ github.event_name == 'workflow_dispatch' && inputs.fault_injection || '' }}"
)


def _workflow_trigger_inputs(workflow_name: str, trigger: str) -> dict[str, object]:
"""Return one trigger's inputs; YAML 1.1 loads the `on:` key as `True`."""
loaded: object = yaml.safe_load(_workflow_text(workflow_name))
assert isinstance(loaded, dict), workflow_name
raw = cast(dict[object, object], loaded)
triggers = _as_str_object_dict(raw.get("on", raw.get(True)))
assert triggers is not None, workflow_name
trigger_config = _as_str_object_dict(triggers.get(trigger))
assert trigger_config is not None, (workflow_name, trigger)
inputs = _as_str_object_dict(trigger_config.get("inputs"))
assert inputs is not None, (workflow_name, trigger)
return inputs


def _job_step(workflow_name: str, job_id: str, step_name: str) -> dict[str, object]:
"""Return one named step of one job."""
job = _as_str_object_dict(_workflow_jobs(workflow_name)[job_id])
assert job is not None, job_id
steps = job.get("steps")
assert isinstance(steps, list), job_id
for raw_step in cast(list[object], steps):
step = _as_str_object_dict(raw_step)
if step is not None and step.get("name") == step_name:
return step
raise AssertionError(f"{workflow_name}:{job_id} has no step {step_name!r}")


def test_fault_injection_cannot_be_enabled_on_push_to_main() -> None:
"""The fault drill reaches the scenario only from a manual dispatch off main."""
leaked_push_env = {
FAULT_INJECTION_ENV: next(iter(KNOWN_FAULTS)),
"GITHUB_EVENT_NAME": "push",
"GITHUB_REF": "refs/heads/main",
}
assert requested_fault(leaked_push_env) is None

core_input = _as_str_object_dict(
_workflow_trigger_inputs("fresh-host-core.yml", "workflow_call").get("fault_injection")
)
assert core_input is not None
assert core_input["default"] == ""
assert _workflow_text("fresh-host-core.yml").count("inputs.fault_injection") == 1
scenario_step = _job_step(
"fresh-host-core.yml", "macos-fresh-host", "Run macOS fresh-host scenario"
)
assert scenario_step.get("env") == {FAULT_INJECTION_ENV: _CORE_FAULT_GATE}

assert "fault_injection" not in _workflow_trigger_inputs(
"fresh-host-acceptance.yml", "workflow_call"
)
dispatch_input = _as_str_object_dict(
_workflow_trigger_inputs("fresh-host-acceptance.yml", "workflow_dispatch").get(
"fault_injection"
)
)
assert dispatch_input is not None
assert dispatch_input["type"] == "choice"
assert dispatch_input["default"] == FAULT_INJECTION_OFF
assert dispatch_input["options"] == [FAULT_INJECTION_OFF, *sorted(KNOWN_FAULTS)]
acceptance_job = _as_str_object_dict(
_workflow_jobs("fresh-host-acceptance.yml")["fresh-host-acceptance"]
)
assert acceptance_job is not None
acceptance_with = _as_str_object_dict(acceptance_job.get("with"))
assert acceptance_with is not None
assert acceptance_with["fault_injection"] == _ACCEPTANCE_FAULT_FORWARD

for caller in ("ci-gate.yml", "nightly.yml", "release.yml", "e2e-acceptance.yml"):
assert "fault_injection" not in _workflow_text(caller), caller
3 changes: 3 additions & 0 deletions tests/suites/unit/ci/test_fresh_host.py
Original file line number Diff line number Diff line change
Expand Up @@ -374,9 +374,12 @@ def test_run_scenario_records_successful_phase_sequence(
def fake_run_named_phase(
loaded_context: fresh_host.FreshHostContext,
phase_name: str,
*,
recovery_window: object = None,
) -> list[str]:
calls.append(phase_name)
assert loaded_context.scenario_id == "linux"
assert recovery_window is None, "Linux scenarios never arm runtime-loss recovery"
return ["echo", phase_name]

test_context.patch.patch_object(
Expand Down
34 changes: 34 additions & 0 deletions tests/suites/unit/ci/test_fresh_host_budget.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
phase_deadline,
record_job_budget,
runtime_phase_deadline,
scenario_phase_deadline,
)
from tests.utils.helpers._fresh_host.models import FreshHostError
from tests.utils.helpers.clock import FakeClock
Expand Down Expand Up @@ -300,3 +301,36 @@ def test_initialize_budget_cli_writes_env_once(tmp_path: Path, test_context: Tes
)
assert JobBudget.from_env(exported) is not None
assert fresh_host_script.main([*argv[:2], "5", *argv[3:]]) == 1


def test_scenario_phase_deadline_follows_the_scenario_step_timeout_when_the_job_has_room() -> None:
clock = FakeClock()
budget = _macos_budget(clock)
clock.advance(600)

deadline = scenario_phase_deadline(budget, clock=clock)

assert deadline.remaining(clock) == budget.scenario_reserve_seconds
assert deadline.limited_by == "phase-cap"


def test_scenario_phase_deadline_keeps_the_observer_reserve_when_the_scenario_starts_late() -> None:
clock = FakeClock()
budget = _macos_budget(clock)
observer_limit = budget.deadline_epoch - budget.observer_reserve_seconds
clock.advance(observer_limit - clock.time() - 300)

deadline = scenario_phase_deadline(budget, clock=clock)

assert deadline.wall_deadline_epoch == observer_limit
assert deadline.remaining(clock) == 300
assert deadline.limited_by == "job-reserve"


def test_scenario_phase_deadline_fails_fast_inside_the_observer_reserve() -> None:
clock = FakeClock()
budget = _macos_budget(clock)
clock.advance(budget.deadline_epoch - budget.observer_reserve_seconds - clock.time())

with pytest.raises(BudgetExhaustedError, match="no scenario time remains"):
scenario_phase_deadline(budget, clock=clock)
110 changes: 110 additions & 0 deletions tests/suites/unit/ci/test_fresh_host_faults.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
"""Unit coverage for the CI-only fresh-host fault-injection switch."""

from __future__ import annotations

import json
from pathlib import Path

import pytest

from tests.plugins.infrastructure.context import TestContext
from tests.utils.helpers import fresh_host
from tests.utils.helpers._fresh_host import faults
from tests.utils.helpers._fresh_host.models import FreshHostContext, FreshHostError

_FAULT = faults.ORBSTACK_STOP_DURING_SIDECARS
_BRANCH_REF = "refs/heads/fix/runtime-loss"


def _environ(fault: str, *, event: str, ref: str) -> dict[str, str]:
return {faults.FAULT_INJECTION_ENV: fault, "GITHUB_EVENT_NAME": event, "GITHUB_REF": ref}


@pytest.mark.parametrize(
("event", "ref"),
[
("push", "refs/heads/main"),
("push", _BRANCH_REF),
("pull_request", "refs/pull/252/merge"),
("schedule", "refs/heads/main"),
("workflow_dispatch", "refs/heads/main"),
("", ""),
],
)
def test_fault_requests_are_ignored_outside_manual_dispatch_off_main(event: str, ref: str) -> None:
assert faults.requested_fault(_environ(_FAULT, event=event, ref=ref)) is None


def test_fault_requests_are_honored_on_manual_dispatch_off_main() -> None:
environ = _environ(_FAULT, event="workflow_dispatch", ref=_BRANCH_REF)

assert faults.requested_fault(environ) == _FAULT


@pytest.mark.parametrize("value", ["", " ", faults.FAULT_INJECTION_OFF])
def test_unset_or_off_fault_requests_inject_nothing(value: str) -> None:
environ = _environ(value, event="workflow_dispatch", ref=_BRANCH_REF)

assert faults.requested_fault(environ) is None


def test_unknown_fault_requests_fail_loudly() -> None:
environ = _environ("kill-everything", event="workflow_dispatch", ref=_BRANCH_REF)

with pytest.raises(FreshHostError, match="is not one of"):
faults.requested_fault(environ)


def _macos_context(tmp_path: Path, test_context: TestContext) -> FreshHostContext:
workspace = tmp_path / "workspace"
workspace.mkdir()
test_context.apply_profiles("fresh_host_macos_orbstack")
return fresh_host.prepare_context(
scenario_id="macos-sidecars",
repo_root=workspace,
runner_temp=tmp_path / "runner-temp",
workspace=workspace,
github_env_file=tmp_path / "github.env",
)


def _record_commands(test_context: TestContext) -> list[list[str]]:
commands: list[list[str]] = []

def fake_run_command(command: list[str], **kwargs: object) -> None:
del kwargs
commands.append(list(command))

test_context.patch.patch_object(faults, "run_command", new=fake_run_command)
return commands


def test_sidecars_runtime_loss_fires_orb_stop_exactly_once(
tmp_path: Path, test_context: TestContext
) -> None:
context = _macos_context(tmp_path, test_context)
commands = _record_commands(test_context)
environ = _environ(_FAULT, event="workflow_dispatch", ref=_BRANCH_REF)

first = faults.inject_sidecars_runtime_loss(context, environ=environ)
second = faults.inject_sidecars_runtime_loss(context, environ=environ)

assert (first, second) == (True, False)
assert commands == [["orb", "stop"]]
marker = Path(context.diagnostics_dir) / "fault-injection" / f"{_FAULT}.json"
assert json.loads(marker.read_text(encoding="utf-8"))["fault"] == _FAULT


def test_sidecars_runtime_loss_is_inert_on_push_to_main(
tmp_path: Path, test_context: TestContext
) -> None:
context = _macos_context(tmp_path, test_context)
commands = _record_commands(test_context)

fired = faults.inject_sidecars_runtime_loss(
context, environ=_environ(_FAULT, event="push", ref="refs/heads/main")
)

assert fired is False
assert commands == []
assert not (Path(context.diagnostics_dir) / "fault-injection").exists()
Loading
Loading