Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/fresh-host-acceptance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ on:
options:
- none
- orbstack-stop-during-sidecars
- orbstack-stop-during-sidecars-health

concurrency:
group: >-
Expand Down
3 changes: 3 additions & 0 deletions platform/docs/OBSERVABILITY.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,14 @@ Those signals reuse the shared ClawOps telemetry path instead of adding a separa
- `clawops.ops.sidecars.wait.start`
- `clawops.ops.sidecars.wait.ready`
- `clawops.ops.sidecars.wait.timeout`
- `clawops.ops.sidecars.wait.backend_lost`
- `clawops.ops.sidecars.ready`
- `clawops.ops.sidecars.status`

Wait-timeout events include `service`, `target`, `observed`, and `timeout_seconds` so failed dependencies can be diagnosed without replaying the command interactively.

A wait whose `docker compose ps` fails probes the Docker backend. When the backend socket is gone, or `docker info` fails three probes in a row, the wait stops early with `wait.backend_lost` (`service`, `target`, `detail`, `elapsed_ms`) instead of polling out its timeout.

## Host Service Activation Events

`clawops services install --activate` now emits:
Expand Down
3 changes: 3 additions & 0 deletions src/clawops/assets/platform/docs/OBSERVABILITY.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,14 @@ Those signals reuse the shared ClawOps telemetry path instead of adding a separa
- `clawops.ops.sidecars.wait.start`
- `clawops.ops.sidecars.wait.ready`
- `clawops.ops.sidecars.wait.timeout`
- `clawops.ops.sidecars.wait.backend_lost`
- `clawops.ops.sidecars.ready`
- `clawops.ops.sidecars.status`

Wait-timeout events include `service`, `target`, `observed`, and `timeout_seconds` so failed dependencies can be diagnosed without replaying the command interactively.

A wait whose `docker compose ps` fails probes the Docker backend. When the backend socket is gone, or `docker info` fails three probes in a row, the wait stops early with `wait.backend_lost` (`service`, `target`, `detail`, `elapsed_ms`) instead of polling out its timeout.

## Host Service Activation Events

`clawops services install --activate` now emits:
Expand Down
41 changes: 40 additions & 1 deletion src/clawops/strongclaw_ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,11 @@
from clawops.strongclaw_runtime import (
VARLOCK_ENV_MODE_MANAGED,
CommandError,
DockerBackendLostError,
DockerBackendWatch,
ensure_docker_backend_ready,
load_env_assignments,
probe_docker_backend,
rendered_openclaw_hypermemory_config_path,
rendered_openclaw_uses_hypermemory,
rendered_openclaw_uses_qmd,
Expand Down Expand Up @@ -346,7 +349,11 @@ def _wait_for_compose_service(
health: str | None = None,
timeout_seconds: int,
) -> None:
"""Wait for one compose service to reach the requested state."""
"""Wait for one compose service to reach the requested state.

A failed status inspection probes the Docker backend; once it is confirmed lost the wait
raises `DockerBackendLostError` instead of polling out the rest of *timeout_seconds*.
"""
started_at = time.monotonic()
target = state if health is None else f"{state}/{health}"
emit_structured_log(
Expand All @@ -360,12 +367,31 @@ def _wait_for_compose_service(
deadline = time.monotonic() + timeout_seconds
last_status: _ComposeServiceStatus | None = None
last_error: CommandError | None = None
backend = DockerBackendWatch(lambda: probe_docker_backend(env=execution.env, cwd=execution.cwd))
while True:
try:
last_status = _compose_service_statuses(execution).get(service_name)
last_error = None
backend.reset()
except CommandError as exc:
last_error = exc
lost_detail = backend.confirm_lost()
if lost_detail is not None:
emit_structured_log(
"clawops.ops.sidecars.wait.backend_lost",
{
"service": service_name,
"target": target,
"detail": lost_detail,
"elapsed_ms": int((time.monotonic() - started_at) * 1000),
"statusInspectionError": str(exc),
},
)
raise DockerBackendLostError(
f"Docker backend became unreachable while waiting for compose service "
f"'{service_name}' to reach {target}: {lost_detail}; last status inspection "
f"error: {exc}. Restart the Docker runtime, then rerun the operation."
) from exc
if time.monotonic() >= deadline:
break
time.sleep(COMPOSE_POLL_INTERVAL_SECONDS)
Expand Down Expand Up @@ -763,6 +789,19 @@ def _record_step(step: str, exit_code: int) -> int:
health="healthy",
timeout_seconds=POSTGRES_HEALTH_TIMEOUT_SECONDS,
)
except DockerBackendLostError as exc:
# Recreating Postgres cannot help once the backend itself is gone.
if not suppress_wait_failures:
raise
return _sidecars_report_payload(
execution,
profile_flags=profile_flags,
targets=targets,
command_steps=command_steps,
command_exit_code=1,
error=str(exc),
failed_service=POSTGRES_SERVICE_NAME,
)
except CommandError as exc:
postgres_wait_error = exc
if postgres_wait_error is not None:
Expand Down
91 changes: 90 additions & 1 deletion src/clawops/strongclaw_runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
import subprocess
import sys
import time
from collections.abc import Generator, Iterable, Mapping, Sequence
from collections.abc import Callable, Generator, Iterable, Mapping, Sequence
from contextlib import contextmanager
from typing import Any, Final, Literal, cast

Expand Down Expand Up @@ -1201,6 +1201,95 @@ def ensure_docker_backend_ready() -> None:
)


DOCKER_BACKEND_PROBE_TIMEOUT_SECONDS: Final[int] = 15
DOCKER_BACKEND_LOSS_PROBE_LIMIT: Final[int] = 3


class DockerBackendLostError(CommandError):
"""Raised when the Docker backend disappears while a command is waiting on it."""


@dataclasses.dataclass(frozen=True, slots=True)
class DockerBackendProbe:
"""One live Docker backend probe; a `definitive` failure confirms loss without repeats."""

reachable: bool
detail: str = ""
definitive: bool = False


def _docker_host_socket(docker_host: str) -> pathlib.Path | None:
"""Return the Unix socket a `DOCKER_HOST` value points at, when it is one."""
if not docker_host.startswith("unix://"):
return None
return pathlib.Path(docker_host.removeprefix("unix://"))


def probe_docker_backend(
*,
env: Mapping[str, str] | None = None,
cwd: pathlib.Path | None = None,
timeout_seconds: int = DOCKER_BACKEND_PROBE_TIMEOUT_SECONDS,
) -> DockerBackendProbe:
"""Probe the Docker backend *env* targets: its Unix socket first, then `docker info`."""
docker_host = (os.environ if env is None else env).get("DOCKER_HOST", "").strip()
socket_path = _docker_host_socket(docker_host)
if socket_path is not None and not socket_path.is_socket():
return DockerBackendProbe(
reachable=False, detail=f"Docker socket {socket_path} is missing", definitive=True
)
result = run_command(["docker", "info"], cwd=cwd, env=env, timeout_seconds=timeout_seconds)
if result.ok:
return DockerBackendProbe(reachable=True)
if result.timed_out:
return DockerBackendProbe(
reachable=False, detail=f"`docker info` timed out after {timeout_seconds}s"
)
# `docker info` prints client details to stdout even when the daemon is gone.
error = " ".join(result.stderr.split())
return DockerBackendProbe(
reachable=False,
detail=f"`docker info` failed: {error}" if error else "`docker info` failed",
)


class DockerBackendWatch:
"""Tell a lost Docker backend from a slow one across a status-polling loop.

Call `confirm_lost` after a status inspection fails and `reset` after one succeeds. A
missing socket confirms loss at once; otherwise *probe_limit* failed probes in a row do.
"""

def __init__(
self,
probe: Callable[[], DockerBackendProbe],
*,
probe_limit: int = DOCKER_BACKEND_LOSS_PROBE_LIMIT,
) -> None:
if probe_limit < 1:
raise ValueError("probe_limit must be positive")
self._probe = probe
self._probe_limit = probe_limit
self._failures = 0

def reset(self) -> None:
"""Record that the backend answered, clearing the failure streak."""
self._failures = 0

def confirm_lost(self) -> str | None:
"""Probe once; return why the backend is lost, or `None` while it may still answer."""
probe = self._probe()
if probe.reachable:
self._failures = 0
return None
self._failures += 1
if probe.definitive:
return probe.detail
if self._failures >= self._probe_limit:
return f"{probe.detail} ({self._failures} failed probes in a row)"
return None


def repair_linux_runtime_user_docker_access(runtime_user: str) -> None:
"""Ensure the runtime user can talk to Docker on Linux."""
group_result = run_command(["getent", "group", "docker"], timeout_seconds=10)
Expand Down
15 changes: 8 additions & 7 deletions tests/suites/contracts/repo/test_ci_workflow_surfaces.py
Original file line number Diff line number Diff line change
Expand Up @@ -1016,13 +1016,14 @@ def _job_step(workflow_name: str, job_id: str, step_name: str) -> dict[str, obje


def test_fault_injection_cannot_be_enabled_on_push_to_main() -> None:
"""The fault drill reaches the scenario only from a manual dispatch off main."""
leaked_push_env = {
FAULT_INJECTION_ENV: next(iter(KNOWN_FAULTS)),
"GITHUB_EVENT_NAME": "push",
"GITHUB_REF": "refs/heads/main",
}
assert requested_fault(leaked_push_env) is None
"""Every fault drill reaches the scenario only from a manual dispatch off main."""
for fault in sorted(KNOWN_FAULTS):
leaked_push_env = {
FAULT_INJECTION_ENV: fault,
"GITHUB_EVENT_NAME": "push",
"GITHUB_REF": "refs/heads/main",
}
assert requested_fault(leaked_push_env) is None, fault

core_input = _as_str_object_dict(
_workflow_trigger_inputs("fresh-host-core.yml", "workflow_call").get("fault_injection")
Expand Down
Loading
Loading