From 85b804d3efa477a872171f4ed2bf0dd771d9ddb4 Mon Sep 17 00:00:00 2001 From: Juan Sugg Date: Wed, 11 Nov 2026 12:26:12 -0300 Subject: [PATCH 1/5] test(e2e): add per-feature verification suite and workflow Add tests/feature_matrix.yaml with one entry per product feature (call sites, test, lanes), a lane-scoped pytest suite that drives the real clawops/openclaw/acpx install, a deterministic local model server, an ACP stub agent for vendor adapters, and a feature-verification workflow that reports PASS/FAIL/UNVERIFIABLE per feature. --- .github/workflows/feature-verification.yml | 287 ++++ pyproject.toml | 1 + tests/conftest.py | 2 + tests/feature_matrix.yaml | 874 +++++++++++ tests/fixtures/__init__.py | 1 + .../fixtures/feature_verification/__init__.py | 3 + tests/fixtures/feature_verification/lane.py | 93 ++ tests/scripts/feature_report.py | 61 + tests/scripts/feature_verification.py | 155 ++ tests/scripts/fresh_host.py | 6 +- .../repo/test_feature_matrix_contract.py | 154 ++ .../test_fixture_plugin_registration.py | 4 + .../contracts/taf/test_framework_health.py | 1 + .../e2e/feature_verification/test_acp.py | 255 ++++ .../test_context_workflows.py | 124 ++ .../test_devflow_planning.py | 129 ++ .../test_host_lifecycle.py | 139 ++ .../test_memory_hypermemory.py | 202 +++ .../test_memory_lancedb.py | 230 +++ .../test_model_gateway.py | 150 ++ .../test_operator_tools.py | 1307 +++++++++++++++++ .../e2e/feature_verification/test_recovery.py | 117 ++ .../test_setup_profiles.py | 157 ++ .../e2e/feature_verification/test_sidecars.py | 249 ++++ .../unit/ci/test_feature_verification.py | 335 +++++ .../helpers/_feature_verification/__init__.py | 1 + .../_feature_verification/acp_stub_agent.py | 163 ++ .../_feature_verification/diagnostics.py | 97 ++ .../helpers/_feature_verification/envvars.py | 10 + .../_feature_verification/fake_model.py | 460 ++++++ .../helpers/_feature_verification/host.py | 288 ++++ .../helpers/_feature_verification/http.py | 53 + .../helpers/_feature_verification/lane.py | 139 ++ .../helpers/_feature_verification/manifest.py | 235 +++ .../_feature_verification/processes.py | 133 ++ .../helpers/_feature_verification/recorder.py | 153 ++ .../_feature_verification/reporting.py | 237 +++ .../helpers/_feature_verification/results.py | 121 ++ .../helpers/_feature_verification/runtime.py | 218 +++ tests/utils/helpers/_fresh_host/context.py | 16 +- tests/utils/helpers/_fresh_host/linux.py | 3 +- tests/utils/helpers/_fresh_host/models.py | 71 +- tests/utils/helpers/_fresh_host/shell.py | 2 +- tests/utils/helpers/_fresh_host/storage.py | 4 + tests/utils/helpers/feature_verification.py | 106 ++ tests/utils/helpers/fresh_host.py | 2 + 46 files changed, 7541 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/feature-verification.yml create mode 100644 tests/feature_matrix.yaml create mode 100644 tests/fixtures/feature_verification/__init__.py create mode 100644 tests/fixtures/feature_verification/lane.py create mode 100755 tests/scripts/feature_report.py create mode 100755 tests/scripts/feature_verification.py create mode 100644 tests/suites/contracts/repo/test_feature_matrix_contract.py create mode 100644 tests/suites/e2e/feature_verification/test_acp.py create mode 100644 tests/suites/e2e/feature_verification/test_context_workflows.py create mode 100644 tests/suites/e2e/feature_verification/test_devflow_planning.py create mode 100644 tests/suites/e2e/feature_verification/test_host_lifecycle.py create mode 100644 tests/suites/e2e/feature_verification/test_memory_hypermemory.py create mode 100644 tests/suites/e2e/feature_verification/test_memory_lancedb.py create mode 100644 tests/suites/e2e/feature_verification/test_model_gateway.py create mode 100644 tests/suites/e2e/feature_verification/test_operator_tools.py create mode 100644 tests/suites/e2e/feature_verification/test_recovery.py create mode 100644 tests/suites/e2e/feature_verification/test_setup_profiles.py create mode 100644 tests/suites/e2e/feature_verification/test_sidecars.py create mode 100644 tests/suites/unit/ci/test_feature_verification.py create mode 100644 tests/utils/helpers/_feature_verification/__init__.py create mode 100644 tests/utils/helpers/_feature_verification/acp_stub_agent.py create mode 100644 tests/utils/helpers/_feature_verification/diagnostics.py create mode 100644 tests/utils/helpers/_feature_verification/envvars.py create mode 100644 tests/utils/helpers/_feature_verification/fake_model.py create mode 100644 tests/utils/helpers/_feature_verification/host.py create mode 100644 tests/utils/helpers/_feature_verification/http.py create mode 100644 tests/utils/helpers/_feature_verification/lane.py create mode 100644 tests/utils/helpers/_feature_verification/manifest.py create mode 100644 tests/utils/helpers/_feature_verification/processes.py create mode 100644 tests/utils/helpers/_feature_verification/recorder.py create mode 100644 tests/utils/helpers/_feature_verification/reporting.py create mode 100644 tests/utils/helpers/_feature_verification/results.py create mode 100644 tests/utils/helpers/_feature_verification/runtime.py create mode 100644 tests/utils/helpers/feature_verification.py diff --git a/.github/workflows/feature-verification.yml b/.github/workflows/feature-verification.yml new file mode 100644 index 00000000..bac6b7a3 --- /dev/null +++ b/.github/workflows/feature-verification.yml @@ -0,0 +1,287 @@ +name: Feature Verification + +# Verifies every feature in tests/feature_matrix.yaml against the OpenClaw/ACPX versions that +# `clawops bootstrap` installs from platform/node-tools, on provisioned Linux and macOS hosts. +on: + workflow_dispatch: + inputs: + platforms: + description: Which lanes to run. + required: true + type: choice + default: all + options: + - all + - linux + - macos + fail_on_feature_failures: + description: Fail the run when any feature is FAIL. + required: true + type: boolean + default: true + schedule: + - cron: "23 5 * * 1" + push: + branches: + - test/feature-verification-suite + +concurrency: + group: feature-verification-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + FRESH_HOST_CACHE_ROOT: ${{ github.workspace }}/.github-cache + UV_CACHE_DIR: ${{ github.workspace }}/.github-cache/uv + npm_config_cache: ${{ github.workspace }}/.github-cache/npm + +jobs: + linux-lanes: + name: Linux Features (${{ matrix.lane }}) + if: ${{ github.event_name == 'schedule' || inputs.platforms != 'macos' }} + runs-on: ubuntu-latest + timeout-minutes: 150 + strategy: + fail-fast: false + matrix: + include: + - lane: linux-core + profile: hypermemory + scenario: linux-features + # LiteLLM reaches the fake Ollama upstream from its container via host-gateway. + fake_model_host: 0.0.0.0 + fake_model_ports: --port 8000 --port 11434 + - lane: linux-lancedb + profile: memory-lancedb-pro + scenario: linux-features + fake_model_host: 0.0.0.0 + fake_model_ports: --port 8000 --port 11434 + - lane: linux-profiles + profile: openclaw-default + scenario: linux-features-local + fake_model_host: 127.0.0.1 + # 11434 stays free for the optional Ollama compose stack this lane starts. + fake_model_ports: --port 8000 + env: + FRESH_HOST_SELECTED_DOCKER_PULL_PARALLELISM: "2" + FRESH_HOST_SELECTED_DOCKER_PULL_MAX_ATTEMPTS: "5" + FEATURE_RESULTS_DIR: ${{ github.workspace }}/.feature-results + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + # Reserves equal the scenario+lane step timeouts and the summed observer step timeouts. + - name: Initialize Linux feature job budget + run: ./tests/scripts/fresh_host.py initialize-budget --job-timeout-minutes 150 --scenario-reserve-minutes 105 --observer-reserve-minutes 10 --github-env-file "${GITHUB_ENV}" + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: "22.22.1" + package-manager-cache: false + - name: Restore package download caches + timeout-minutes: 5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + .github-cache/npm + .github-cache/uv + key: fresh-host-packages-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('uv.lock', 'platform/node-tools/package-lock.json', 'platform/plugins/memory-lancedb-pro/package-lock.json') }} + - name: Start the systemd user manager + if: ${{ matrix.scenario == 'linux-features' }} + timeout-minutes: 3 + run: ./tests/scripts/feature_verification.py enable-systemd-user + - name: Prepare Linux feature context + run: ./tests/scripts/fresh_host.py prepare-context --scenario "${{ matrix.scenario }}" --profile "${{ matrix.profile }}" --repo-root . --runner-temp "${RUNNER_TEMP}" --workspace "${GITHUB_WORKSPACE}" --github-env-file "${GITHUB_ENV}" + - name: Ensure Linux sidecar images + if: ${{ matrix.scenario == 'linux-features' }} + timeout-minutes: 32 + run: ./tests/scripts/hosted_docker.py ensure-images --context "${FRESH_HOST_CONTEXT}" + - name: Start the fake model provider + timeout-minutes: 2 + run: ./tests/scripts/feature_verification.py start-fake-model --host "${{ matrix.fake_model_host }}" ${{ matrix.fake_model_ports }} --state-dir "${RUNNER_TEMP}/fake-model" + - name: Point the env contract at the fake model provider + run: ./tests/scripts/feature_verification.py configure-model-provider --context "${FRESH_HOST_CONTEXT}" + # Bootstrap + setup are themselves verified features; a failure here is reported per + # feature by the lane, which falls back to a harness gateway for the remaining features. + - name: Bootstrap and set up the host + continue-on-error: true + timeout-minutes: 45 + run: ./tests/scripts/fresh_host.py run-scenario --context "${FRESH_HOST_CONTEXT}" + - name: Run lane features + timeout-minutes: 60 + run: ./tests/scripts/feature_verification.py run-lane --context "${FRESH_HOST_CONTEXT}" --lane "${{ matrix.lane }}" --results "${FEATURE_RESULTS_DIR}/feature-results-${{ matrix.lane }}.json" --fake-model-dir "${RUNNER_TEMP}/fake-model" + - name: Collect lane diagnostics + if: ${{ always() && env.FRESH_HOST_CONTEXT != '' }} + continue-on-error: true + timeout-minutes: 3 + run: ./tests/scripts/feature_verification.py collect-diagnostics --context "${FRESH_HOST_CONTEXT}" --output-dir "${FEATURE_RESULTS_DIR}/diagnostics" + - name: Collect fresh-host diagnostics + if: ${{ always() && env.FRESH_HOST_CONTEXT != '' }} + continue-on-error: true + timeout-minutes: 3 + run: ./tests/scripts/fresh_host.py collect-diagnostics --context "${FRESH_HOST_CONTEXT}" + - name: Stop the fake model provider + if: always() + continue-on-error: true + timeout-minutes: 1 + run: ./tests/scripts/feature_verification.py stop-fake-model --state-dir "${RUNNER_TEMP}/fake-model" + - name: Upload lane results + if: always() + continue-on-error: true + timeout-minutes: 3 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: feature-results-${{ matrix.lane }} + path: | + ${{ github.workspace }}/.feature-results + ${{ runner.temp }}/fresh-host-reports + ${{ runner.temp }}/fake-model/fake-model.log + include-hidden-files: true + + macos-lane: + name: macOS Features (macos-launchd) + if: ${{ github.event_name == 'schedule' || inputs.platforms != 'linux' }} + runs-on: macos-15-intel + timeout-minutes: 150 + env: + FRESH_HOST_SELECTED_DOCKER_PULL_PARALLELISM: "2" + FRESH_HOST_SELECTED_DOCKER_PULL_MAX_ATTEMPTS: "5" + FEATURE_RESULTS_DIR: ${{ github.workspace }}/.feature-results + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + - name: Initialize macOS feature job budget + run: ./tests/scripts/fresh_host.py initialize-budget --job-timeout-minutes 150 --scenario-reserve-minutes 75 --observer-reserve-minutes 12 --github-env-file "${GITHUB_ENV}" + - name: Restore OrbStack DMG cache + id: orbstack_dmg_cache + timeout-minutes: 5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: /tmp/orbstack.dmg + key: orbstack-v1.5.1-16857-amd64 + - name: Set up Docker via OrbStack + timeout-minutes: 10 + # v1.6.0+ panics on GHA macos-15-intel (Skylake CPU check); v1.5.1 is safe. + env: + ORBSTACK_DMG_PATH: /tmp/orbstack.dmg + ORBSTACK_DMG_SHA256: fb95108ded54a27603b68184a13f7e666e0e758167652c0b65cd4dc5eff94617 + ORBSTACK_DMG_URL: https://cdn-updates.orbstack.dev/amd64/OrbStack_v1.5.1_16857_amd64.dmg + run: >- + ./tests/scripts/hosted_docker.py setup-orbstack + --dmg-path "${ORBSTACK_DMG_PATH}" + --url "${ORBSTACK_DMG_URL}" + --sha256 "${ORBSTACK_DMG_SHA256}" + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: "22.22.1" + package-manager-cache: false + - name: Restore package download caches + timeout-minutes: 5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + .github-cache/npm + .github-cache/uv + key: fresh-host-packages-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('uv.lock', 'platform/node-tools/package-lock.json', 'platform/plugins/memory-lancedb-pro/package-lock.json') }} + - name: Prepare macOS feature context + run: >- + ./tests/scripts/fresh_host.py prepare-context + --scenario macos-features + --profile openclaw-default + --repo-root . + --runner-temp "${RUNNER_TEMP}" + --workspace "${GITHUB_WORKSPACE}" + --github-env-file "${GITHUB_ENV}" + # Backstop only: the helper ends at its own deadline with recovery reserved inside it. + - name: Wait for hosted macOS Docker runtime + timeout-minutes: 12 + run: >- + ./tests/scripts/hosted_docker.py wait-runtime-ready + --context "${FRESH_HOST_CONTEXT}" + --github-env-file "${GITHUB_ENV}" + - name: Ensure hosted macOS Docker images + timeout-minutes: 32 + run: ./tests/scripts/hosted_docker.py ensure-images --context "${FRESH_HOST_CONTEXT}" + - name: Start the fake model provider + timeout-minutes: 2 + run: ./tests/scripts/feature_verification.py start-fake-model --host 127.0.0.1 --port 8000 --port 11434 --state-dir "${RUNNER_TEMP}/fake-model" + - name: Point the env contract at the fake model provider + run: ./tests/scripts/feature_verification.py configure-model-provider --context "${FRESH_HOST_CONTEXT}" + # The scenario recovers a Docker runtime lost mid-run inside its own budget. + - name: Bootstrap and set up the host + continue-on-error: true + timeout-minutes: 40 + run: ./tests/scripts/fresh_host.py run-scenario --context "${FRESH_HOST_CONTEXT}" + - name: Run lane features + timeout-minutes: 35 + run: ./tests/scripts/feature_verification.py run-lane --context "${FRESH_HOST_CONTEXT}" --lane macos-launchd --results "${FEATURE_RESULTS_DIR}/feature-results-macos-launchd.json" --fake-model-dir "${RUNNER_TEMP}/fake-model" + - name: Collect lane diagnostics + if: ${{ always() && env.FRESH_HOST_CONTEXT != '' }} + continue-on-error: true + timeout-minutes: 3 + run: ./tests/scripts/feature_verification.py collect-diagnostics --context "${FRESH_HOST_CONTEXT}" --output-dir "${FEATURE_RESULTS_DIR}/diagnostics" + - name: Collect hosted macOS Docker diagnostics + if: ${{ always() && env.FRESH_HOST_CONTEXT != '' }} + continue-on-error: true + timeout-minutes: 4 + run: ./tests/scripts/hosted_docker.py collect-diagnostics --context "${FRESH_HOST_CONTEXT}" + - name: Stop the fake model provider + if: always() + continue-on-error: true + timeout-minutes: 1 + run: ./tests/scripts/feature_verification.py stop-fake-model --state-dir "${RUNNER_TEMP}/fake-model" + - name: Upload lane results + if: always() + continue-on-error: true + timeout-minutes: 3 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: feature-results-macos-launchd + path: | + ${{ github.workspace }}/.feature-results + ${{ runner.temp }}/fresh-host-reports + ${{ runner.temp }}/fake-model/fake-model.log + include-hidden-files: true + - name: Clean up macOS feature context + if: ${{ always() && env.FRESH_HOST_CONTEXT != '' }} + continue-on-error: true + timeout-minutes: 2 + run: ./tests/scripts/fresh_host.py cleanup --context "${FRESH_HOST_CONTEXT}" + + report: + name: Feature Verification Report + needs: + - linux-lanes + - macos-lane + if: always() + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version: "0.10.9" + enable-cache: true + - run: uv sync --locked + - name: Download lane results + continue-on-error: true + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + pattern: feature-results-* + path: lane-results + - name: Write the per-feature report + run: uv run python3 ./tests/scripts/feature_report.py --results-dir lane-results --output feature-report/feature-verification.json --summary-file "${GITHUB_STEP_SUMMARY}" ${{ (github.event_name == 'schedule' || inputs.fail_on_feature_failures) && '--fail-on-failures' || '' }} + - name: Upload the per-feature report + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: feature-verification-report + path: feature-report/feature-verification.json diff --git a/pyproject.toml b/pyproject.toml index 28949de0..81c2c020 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -167,6 +167,7 @@ markers = [ "integration: cross-module and service-shaped tests", "contract: repository and artifact contract checks", "e2e: black-box end-to-end orchestration tests", + "feature_verification: per-feature verification against a provisioned host (lane-only)", "devflow: production devflow coverage", "framework: explicit validation of the pytest framework itself", "test_profile: named infrastructure runtime profile declarations", diff --git a/tests/conftest.py b/tests/conftest.py index d5588629..0cfcfcfb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,6 +56,8 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item item.add_marker(pytest.mark.e2e) if "framework" in parts: item.add_marker(pytest.mark.framework) + if "feature_verification" in parts: + item.add_marker(pytest.mark.feature_verification) if "hypermemory" in parts: item.add_marker(pytest.mark.hypermemory) if rel_path_posix in _DEVFLOW_MARKED_FILES: diff --git a/tests/feature_matrix.yaml b/tests/feature_matrix.yaml new file mode 100644 index 00000000..e52fdff7 --- /dev/null +++ b/tests/feature_matrix.yaml @@ -0,0 +1,874 @@ +# StrongClaw end-to-end feature matrix. +# +# One entry per product feature: where it lives (call sites), the single test that verifies it, +# and the lanes (provisioned CI hosts) it runs on; platforms are derived from the lanes. +# `verification` says how deep the check goes: +# live - real StrongClaw + real OpenClaw/acpx + real sidecars, no stand-ins +# fake-model - live, with the local deterministic model server as the model provider +# stub - a third-party endpoint (vendor CLI/adapter, messaging API) is stood in for +# `status: unverifiable` features cannot be exercised without third-party live services; they +# carry the reason and are reported UNVERIFIABLE, never PASS. +# Contract: tests/suites/contracts/repo/test_feature_matrix_contract.py. +schema_version: 1 + +lanes: + linux-core: + platform: linux + profile: hypermemory + scenario: linux-features + activate_services: true + description: Default product profile with systemd user services, sidecars, and the gateway. + linux-lancedb: + platform: linux + profile: memory-lancedb-pro + scenario: linux-features + activate_services: true + description: Vendored memory-lancedb-pro profile with systemd services and sidecars. + linux-profiles: + platform: linux + profile: openclaw-default + scenario: linux-features-local + activate_services: false + description: Profile matrix plus CLI-only operator tools on a harness-managed gateway. + macos-launchd: + platform: macos + profile: openclaw-default + scenario: macos-features + activate_services: true + description: macOS bootstrap and launchd-supervised runtime. + +features: + # --- host lifecycle ------------------------------------------------------------------------- + - id: bootstrap.host + category: host-lifecycle + title: clawops bootstrap installs the locked OpenClaw/ACPX toolchain, varlock and the venv + call_sites: + - src/clawops/strongclaw_bootstrap.py:576 + - src/clawops/strongclaw_node_tools.py:715 + lanes: [linux-core, macos-launchd] + verification: live + test: tests/suites/e2e/feature_verification/test_host_lifecycle.py::test_bootstrap_installs_the_locked_toolchain + - id: setup.guided + category: host-lifecycle + title: clawops setup (non-interactive) renders config, applies model auth, activates services + call_sites: + - src/clawops/setup_cli.py:310 + - src/clawops/setup_cli.py:139 + lanes: [linux-core, macos-launchd] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_host_lifecycle.py::test_guided_setup_completes_with_service_activation + - id: services.systemd + category: host-lifecycle + title: systemd user units and timers are installed, enabled and active on Linux + call_sites: + - src/clawops/strongclaw_services.py:51 + - src/clawops/strongclaw_services.py:398 + - platform/systemd/openclaw-gateway.service:16 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_host_lifecycle.py::test_systemd_user_units_are_installed_and_active + - id: services.launchd + category: host-lifecycle + title: launchd agents are bootstrapped and running on macOS + call_sites: + - src/clawops/strongclaw_services.py:27 + - src/clawops/strongclaw_services.py:233 + - platform/launchd/ai.openclaw.gateway.plist.template:7 + lanes: [macos-launchd] + verification: live + test: tests/suites/e2e/feature_verification/test_host_lifecycle.py::test_launchd_agents_are_loaded_and_running + - id: varlock.env-contract + category: host-lifecycle + title: varlock-env configure/check generates and validates the operator env contract + call_sites: + - src/clawops/strongclaw_varlock_env.py:641 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_host_lifecycle.py::test_varlock_env_contract_is_valid + - id: doctor.host + category: host-lifecycle + title: clawops doctor-host validates the toolchain and the rendered config + call_sites: + - src/clawops/setup_cli.py:127 + lanes: [linux-core, macos-launchd] + verification: live + test: tests/suites/e2e/feature_verification/test_host_lifecycle.py::test_doctor_host_reports_toolchain + - id: doctor.deep + category: host-lifecycle + title: clawops doctor runs every readiness check (model probe, audits, memory, sidecars) + call_sites: + - src/clawops/setup_cli.py:392 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_host_lifecycle.py::test_deep_doctor_passes_every_check + - id: baseline.verify + category: host-lifecycle + title: clawops baseline verify passes with runtime probes + call_sites: + - src/clawops/strongclaw_baseline.py:138 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_host_lifecycle.py::test_baseline_verify_passes_with_runtime_probes + - id: host.runtime-user-provisioning + category: host-lifecycle + title: Dedicated runtime-user provisioning per the setup runbooks + call_sites: + - SETUP_GUIDE.md:12 + status: unverifiable + reason: Manual operator runbook (OS account creation and service-user SSH setup); StrongClaw ships no code path that performs it, so there is nothing to execute. + + # --- model auth ----------------------------------------------------------------------------- + - id: model-auth.agents-list + category: model-auth + title: openclaw agents list --json reports every configured agent + call_sites: + - src/clawops/strongclaw_model_auth.py:114 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_agents_list_matches_rendered_config + - id: model-auth.ensure + category: model-auth + title: clawops model-auth ensure applies the env-driven model chain to every agent + call_sites: + - src/clawops/strongclaw_model_auth.py:343 + - src/clawops/strongclaw_model_auth.py:239 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_model_auth_ensure_applies_env_driven_chain + - id: model-auth.probe + category: model-auth + title: models status --check --probe reaches the provider for every agent + call_sites: + - src/clawops/strongclaw_model_auth.py:170 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_model_probe_reaches_the_provider + - id: model-auth.models-list + category: model-auth + title: openclaw models --agent X list --json reports an available model (fallback path) + call_sites: + - src/clawops/strongclaw_model_auth.py:141 + lanes: [linux-profiles] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_models_list_reports_available_models + - id: model-auth.configure-wizard + category: model-auth + title: Interactive model wizard (openclaw configure --section model) + call_sites: + - src/clawops/strongclaw_model_auth.py:393 + status: unverifiable + reason: Runs only on an interactive TTY and walks a vendor sign-in/provider picker; the non-interactive env-driven chain (model-auth.ensure) covers the same outcome. + + # --- gateway -------------------------------------------------------------------------------- + - id: gateway.status + category: gateway + title: openclaw gateway status --json reports a loopback RPC endpoint + call_sites: + - src/clawops/setup_cli.py:493 + - src/clawops/strongclaw_ops.py:977 + lanes: [linux-core, macos-launchd] + verification: live + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_gateway_status_reports_loopback_rpc + - id: gateway.agent-turn + category: gateway + title: The supervised gateway runs a full agent turn through the configured provider + call_sites: + - src/clawops/strongclaw_ops.py:977 + - platform/systemd/openclaw-gateway.service:16 + lanes: [linux-core, macos-launchd] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_gateway_runs_an_agent_turn_through_the_provider + - id: gateway.loopback-bind + category: gateway + title: The gateway listens on loopback only + call_sites: + - platform/configs/openclaw/00-baseline.json5:116 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_gateway_binds_only_loopback + - id: openclaw.doctor + category: gateway + title: openclaw doctor --non-interactive is clean on the rendered profile + call_sites: + - src/clawops/strongclaw_baseline.py:156 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_openclaw_doctor_is_clean + - id: security.audit + category: security + title: openclaw security audit --deep passes + call_sites: + - src/clawops/strongclaw_baseline.py:157 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_security_audit_passes + - id: secrets.audit + category: security + title: openclaw secrets audit --check passes + call_sites: + - src/clawops/strongclaw_baseline.py:158 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_secrets_audit_passes + - id: secrets.external-backends + category: security + title: Varlock secret backends other than local (1Password, AWS, Azure, Bitwarden, GCP, Infisical) + call_sites: + - platform/configs/varlock/.env.schema:15 + - platform/docs/SECRETS_AND_ENV.md:22 + status: unverifiable + reason: Each backend resolves secrets from a third-party vault account; no account or credentials exist in CI. + + # --- memory: hypermemory -------------------------------------------------------------------- + - id: profile.hypermemory + category: memory + title: hypermemory profile loads strongclaw-hypermemory and lossless-claw into their slots + call_sites: + - src/clawops/memory_profiles.py:23 + - src/clawops/strongclaw_bootstrap.py:425 + - platform/configs/openclaw/77-hypermemory.example.json5:12 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_hypermemory_profile_plugins_are_loaded + - id: memory.openclaw-status + category: memory + title: openclaw memory status --deep reports the active memory backend + call_sites: + - src/clawops/setup_cli.py:501 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_openclaw_memory_status_is_healthy + - id: memory.openclaw-search + category: memory + title: openclaw memory search returns corpus hits through the memory plugin + call_sites: + - src/clawops/setup_cli.py:510 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_openclaw_memory_search_returns_corpus_hits + - id: hypermemory.index-search + category: memory + title: hypermemory index + sparse/dense hybrid search over Qdrant with real embeddings route + call_sites: + - src/clawops/hypermemory/cli.py:37 + - src/clawops/hypermemory/cli.py:40 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_hypermemory_index_and_hybrid_search + - id: hypermemory.verify + category: memory + title: hypermemory verify reports a consistent Qdrant index + call_sites: + - src/clawops/hypermemory/cli.py:31 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_hypermemory_verify_reports_consistent_index + - id: hypermemory.store-forget + category: memory + title: hypermemory store makes a fact searchable and forget removes it + call_sites: + - src/clawops/hypermemory/cli.py:85 + - src/clawops/hypermemory/cli.py:125 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_hypermemory_store_search_and_forget + - id: hypermemory.maintenance + category: memory + title: hypermemory reflect/lifecycle/list-facts/flush-metadata run against the live index + call_sites: + - src/clawops/hypermemory/cli.py:108 + - src/clawops/hypermemory/cli.py:174 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_hypermemory_maintenance_operations + - id: hypermemory.export-memory-pro + category: memory + title: hypermemory export-memory-pro writes a memory-pro import snapshot + call_sites: + - src/clawops/hypermemory/cli.py:195 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_hypermemory_exports_memory_pro_snapshot + - id: hypermemory.plugin-recall + category: memory + title: strongclaw-hypermemory autoRecall injects stored memory into gateway agent turns + call_sites: + - platform/plugins/strongclaw-hypermemory/index.js:884 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_hypermemory_plugin_recalls_memory_into_turns + - id: qdrant.storage-stamp + category: memory + title: sidecars up stamps Qdrant storage with the pinned release line (#237) + call_sites: + - src/clawops/qdrant_storage.py:278 + - src/clawops/strongclaw_ops.py:677 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_qdrant_storage_stamp_tracks_pinned_release + - id: qdrant.reset-reindex + category: memory + title: A Qdrant release-line change resets storage and reindexes hypermemory (#237) + call_sites: + - src/clawops/qdrant_storage.py:318 + - src/clawops/strongclaw_ops.py:710 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_qdrant_release_change_resets_storage_and_reindexes + + # --- memory: memory-lancedb-pro ------------------------------------------------------------- + - id: profile.memory-lancedb-pro + category: memory + title: memory-lancedb-pro profile installs the vendored plugin and loads it into the memory slot + call_sites: + - src/clawops/memory_profiles.py:41 + - src/clawops/strongclaw_bootstrap.py:391 + - platform/configs/openclaw/75-memory-lancedb-pro.local.json5:11 + lanes: [linux-lancedb] + verification: live + test: tests/suites/e2e/feature_verification/test_memory_lancedb.py::test_lancedb_plugin_is_loaded_in_the_memory_slot + - id: memory.migrate-hypermemory-to-pro + category: memory + title: clawops memory migrate-hypermemory-to-pro exports a scope into memory-pro import JSON + call_sites: + - src/clawops/memory_tools.py:147 + lanes: [linux-lancedb] + verification: live + test: tests/suites/e2e/feature_verification/test_memory_lancedb.py::test_migrate_hypermemory_exports_memory_pro_snapshot + - id: memory.import-pro-snapshot + category: memory + title: clawops memory import-pro-snapshot imports through openclaw memory-pro import + call_sites: + - src/clawops/memory_tools.py:174 + lanes: [linux-lancedb] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_lancedb.py::test_import_pro_snapshot_lands_in_lancedb + - id: memory.verify-pro-parity + category: memory + title: clawops memory verify-pro-parity compares hypermemory hits with live memory-pro search + call_sites: + - src/clawops/memory_tools.py:476 + lanes: [linux-lancedb] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_lancedb.py::test_verify_pro_parity_against_live_plugin + - id: lancedb.smart-extraction + category: memory + title: memory-lancedb-pro autoCapture runs smart extraction and embeddings after a turn + call_sites: + - platform/plugins/memory-lancedb-pro/index.ts:2725 + lanes: [linux-lancedb] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_lancedb.py::test_smart_extraction_calls_the_extraction_model + - id: lancedb.reflection-embedded-runner + category: memory + title: memory reflection runs through extensionAPI.js runEmbeddedPiAgent on session reset + call_sites: + - platform/plugins/memory-lancedb-pro/index.ts:399 + - platform/plugins/memory-lancedb-pro/index.ts:1212 + - platform/plugins/memory-lancedb-pro/index.ts:3252 + lanes: [linux-lancedb] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_lancedb.py::test_reflection_runs_through_the_embedded_runner + + # --- setup profiles ------------------------------------------------------------------------- + - id: profile.openclaw-default + category: profiles + title: openclaw-default profile renders, validates and boots the gateway with memory-core + call_sites: + - src/clawops/memory_profiles.py:30 + - src/clawops/config_cli.py:86 + lanes: [linux-profiles] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_setup_profiles.py::test_openclaw_default_profile_boots_with_memory_core + - id: profile.openclaw-qmd + category: profiles + title: openclaw-qmd profile installs QMD and boots the gateway on the qmd memory backend + call_sites: + - src/clawops/memory_profiles.py:35 + - src/clawops/strongclaw_bootstrap.py:342 + - platform/configs/openclaw/40-qmd-context.json5:3 + lanes: [linux-profiles] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_setup_profiles.py::test_openclaw_qmd_profile_installs_and_uses_qmd + - id: profile.acp + category: profiles + title: acp profile registers the ACP worker agents and boots the gateway + call_sites: + - src/clawops/memory_profiles.py:48 + - platform/configs/openclaw/20-acp-workers.json5:11 + lanes: [linux-profiles] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_setup_profiles.py::test_acp_profile_registers_acp_worker_agents + - id: profile.browser-lab + category: profiles + title: browser-lab profile renders the CDP browser profile and boots the gateway + call_sites: + - src/clawops/memory_profiles.py:55 + - platform/configs/openclaw/60-browser-lab.json5:8 + lanes: [linux-profiles] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_setup_profiles.py::test_browser_lab_profile_points_at_the_cdp_endpoint + + # --- ACP workers ---------------------------------------------------------------------------- + - id: acp.worker-agent-turn + category: acp + title: An OpenClaw ACP worker agent (acp profile) runs a turn through OpenClaw's acpx backend + call_sites: + - platform/configs/openclaw/20-acp-workers.json5:11 + lanes: [linux-profiles] + verification: stub + limitations: The codex adapter behind acpx is the ACP stub agent; a live vendor session is acp.vendor-live-sessions. + test: tests/suites/e2e/feature_verification/test_setup_profiles.py::test_acp_worker_agent_runs_a_turn_through_acpx + - id: acp.openclaw-local + category: acp + title: acp-runner --backend openclaw --auth-mode local runs a live turn (acpx -> openclaw acp) + call_sites: + - src/clawops/acp_runner.py:394 + - src/clawops/acpx_adapter.py:49 + - src/clawops/backend_registry.py:109 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_acp.py::test_openclaw_backend_runs_a_live_acp_turn + - id: acp.codex-api + category: acp + title: codex backend in api mode keeps OPENAI_API_KEY and acpx authenticates with it + call_sites: + - src/clawops/backend_registry.py:69 + - src/clawops/credential_broker.py:160 + lanes: [linux-core] + verification: stub + limitations: Real acpx; the codex-acp adapter is the ACP stub agent (same advertised auth methods). + test: tests/suites/e2e/feature_verification/test_acp.py::test_codex_api_mode_passes_the_key_and_authenticates + - id: acp.codex-subscription + category: acp + title: codex backend in subscription mode strips API keys before dispatch + call_sites: + - src/clawops/backend_registry.py:69 + lanes: [linux-core] + verification: stub + limitations: Vendor `codex login status` and the adapter are stubs. + test: tests/suites/e2e/feature_verification/test_acp.py::test_codex_subscription_mode_strips_api_keys + - id: acp.claude-api + category: acp + title: claude backend in api mode passes ANTHROPIC_API_KEY to the agent + call_sites: + - src/clawops/backend_registry.py:82 + lanes: [linux-core] + verification: stub + limitations: Real acpx; the claude-agent-acp adapter is the ACP stub agent. + test: tests/suites/e2e/feature_verification/test_acp.py::test_claude_api_mode_passes_the_key + - id: acp.claude-subscription + category: acp + title: claude backend in subscription mode strips API keys before dispatch + call_sites: + - src/clawops/backend_registry.py:82 + lanes: [linux-core] + verification: stub + limitations: Vendor `claude auth status` and the adapter are stubs. + test: tests/suites/e2e/feature_verification/test_acp.py::test_claude_subscription_mode_strips_api_keys + - id: acp.claude-cloud-provider + category: acp + title: claude backend in cloud-provider mode keeps cloud credentials and strips the API key + call_sites: + - src/clawops/backend_registry.py:82 + lanes: [linux-core] + verification: stub + limitations: Cloud credentials are presence-only markers; no cloud account is used. + test: tests/suites/e2e/feature_verification/test_acp.py::test_claude_cloud_provider_mode_keeps_cloud_credentials + - id: acp.local-backends + category: acp + title: gemini/qwen/copilot/cursor local backends dispatch through acpx + call_sites: + - src/clawops/backend_registry.py:105 + lanes: [linux-core] + verification: stub + limitations: The vendor CLIs' ACP servers are the ACP stub agent. + test: tests/suites/e2e/feature_verification/test_acp.py::test_local_backends_dispatch_through_acpx + - id: acp.auth-policy + category: acp + title: acp-runner refuses unsupported and unready auth modes at preflight + call_sites: + - src/clawops/credential_broker.py:236 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_acp.py::test_auth_policy_blocks_unsupported_and_unready_modes + - id: acp.credential-status + category: acp + title: clawops credential-status reports machine-readable readiness per auth mode + call_sites: + - src/clawops/credential_broker.py:292 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_acp.py::test_credential_status_reports_each_mode + - id: acp.vendor-live-sessions + category: acp + title: Live Codex/Claude/Gemini/Qwen/Copilot/Cursor coding sessions through acpx + call_sites: + - src/clawops/backend_registry.py:69 + - src/clawops/backend_registry.py:82 + status: unverifiable + reason: Requires signed-in vendor accounts or paid API keys for each coding agent; CI has none. Routing is covered at stub level by the acp.* features. + + # --- sidecars and platform ------------------------------------------------------------------ + - id: sidecars.postgres + category: sidecars + title: Postgres sidecar is up and serves the LiteLLM database + call_sites: + - platform/compose/docker-compose.aux-stack.yaml:2 + - src/clawops/strongclaw_ops.py:495 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_postgres_sidecar_accepts_queries + - id: sidecars.litellm + category: sidecars + title: LiteLLM routes the hypermemory-embedding alias to its upstream + call_sites: + - platform/compose/docker-compose.aux-stack.yaml:25 + - platform/configs/litellm/config.yaml:22 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_litellm_routes_embeddings_to_the_upstream + - id: sidecars.qdrant + category: sidecars + title: Qdrant sidecar creates collections, upserts and searches vectors + call_sites: + - platform/compose/docker-compose.aux-stack.yaml:90 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_qdrant_sidecar_round_trips_vectors + - id: sidecars.neo4j + category: sidecars + title: Neo4j sidecar accepts authenticated Cypher writes and reads + call_sites: + - platform/compose/docker-compose.aux-stack.yaml:111 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_neo4j_sidecar_round_trips_cypher + - id: observability.otel + category: observability + title: OTel collector accepts OTLP traces and exports Prometheus metrics; verify-platform observability + call_sites: + - platform/compose/docker-compose.aux-stack.yaml:69 + - src/clawops/platform_verify.py:383 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_observability_pipeline_accepts_telemetry + - id: verify-platform.sidecars + category: sidecars + title: clawops verify-platform sidecars passes with runtime probes + call_sites: + - src/clawops/platform_verify.py:261 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_verify_platform_sidecars_runtime + - id: ops.status + category: sidecars + title: clawops ops status reports every required sidecar ready + call_sites: + - src/clawops/strongclaw_ops.py:1024 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_ops_status_reports_required_sidecars_ready + - id: ops.prune-qdrant-test-collections + category: sidecars + title: clawops ops prune-qdrant-test-collections removes prefixed collections (dry-run first) + call_sites: + - src/clawops/strongclaw_ops.py:1174 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_prune_qdrant_test_collections + - id: browser-lab.runtime + category: sidecars + title: Browser lab starts, exposes CDP on 9222 and the egress proxy on 3128, then stops + call_sites: + - src/clawops/strongclaw_ops.py:1004 + - src/clawops/platform_verify.py:678 + - platform/compose/docker-compose.browser-lab.yaml:12 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_browser_lab_starts_and_exposes_cdp + - id: ops.reset-compose-state + category: sidecars + title: clawops ops reset-compose-state clears one component's state directory + call_sites: + - src/clawops/strongclaw_ops.py:1114 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_reset_compose_state_clears_one_component + - id: sidecars.ollama-optional + category: sidecars + title: Optional Ollama compose stack serves the Ollama API on loopback + call_sites: + - platform/compose/docker-compose.ollama.optional.yaml:2 + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_optional_ollama_compose_serves_the_api + - id: observability.langfuse-optional + category: observability + title: Optional Langfuse OTLP exporter stack + call_sites: + - platform/compose/docker-compose.langfuse.optional.yaml:2 + - platform/docs/OBSERVABILITY.md:14 + status: unverifiable + reason: The optional compose file only runs langfuse-web/worker and expects an operator-provided Postgres, Redis and ClickHouse (LANGFUSE_* URLs); StrongClaw ships none of them. + + # --- recovery ------------------------------------------------------------------------------- + - id: recovery.backup-create + category: recovery + title: recovery backup-create writes an archive through openclaw backup create + call_sites: + - src/clawops/strongclaw_recovery.py:144 + - src/clawops/recovery/backends.py:49 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_recovery.py::test_backup_create_uses_the_openclaw_backend + - id: recovery.backup-verify + category: recovery + title: recovery backup-verify validates the latest archive with openclaw backup verify + call_sites: + - src/clawops/strongclaw_recovery.py:199 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_recovery.py::test_backup_verify_accepts_latest_archive + - id: recovery.restore + category: recovery + title: recovery restore extracts every archived file safely into a destination + call_sites: + - src/clawops/strongclaw_recovery.py:259 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_recovery.py::test_restore_extracts_openclaw_state + - id: recovery.prune-retention + category: recovery + title: recovery prune-retention deletes only archives past retention + call_sites: + - src/clawops/strongclaw_recovery.py:274 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_recovery.py::test_prune_retention_removes_only_stale_archives + - id: recovery.rotate-secrets + category: recovery + title: recovery rotate-secrets prints the rotation runbook steps + call_sites: + - src/clawops/strongclaw_recovery.py:302 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_recovery.py::test_rotate_secrets_guidance_lists_rotation_targets + - id: recovery.scheduled-jobs + category: recovery + title: The scheduled backup job (systemd timer / launchd agent) produces an archive + call_sites: + - platform/systemd/openclaw-backup-create.timer:5 + - platform/launchd/ai.openclaw.backup-create.plist.template:7 + lanes: [linux-core, macos-launchd] + verification: live + test: tests/suites/e2e/feature_verification/test_recovery.py::test_scheduled_backup_job_produces_an_archive + + # --- context, workflows, channels ----------------------------------------------------------- + - id: context.codebase-graph + category: context + title: context codebase index/query/pack at large scale writes the Neo4j graph + call_sites: + - src/clawops/context/codebase/service.py:1686 + - src/clawops/context/codebase/service.py:3504 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_context_workflows.py::test_codebase_context_indexes_into_the_neo4j_graph + - id: workflow.daily-healthcheck + category: operator-tools + title: The shipped daily-healthcheck workflow runs openclaw doctor and security audit + call_sites: + - src/clawops/workflow_runner.py:1168 + - platform/configs/workflows/daily_healthcheck.yaml:1 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_context_workflows.py::test_daily_healthcheck_workflow_runs_openclaw_checks + - id: channels.openclaw-config + category: channels + title: The channels overlay renders and validates under OpenClaw with Telegram and WhatsApp configured + call_sites: + - platform/configs/openclaw/30-channels.json5:12 + - src/clawops/openclaw_config.py:338 + lanes: [linux-profiles] + verification: stub + limitations: Telegram uses a placeholder bot token; no message reaches a messaging network. + test: tests/suites/e2e/feature_verification/test_context_workflows.py::test_channels_overlay_validates_under_openclaw + - id: channels.telegram-live + category: channels + title: Live Telegram bot messaging through the gateway + call_sites: + - platform/docs/CHANNELS.md:3 + status: unverifiable + reason: Needs a real Telegram bot token and a Telegram account to message it; neither exists in CI. + - id: channels.whatsapp-live + category: channels + title: Live WhatsApp messaging through the gateway + call_sites: + - platform/docs/channels/whatsapp.md:1 + status: unverifiable + reason: Needs a WhatsApp account paired by QR code on a real phone. + - id: ops.remote-operator-access + category: operator-tools + title: Remote operator access to the gateway over an SSH tunnel + call_sites: + - USAGE_GUIDE.md:65 + status: unverifiable + reason: Operator procedure between two hosts; StrongClaw ships no code that opens the tunnel. + + # --- operator tools and governance (CLI-only) ----------------------------------------------- + - id: policy.evaluate + category: governance + title: Policy engine evaluates payloads against the shipped YAML rules + call_sites: [src/clawops/policy_engine.py:261, src/clawops/policy_engine.py:267] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_policy_evaluates_shipped_rules + - id: op-journal.lifecycle + category: governance + title: Operation journal records begin, transition, approval, and stuck listings + call_sites: [src/clawops/op_journal.py:1012, src/clawops/op_journal.py:1048] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_op_journal_tracks_operation_lifecycle + - id: approvals.review-queue + category: governance + title: Approval queue lists, approves, rejects, delegates, and ingests reviews + call_sites: [src/clawops/approvals.py:32, src/clawops/approvals.py:82] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_approvals_review_queue_resolves_operations + - id: skills.intake + category: governance + title: Skill intake scans, quarantines, promotes, and demotes bundles + call_sites: [src/clawops/skill_scanner.py:236, src/clawops/skill_scanner.py:298] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_skills_intake_moves_bundle_through_stages + - id: wrapper.webhook + category: governance + title: Webhook wrapper posts exactly once after journaled approval + call_sites: [src/clawops/wrappers/webhook.py:104, src/clawops/wrappers/webhook.py:119] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_webhook_wrapper_posts_once_after_approval + - id: wrapper.github + category: governance + title: GitHub wrapper dry run journals planned comment and label operations + call_sites: [src/clawops/wrappers/github.py:327, src/clawops/wrappers/github.py:359] + lanes: [linux-profiles] + verification: stub + limitations: Dry run only; executing against the live GitHub API needs a repo token. + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_github_wrapper_dry_run_plans_without_executing + - id: harness.regression-suites + category: operator-tools + title: YAML regression harness runs the shipped policy and security suites + call_sites: [src/clawops/harness.py:102, src/clawops/harness.py:110] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_harness_regression_suites_pass + - id: charts.render + category: operator-tools + title: Charts render harness results to a PNG + call_sites: [src/clawops/charts.py:26, src/clawops/charts.py:34] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_charts_render_harness_results + - id: merge-json.overlays + category: operator-tools + title: merge-json deep-merges JSON5 overlays onto a base document + call_sites: [src/clawops/json_merge.py:20, src/clawops/json_merge.py:29] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_merge_json_deep_merges_overlays + - id: allowlists.render + category: channels + title: Channel allowlists render a normalized channels fragment + call_sites: [src/clawops/allowlist_sync.py:59, src/clawops/allowlist_sync.py:67] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_allowlists_render_channel_fragment + - id: worktree.lifecycle + category: operator-tools + title: Managed git worktrees are created, listed, and pruned + call_sites: [src/clawops/repo_tools.py:272, src/clawops/repo_tools.py:298] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_worktree_lifecycle_creates_lists_and_prunes + - id: repo.doctor + category: operator-tools + title: Repo doctor validates the repo/upstream and repo/worktrees contract + call_sites: [src/clawops/repo_tools.py:267, src/clawops/repo_tools.py:286] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_repo_doctor_accepts_valid_layout + - id: workflow.code-review + category: operator-tools + title: Workflow runner executes the shipped code-review workflow and its dry run + call_sites: [src/clawops/workflow_runner.py:1158, src/clawops/workflow_runner.py:1168] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_workflow_code_review_records_step_results + - id: supply-chain.inventory + category: operator-tools + title: Supply-chain inventory lists pinned workflow actions and compose digests + call_sites: [src/clawops/supply_chain.py:550, src/clawops/supply_chain.py:592] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_supply_chain_inventory_lists_pins + - id: supply-chain.propose-refresh + category: operator-tools + title: Supply-chain propose-refresh dry run plans pin updates without side effects + call_sites: [src/clawops/supply_chain.py:565, src/clawops/supply_chain.py:423] + lanes: [linux-profiles] + verification: stub + limitations: Runs against a scratch repo with a docker shim; opening a real refresh PR needs repo write access. + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_supply_chain_propose_refresh_dry_run_plans_updates + - id: observability.structured-logs + category: observability + title: CLAWOPS_STRUCTURED_LOGS emits JSON event lines on stderr + call_sites: [src/clawops/observability.py:37, src/clawops/observability.py:61] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_structured_logs_emit_json_events + - id: baseline.harness-smoke + category: host-lifecycle + title: Baseline harness-smoke runs both regression suites into a runs dir + call_sites: [src/clawops/strongclaw_baseline.py:347, src/clawops/strongclaw_baseline.py:34] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_baseline_harness_smoke_writes_results + - id: channels.verify-platform + category: channels + title: verify-platform channels checks overlays, docs, and allowlist parity + call_sites: [src/clawops/platform_verify.py:751, src/clawops/platform_verify.py:528] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_verify_platform_channels_passes + - id: config.memory-profiles + category: profiles + title: config memory lists every managed memory profile + call_sites: [src/clawops/config_cli.py:84, src/clawops/config_cli.py:110] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_config_lists_managed_memory_profiles + - id: render.overlays + category: profiles + title: render-openclaw-config merges operator overlays and renders exec approvals + call_sites: [src/clawops/openclaw_config.py:386, src/clawops/openclaw_config.py:431] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_render_openclaw_config_merges_overlay + - id: services.render + category: host-lifecycle + title: services render materializes every systemd and launchd template + call_sites: [src/clawops/strongclaw_services.py:421, src/clawops/strongclaw_services.py:113] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_services_render_every_template + - id: devflow.plan-status-audit + category: devflow + title: devflow plans stages and roles; status/audit reject unknown run ids + call_sites: [src/clawops/devflow.py:651, src/clawops/devflow.py:671, src/clawops/devflow.py:705] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_devflow_planning.py::test_devflow_plan_status_audit diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py index 1b5f1748..6500a789 100644 --- a/tests/fixtures/__init__.py +++ b/tests/fixtures/__init__.py @@ -2,6 +2,7 @@ pytest_plugins = ( "tests.fixtures.core", + "tests.fixtures.feature_verification", "tests.fixtures.hypermemory", "tests.fixtures.platform", ) diff --git a/tests/fixtures/feature_verification/__init__.py b/tests/fixtures/feature_verification/__init__.py new file mode 100644 index 00000000..d6fdca23 --- /dev/null +++ b/tests/fixtures/feature_verification/__init__.py @@ -0,0 +1,3 @@ +"""Feature-verification suite fixtures and lane hooks.""" + +pytest_plugins = ("tests.fixtures.feature_verification.lane",) diff --git a/tests/fixtures/feature_verification/lane.py b/tests/fixtures/feature_verification/lane.py new file mode 100644 index 00000000..a436d656 --- /dev/null +++ b/tests/fixtures/feature_verification/lane.py @@ -0,0 +1,93 @@ +"""Pytest hooks and fixtures for the end-to-end feature-verification suite. + +Outside a provisioned lane (``FEATURE_VERIFICATION_LANE`` unset) the suite is deselected, so +``make test`` and nested product runs such as ``clawops baseline verify`` never touch it. +Inside a lane, only the features the matrix schedules on that lane run, and every outcome is +written to the lane results file the reporter joins with the matrix. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Final + +import pytest + +from tests.utils.helpers.feature_verification import ( + FeatureHost, + GatewayController, + LaneRecorder, + is_feature_suite_item, +) + +_RECORDER_KEY: Final[pytest.StashKey[LaneRecorder]] = pytest.StashKey[LaneRecorder]() +# The logreport hook receives no config, so the active recorder is also held here. +_ACTIVE_RECORDERS: Final[dict[str, LaneRecorder]] = {} + + +def pytest_configure(config: pytest.Config) -> None: + """Attach the lane recorder when running inside a provisioned lane.""" + recorder = LaneRecorder.from_environment() + if recorder is not None: + config.stash[_RECORDER_KEY] = recorder + _ACTIVE_RECORDERS["lane"] = recorder + + +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + """Deselect the suite outside a lane, and features scheduled on other lanes inside one.""" + suite_items = [item for item in items if is_feature_suite_item(item)] + if not suite_items: + return + recorder = config.stash.get(_RECORDER_KEY, None) + if recorder is None: + deselected = suite_items + else: + _, deselected = recorder.select(suite_items) + if deselected: + dropped = {id(item) for item in deselected} + config.hook.pytest_deselected(items=deselected) + items[:] = [item for item in items if id(item) not in dropped] + + +def pytest_runtest_logreport(report: pytest.TestReport) -> None: + """Record feature outcomes as phases finish.""" + recorder = _ACTIVE_RECORDERS.get("lane") + if recorder is not None: + recorder.record(report) + + +def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: + """Persist the lane results file.""" + del exitstatus + recorder = session.config.stash.get(_RECORDER_KEY, None) + if recorder is not None: + recorder.write() + _ACTIVE_RECORDERS.pop("lane", None) + + +@pytest.fixture(scope="session") +def feature_host(request: pytest.FixtureRequest) -> FeatureHost: + """Return the provisioned host this lane verifies, recording tool versions once.""" + host = FeatureHost.from_environment() + recorder = request.config.stash.get(_RECORDER_KEY, None) + if recorder is not None and not recorder.versions: + for tool in ("openclaw", "acpx"): + outcome = host.run([tool, "--version"], timeout=60) + recorder.versions[tool] = ( + outcome.stdout.strip().splitlines()[-1] + if outcome.ok and outcome.stdout.strip() + else f"unavailable (exit {outcome.returncode})" + ) + return host + + +@pytest.fixture(scope="session") +def gateway(feature_host: FeatureHost) -> Iterator[GatewayController]: + """Return a running gateway, started by the harness when no service supervises one.""" + controller = GatewayController(feature_host) + controller.ensure_running() + try: + yield controller + finally: + controller.stop() diff --git a/tests/scripts/feature_report.py b/tests/scripts/feature_report.py new file mode 100755 index 00000000..0c0bc8b2 --- /dev/null +++ b/tests/scripts/feature_report.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Join feature-verification lane results with the feature matrix into a verdict report.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +SRC_ROOT = REPO_ROOT / "src" +for import_root in (SRC_ROOT, REPO_ROOT): + if str(import_root) not in sys.path: + sys.path.insert(0, str(import_root)) + +from tests.utils.helpers.feature_verification import ( # noqa: E402 + FeatureVerificationError, + write_lane_report, +) + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--results-dir", type=Path, required=True) + parser.add_argument( + "--manifest", type=Path, default=REPO_ROOT / "tests" / "feature_matrix.yaml" + ) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--summary-file", type=Path) + parser.add_argument( + "--fail-on-failures", + action="store_true", + help="Exit non-zero when any feature is FAIL.", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + """Write the report and optionally fail on FAIL verdicts.""" + args = _parse_args(argv) + try: + failures = write_lane_report( + results_dir=Path(args.results_dir).expanduser().resolve(), + manifest_path=Path(args.manifest).expanduser().resolve(), + output_path=Path(args.output).expanduser().resolve(), + summary_file=( + Path(args.summary_file).expanduser().resolve() + if args.summary_file is not None + else None + ), + ) + except FeatureVerificationError as exc: + print(f"feature-report error: {exc}", file=sys.stderr) + return 1 + print(f"feature verification: {failures} feature(s) FAIL") + return 1 if failures and bool(args.fail_on_failures) else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/scripts/feature_verification.py b/tests/scripts/feature_verification.py new file mode 100755 index 00000000..b252f8b2 --- /dev/null +++ b/tests/scripts/feature_verification.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Semantic CLI for the host-side steps of the feature-verification workflow. + +These steps run on a fresh runner's bare Python (before ``clawops bootstrap`` creates the +managed environment), so this CLI only imports stdlib-backed helpers; the matrix report lives +in ``feature_report.py``. +""" + +from __future__ import annotations + +import argparse +import json +import signal +import sys +import threading +from pathlib import Path +from types import FrameType + +REPO_ROOT = Path(__file__).resolve().parents[2] +SRC_ROOT = REPO_ROOT / "src" +for import_root in (SRC_ROOT, REPO_ROOT): + if str(import_root) not in sys.path: + sys.path.insert(0, str(import_root)) + +from tests.utils.helpers._feature_verification.diagnostics import ( # noqa: E402 + collect_lane_diagnostics, +) +from tests.utils.helpers._feature_verification.fake_model import FakeModelServer # noqa: E402 +from tests.utils.helpers._feature_verification.lane import ( # noqa: E402 + LaneError, + configure_model_provider, + enable_systemd_user_manager, + run_lane, +) +from tests.utils.helpers._feature_verification.processes import ( # noqa: E402 + FakeModelProcessError, + start_detached, + stop_detached, +) + + +def _resolved(path: Path) -> Path: + return path.expanduser().resolve() + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + serve = subparsers.add_parser("serve-fake-model", help="Serve the fake model API (blocks).") + serve.add_argument("--host", default="127.0.0.1") + serve.add_argument("--port", type=int, action="append", required=True) + + start = subparsers.add_parser("start-fake-model", help="Start the fake model API detached.") + start.add_argument("--host", default="127.0.0.1") + start.add_argument("--port", type=int, action="append", required=True) + start.add_argument("--state-dir", type=Path, required=True) + + stop = subparsers.add_parser("stop-fake-model", help="Stop the detached fake model API.") + stop.add_argument("--state-dir", type=Path, required=True) + + provider = subparsers.add_parser( + "configure-model-provider", + help="Point the operator env contract at the fake model server.", + ) + provider.add_argument("--context", type=Path, required=True) + + subparsers.add_parser( + "enable-systemd-user", + help="Start the runner's systemd user manager so user units can be activated.", + ) + + lane = subparsers.add_parser("run-lane", help="Run one lane's feature tests.") + lane.add_argument("--context", type=Path, required=True) + lane.add_argument("--lane", required=True) + lane.add_argument("--results", type=Path, required=True) + lane.add_argument("--fake-model-dir", type=Path) + + diagnostics = subparsers.add_parser( + "collect-diagnostics", help="Capture service-manager and gateway diagnostics." + ) + diagnostics.add_argument("--context", type=Path, required=True) + diagnostics.add_argument("--output-dir", type=Path, required=True) + return parser.parse_args(argv) + + +def _serve(host: str, ports: list[int]) -> int: + """Serve until SIGTERM or SIGINT.""" + server = FakeModelServer(host=host, ports=ports) + stopped = threading.Event() + + def _stop(signum: int, frame: FrameType | None) -> None: + del signum, frame + stopped.set() + + signal.signal(signal.SIGTERM, _stop) + signal.signal(signal.SIGINT, _stop) + server.start() + print(f"fake model server listening on {host}:{server.ports}", flush=True) + stopped.wait() + server.stop() + return 0 + + +def main(argv: list[str] | None = None) -> int: + """Run the requested feature-verification subcommand.""" + args = _parse_args(argv) + try: + if args.command == "serve-fake-model": + return _serve(str(args.host), [int(port) for port in args.port]) + if args.command == "start-fake-model": + state = start_detached( + script_path=Path(__file__).resolve(), + host=str(args.host), + ports=[int(port) for port in args.port], + state_dir=_resolved(args.state_dir), + ) + print(json.dumps(state, sort_keys=True)) + return 0 + if args.command == "stop-fake-model": + print(json.dumps({"stopped": stop_detached(_resolved(args.state_dir))})) + return 0 + if args.command == "configure-model-provider": + env_file = configure_model_provider(_resolved(args.context)) + print(f"configured the fake model provider in {env_file}") + return 0 + if args.command == "enable-systemd-user": + print(f"systemd user manager ready at {enable_systemd_user_manager()}") + return 0 + if args.command == "run-lane": + exit_code = run_lane( + context_file=_resolved(args.context), + lane_id=str(args.lane), + results_path=_resolved(args.results), + fake_model_dir=( + _resolved(args.fake_model_dir) if args.fake_model_dir is not None else None + ), + ) + # Feature failures are report data; only a missing results file fails this step. + print(f"lane pytest exit code: {exit_code}") + return 0 + if args.command == "collect-diagnostics": + collect_lane_diagnostics( + context_file=_resolved(args.context), output_dir=_resolved(args.output_dir) + ) + return 0 + except (FakeModelProcessError, LaneError) as exc: + print(f"feature-verification error: {exc}", file=sys.stderr) + return 1 + raise AssertionError(f"Unhandled command: {args.command}") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/scripts/fresh_host.py b/tests/scripts/fresh_host.py index 42094e55..d3f10717 100755 --- a/tests/scripts/fresh_host.py +++ b/tests/scripts/fresh_host.py @@ -14,7 +14,9 @@ if str(import_root) not in sys.path: sys.path.insert(0, str(import_root)) +from clawops.memory_profiles import MANAGED_MEMORY_PROFILE_IDS # noqa: E402 from tests.utils.helpers.fresh_host import ( # noqa: E402 + SCENARIO_SPECS, FreshHostError, cleanup, collect_diagnostics, @@ -44,13 +46,13 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: prepare_parser.add_argument( "--scenario", required=True, - choices=("linux", "macos-sidecars", "macos-browser-lab"), + choices=tuple(SCENARIO_SPECS), ) prepare_parser.add_argument("--repo-root", type=Path, default=REPO_ROOT) prepare_parser.add_argument( "--profile", default="openclaw-default", - choices=("openclaw-default", "hypermemory"), + choices=MANAGED_MEMORY_PROFILE_IDS, ) prepare_parser.add_argument("--runner-temp", type=Path, required=True) prepare_parser.add_argument("--workspace", type=Path, default=REPO_ROOT) diff --git a/tests/suites/contracts/repo/test_feature_matrix_contract.py b/tests/suites/contracts/repo/test_feature_matrix_contract.py new file mode 100644 index 00000000..41ad669b --- /dev/null +++ b/tests/suites/contracts/repo/test_feature_matrix_contract.py @@ -0,0 +1,154 @@ +"""Contracts binding the feature matrix to the product surface, the suite, and the workflow.""" + +from __future__ import annotations + +import ast +from pathlib import Path +from typing import cast + +import yaml + +from clawops.backend_registry import BACKEND_REGISTRY +from clawops.cli import COMMANDS, WRAPPER_COMMANDS +from clawops.memory_profiles import MEMORY_PROFILES +from tests.utils.helpers.feature_verification import FeatureManifest, load_manifest +from tests.utils.helpers.fresh_host import SCENARIO_SPECS +from tests.utils.helpers.repo import REPO_ROOT + +_MANIFEST = REPO_ROOT / "tests" / "feature_matrix.yaml" +_SUITE_DIR = REPO_ROOT / "tests" / "suites" / "e2e" / "feature_verification" +_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "feature-verification.yml" +_COMPOSE_DIR = REPO_ROOT / "platform" / "compose" + + +def _manifest() -> FeatureManifest: + return load_manifest(_MANIFEST) + + +def _suite_tests() -> set[str]: + node_ids: set[str] = set() + for path in sorted(_SUITE_DIR.glob("test_*.py")): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=path.as_posix()) + rel_path = path.relative_to(REPO_ROOT).as_posix() + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name.startswith("test_"): + node_ids.add(f"{rel_path}::{node.name}") + return node_ids + + +def _cited_files(manifest: FeatureManifest) -> set[str]: + return { + call_site.rsplit(":", 1)[0] + for feature in manifest.features + for call_site in feature.call_sites + } + + +def _module_path(import_path: str) -> str: + candidate = Path("src", *import_path.split(".")) + return candidate.as_posix() if (REPO_ROOT / candidate).is_dir() else f"{candidate}.py" + + +def test_every_feature_test_exists_in_the_suite() -> None: + referenced = set(_manifest().feature_by_test()) + missing = sorted(referenced - _suite_tests()) + assert not missing, f"matrix references tests that do not exist: {missing}" + + +def test_every_suite_test_maps_to_exactly_one_feature() -> None: + unmapped = sorted(_suite_tests() - set(_manifest().feature_by_test())) + assert not unmapped, f"suite tests missing from the matrix: {unmapped}" + + +def test_every_call_site_resolves_to_a_real_line() -> None: + for feature in _manifest().features: + for call_site in feature.call_sites: + rel_path, line_text = call_site.rsplit(":", 1) + path = REPO_ROOT / rel_path + assert path.is_file(), f"{feature.feature_id}: {rel_path} does not exist" + line_count = len(path.read_text(encoding="utf-8").splitlines()) + assert int(line_text) <= line_count, f"{feature.feature_id}: {call_site} is past EOF" + + +def test_every_clawops_command_is_covered() -> None: + cited = _cited_files(_manifest()) + modules = {spec.name: _module_path(spec.import_path) for spec in COMMANDS} + modules.update( + {f"wrapper {name}": _module_path(module) for name, (module, _) in WRAPPER_COMMANDS.items()} + ) + uncovered = sorted( + command + for command, module in modules.items() + if not any(path == module or path.startswith(f"{module}/") for path in cited) + ) + assert not uncovered, f"clawops commands without a feature: {uncovered}" + + +def test_every_profile_has_a_profile_feature() -> None: + feature_ids = {feature.feature_id for feature in _manifest().features} + missing = sorted( + profile for profile in MEMORY_PROFILES if f"profile.{profile}" not in feature_ids + ) + assert not missing, f"profiles without a profile. feature: {missing}" + + +def test_every_backend_auth_mode_is_covered() -> None: + feature_ids = {feature.feature_id for feature in _manifest().features} + missing: list[str] = [] + for name, backend in BACKEND_REGISTRY.items(): + for mode in backend.supported_auth_modes: + if mode == "local": + expected = "acp.openclaw-local" if name == "openclaw" else "acp.local-backends" + else: + expected = f"acp.{name}-{mode}" + if expected not in feature_ids: + missing.append(f"{name}/{mode} -> {expected}") + assert not missing, f"backend auth modes without a feature: {missing}" + + +def test_every_shipped_compose_stack_is_covered() -> None: + cited = _cited_files(_manifest()) + stacks = sorted( + path.relative_to(REPO_ROOT).as_posix() + for path in _COMPOSE_DIR.glob("docker-compose.*.yaml") + if ".ci-hosted-" not in path.name + ) + uncovered = [stack for stack in stacks if stack not in cited] + assert not uncovered, f"compose stacks without a feature: {uncovered}" + + +def test_lanes_match_fresh_host_scenarios() -> None: + for lane in _manifest().lanes.values(): + spec = SCENARIO_SPECS.get(lane.scenario) # type: ignore[call-overload] + assert spec is not None, f"lane {lane.lane_id} uses unknown scenario {lane.scenario}" + assert spec.platform == lane.platform, lane + assert spec.activate_services == lane.activate_services, lane + assert spec.model_setup_mode == "auto", lane + + +def test_workflow_runs_every_lane_and_nothing_else() -> None: + workflow = cast(dict[str, object], yaml.safe_load(_WORKFLOW.read_text(encoding="utf-8"))) + jobs = cast(dict[str, dict[str, object]], workflow["jobs"]) + linux_matrix = cast(dict[str, object], jobs["linux-lanes"]["strategy"])["matrix"] + entries = cast(list[dict[str, str]], cast(dict[str, object], linux_matrix)["include"]) + workflow_lanes = {entry["lane"]: entry for entry in entries} + workflow_text = _WORKFLOW.read_text(encoding="utf-8") + manifest = _manifest() + for lane_id, lane in manifest.lanes.items(): + if lane.platform == "linux": + assert lane_id in workflow_lanes, f"workflow does not run lane {lane_id}" + entry = workflow_lanes[lane_id] + assert (entry["profile"], entry["scenario"]) == (lane.profile, lane.scenario), entry + else: + assert f"--lane {lane_id}" in workflow_text, f"workflow does not run lane {lane_id}" + assert f"--scenario {lane.scenario}" in workflow_text + assert f"--profile {lane.profile}" in workflow_text + assert set(workflow_lanes) <= set(manifest.lanes), sorted( + set(workflow_lanes) - set(manifest.lanes) + ) + + +def test_unverifiable_features_explain_why() -> None: + for feature in _manifest().features: + if feature.status == "unverifiable": + assert feature.reason is not None and len(feature.reason) >= 40, feature.feature_id diff --git a/tests/suites/contracts/taf/framework/test_fixture_plugin_registration.py b/tests/suites/contracts/taf/framework/test_fixture_plugin_registration.py index a013372b..bca595cd 100644 --- a/tests/suites/contracts/taf/framework/test_fixture_plugin_registration.py +++ b/tests/suites/contracts/taf/framework/test_fixture_plugin_registration.py @@ -41,10 +41,14 @@ def test_recursive_fixture_plugin_registries_are_explicit() -> None: fixture_root / "__init__.py": ( "tests.fixtures.core", + "tests.fixtures.feature_verification", "tests.fixtures.hypermemory", "tests.fixtures.platform", ), fixture_root + / "feature_verification" + / "__init__.py": ("tests.fixtures.feature_verification.lane",), + fixture_root / "core" / "__init__.py": ( "tests.fixtures.core.cli", diff --git a/tests/suites/contracts/taf/test_framework_health.py b/tests/suites/contracts/taf/test_framework_health.py index 94436a57..602542f1 100644 --- a/tests/suites/contracts/taf/test_framework_health.py +++ b/tests/suites/contracts/taf/test_framework_health.py @@ -15,6 +15,7 @@ "contract", "devflow", "e2e", + "feature_verification", "framework", "hypermemory", "integration", diff --git a/tests/suites/e2e/feature_verification/test_acp.py b/tests/suites/e2e/feature_verification/test_acp.py new file mode 100644 index 00000000..b1481a25 --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_acp.py @@ -0,0 +1,255 @@ +"""ACP worker features: acp-runner through the real acpx in every backend auth mode. + +The ``openclaw`` backend runs fully live (acpx -> ``openclaw acp`` -> gateway -> provider). +Vendor backends replace only the vendor adapter with an ACP stub agent through acpx's own +agent-override config, so StrongClaw's credential broker, env sanitization and the real acpx +client are exercised without vendor accounts. +""" + +from __future__ import annotations + +import json +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Final, cast + +from tests.utils.helpers.feature_verification import ( + ACP_STUB_REPLY_PREFIX, + REPLY_PREFIX, + FeatureHost, + GatewayController, +) + +STUB_AGENT: Final[Path] = Path("tests/utils/helpers/_feature_verification/acp_stub_agent.py") +STUBBED_AGENTS: Final[tuple[str, ...]] = ("codex", "claude", "gemini", "qwen", "copilot", "cursor") +FAKE_KEY: Final[str] = "featver-not-a-real-key" + + +@dataclass(frozen=True, slots=True) +class AcpSession: + """One finished acp-runner session.""" + + summary: dict[str, object] + stdout: str + + def stub_reply(self) -> str: + """Return the stub agent's reply line from the session transcript.""" + for line in self.stdout.splitlines(): + if ACP_STUB_REPLY_PREFIX in line: + return line + raise AssertionError(f"no stub reply in acpx output: {self.stdout[-2000:]}") + + +def _workspace(feature_host: FeatureHost, *, stubbed: bool) -> tuple[Path, Path]: + root = feature_host.scratch("acp") + project = root / "project" + workspace = project / "workspace" + workspace.mkdir(parents=True) + if stubbed: + stub = feature_host.repo_root / STUB_AGENT + agents = { + name: {"command": f"{feature_host.python} {stub} --persona {name}"} + for name in STUBBED_AGENTS + } + (workspace / ".acpxrc.json").write_text(json.dumps({"agents": agents}), encoding="utf-8") + return project, workspace + + +def _vendor_cli_stubs(feature_host: FeatureHost) -> Path: + """Stand-in vendor CLIs that report a logged-in subscription, as the real ones do.""" + bin_dir = feature_host.scratch("vendor-bin") + scripts = { + "codex": "#!/bin/sh\necho 'Logged in using ChatGPT'\n", + "claude": '#!/bin/sh\necho \'{"loggedIn": true, "authMethod": "claude.ai"}\'\n', + } + for name, body in scripts.items(): + path = bin_dir / name + path.write_text(body, encoding="utf-8") + path.chmod(0o755) + return bin_dir + + +def _run_session( + feature_host: FeatureHost, + *, + backend: str, + auth_mode: str, + prompt: str, + stubbed: bool = True, + env: dict[str, str] | None = None, +) -> AcpSession: + project, workspace = _workspace(feature_host, stubbed=stubbed) + outcome = feature_host.clawops( + "acp-runner", + "--backend", + backend, + "--auth-mode", + auth_mode, + "--prompt", + prompt, + "--project-root", + str(project), + "--workspace", + str(workspace), + "--allowed-workspace-root", + str(project), + "--state-dir", + str(project / "state"), + "--permissions-mode", + "approve-reads", + "--output-format", + "json", + "--timeout-seconds", + "240", + timeout=300, + env=env, + ) + summary = cast(dict[str, object], outcome.json()) + stdout_path = summary.get("stdout_path") + stdout = Path(str(stdout_path)).read_text(encoding="utf-8") if stdout_path else "" + assert summary.get("status") == "succeeded", f"{outcome.describe()}\n{stdout[-2000:]}" + return AcpSession(summary=summary, stdout=stdout) + + +def _env_with(feature_host: FeatureHost, **values: str) -> dict[str, str]: + env = {"PATH": f"{_vendor_cli_stubs(feature_host)}:{feature_host.product_env()['PATH']}"} + env.update(values) + return env + + +def test_openclaw_backend_runs_a_live_acp_turn( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + del gateway + token = f"FV-acp-{uuid.uuid4().hex[:10]}" + session = _run_session( + feature_host, backend="openclaw", auth_mode="local", prompt=f"Reply {token}", stubbed=False + ) + assert ( + f"{REPLY_PREFIX} {token}" in session.stdout + ), f"openclaw acp did not return the provider reply:\n{session.stdout[-3000:]}" + + +def test_codex_api_mode_passes_the_key_and_authenticates(feature_host: FeatureHost) -> None: + session = _run_session( + feature_host, + backend="codex", + auth_mode="api", + prompt="FV-codex-api", + env=_env_with(feature_host, OPENAI_API_KEY=FAKE_KEY), + ) + reply = session.stub_reply() + assert "OPENAI_API_KEY:present" in reply, reply + assert "auth=openai-api-key" in reply, f"acpx did not select the env-backed auth: {reply}" + assert session.summary["credential_source_class"] == "process-env", session.summary + + +def test_codex_subscription_mode_strips_api_keys(feature_host: FeatureHost) -> None: + session = _run_session( + feature_host, + backend="codex", + auth_mode="subscription", + prompt="FV-codex-sub", + env=_env_with(feature_host, OPENAI_API_KEY=FAKE_KEY), + ) + reply = session.stub_reply() + assert "OPENAI_API_KEY:absent" in reply and "auth=none" in reply, reply + assert session.summary["credential_source_class"] == "vendor-cli-state", session.summary + + +def test_claude_api_mode_passes_the_key(feature_host: FeatureHost) -> None: + session = _run_session( + feature_host, + backend="claude", + auth_mode="api", + prompt="FV-claude-api", + env=_env_with(feature_host, ANTHROPIC_API_KEY=FAKE_KEY), + ) + assert "ANTHROPIC_API_KEY:present" in session.stub_reply() + + +def test_claude_subscription_mode_strips_api_keys(feature_host: FeatureHost) -> None: + session = _run_session( + feature_host, + backend="claude", + auth_mode="subscription", + prompt="FV-claude-sub", + env=_env_with(feature_host, ANTHROPIC_API_KEY=FAKE_KEY), + ) + assert "ANTHROPIC_API_KEY:absent" in session.stub_reply() + assert session.summary["credential_source_class"] == "vendor-cli-state", session.summary + + +def test_claude_cloud_provider_mode_keeps_cloud_credentials(feature_host: FeatureHost) -> None: + session = _run_session( + feature_host, + backend="claude", + auth_mode="cloud-provider", + prompt="FV-claude-cloud", + env=_env_with(feature_host, ANTHROPIC_API_KEY=FAKE_KEY, AWS_PROFILE="featver"), + ) + reply = session.stub_reply() + assert "ANTHROPIC_API_KEY:absent" in reply and "AWS_PROFILE:present" in reply, reply + assert session.summary["credential_source_class"] == "cloud-env", session.summary + + +def test_local_backends_dispatch_through_acpx(feature_host: FeatureHost) -> None: + for backend in ("gemini", "qwen", "copilot", "cursor"): + session = _run_session( + feature_host, backend=backend, auth_mode="local", prompt=f"FV-{backend}" + ) + reply = session.stub_reply() + assert f"persona={backend}" in reply and f"prompt=FV-{backend}" in reply, reply + + +def test_auth_policy_blocks_unsupported_and_unready_modes(feature_host: FeatureHost) -> None: + project, workspace = _workspace(feature_host, stubbed=True) + common = ( + "--prompt", + "FV-policy", + "--project-root", + str(project), + "--workspace", + str(workspace), + "--allowed-workspace-root", + str(project), + "--state-dir", + str(project / "state"), + ) + unsupported = feature_host.clawops( + "acp-runner", "--backend", "codex", "--auth-mode", "cloud-provider", *common + ) + payload = cast(dict[str, object], unsupported.json()) + assert not unsupported.ok and payload["status"] == "preflight_failed", unsupported.describe() + assert payload["credential_state"] == "forbidden_by_policy", payload + env = _env_with(feature_host) + env["PATH"] = f"{feature_host.scratch('empty-bin')}:{feature_host.product_env()['PATH']}" + unready = feature_host.clawops( + "acp-runner", + "--backend", + "codex", + "--auth-mode", + "api", + *common, + env={**env, "OPENAI_API_KEY": ""}, + ) + unready_payload = cast(dict[str, object], unready.json()) + assert unready_payload["status"] == "preflight_failed", unready.describe() + + +def test_credential_status_reports_each_mode(feature_host: FeatureHost) -> None: + ready = feature_host.clawops( + "credential-status", + "--backend", + "claude", + "--auth-mode", + "cloud-provider", + env={"AWS_PROFILE": "featver"}, + ) + ready_payload = cast(dict[str, object], ready.json()) + assert ready_payload["state"] == "ready" and ready_payload["source_class"] == "cloud-env" + missing = feature_host.clawops( + "credential-status", "--backend", "codex", "--auth-mode", "api", env={"OPENAI_API_KEY": ""} + ) + assert cast(dict[str, object], missing.json())["state"] == "misconfigured", missing.describe() diff --git a/tests/suites/e2e/feature_verification/test_context_workflows.py b/tests/suites/e2e/feature_verification/test_context_workflows.py new file mode 100644 index 00000000..0a95a45e --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_context_workflows.py @@ -0,0 +1,124 @@ +"""Context-service, workflow-runner and channel-config features against the real runtime.""" + +from __future__ import annotations + +import json +import subprocess +import uuid +from pathlib import Path +from typing import cast + +from tests.utils.helpers.feature_verification import FeatureHost, basic_auth, http_request + +CONTEXT_CONFIG = "platform/configs/context/codebase.yaml" +CHANNELS_OVERLAY = "platform/configs/openclaw/30-channels.json5" +NEO4J_HTTP = "http://127.0.0.1:7474/db/neo4j/tx/commit" + + +def _sample_repo(feature_host: FeatureHost, marker: str) -> Path: + repo = feature_host.scratch("context-repo") + (repo / "pkg").mkdir() + (repo / "pkg" / "ledger.py").write_text( + f'"""Ledger module {marker}."""\n\n\n' + "class LedgerReconciler:\n" + " def reconcile(self, entries: list[int]) -> int:\n" + " return sum(entries)\n", + encoding="utf-8", + ) + (repo / "README.md").write_text(f"# Ledger\n\nReconciliation notes {marker}.\n", "utf-8") + for command in ( + ["git", "init", "-q", "-b", "main"], + ["git", "add", "."], + ["git", "-c", "user.email=fv@example.com", "-c", "user.name=fv", "commit", "-qm", "seed"], + ): + subprocess.run(command, cwd=repo, check=True, capture_output=True) + return repo + + +def _context_env(feature_host: FeatureHost) -> dict[str, str]: + values = feature_host.env_values() + keys = ( + "NEO4J_USERNAME", + "NEO4J_PASSWORD", + "HYPERMEMORY_QDRANT_URL", + "HYPERMEMORY_EMBEDDING_MODEL", + ) + return {key: values[key] for key in keys if values.get(key)} + + +def _neo4j_node_count(feature_host: FeatureHost) -> int: + values = feature_host.env_values() + response = http_request( + "POST", + NEO4J_HTTP, + body={"statements": [{"statement": "MATCH (n) RETURN count(n) AS nodes"}]}, + headers=basic_auth(values.get("NEO4J_USERNAME", "neo4j"), values["NEO4J_PASSWORD"]), + timeout=60, + ) + assert response.status == 200, response.body[:1000] + results = cast(list[dict[str, object]], cast(dict[str, object], response.json())["results"]) + data = cast(list[dict[str, object]], results[0]["data"]) + return int(cast(list[int], data[0]["row"])[0]) + + +def test_codebase_context_indexes_into_the_neo4j_graph(feature_host: FeatureHost) -> None: + marker = f"fv{uuid.uuid4().hex[:8]}" + repo = _sample_repo(feature_host, marker) + env = _context_env(feature_host) + common = ("--config", CONTEXT_CONFIG, "--repo", str(repo), "--scale", "large") + index = feature_host.clawops("context", "codebase", "index", *common, "--json", env=env) + stats = cast(dict[str, object], index.require_ok().json()) + assert int(cast(int, stats["total_files"])) >= 2, stats + assert _neo4j_node_count(feature_host) > 0, "large-scale indexing wrote nothing to neo4j" + query = feature_host.clawops( + "context", "codebase", "query", *common, "--query", "LedgerReconciler reconcile", env=env + ).require_ok() + assert "pkg/ledger.py" in query.stdout, query.describe() + pack_path = feature_host.scratch("context-pack") / "pack.md" + feature_host.clawops( + "context", + "codebase", + "pack", + *common, + "--query", + "reconcile", + "--output", + str(pack_path), + env=env, + ).require_ok() + assert "LedgerReconciler" in pack_path.read_text(encoding="utf-8") + + +def test_daily_healthcheck_workflow_runs_openclaw_checks(feature_host: FeatureHost) -> None: + outcome = feature_host.clawops( + "workflow", "--workflow", "platform/configs/workflows/daily_healthcheck.yaml", timeout=1200 + ) + rows = [line.split("\t") for line in outcome.stdout.splitlines() if "\t" in line] + results = {row[0]: row[1] for row in rows if len(row) >= 2} + assert results == {"openclaw doctor": "ok", "security audit": "ok"}, outcome.describe() + assert outcome.ok, outcome.describe() + + +def test_channels_overlay_validates_under_openclaw(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("channels") + rendered = scratch / "openclaw.json" + feature_host.clawops( + "render-openclaw-config", + "--asset-root", + ".", + "--home-dir", + str(feature_host.home), + "--profile", + "openclaw-default", + "--overlay", + CHANNELS_OVERLAY, + "--output", + str(rendered), + "--exec-approvals-output", + str(scratch / "exec-approvals.json"), + ).require_ok() + env = {"OPENCLAW_CONFIG_PATH": str(rendered), "TELEGRAM_BOT_TOKEN": "123456:featver-stub-token"} + feature_host.openclaw("config", "validate", env=env, timeout=120).require_ok() + listed = feature_host.openclaw("channels", "list", "--no-usage", "--json", env=env, timeout=180) + payload = json.dumps(listed.require_ok().json()) + assert "telegram" in payload and "whatsapp" in payload, payload[:2000] diff --git a/tests/suites/e2e/feature_verification/test_devflow_planning.py b/tests/suites/e2e/feature_verification/test_devflow_planning.py new file mode 100644 index 00000000..af7e2430 --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_devflow_planning.py @@ -0,0 +1,129 @@ +"""Devflow operator CLI: plan a run for a scratch project and inspect its state and audit surface. + +``devflow run`` dispatches real ACP workers, so it is out of scope here. ``devflow plan`` is a +pure planner that persists nothing, which makes ``status`` and ``audit`` for the planned run id +observable only through their unknown-run contract. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Final, cast + +import yaml + +from tests.utils.helpers.feature_verification import CommandOutcome, FeatureHost + +type JsonObject = dict[str, object] + +DEVFLOW_CONFIG_DIR: Final[Path] = Path("platform/configs/devflow") +RUN_ID_PATTERN: Final[re.Pattern[str]] = re.compile(r"^df_[0-9a-f]{12}$") +GIT_CONFIG_ARGS: Final[tuple[str, ...]] = ( + "-c", + "user.name=Feature Verification", + "-c", + "user.email=featver@example.invalid", + "-c", + "commit.gpgsign=false", + "-c", + "core.hooksPath=/dev/null", +) + + +def _as_object(value: object) -> JsonObject: + assert isinstance(value, dict), f"expected a JSON object, got {value!r}" + return cast(JsonObject, value) + + +def _as_objects(value: object) -> list[JsonObject]: + assert isinstance(value, list), f"expected a JSON array, got {value!r}" + return [_as_object(item) for item in cast(list[object], value)] + + +def _load_yaml(path: Path) -> JsonObject: + return _as_object(yaml.safe_load(path.read_text(encoding="utf-8"))) + + +def _scratch_project(feature_host: FeatureHost) -> Path: + project = feature_host.scratch("devflow-project") + (project / "pyproject.toml").write_text( + '[project]\nname = "featver-demo"\nversion = "0.1.0"\n', encoding="utf-8" + ) + (project / "app.py").write_text("def health() -> str:\n return 'ok'\n", encoding="utf-8") + for arguments in (("init", "-q"), ("add", "-A"), ("commit", "-q", "-m", "initial commit")): + feature_host.run( + ["git", *GIT_CONFIG_ARGS, "-C", str(project), *arguments], timeout=60 + ).require_ok() + return project + + +def _devflow(feature_host: FeatureHost, project: Path, *arguments: str) -> CommandOutcome: + command, *rest = arguments + return feature_host.clawops("devflow", command, "--project-root", str(project), *rest) + + +def test_devflow_plan_status_audit(feature_host: FeatureHost) -> None: + project = _scratch_project(feature_host) + goal = "Add a /health endpoint" + plan_outcome = _devflow(feature_host, project, "plan", "--goal", goal).require_ok() + plan = _as_object(plan_outcome.json()) + run_id = str(plan["run_id"]) + assert RUN_ID_PATTERN.fullmatch(run_id), plan + assert plan["goal"] == goal and plan["lane"] == "default", plan + assert plan["repo_root"] == project.resolve().as_posix(), plan + assert plan["bootstrap_profile"] == "python-basic", plan + commands = _as_object(plan["bootstrap_commands"]) + assert commands["test_unit"] == [["python", "-m", "pytest", "-q"]], commands + + workflow = _load_yaml(feature_host.repo_root / DEVFLOW_CONFIG_DIR / "workflows/production.yaml") + roles = _as_object( + _load_yaml(feature_host.repo_root / DEVFLOW_CONFIG_DIR / "roles.yaml")["roles"] + ) + assert plan["run_profile"] == workflow["run_profile"], plan + stages = _as_objects(plan["stages"]) + assert [stage["name"] for stage in stages] == workflow["stage_order"], stages + for stage in stages: + role = _as_object(roles[str(stage["role"])]) + assert stage["role"] == stage["name"], stage + assert stage["backend"] == role["default_backend"], stage + assert stage["permissions_mode"] == role["permissions_mode"], stage + assert stage["workspace_mode"] == role["workspace_mode"], stage + assert stage["required_auth_mode"] == role["required_auth_mode"], stage + assert stage["approval_required"] is bool(role.get("approval_required", False)), stage + assert stage["expected_artifacts"] == role["expected_artifacts"], stage + prompt = Path(str(stage["worker_prompt"])) + assert prompt.as_posix().endswith(str(role["worker_prompt"])), stage + assert prompt.is_file(), f"planned worker prompt {prompt} does not exist" + assert [stage["name"] for stage in stages if stage["approval_required"]] == ["lead"], stages + + replanned = _as_object( + _devflow(feature_host, project, "plan", "--goal", goal).require_ok().json() + ) + assert replanned == plan, "planning the same goal must be deterministic" + other_lane = _as_object( + _devflow(feature_host, project, "plan", "--goal", goal, "--lane", "hotfix") + .require_ok() + .json() + ) + assert other_lane["run_id"] != run_id and other_lane["lane"] == "hotfix", other_lane + run_root = project / ".clawops" / "devflow" / run_id + assert not run_root.exists(), "devflow plan must not materialize a run" + + status = _devflow(feature_host, project, "status", "--run-id", run_id) + assert status.returncode == 2, status.describe() + status_payload = _as_object(status.json()) + assert status_payload["ok"] is False and run_id in str( + status_payload["message"] + ), status_payload + stuck = _as_object( + _devflow(feature_host, project, "status", "--stuck-only").require_ok().json() + ) + assert stuck == {"stuck_runs": []}, stuck + + audit = _devflow(feature_host, project, "audit", "--run-id", run_id) + assert not audit.ok, audit.describe() + assert f"unknown devflow run: {run_id}" in audit.output, audit.describe() + assert not (run_root / "audit" / "bundle.json").exists(), "no audit bundle for an unknown run" + assert json.loads(plan_outcome.stdout) == plan, "plan output must be a single JSON document" diff --git a/tests/suites/e2e/feature_verification/test_host_lifecycle.py b/tests/suites/e2e/feature_verification/test_host_lifecycle.py new file mode 100644 index 00000000..39428635 --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_host_lifecycle.py @@ -0,0 +1,139 @@ +"""Host lifecycle features: bootstrap, guided setup, service managers, env contract, doctors.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import cast + +from clawops.platform_compat import DEFAULT_ACPX_VERSION, DEFAULT_OPENCLAW_VERSION +from clawops.strongclaw_runtime import load_bootstrap_state +from tests.utils.helpers.feature_verification import FeatureHost +from tests.utils.helpers.fresh_host import load_report + +SYSTEMD_PERSISTENT_UNITS = ("openclaw-gateway.service",) +SYSTEMD_ONESHOT_UNITS = ("openclaw-sidecars.service",) +SYSTEMD_TIMERS = ( + "openclaw-backup-create.timer", + "openclaw-backup-verify.timer", + "openclaw-maintenance.timer", +) +LAUNCHD_JOBS = ( + "ai.openclaw.backup-create", + "ai.openclaw.backup-verify", + "ai.openclaw.maintenance", +) + + +def _phase_status(feature_host: FeatureHost, phase: str) -> tuple[str, str]: + report = load_report(Path(feature_host.context.report_path)) + for result in report.phases: + if result.name == phase: + return result.status, result.failure_reason or "" + return "missing", f"phase {phase!r} never ran; scenario status={report.status}" + + +def test_bootstrap_installs_the_locked_toolchain(feature_host: FeatureHost) -> None: + status, reason = _phase_status(feature_host, "bootstrap") + assert status == "success", f"clawops bootstrap failed: {reason}" + state = load_bootstrap_state() + assert state is not None, "bootstrap did not persist its completion state" + assert state.profile == feature_host.lane.profile + openclaw_version = feature_host.run(["openclaw", "--version"]).require_ok().stdout + acpx_version = feature_host.run(["acpx", "--version"]).require_ok().stdout + assert DEFAULT_OPENCLAW_VERSION in openclaw_version, openclaw_version + assert DEFAULT_ACPX_VERSION in acpx_version, acpx_version + assert feature_host.python.is_file(), "bootstrap did not create the managed venv" + + +def test_guided_setup_completes_with_service_activation(feature_host: FeatureHost) -> None: + status, reason = _phase_status(feature_host, "setup") + assert status == "success", f"clawops setup (with service activation) failed: {reason}" + config = feature_host.rendered_config() + agents = cast(dict[str, object], config["agents"]) + defaults = cast(dict[str, object], agents["defaults"]) + model = cast(dict[str, object], defaults["model"]) + assert str(model["primary"]).startswith("vllm/"), model + feature_host.openclaw("config", "validate", timeout=120).require_ok() + + +def test_systemd_user_units_are_installed_and_active(feature_host: FeatureHost) -> None: + unit_dir = feature_host.home / ".config" / "systemd" / "user" + for unit in (*SYSTEMD_PERSISTENT_UNITS, *SYSTEMD_ONESHOT_UNITS, *SYSTEMD_TIMERS): + rendered = unit_dir / unit + assert rendered.is_file(), f"{rendered} was not rendered" + assert "__REPO_ROOT__" not in rendered.read_text(encoding="utf-8") + enabled = feature_host.run(["systemctl", "--user", "is-enabled", unit], timeout=30) + assert enabled.stdout.strip() == "enabled", enabled.describe() + for unit in (*SYSTEMD_PERSISTENT_UNITS, *SYSTEMD_TIMERS): + active = feature_host.run(["systemctl", "--user", "is-active", unit], timeout=30) + assert active.stdout.strip() == "active", active.describe() + sidecars = feature_host.run( + ["systemctl", "--user", "show", "--property=ActiveState,Result", *SYSTEMD_ONESHOT_UNITS], + timeout=30, + ).require_ok() + assert ( + "ActiveState=active" in sidecars.stdout and "Result=success" in sidecars.stdout + ), sidecars.describe() + + +def test_launchd_agents_are_loaded_and_running(feature_host: FeatureHost) -> None: + domain = f"gui/{os.getuid()}" + gateway = feature_host.run(["launchctl", "print", f"{domain}/ai.openclaw.gateway"], timeout=30) + assert gateway.ok and "state = running" in gateway.stdout, gateway.describe() + sidecars = feature_host.run( + ["launchctl", "print", f"{domain}/ai.openclaw.sidecars"], timeout=30 + ) + assert sidecars.ok and "last exit code = 0" in sidecars.stdout, sidecars.describe() + for label in LAUNCHD_JOBS: + job = feature_host.run(["launchctl", "print", f"{domain}/{label}"], timeout=30) + assert job.ok, f"{label} is not loaded:\n{job.describe()}" + plist_dir = feature_host.home / "Library" / "LaunchAgents" + for label in ("ai.openclaw.gateway", *LAUNCHD_JOBS): + plist = plist_dir / f"{label}.plist" + assert plist.is_file(), f"{plist} was not rendered" + lint = feature_host.run(["plutil", "-lint", str(plist)], timeout=30) + assert lint.ok, lint.describe() + + +def test_varlock_env_contract_is_valid(feature_host: FeatureHost) -> None: + check = feature_host.clawops("varlock-env", "--asset-root", ".", "check").require_ok() + payload = cast(dict[str, object], check.json()) + assert payload["ok"] is True and payload["checkOnly"] is True, payload + assert payload["varlockValidated"] is True, payload + values = feature_host.env_values() + for key in ("OPENCLAW_GATEWAY_TOKEN", "LITELLM_MASTER_KEY", "NEO4J_PASSWORD"): + assert len(values.get(key, "")) >= 16, f"{key} was not generated by the env contract" + mode = feature_host.env_file().stat().st_mode & 0o777 + assert mode == 0o600, f"{feature_host.env_file()} has mode {oct(mode)}" + reconfigure = feature_host.clawops( + "varlock-env", "--asset-root", ".", "configure", "--non-interactive" + ).require_ok() + assert cast(dict[str, object], reconfigure.json())["created"] is False + + +def test_doctor_host_reports_toolchain(feature_host: FeatureHost) -> None: + outcome = feature_host.clawops("doctor-host", "--asset-root", ".", timeout=300).require_ok() + payload = cast(dict[str, object], outcome.json()) + assert payload["ok"] is True, payload + assert DEFAULT_OPENCLAW_VERSION in str(payload["openclawVersion"]), payload + assert DEFAULT_ACPX_VERSION in str(payload["acpxVersion"]), payload + assert Path(str(payload["config"])).is_file(), payload + + +def test_deep_doctor_passes_every_check(feature_host: FeatureHost) -> None: + outcome = feature_host.clawops("doctor", "--asset-root", ".", timeout=1800) + payload = cast(dict[str, object], outcome.json()) + checks = cast(list[dict[str, object]], payload["checks"]) + failed = { + str(check["name"]): str(check["message"])[:600] for check in checks if not check["ok"] + } + assert payload["status"] == "pass" and outcome.ok, json.dumps(failed, indent=2) + + +def test_baseline_verify_passes_with_runtime_probes(feature_host: FeatureHost) -> None: + outcome = feature_host.clawops("baseline", "verify", timeout=3000) + assert outcome.ok, outcome.describe() + payload = cast(dict[str, object], outcome.json()) + assert payload["ok"] is True and payload["verificationMode"] == "runtime", payload diff --git a/tests/suites/e2e/feature_verification/test_memory_hypermemory.py b/tests/suites/e2e/feature_verification/test_memory_hypermemory.py new file mode 100644 index 00000000..a9fa086d --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_memory_hypermemory.py @@ -0,0 +1,202 @@ +"""Hypermemory features: OpenClaw memory surface, hypermemory engine, plugin recall, Qdrant upkeep.""" + +from __future__ import annotations + +import re +import time +import uuid +from pathlib import Path +from typing import cast + +from clawops.qdrant_storage import QdrantRelease, StorageStamp, read_stamp, stamp_path, write_stamp +from clawops.strongclaw_runtime import ( + rendered_openclaw_hypermemory_config_path, + resolve_compose_state_dir, + resolve_openclaw_config_path, +) +from tests.utils.helpers.feature_verification import ( + CommandOutcome, + FeatureHost, + GatewayController, + plugin_statuses, +) + +AUX_COMPOSE = Path("platform/compose/docker-compose.aux-stack.yaml") +_QDRANT_IMAGE = re.compile(r"^\s*image:\s*(qdrant/qdrant:\S+)", re.MULTILINE) + + +def _config_path(feature_host: FeatureHost) -> Path: + path = rendered_openclaw_hypermemory_config_path( + resolve_openclaw_config_path(feature_host.repo_root) + ) + assert path is not None and path.is_file(), "the hypermemory profile has no rendered config" + return path + + +def _hypermemory( + feature_host: FeatureHost, *arguments: str, timeout: float = 600 +) -> CommandOutcome: + return feature_host.clawops( + "hypermemory", "--config", str(_config_path(feature_host)), *arguments, timeout=timeout + ) + + +def _status(feature_host: FeatureHost) -> dict[str, object]: + return cast( + dict[str, object], _hypermemory(feature_host, "status", "--json").require_ok().json() + ) + + +def _search(feature_host: FeatureHost, query: str) -> list[dict[str, object]]: + payload = ( + _hypermemory(feature_host, "search", "--json", "--query", query, "--max-results", "5") + .require_ok() + .json() + ) + return cast(list[dict[str, object]], cast(dict[str, object], payload)["results"]) + + +def test_hypermemory_profile_plugins_are_loaded(feature_host: FeatureHost) -> None: + statuses = plugin_statuses(feature_host) + for plugin_id in ("strongclaw-hypermemory", "lossless-claw"): + assert statuses.get(plugin_id) == "loaded", f"{plugin_id}: {statuses.get(plugin_id)}" + plugins = cast(dict[str, object], feature_host.rendered_config()["plugins"]) + slots = cast(dict[str, object], plugins["slots"]) + assert slots == {"contextEngine": "lossless-claw", "memory": "strongclaw-hypermemory"}, slots + + +def test_openclaw_memory_status_is_healthy(feature_host: FeatureHost) -> None: + outcome = feature_host.openclaw("memory", "status", "--deep", timeout=600).require_ok() + assert ( + "strongclaw-hypermemory" in outcome.output or "hypermemory" in outcome.output + ), outcome.describe() + + +def test_openclaw_memory_search_returns_corpus_hits(feature_host: FeatureHost) -> None: + outcome = feature_host.openclaw( + "memory", "search", "--query", "ClawOps", "--max-results", "1", timeout=600 + ).require_ok() + assert ".md" in outcome.stdout, f"memory search returned no corpus hit\n{outcome.describe()}" + + +def test_hypermemory_index_and_hybrid_search(feature_host: FeatureHost) -> None: + index = cast( + dict[str, object], _hypermemory(feature_host, "index", "--json").require_ok().json() + ) + assert int(cast(int, index["files"])) > 0 and int(cast(int, index["chunks"])) > 0, index + status = _status(feature_host) + assert status["backendActive"] == "qdrant_sparse_dense_hybrid", status + assert int(cast(int, status["vectorItems"])) > 0, status + assert int(cast(int, status["sparseVectorItems"])) > 0, status + hits = _search(feature_host, "backup and recovery") + assert hits, "hybrid search returned no hits for a shipped runbook topic" + assert any("BACKUP" in str(hit["path"]).upper() for hit in hits), hits + + +def test_hypermemory_verify_reports_consistent_index(feature_host: FeatureHost) -> None: + payload = cast( + dict[str, object], _hypermemory(feature_host, "verify", "--json").require_ok().json() + ) + assert payload["ok"] is True, payload + + +def test_hypermemory_store_search_and_forget(feature_host: FeatureHost) -> None: + token = f"FV-store-{uuid.uuid4().hex[:10]}" + text = f"The feature verification vault code is {token}." + stored = _hypermemory(feature_host, "store", "--json", "--type", "fact", "--text", text) + assert cast(dict[str, object], stored.require_ok().json()).get("ok", True) is not False + assert any(token in str(hit.get("snippet")) for hit in _search(feature_host, token)), token + forgot = _hypermemory(feature_host, "forget", "--json", "--entry-text", text).require_ok() + assert cast(dict[str, object], forgot.json()).get("ok", True) is not False, forgot.describe() + assert not any(token in str(hit.get("snippet")) for hit in _search(feature_host, token)) + + +def test_hypermemory_maintenance_operations(feature_host: FeatureHost) -> None: + for arguments in ( + ("reflect", "--json", "--mode", "safe"), + ("lifecycle", "--json"), + ("list-facts", "--json"), + ("flush-metadata", "--json"), + ): + outcome = _hypermemory(feature_host, *arguments) + assert outcome.ok, outcome.describe() + assert isinstance(outcome.json(), dict | list), outcome.describe() + + +def test_hypermemory_exports_memory_pro_snapshot(feature_host: FeatureHost) -> None: + output = feature_host.scratch("hypermemory-export") / "memory-pro.json" + _hypermemory(feature_host, "export-memory-pro", "--json", "--output", str(output)).require_ok() + assert output.is_file() and output.stat().st_size > 2, f"no export written to {output}" + + +def test_hypermemory_plugin_recalls_memory_into_turns( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + del gateway + token = f"FV-recall-{uuid.uuid4().hex[:10]}" + _hypermemory( + feature_host, + "store", + "--json", + "--type", + "fact", + "--text", + f"The featver lighthouse passphrase is {token}.", + ).require_ok() + started = time.time() + feature_host.openclaw( + "agent", + "--agent", + "admin", + "--session-id", + f"featver-recall-{token}", + "--message", + "What is the featver lighthouse passphrase?", + "--json", + timeout=300, + ).require_ok() + prompts = [ + str(request.get("prompt", "")) + for request in feature_host.fake_model_requests() + if float(cast(float, request.get("at", 0.0))) >= started + ] + assert any( + token in prompt for prompt in prompts + ), "strongclaw-hypermemory autoRecall did not inject the stored fact into the model prompt" + + +def test_qdrant_storage_stamp_tracks_pinned_release(feature_host: FeatureHost) -> None: + image_match = _QDRANT_IMAGE.search((feature_host.repo_root / AUX_COMPOSE).read_text("utf-8")) + assert image_match is not None + target = QdrantRelease.from_image(image_match.group(1)) + stamp = read_stamp(stamp_path(resolve_compose_state_dir())) + assert stamp is not None, f"no qdrant stamp under {resolve_compose_state_dir()}" + assert stamp.release_line == target.line and stamp.reindex_pending is False, stamp + + +def test_qdrant_release_change_resets_storage_and_reindexes(feature_host: FeatureHost) -> None: + stamp_file = stamp_path(resolve_compose_state_dir()) + current = read_stamp(stamp_file) + assert current is not None, f"no qdrant stamp at {stamp_file}" + write_stamp( + stamp_file, + StorageStamp( + release_line="0.0", + image="qdrant/qdrant:v0.0.0", + reindex_pending=False, + updated_at=current.updated_at, + ), + ) + outcome = feature_host.clawops( + "ops", "--asset-root", ".", "sidecars", "up", "--json", timeout=1500 + ).require_ok() + assert "Resetting qdrant storage" in outcome.stderr, outcome.describe() + command = cast(dict[str, object], cast(dict[str, object], outcome.json())["command"]) + steps = { + str(cast(dict[str, object], step)["step"]) for step in cast(list[object], command["steps"]) + } + assert {"reset-qdrant-storage", "reindex-hypermemory"} <= steps, steps + restored = read_stamp(stamp_file) + assert restored is not None and restored.release_line == current.release_line, restored + assert restored.reindex_pending is False, restored + assert int(cast(int, _status(feature_host)["vectorItems"])) > 0 diff --git a/tests/suites/e2e/feature_verification/test_memory_lancedb.py b/tests/suites/e2e/feature_verification/test_memory_lancedb.py new file mode 100644 index 00000000..c23f3ec7 --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_memory_lancedb.py @@ -0,0 +1,230 @@ +"""memory-lancedb-pro features: plugin load, memory-pro CLI, migration tools, extraction, reflection.""" + +from __future__ import annotations + +import json +import time +import uuid +from datetime import UTC, datetime +from pathlib import Path +from typing import cast + +from clawops.runtime_assets import resolve_runtime_layout +from tests.utils.helpers.feature_verification import ( + CommandOutcome, + FeatureHost, + GatewayController, + plugin_statuses, +) + +PLUGIN_ID = "memory-lancedb-pro" +SCOPE = "project:featver" +STRATEGY_KEY = f"plugins.entries.{PLUGIN_ID}.config.sessionStrategy" +REFLECTION_TIMEOUT_SECONDS = 240.0 +_POLL_SECONDS = 3.0 + + +def _sqlite_config(feature_host: FeatureHost) -> Path: + path = resolve_runtime_layout(repo_root=feature_host.repo_root).hypermemory_sqlite_config_path + assert path.is_file(), f"setup did not materialize {path}" + return path + + +def _seed_hypermemory_fact(feature_host: FeatureHost, token: str) -> None: + feature_host.clawops( + "hypermemory", + "--config", + str(_sqlite_config(feature_host)), + "store", + "--json", + "--type", + "fact", + "--scope", + SCOPE, + "--text", + f"The featver orchard code is {token}.", + ).require_ok() + + +def _migrate(feature_host: FeatureHost, token: str) -> tuple[Path, dict[str, object]]: + _seed_hypermemory_fact(feature_host, token) + scratch = feature_host.scratch("memory-pro-migration") + snapshot = scratch / "snapshot.json" + outcome = feature_host.clawops( + "memory", + "migrate-hypermemory-to-pro", + "--config", + str(_sqlite_config(feature_host)), + "--scope", + SCOPE, + "--output", + str(snapshot), + "--report", + str(scratch / "report.json"), + ).require_ok() + return snapshot, cast(dict[str, object], outcome.json()) + + +def _memory_pro_search(feature_host: FeatureHost, query: str) -> CommandOutcome: + return feature_host.openclaw( + "memory-pro", "search", query, "--scope", SCOPE, "--limit", "5", "--json", timeout=300 + ) + + +def test_lancedb_plugin_is_loaded_in_the_memory_slot(feature_host: FeatureHost) -> None: + assert plugin_statuses(feature_host).get(PLUGIN_ID) == "loaded" + plugins = cast(dict[str, object], feature_host.rendered_config()["plugins"]) + assert cast(dict[str, object], plugins["slots"])["memory"] == PLUGIN_ID + + +def test_migrate_hypermemory_exports_memory_pro_snapshot(feature_host: FeatureHost) -> None: + token = f"FV-migrate-{uuid.uuid4().hex[:10]}" + snapshot, summary = _migrate(feature_host, token) + assert summary["dryRun"] is False and summary["scope"] == SCOPE, summary + assert token in snapshot.read_text(encoding="utf-8"), "snapshot is missing the seeded fact" + + +def test_import_pro_snapshot_lands_in_lancedb(feature_host: FeatureHost) -> None: + token = f"FV-import-{uuid.uuid4().hex[:10]}" + snapshot, _ = _migrate(feature_host, token) + imported = feature_host.clawops( + "memory", "import-pro-snapshot", "--input", str(snapshot), "--scope", SCOPE, timeout=600 + ) + assert cast(dict[str, object], imported.json())["ok"] is True, imported.describe() + search = _memory_pro_search(feature_host, token).require_ok() + assert token in search.stdout, f"memory-pro search cannot find the import\n{search.describe()}" + + +def test_verify_pro_parity_against_live_plugin(feature_host: FeatureHost) -> None: + token = f"FV-parity-{uuid.uuid4().hex[:10]}" + snapshot, _ = _migrate(feature_host, token) + feature_host.clawops( + "memory", "import-pro-snapshot", "--input", str(snapshot), "--scope", SCOPE, timeout=600 + ).require_ok() + parity = feature_host.clawops( + "memory", + "verify-pro-parity", + "--config", + str(_sqlite_config(feature_host)), + "--scope", + SCOPE, + "--mode", + "openclaw", + "--query", + token, + timeout=600, + ) + payload = cast(dict[str, object], parity.json()) + assert parity.ok and payload["ok"] is True, parity.describe() + + +def test_smart_extraction_calls_the_extraction_model( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + del gateway + started = time.time() + feature_host.openclaw( + "agent", + "--agent", + "admin", + "--session-id", + f"featver-extract-{uuid.uuid4().hex[:8]}", + "--message", + "Remember that I prefer feature verification reports in tables.", + "--json", + timeout=300, + ).require_ok() + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + requests = [ + request + for request in feature_host.fake_model_requests() + if float(cast(float, request.get("at", 0.0))) >= started + ] + if any( + "extract memories worth long-term preservation" in str(r.get("prompt")) + for r in requests + ): + assert any(r.get("path") == "/v1/embeddings" for r in requests), "no embedding call" + return + time.sleep(_POLL_SECONDS) + raise AssertionError("autoCapture smart extraction never called the extraction model") + + +def _session_key(feature_host: FeatureHost, session_id: str) -> str: + payload = ( + feature_host.openclaw("sessions", "--agent", "admin", "--json", timeout=120) + .require_ok() + .json() + ) + raw_sessions = cast( + list[object], + payload if isinstance(payload, list) else cast(dict[str, object], payload).get("sessions"), + ) + for raw in raw_sessions or []: + entry = cast(dict[str, object], raw) + if entry.get("sessionId") == session_id or session_id in str(entry.get("key", "")): + return str(entry["key"]) + raise AssertionError(f"session {session_id} not found in {json.dumps(payload)[:2000]}") + + +def _reflection_files(feature_host: FeatureHost) -> list[Path]: + agents = cast(dict[str, object], feature_host.rendered_config()["agents"]) + admin = next( + cast(dict[str, object], agent) + for agent in cast(list[object], agents["list"]) + if cast(dict[str, object], agent)["id"] == "admin" + ) + day = datetime.now(tz=UTC).strftime("%Y-%m-%d") + return sorted((Path(str(admin["workspace"])) / "memory" / "reflections" / day).glob("*.md")) + + +def test_reflection_runs_through_the_embedded_runner( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + feature_host.openclaw("config", "set", STRATEGY_KEY, "memoryReflection").require_ok() + try: + gateway.restart() + session_id = f"featver-reflect-{uuid.uuid4().hex[:8]}" + feature_host.openclaw( + "agent", + "--agent", + "admin", + "--session-id", + session_id, + "--message", + "We decided feature verification must use the real runtime.", + "--json", + timeout=300, + ).require_ok() + started = time.time() + params = json.dumps({"key": _session_key(feature_host, session_id), "reason": "new"}) + feature_host.openclaw( + "gateway", "call", "sessions.reset", "--params", params, "--json", timeout=300 + ).require_ok() + deadline = time.monotonic() + REFLECTION_TIMEOUT_SECONDS + while time.monotonic() < deadline: + reflected = [ + path + for path in _reflection_files(feature_host) + if "FEATVER_REFLECTION_INVARIANT" in path.read_text(encoding="utf-8") + ] + if reflected: + break + time.sleep(_POLL_SECONDS) + else: + raise AssertionError("no reflection file was written after sessions.reset") + prompts = [ + str(request.get("prompt")) + for request in feature_host.fake_model_requests() + if float(cast(float, request.get("at", 0.0))) >= started + ] + assert any("MEMORY REFLECTION" in prompt for prompt in prompts), "no reflection prompt" + logs = gateway.log_text() + assert "runner=embedded" in logs, ( + "reflection did not run through extensionAPI runEmbeddedPiAgent " + f"(runner line: {[line for line in logs.splitlines() if 'runner=' in line][-3:]})" + ) + finally: + feature_host.openclaw("config", "set", STRATEGY_KEY, "none") + gateway.restart() diff --git a/tests/suites/e2e/feature_verification/test_model_gateway.py b/tests/suites/e2e/feature_verification/test_model_gateway.py new file mode 100644 index 00000000..0a33401d --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_model_gateway.py @@ -0,0 +1,150 @@ +"""Model auth and gateway features against the real OpenClaw runtime and the fake model server.""" + +from __future__ import annotations + +import time +import uuid +from typing import cast + +from tests.utils.helpers.feature_verification import ( + FAKE_CHAT_MODEL_REF, + REPLY_PREFIX, + FeatureHost, + GatewayController, + gateway_status, +) + +TURN_TIMEOUT_SECONDS = 300 + + +def _agent_ids(feature_host: FeatureHost) -> list[str]: + agents = cast(dict[str, object], feature_host.rendered_config()["agents"]) + return [ + str(cast(dict[str, object], agent)["id"]) for agent in cast(list[object], agents["list"]) + ] + + +def _token(label: str) -> str: + return f"FV-{label}-{uuid.uuid4().hex[:10]}" + + +def _requests_since(feature_host: FeatureHost, started: float) -> list[dict[str, object]]: + return [ + request + for request in feature_host.fake_model_requests() + if float(cast(float, request.get("at", 0.0))) >= started + ] + + +def test_agents_list_matches_rendered_config(feature_host: FeatureHost) -> None: + payload = feature_host.openclaw("agents", "list", "--json", timeout=120).require_ok().json() + listed = [str(cast(dict[str, object], agent)["id"]) for agent in cast(list[object], payload)] + assert sorted(listed) == sorted(_agent_ids(feature_host)), (listed, _agent_ids(feature_host)) + + +def test_model_auth_ensure_applies_env_driven_chain(feature_host: FeatureHost) -> None: + outcome = feature_host.clawops("model-auth", "--asset-root", ".", "ensure", timeout=600) + payload = cast(dict[str, object], outcome.require_ok().json()) + assert payload["ok"] is True and payload["missingAgents"] == [], payload + agents = cast(dict[str, object], feature_host.rendered_config()["agents"]) + defaults = cast(dict[str, object], agents["defaults"]) + assert cast(dict[str, object], defaults["model"])["primary"] == FAKE_CHAT_MODEL_REF + for raw_agent in cast(list[object], agents["list"]): + agent = cast(dict[str, object], raw_agent) + assert cast(dict[str, object], agent["model"])["primary"] == FAKE_CHAT_MODEL_REF, agent + + +def test_model_probe_reaches_the_provider(feature_host: FeatureHost) -> None: + started = time.time() + outcome = feature_host.clawops( + "model-auth", + "--asset-root", + ".", + "check", + "--probe", + "--probe-max-tokens", + "8", + timeout=900, + ) + payload = cast(dict[str, object], outcome.require_ok().json()) + assert payload["ok"] is True and payload["checkedOnly"] is True, payload + probes = [ + request + for request in _requests_since(feature_host, started) + if request.get("path") == "/v1/chat/completions" + ] + assert len(probes) >= len( + _agent_ids(feature_host) + ), f"expected one live probe per agent, fake provider saw {len(probes)}" + + +def test_models_list_reports_available_models(feature_host: FeatureHost) -> None: + for agent_id in _agent_ids(feature_host): + payload = ( + feature_host.openclaw("models", "--agent", agent_id, "list", "--json", timeout=120) + .require_ok() + .json() + ) + models = cast(list[dict[str, object]], cast(dict[str, object], payload)["models"]) + available = [model for model in models if model.get("available") is True] + assert available, f"agent {agent_id} has no available model: {models}" + + +def test_gateway_status_reports_loopback_rpc( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + del gateway + status = gateway_status(feature_host) + rpc = cast(dict[str, object], status["rpc"]) + bind = cast(dict[str, object], status["gateway"]) + assert rpc["ok"] is True, status + assert bind["bindMode"] == "loopback" and bind["bindHost"] == "127.0.0.1", bind + + +def test_gateway_runs_an_agent_turn_through_the_provider( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + del gateway + token = _token("turn") + started = time.time() + outcome = feature_host.openclaw( + "agent", + "--agent", + _agent_ids(feature_host)[0], + "--session-id", + f"featver-{token}", + "--message", + f"Feature verification turn {token}", + "--json", + timeout=TURN_TIMEOUT_SECONDS, + ).require_ok() + turns = [ + request + for request in _requests_since(feature_host, started) + if token in str(request.get("latestUser", "")) + ] + assert turns, f"the gateway never sent the turn to the provider\n{outcome.describe()}" + assert f"{REPLY_PREFIX} {token}" in outcome.stdout, outcome.describe() + + +def test_gateway_binds_only_loopback(feature_host: FeatureHost, gateway: GatewayController) -> None: + del gateway + listeners = feature_host.run(["ss", "-H", "-ltn", "sport = :18789"], timeout=30).require_ok() + addresses = [line.split()[3] for line in listeners.stdout.splitlines() if line.strip()] + assert addresses, listeners.describe() + for address in addresses: + host = address.rsplit(":", 1)[0].strip("[]") + assert host in {"127.0.0.1", "::1"}, f"gateway listens on non-loopback {address}" + + +def test_openclaw_doctor_is_clean(feature_host: FeatureHost) -> None: + feature_host.openclaw("doctor", "--non-interactive", timeout=600).require_ok() + + +def test_security_audit_passes(feature_host: FeatureHost, gateway: GatewayController) -> None: + del gateway + feature_host.openclaw("security", "audit", "--deep", timeout=600).require_ok() + + +def test_secrets_audit_passes(feature_host: FeatureHost) -> None: + feature_host.openclaw("secrets", "audit", "--check", timeout=600).require_ok() diff --git a/tests/suites/e2e/feature_verification/test_operator_tools.py b/tests/suites/e2e/feature_verification/test_operator_tools.py new file mode 100644 index 00000000..0bddee16 --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_operator_tools.py @@ -0,0 +1,1307 @@ +"""CLI-only operator tools: governance, journaling, harness, intake, repo, and render surfaces. + +Every test drives the real ``clawops`` entrypoint against scratch inputs and asserts on the +command's own observable outputs: exit codes, JSON payloads, files it writes, journal rows read +back through the CLI, and HTTP requests received by a loopback listener. +""" + +from __future__ import annotations + +import contextlib +import copy +import json +import os +import plistlib +import re +import shutil +import threading +from collections.abc import Generator +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Final, cast + +import yaml + +from clawops.memory_profiles import MANAGED_MEMORY_PROFILE_IDS, MEMORY_PROFILES +from tests.utils.helpers.feature_verification import CommandOutcome, FeatureHost + +type JsonObject = dict[str, object] + +SHIPPED_POLICY: Final[Path] = Path("platform/configs/policy/policy.yaml") +HARNESS_DIR: Final[Path] = Path("platform/configs/harness") +HARNESS_SUITES: Final[tuple[str, ...]] = ("policy_regressions.yaml", "security_regressions.yaml") +CODE_REVIEW_WORKFLOW: Final[Path] = Path("platform/configs/workflows/code_review.yaml") +CODEBASE_CONTEXT_CONFIG: Final[Path] = Path("platform/configs/context/codebase.yaml") +ALLOWLIST_EXAMPLE: Final[Path] = Path("platform/configs/source-allowlists.example.yaml") +PLACEHOLDER_PATTERN: Final[re.Pattern[str]] = re.compile(r"__[A-Z][A-Z0-9_]*__") +SHA1_PATTERN: Final[re.Pattern[str]] = re.compile(r"^[0-9a-f]{40}$") +SHA256_DIGEST_PATTERN: Final[re.Pattern[str]] = re.compile(r"^sha256:[0-9a-f]{64}$") +PNG_SIGNATURE: Final[bytes] = b"\x89PNG\r\n\x1a\n" +LOOPBACK_ENV: Final[dict[str, str]] = {"NO_PROXY": "127.0.0.1,localhost", "no_proxy": "127.0.0.1"} +GIT_CONFIG_ARGS: Final[tuple[str, ...]] = ( + "-c", + "user.name=Feature Verification", + "-c", + "user.email=featver@example.invalid", + "-c", + "commit.gpgsign=false", + "-c", + "core.hooksPath=/dev/null", + "-c", + "init.defaultBranch=main", +) +CHANNEL_CHECKS: Final[frozenset[str]] = frozenset( + { + "telegram-dm-policy", + "telegram-group-policy", + "telegram-allowlist", + "whatsapp-dm-policy", + "whatsapp-group-policy", + "default-group-policy", + "channel-docs-pairing", + "channel-docs-allowlists", + "telegram-guidance-pairing", + "whatsapp-guidance-pairing", + "rendered-telegram-allowlist", + "rendered-whatsapp-allowlist", + } +) + + +def _as_object(value: object) -> JsonObject: + assert isinstance(value, dict), f"expected a JSON object, got {value!r}" + return cast(JsonObject, value) + + +def _as_objects(value: object) -> list[JsonObject]: + assert isinstance(value, list), f"expected a JSON array, got {value!r}" + return [_as_object(item) for item in cast(list[object], value)] + + +def _ok_object(outcome: CommandOutcome) -> JsonObject: + return _as_object(outcome.require_ok().json()) + + +def _read_json(path: Path) -> object: + assert path.is_file(), f"{path} was not written" + return cast(object, json.loads(path.read_text(encoding="utf-8"))) + + +def _read_jsonl(path: Path) -> list[JsonObject]: + assert path.is_file(), f"{path} was not written" + lines = path.read_text(encoding="utf-8").splitlines() + return [_as_object(json.loads(line)) for line in lines if line.strip()] + + +def _write_json(path: Path, payload: object) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2), encoding="utf-8") + return path + + +def _write_yaml(path: Path, payload: object) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + return path + + +def _git(feature_host: FeatureHost, repo: Path, *arguments: str) -> CommandOutcome: + return feature_host.run( + ["git", *GIT_CONFIG_ARGS, "-C", str(repo), *arguments], timeout=60 + ).require_ok() + + +def _init_git_repo(feature_host: FeatureHost, repo: Path) -> None: + repo.mkdir(parents=True, exist_ok=True) + readme = repo / "README.md" + if not readme.exists(): + readme.write_text("# feature verification fixture\n", encoding="utf-8") + _git(feature_host, repo, "init", "-q") + _git(feature_host, repo, "add", "-A") + _git(feature_host, repo, "commit", "-q", "-m", "initial commit") + + +def _evaluate_policy(feature_host: FeatureHost, policy: Path, payload: Path) -> JsonObject: + return _ok_object( + feature_host.clawops("policy", "--policy", str(policy), "--input", str(payload)) + ) + + +def _journal_begin( + feature_host: FeatureHost, db: Path, *, target: str, payload: Path +) -> JsonObject: + return _ok_object( + feature_host.clawops( + "op-journal", + "begin", + "--db", + str(db), + "--scope", + "featver", + "--kind", + "webhook_post", + "--trust-zone", + "automation", + "--target", + target, + "--payload-file", + str(payload), + ) + ) + + +def _journal_transition( + feature_host: FeatureHost, db: Path, op_id: str, status: str +) -> CommandOutcome: + return feature_host.clawops( + "op-journal", "transition", "--db", str(db), "--op-id", op_id, "--status", status + ) + + +def _list_stuck_ids(feature_host: FeatureHost, db: Path, older_than_ms: int) -> set[str]: + stuck = feature_host.clawops( + "op-journal", "list-stuck", "--db", str(db), "--older-than-ms", str(older_than_ms) + ).require_ok() + return {str(row["op_id"]) for row in _as_objects(stuck.json())} + + +def _approvals( + feature_host: FeatureHost, command: str, db: Path, *arguments: str +) -> CommandOutcome: + return feature_host.clawops("approvals", command, "--db", str(db), *arguments) + + +def _run_harness_suite(feature_host: FeatureHost, suite: str, output: Path) -> list[JsonObject]: + outcome = feature_host.clawops( + "harness", "--suite", str(HARNESS_DIR / suite), "--output", str(output), timeout=900 + ).require_ok() + results = _read_jsonl(output) + assert f"passed={len(results)} total={len(results)}" in outcome.stdout, outcome.describe() + return results + + +def _suite_case_ids(feature_host: FeatureHost, suite: str) -> list[str]: + document = _as_object( + yaml.safe_load((feature_host.repo_root / HARNESS_DIR / suite).read_text("utf-8")) + ) + return [str(case["id"]) for case in _as_objects(document["cases"])] + + +@dataclass(frozen=True, slots=True) +class _ReceivedRequest: + method: str + path: str + content_type: str + body: bytes + + +class _RecordingServer(ThreadingHTTPServer): + """Loopback HTTP endpoint on an ephemeral port that records every request it answers.""" + + def __init__(self) -> None: + super().__init__(("127.0.0.1", 0), _RecordingHandler) + self._lock = threading.Lock() + self._received: list[_ReceivedRequest] = [] + + def url(self, path: str) -> str: + return f"http://127.0.0.1:{self.server_port}{path}" + + def record(self, request: _ReceivedRequest) -> None: + with self._lock: + self._received.append(request) + + def received(self) -> list[_ReceivedRequest]: + with self._lock: + return list(self._received) + + +class _RecordingHandler(BaseHTTPRequestHandler): + def do_POST(self) -> None: + body = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + cast(_RecordingServer, self.server).record( + _ReceivedRequest( + method="POST", + path=self.path, + content_type=self.headers.get("Content-Type", ""), + body=body, + ) + ) + reply = b'{"received": true}' + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(reply))) + self.end_headers() + self.wfile.write(reply) + + def log_message(self, format: str, *args: object) -> None: + del format, args + + +@contextlib.contextmanager +def _recording_server() -> Generator[_RecordingServer, None, None]: + server = _RecordingServer() + thread = threading.Thread(target=server.serve_forever, name="featver-webhook", daemon=True) + thread.start() + try: + yield server + finally: + server.shutdown() + server.server_close() + thread.join(timeout=10) + + +def test_policy_evaluates_shipped_rules(feature_host: FeatureHost) -> None: + fixtures = HARNESS_DIR / "fixtures" + approval = _evaluate_policy( + feature_host, SHIPPED_POLICY, fixtures / "policy_github_approval.json" + ) + assert approval["decision"] == "require_approval", approval + assert approval["review_policy_id"] == "approval.actions.github.comment.create", approval + assert approval["review_mode"] == "manual", approval + assert approval["reasons"] == ["approval required by approval matrix"], approval + + webhook = _evaluate_policy(feature_host, SHIPPED_POLICY, fixtures / "policy_webhook_deny.json") + assert webhook["decision"] == "deny", webhook + assert webhook["reasons"] == ["target:webhook_url:not allowlisted"], webhook + assert "review_mode" not in webhook, webhook + + browser = _evaluate_policy(feature_host, SHIPPED_POLICY, fixtures / "policy_browser_deny.json") + assert browser["decision"] == "deny", browser + assert browser["reasons"] == ["zone:coder:not in allowlist"], browser + + scratch = feature_host.scratch("policy") + prod_webhook = _evaluate_policy( + feature_host, + SHIPPED_POLICY, + _write_json( + scratch / "prod-webhook.json", + { + "trust_zone": "automation", + "action": "webhook.post", + "category": "external_write", + "target_kind": "webhook_url", + "target": "https://example.internal/hooks/deploy", + "environment": "prod", + }, + ), + ) + assert prod_webhook["decision"] == "require_approval", prod_webhook + assert prod_webhook["matched_rules"] == ["approval-for-prod-webhook"], prod_webhook + assert prod_webhook["review_reason"] == "production webhooks always require approval" + + browser_rule = _evaluate_policy( + feature_host, + SHIPPED_POLICY, + _write_json( + scratch / "browser-main-host.json", + { + "trust_zone": "admin", + "action": "browser.navigate", + "host_boundary": "main-control-plane", + }, + ), + ) + assert browser_rule["decision"] == "deny", browser_rule + assert browser_rule["matched_rules"] == ["deny-browser-on-main-host"], browser_rule + + read_only = _write_json( + scratch / "reader-search.json", + {"trust_zone": "reader", "action": "memory.search", "category": "read_only"}, + ) + default_deny = _evaluate_policy(feature_host, SHIPPED_POLICY, read_only) + assert default_deny == {"decision": "deny", "matched_rules": [], "reasons": ["default"]} + + # The shipped policy has no allow rule, so the allow path needs an operator default. + shipped = _as_object( + yaml.safe_load((feature_host.repo_root / SHIPPED_POLICY).read_text("utf-8")) + ) + permissive = _write_yaml( + scratch / "policy.yaml", {**shipped, "defaults": {"decision": "allow"}} + ) + allowed = _evaluate_policy(feature_host, permissive, read_only) + assert allowed == {"decision": "allow", "matched_rules": [], "reasons": ["default"]} + still_gated = _evaluate_policy( + feature_host, permissive, fixtures / "policy_github_approval.json" + ) + assert still_gated["decision"] == "require_approval", still_gated + + +def test_op_journal_tracks_operation_lifecycle(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("op-journal") + db = scratch / "journal.sqlite" + feature_host.clawops("op-journal", "init", "--db", str(db)).require_ok() + assert db.is_file(), f"op-journal init did not create {db}" + payload = _write_json(scratch / "payload.json", {"message": "deploy finished"}) + target = "https://example.internal/hooks/deploy" + + proposed = _journal_begin(feature_host, db, target=target, payload=payload) + op_id = str(proposed["op_id"]) + assert proposed["status"] == "proposed" and proposed["attempt"] == 0, proposed + assert json.loads(str(proposed["inputs_json"])) == {"message": "deploy finished"}, proposed + replayed = _journal_begin(feature_host, db, target=target, payload=payload) + assert replayed["op_id"] == op_id, "begin with identical inputs must be idempotent" + + pending = _ok_object(_journal_transition(feature_host, db, op_id, "pending_approval")) + assert pending["status"] == "pending_approval", pending + approved = _ok_object( + feature_host.clawops( + "op-journal", + "approve", + "--db", + str(db), + "--op-id", + op_id, + "--approved-by", + "operator", + "--note", + "change window open", + ) + ) + assert approved["status"] == "approved" and approved["approved_by"] == "operator", approved + assert approved["approval_note"] == "change window open", approved + running = _ok_object(_journal_transition(feature_host, db, op_id, "running")) + assert running["status"] == "running" and running["attempt"] == 1, running + + assert op_id in _list_stuck_ids(feature_host, db, 0), "running op must be listed as stuck" + assert op_id not in _list_stuck_ids(feature_host, db, 3_600_000), "fresh op is not stale" + + succeeded = _ok_object(_journal_transition(feature_host, db, op_id, "succeeded")) + assert succeeded["status"] == "succeeded", succeeded + assert op_id not in _list_stuck_ids(feature_host, db, 0), "terminal ops are never stuck" + + invalid = _journal_transition(feature_host, db, op_id, "running") + assert not invalid.ok, invalid.describe() + assert ( + "invalid operation transition: succeeded -> running" in invalid.stderr + ), invalid.describe() + + +def test_approvals_review_queue_resolves_operations(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("approvals") + db = scratch / "journal.sqlite" + payload = _write_json(scratch / "payload.json", {"message": "review me"}) + op_ids: list[str] = [] + for name in ("approve", "reject", "delegate"): + op = _journal_begin( + feature_host, db, target=f"https://example.internal/hooks/{name}", payload=payload + ) + _journal_transition(feature_host, db, str(op["op_id"]), "pending_approval").require_ok() + op_ids.append(str(op["op_id"])) + approve_id, reject_id, delegate_id = op_ids + + queue_file = scratch / "queue.json" + queue_outcome = _approvals(feature_host, "queue", db, "--output", str(queue_file)).require_ok() + queue = _as_objects(queue_outcome.json()) + assert [str(row["op_id"]) for row in queue] == op_ids, queue + assert all(row["status"] == "pending_approval" for row in queue), queue + assert _read_json(queue_file) == queue, "queue --output must match the printed queue" + + shown = _ok_object(_approvals(feature_host, "show", db, "--op-id", approve_id)) + assert shown["op_id"] == approve_id and shown["status"] == "pending_approval", shown + + approved = _ok_object( + _approvals(feature_host, "approve", db, "--op-id", approve_id, "--approved-by", "alice") + ) + assert approved["status"] == "approved" and approved["review_status"] == "approved", approved + assert approved["approved_by"] == "alice" and approved["reviewed_by"] == "alice", approved + + rejected = _ok_object( + _approvals( + feature_host, + "reject", + db, + "--op-id", + reject_id, + "--reviewed-by", + "bob", + "--note", + "target out of scope", + ) + ) + assert rejected["status"] == "rejected" and rejected["review_status"] == "rejected", rejected + assert rejected["last_error"] == "target out of scope", rejected + + delegated = _ok_object( + _approvals( + feature_host, + "delegate", + db, + "--op-id", + delegate_id, + "--reviewed-by", + "carol", + "--to", + "reviewer-acp-claude", + ) + ) + assert delegated["status"] == "pending_approval", delegated + assert delegated["review_status"] == "delegated", delegated + assert delegated["review_target"] == "reviewer-acp-claude", delegated + + review_payload = _write_json(scratch / "review.json", {"summary": "change is safe"}) + review_artifact = scratch / "review.md" + review_artifact.write_text("LGTM\n", encoding="utf-8") + ingested = _ok_object( + _approvals( + feature_host, + "ingest-review", + db, + "--op-id", + delegate_id, + "--reviewed-by", + "reviewer-acp-claude", + "--decision", + "allow", + "--payload-file", + str(review_payload), + "--artifact-path", + str(review_artifact), + ) + ) + assert ingested["status"] == "approved", ingested + assert ingested["approved_by"] == "reviewer-acp-claude", ingested + assert ingested["review_artifact_path"] == str(review_artifact), ingested + assert json.loads(str(ingested["review_payload_json"])) == { + "delegate_to": "reviewer-acp-claude", + "review_decision": "allow", + "summary": "change is safe", + }, ingested + + final_queue = _approvals(feature_host, "queue", db).require_ok() + assert _as_objects(final_queue.json()) == [], final_queue.describe() + reapprove = _approvals( + feature_host, "approve", db, "--op-id", reject_id, "--approved-by", "alice" + ) + assert not reapprove.ok, "a rejected operation must not be approvable" + + +def test_harness_regression_suites_pass(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("harness") + for suite in HARNESS_SUITES: + results = _run_harness_suite(feature_host, suite, scratch / f"{Path(suite).stem}.jsonl") + assert [str(item["id"]) for item in results] == _suite_case_ids(feature_host, suite) + failed = { + str(item["id"]): item["failures"] for item in results if item["passed"] is not True + } + assert not failed, json.dumps(failed, indent=2) + assert all(item["failures"] == [] for item in results), results + + +def test_charts_render_harness_results(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("charts") + results_path = scratch / "policy.jsonl" + results = _run_harness_suite(feature_host, "policy_regressions.yaml", results_path) + chart = scratch / "nested" / "harness.png" + feature_host.clawops( + "charts", "--input", str(results_path), "--output", str(chart) + ).require_ok() + assert chart.is_file(), f"charts did not write {chart}" + data = chart.read_bytes() + assert data.startswith(PNG_SIGNATURE), "chart output is not a PNG" + assert len(data) > 4096, f"chart is suspiciously small ({len(data)} bytes)" + width, height = int.from_bytes(data[16:20], "big"), int.from_bytes(data[20:24], "big") + expected_ratio = max(6.0, len(results) * 1.2) / 4.0 + assert abs(width / height - expected_ratio) < 0.02, (width, height, expected_ratio) + + +def test_merge_json_deep_merges_overlays(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("merge-json") + base = _write_json( + scratch / "base.json", + { + "gateway": {"port": 18789, "bind": "loopback", "auth": {"mode": "token"}}, + "agents": {"list": [{"id": "admin"}, {"id": "reader"}]}, + "keep": True, + }, + ) + first = scratch / "10-gateway.json5" + first.write_text( + "{\n" + " // operator overlay: json5 comments and trailing commas are accepted\n" + " gateway: { port: 19001, auth: { token: 'from-overlay' } },\n" + " agents: { list: [{ id: 'coder' }] },\n" + "}\n", + encoding="utf-8", + ) + second = _write_json(scratch / "20-extra.json", {"gateway": {"bind": "lan"}, "extra": {"n": 1}}) + output = scratch / "merged.json" + feature_host.clawops( + "merge-json", + "--base", + str(base), + "--overlay", + str(first), + str(second), + "--output", + str(output), + ).require_ok() + assert _read_json(output) == { + "gateway": { + "port": 19001, + "bind": "lan", + "auth": {"mode": "token", "token": "from-overlay"}, + }, + "agents": {"list": [{"id": "coder"}]}, + "keep": True, + "extra": {"n": 1}, + } + + duplicate = scratch / "duplicate.json5" + duplicate.write_text('{"gateway": {"port": 1}, "gateway": {"port": 2}}\n', encoding="utf-8") + rejected_output = scratch / "rejected.json" + rejected = feature_host.clawops( + "merge-json", + "--base", + str(base), + "--overlay", + str(duplicate), + "--output", + str(rejected_output), + ) + assert not rejected.ok, "overlays with duplicate keys must be rejected" + assert not rejected_output.exists(), "a rejected merge must not write output" + + +def test_allowlists_render_channel_fragment(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("allowlists") + output = scratch / "channels.json" + outcome = feature_host.clawops( + "allowlists", "--source", str(ALLOWLIST_EXAMPLE), "--output", str(output) + ).require_ok() + expected = { + "channels": { + "telegram": {"allowFrom": ["tg:12345678"]}, + "whatsapp": {"allowFrom": ["+5511999999999"]}, + "modelByChannel": { + "telegram": {"12345678": "messaging"}, + "whatsapp": {"+5511999999999": "messaging"}, + }, + } + } + assert outcome.json() == expected, outcome.describe() + assert _read_json(output) == expected + + invalid = _write_json( + scratch / "invalid.json", {"telegram_allow": ["telegram:42"], "whatsapp_allow": ["12345"]} + ) + rejected = feature_host.clawops( + "allowlists", "--source", str(invalid), "--output", str(scratch / "rejected.json") + ) + assert ( + not rejected.ok and "invalid WhatsApp E.164: 12345" in rejected.stderr + ), rejected.describe() + + +def test_skills_intake_moves_bundle_through_stages(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("skills") + source = scratch / "incoming" / "weather-skill" + (source / "scripts").mkdir(parents=True) + (source / "SKILL.md").write_text("# Weather\nSummarize the forecast.\n", encoding="utf-8") + script = "import subprocess\nsubprocess.run(['curl', 'https://example.invalid'])\n" + (source / "scripts" / "fetch.py").write_text(script, encoding="utf-8") + skills_root = scratch / "skills" + resolved_root = skills_root.resolve() + + scan_report = scratch / "scan.json" + scanned = _ok_object( + feature_host.clawops( + "skills", "scan", "--source", str(source), "--report", str(scan_report) + ) + ) + assert scanned["status"] == "scanned", scanned + assert scanned["bundlePath"] == source.resolve().as_posix(), scanned + assert scanned["findingCounts"] == {"critical": 1, "high": 1}, scanned + findings = _as_objects(scanned["findings"]) + assert {(str(item["rule"]), item["line"]) for item in findings} == { + ("shell exfil", 2), + ("python subprocess", 2), + }, findings + assert {str(item["path"]) for item in _as_objects(scanned["files"])} == { + "SKILL.md", + "scripts/fetch.py", + }, scanned + assert _read_json(scan_report) == scanned + + manifest = scratch / "manifest.json" + quarantined = _ok_object( + feature_host.clawops( + "skills", + "quarantine", + "--source", + str(source), + "--report", + str(manifest), + "--quarantine-root", + str(skills_root / "quarantine"), + ) + ) + quarantine_copy = resolved_root / "quarantine" / "weather-skill" + assert quarantined["status"] == "quarantined", quarantined + assert quarantined["bundlePath"] == quarantine_copy.as_posix(), quarantined + assert (quarantine_copy / "scripts" / "fetch.py").read_text(encoding="utf-8") == script + assert (source / "SKILL.md").is_file(), "quarantine must copy, not move, the source" + + premature = feature_host.clawops( + "skills", + "promote", + "--manifest", + str(manifest), + "--skills-root", + str(skills_root), + "--stage", + "approved", + ) + assert not premature.ok, "quarantined bundles must be reviewed before approval" + assert "cannot transition quarantined -> approved" in premature.stderr, premature.describe() + + transitions = (("promote", "reviewed"), ("promote", "approved"), ("demote", "reviewed")) + previous = quarantine_copy + for command, stage in transitions: + staged = _ok_object( + feature_host.clawops( + "skills", + command, + "--manifest", + str(manifest), + "--skills-root", + str(skills_root), + "--stage", + stage, + ) + ) + destination = resolved_root / stage / "weather-skill" + assert staged["status"] == stage, staged + assert staged["bundlePath"] == destination.as_posix(), staged + assert (destination / "SKILL.md").is_file(), f"{destination} was not populated" + assert not previous.exists(), f"{command} left the bundle behind at {previous}" + previous = destination + + history = _as_objects(_as_object(_read_json(manifest))["stageHistory"]) + assert [(item["status"], item["reason"]) for item in history] == [ + ("quarantined", "initial_scan"), + ("reviewed", "promote"), + ("approved", "promote"), + ("reviewed", "demote"), + ], history + + +def test_worktree_lifecycle_creates_lists_and_prunes(feature_host: FeatureHost) -> None: + root = feature_host.scratch("worktree") + _init_git_repo(feature_host, root / "repo" / "upstream") + managed_path = (root / "repo" / "worktrees" / "feature-x").resolve() + + initial = _ok_object(feature_host.clawops("worktree", "--repo-root", str(root), "list")) + initial_entries = _as_objects(initial["worktrees"]) + assert len(initial_entries) == 1 and initial_entries[0]["managed"] is False, initial + + created = _ok_object( + feature_host.clawops("worktree", "--repo-root", str(root), "new", "--branch", "feature-x") + ) + entry = _as_object(created["created"]) + assert created["branch"] == "feature-x", created + assert Path(str(entry["path"])).resolve() == managed_path, entry + assert entry["branch"] == "feature-x" and entry["managed"] is True, entry + assert (managed_path / "README.md").is_file(), "worktree was not checked out" + + duplicate = feature_host.clawops( + "worktree", "--repo-root", str(root), "new", "--branch", "feature-x" + ) + assert duplicate.returncode == 1, duplicate.describe() + assert _as_object(duplicate.json())["ok"] is False, duplicate.describe() + + listed = _as_objects( + _ok_object(feature_host.clawops("worktree", "--repo-root", str(root), "list"))["worktrees"] + ) + managed = [item for item in listed if item["managed"] is True] + assert len(listed) == 2 and [item["branch"] for item in managed] == ["feature-x"], listed + + shutil.rmtree(managed_path) + pruned = _ok_object(feature_host.clawops("worktree", "--repo-root", str(root), "prune")) + assert pruned["beforeCount"] == 2 and pruned["afterCount"] == 1, pruned + remaining = _as_objects(pruned["worktrees"]) + assert all(item["managed"] is False for item in remaining), remaining + + +def test_repo_doctor_accepts_valid_layout(feature_host: FeatureHost) -> None: + root = feature_host.scratch("repo-doctor") + _init_git_repo(feature_host, root / "repo" / "upstream") + (root / "repo" / "worktrees").mkdir() + + report = _ok_object(feature_host.clawops("repo", "--repo-root", str(root), "doctor")) + checks = _as_object(report["checks"]) + assert report["ok"] is True and "errors" not in report, report + assert checks["gitAvailable"] is True and checks["upstreamExists"] is True, checks + assert checks["worktreesRootExists"] is True and checks["managedWorktrees"] == 0, checks + assert checks["repoRoot"] == root.resolve().as_posix(), checks + assert len(_as_objects(report["worktrees"])) == 1, report + + missing = feature_host.clawops( + "repo", "--repo-root", str(root), "doctor", "--branch", "missing-branch" + ) + missing_report = _as_object(missing.json()) + assert missing.returncode == 1 and missing_report["ok"] is False, missing.describe() + assert any( + "missing managed worktree for branch missing-branch" in str(error) + for error in cast(list[object], missing_report["errors"]) + ), missing_report + + +def _code_review_base(feature_host: FeatureHost, name: str) -> Path: + base = feature_host.scratch(name) + for relative in (SHIPPED_POLICY, CODEBASE_CONTEXT_CONFIG): + (base / relative).parent.mkdir(parents=True, exist_ok=True) + shutil.copy(feature_host.repo_root / relative, base / relative) + notes = base / "docs" / "auth-changes.md" + notes.parent.mkdir(parents=True) + # Verbatim query phrase: multi-term FTS hits are dropped by the contentless files_fts JOIN. + notes.write_text( + "# Auth notes\n\nPlease review recent auth changes before merge.\n", encoding="utf-8" + ) + return base + + +def _workflow_steps(outcome: CommandOutcome) -> list[tuple[str, str, str]]: + rows = [line.split("\t") for line in outcome.stdout.splitlines() if "\t" in line] + assert all(len(row) == 3 for row in rows), outcome.describe() + return [(row[0], row[1], row[2]) for row in rows] + + +def test_workflow_code_review_records_step_results(feature_host: FeatureHost) -> None: + workflow = _as_object( + yaml.safe_load((feature_host.repo_root / CODE_REVIEW_WORKFLOW).read_text(encoding="utf-8")) + ) + step_names = [str(step["name"]) for step in _as_objects(workflow["steps"])] + assert step_names == ["init journal", "policy preflight", "dry context pack"], step_names + + base = _code_review_base(feature_host, "code-review") + state_dir = feature_host.scratch("code-review-state") + run = feature_host.clawops( + "workflow", + "--workflow", + str(CODE_REVIEW_WORKFLOW), + "--base-dir", + str(base), + env={"STRONGCLAW_STATE_DIR": str(state_dir)}, + timeout=600, + ).require_ok() + steps = _workflow_steps(run) + assert [(name, status) for name, status, _ in steps] == [ + (name, "ok") for name in step_names + ], run.describe() + journal_message, decision_message, pack_message = (message for _, _, message in steps) + journal = base.resolve() / ".clawops" / "op_journal.sqlite" + assert journal_message == f"journal={journal}", run.describe() + assert journal.is_file(), f"journal_init did not create {journal}" + assert decision_message == "require_approval", run.describe() + pack_path = Path(pack_message.removeprefix("context packed -> ")) + assert pack_message.startswith("context packed -> "), run.describe() + assert pack_path.resolve().is_relative_to(state_dir.resolve()), pack_path + pack = pack_path.read_text(encoding="utf-8") + assert "- query: review recent auth changes" in pack, pack + assert "### docs/auth-changes.md" in pack, pack + + dry_base = _code_review_base(feature_host, "code-review-dry") + dry_state = feature_host.scratch("code-review-dry-state") + dry_run = feature_host.clawops( + "workflow", + "--workflow", + str(CODE_REVIEW_WORKFLOW), + "--base-dir", + str(dry_base), + "--dry-run", + env={"STRONGCLAW_STATE_DIR": str(dry_state)}, + ).require_ok() + dry_journal = dry_base.resolve() / ".clawops" / "op_journal.sqlite" + assert _workflow_steps(dry_run) == [ + ("init journal", "ok", f"journal={dry_journal}"), + ("policy preflight", "ok", "require_approval"), + ("dry context pack", "ok", "dry-run context pack"), + ], dry_run.describe() + assert not dry_journal.exists(), "--dry-run must not create the journal" + assert not any(dry_state.iterdir()), "--dry-run must not write context packs" + + +def test_webhook_wrapper_posts_once_after_approval(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("webhook") + db = scratch / "journal.sqlite" + body = {"event": "deploy.finished", "build": 42} + payload = _write_json(scratch / "payload.json", body) + with _recording_server() as listener: + url = listener.url("/hooks/deploy") + policy = _write_yaml( + scratch / "policy.yaml", + { + "defaults": {"decision": "deny"}, + "zones": { + "automation": { + "allow_actions": ["webhook.post"], + "allow_categories": ["external_write"], + } + }, + "approval": {"require_for_actions": ["webhook.post"]}, + "allowlists": {"webhook_url": [url]}, + }, + ) + + def wrapper(*arguments: str) -> CommandOutcome: + return feature_host.clawops( + "wrapper", + "webhook", + "--db", + str(db), + "--policy", + str(policy), + *arguments, + env=LOOPBACK_ENV, + timeout=120, + ) + + def propose(target: str) -> CommandOutcome: + return wrapper( + "--scope", + "featver", + "--trust-zone", + "automation", + "--url", + target, + "--payload-file", + str(payload), + ) + + proposal = _ok_object(propose(url)) + op_id = str(proposal["op_id"]) + assert proposal["status"] == "pending_approval", proposal + assert proposal["accepted"] is True and proposal["executed"] is False, proposal + assert _as_object(proposal["decision"])["decision"] == "require_approval", proposal + assert Path(str(proposal["review_artifact_path"])).is_file(), proposal + assert listener.received() == [], "nothing may be sent before approval" + + early = _ok_object(wrapper("--op-id", op_id, "--execute-approved")) + assert early["status"] == "pending_approval" and early["executed"] is False, early + assert listener.received() == [], "an unapproved op must not be sent" + + _approvals( + feature_host, "approve", db, "--op-id", op_id, "--approved-by", "ops" + ).require_ok() + executed = _ok_object(wrapper("--op-id", op_id, "--execute-approved")) + assert executed["ok"] is True and executed["executed"] is True, executed + assert executed["status"] == "succeeded" and executed["status_code"] == 200, executed + assert executed["request_method"] == "POST" and executed["request_url"] == url, executed + + replay = _ok_object(wrapper("--op-id", op_id, "--execute-approved")) + assert replay["status"] == "succeeded" and replay["status_code"] == 200, replay + + denied = propose(listener.url("/hooks/not-allowlisted")) + denied_result = _as_object(denied.json()) + assert denied.returncode == 1 and denied_result["accepted"] is False, denied.describe() + assert denied_result["status"] == "failed", denied_result + + received = listener.received() + assert len(received) == 1, f"listener received {len(received)} requests, expected exactly 1" + request = received[0] + assert (request.method, request.path) == ("POST", "/hooks/deploy"), request + assert request.content_type == "application/json", request + assert json.loads(request.body) == body, request + journal_row = _ok_object(_approvals(feature_host, "show", db, "--op-id", op_id)) + assert journal_row["status"] == "succeeded" and journal_row["attempt"] == 1, journal_row + assert journal_row["result_status_code"] == 200, journal_row + assert journal_row["result_body_excerpt"] == '{"received": true}', journal_row + + +def test_github_wrapper_dry_run_plans_without_executing(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("github") + db = scratch / "journal.sqlite" + repo = "your-org/openclaw-platform" + target = f"github://{repo}/issues/7" + common = ("--db", str(db), "--scope", "featver", "--trust-zone", "coder", "--repo", repo) + + comment = _ok_object( + feature_host.clawops( + "wrapper", + "github", + *common, + "--policy", + str(SHIPPED_POLICY), + "--operation", + "comment", + "--issue-number", + "7", + "--body", + "Feature verification comment", + "--dry-run", + ) + ) + assert comment["status"] == "pending_approval", comment + assert comment["accepted"] is True and comment["executed"] is False, comment + assert _as_object(comment["decision"])["decision"] == "require_approval", comment + + allow_policy = _write_yaml( + scratch / "policy.yaml", + { + "defaults": {"decision": "allow"}, + "zones": {"coder": {"allow_actions": ["github.issue.labels.add"]}}, + "allowlists": {"github_repo": [repo]}, + }, + ) + labels = _ok_object( + feature_host.clawops( + "wrapper", + "github", + *common, + "--policy", + str(allow_policy), + "--operation", + "labels", + "--issue-number", + "7", + "--label", + "needs-review", + "--label", + "featver", + "--dry-run", + ) + ) + assert labels["status"] == "approved" and labels["dry_run"] is True, labels + assert labels["executed"] is False and "status_code" not in labels, labels + + expected_inputs = { + str(comment["op_id"]): ("github_comment", {"body": "Feature verification comment"}), + str(labels["op_id"]): ("github_issue_labels", {"labels": ["needs-review", "featver"]}), + } + for op_id, (kind, inputs) in expected_inputs.items(): + row = _ok_object(_approvals(feature_host, "show", db, "--op-id", op_id)) + assert row["kind"] == kind and row["normalized_target"] == target, row + assert json.loads(str(row["inputs_json"])) == inputs, row + assert row["attempt"] == 0 and row["result_status_code"] is None, row + assert row["result_request_url"] is None and row["result_ok"] is None, row + queued = _as_objects(_approvals(feature_host, "queue", db).require_ok().json()) + assert [row["op_id"] for row in queued] == [comment["op_id"]], queued + + +def test_supply_chain_inventory_lists_pins(feature_host: FeatureHost) -> None: + inventory = _ok_object(feature_host.clawops("supply-chain", "inventory")) + assert inventory["ok"] is True, inventory + assert inventory["repoRoot"] == feature_host.repo_root.resolve().as_posix(), inventory + actions = _as_objects(inventory["workflowActions"]) + images = _as_objects(inventory["composeImages"]) + assert actions and images, inventory + file_lines: dict[str, list[str]] = {} + + def line_at(relative: object, line: object) -> str: + key = str(relative) + if key not in file_lines: + file_lines[key] = ( + (feature_host.repo_root / key).read_text(encoding="utf-8").splitlines() + ) + return file_lines[key][cast(int, line) - 1] + + for pin in actions: + assert str(pin["path"]).startswith(".github/workflows/"), pin + assert SHA1_PATTERN.fullmatch(str(pin["ref"])), f"action not pinned to a commit: {pin}" + assert f"{pin['action']}@{pin['ref']}" in line_at(pin["path"], pin["line"]), pin + for pin in images: + assert str(pin["path"]).startswith("platform/compose/"), pin + assert SHA256_DIGEST_PATTERN.fullmatch(str(pin["digest"])), pin + assert f"{pin['image']}@{pin['digest']}" in line_at(pin["path"], pin["line"]), pin + + +def test_supply_chain_propose_refresh_dry_run_plans_updates(feature_host: FeatureHost) -> None: + root = feature_host.scratch("supply-chain") + old_digest = f"sha256:{'1' * 64}" + new_digest = f"sha256:{'2' * 64}" + workflow = root / ".github" / "workflows" / "ci.yml" + workflow.parent.mkdir(parents=True) + # No version-tag comment, so the action refresh resolves nothing over the network. + workflow.write_text( + f"jobs:\n build:\n steps:\n - uses: actions/checkout@{'a' * 40}\n", + encoding="utf-8", + ) + compose = root / "platform" / "compose" / "docker-compose.yaml" + compose.parent.mkdir(parents=True) + compose_text = f"services:\n db:\n image: postgres:16-alpine@{old_digest}\n" + compose.write_text(compose_text, encoding="utf-8") + _init_git_repo(feature_host, root) + + shim_dir = feature_host.scratch("supply-chain-shim") + shim_log = shim_dir / "docker.log" + docker_shim = shim_dir / "docker" + docker_shim.write_text( + "#!/bin/sh\n" + f"printf '%s\\n' \"$*\" >> '{shim_log}'\n" + f"printf 'Name: %s\\nDigest: {new_digest}\\n' \"$4\"\n", + encoding="utf-8", + ) + docker_shim.chmod(0o755) + path = os.environ.get("PATH", "") + head_before = _git(feature_host, root, "rev-parse", "HEAD").stdout.strip() + + plan = _ok_object( + feature_host.clawops( + "supply-chain", + "--repo-root", + str(root), + "propose-refresh", + "--dry-run", + "--create-pr", + "--refresh-command", + "uv lock", + env={"PATH": f"{shim_dir}:{path}"}, + ) + ) + assert plan["ok"] is True and plan["dryRun"] is True, plan + assert plan["workflowActions"] == [], plan + assert plan["composeImages"] == [ + { + "path": "platform/compose/docker-compose.yaml", + "line": 3, + "image": "postgres:16-alpine", + "from": old_digest, + "to": new_digest, + } + ], plan + assert plan["refreshCommands"] == ["uv lock"], plan + assert "prUrl" not in plan and "branch" not in plan, plan + assert shim_log.read_text(encoding="utf-8").splitlines() == [ + "buildx imagetools inspect postgres:16-alpine" + ] + assert compose.read_text(encoding="utf-8") == compose_text, "dry-run must not rewrite pins" + assert _git(feature_host, root, "status", "--porcelain").stdout.strip() == "" + assert _git(feature_host, root, "rev-parse", "HEAD").stdout.strip() == head_before + branches = _git(feature_host, root, "branch", "--format=%(refname:short)").stdout.split() + assert branches == ["main"], f"dry-run must not create branches: {branches}" + + +def test_structured_logs_emit_json_events(feature_host: FeatureHost) -> None: + project = feature_host.scratch("structured-logs") + (project / "src").mkdir() + (project / "src" / "service.py").write_text("def handler() -> int:\n return 1\n") + (project / "README.md").write_text("# Service\n", encoding="utf-8") + index_args = ( + "context", + "codebase", + "index", + "--config", + str(CODEBASE_CONTEXT_CONFIG), + "--repo", + str(project), + "--scale", + "small", + "--json", + ) + + enabled = feature_host.clawops(*index_args, env={"CLAWOPS_STRUCTURED_LOGS": "1"}) + stats = _ok_object(enabled) + records = [ + _as_object(json.loads(line)) for line in enabled.stderr.splitlines() if line.startswith("{") + ] + index_events = [item for item in records if item["event"] == "clawops.context.codebase.index"] + assert len(index_events) == 1, enabled.describe() + event = index_events[0] + assert event["provider"] == "codebase" and event["scale"] == "small", event + assert event["repo"] == project.resolve().as_posix(), event + assert event["indexed_files"] == stats["indexed_files"] == 2, (event, stats) + assert isinstance(event["elapsed_ms"], int), event + assert "clawops.context.codebase.index" not in enabled.stdout, "logs belong on stderr" + + (project / "src" / "extra.py").write_text("VALUE = 2\n", encoding="utf-8") + disabled = feature_host.clawops(*index_args, env={"CLAWOPS_STRUCTURED_LOGS": "0"}) + assert _ok_object(disabled)["indexed_files"] == 1, disabled.describe() + assert '"event"' not in disabled.stderr, disabled.describe() + + +def test_baseline_harness_smoke_writes_results(feature_host: FeatureHost) -> None: + runs_dir = feature_host.scratch("baseline-runs") + payload = _ok_object( + feature_host.clawops("baseline", "--runs-dir", str(runs_dir), "harness-smoke", timeout=900) + ) + assert payload == {"ok": True, "runsDir": str(runs_dir)}, payload + for output, suite in ( + ("security.jsonl", "security_regressions.yaml"), + ("policy.jsonl", "policy_regressions.yaml"), + ): + results = _read_jsonl(runs_dir / output) + assert [str(item["id"]) for item in results] == _suite_case_ids(feature_host, suite) + assert all(item["passed"] is True for item in results), results + + +def test_verify_platform_channels_passes(feature_host: FeatureHost) -> None: + report = _ok_object(feature_host.clawops("verify-platform", "channels", "--asset-root", ".")) + checks = _as_objects(report["checks"]) + assert report["name"] == "channels" and report["ok"] is True, report + assert {str(check["name"]) for check in checks} == CHANNEL_CHECKS, report + assert all(check["ok"] is True for check in checks), report + + drifted = _write_json( + feature_host.scratch("channels") / "30-channels.json", + { + "channels": { + "telegram": {"dmPolicy": "open", "groupPolicy": "allowlist", "allowFrom": ["tg:1"]}, + "whatsapp": {"dmPolicy": "pairing", "groupPolicy": "allowlist"}, + "defaults": {"groupPolicy": "allowlist"}, + } + }, + ) + failing = feature_host.clawops( + "verify-platform", "channels", "--asset-root", ".", "--overlay", str(drifted) + ) + failing_report = _as_object(failing.json()) + failed = { + str(check["name"]) for check in _as_objects(failing_report["checks"]) if not check["ok"] + } + assert failing.returncode == 1 and failing_report["ok"] is False, failing.describe() + assert failed == {"telegram-dm-policy"}, failing_report + + +def test_config_lists_managed_memory_profiles(feature_host: FeatureHost) -> None: + home = feature_host.scratch("config-home") + listing = _ok_object( + feature_host.clawops( + "config", + "--asset-root", + ".", + "--home-dir", + str(home), + "memory", + "--list-profiles", + "--json", + ) + ) + profiles = _as_objects(listing["profiles"]) + assert [profile["id"] for profile in profiles] == list(MANAGED_MEMORY_PROFILE_IDS), listing + for profile in profiles: + spec = MEMORY_PROFILES[str(profile["id"])] + assert profile["renderProfile"] == spec.render_profile, profile + assert profile["description"] == spec.description, profile + unmanaged = {profile_id for profile_id, spec in MEMORY_PROFILES.items() if not spec.managed} + assert unmanaged.isdisjoint(str(profile["id"]) for profile in profiles), listing + assert not any(home.iterdir()), "listing profiles must not write into the home dir" + + +def test_render_openclaw_config_merges_overlay(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("render") + home = scratch / "home" + home.mkdir() + config_dir = scratch / "strongclaw-config" + isolation = { + "STRONGCLAW_CONFIG_DIR": str(config_dir), + "STRONGCLAW_DATA_DIR": str(scratch / "strongclaw-data"), + "STRONGCLAW_STATE_DIR": str(scratch / "strongclaw-state"), + } + common = ( + "render-openclaw-config", + "--asset-root", + ".", + "--profile", + "openclaw-default", + "--home-dir", + str(home), + "--user-timezone", + "UTC", + ) + base_path = scratch / "base.json" + feature_host.clawops(*common, "--output", str(base_path), env=isolation).require_ok() + base = _as_object(_read_json(base_path)) + + overlay = scratch / "90-operator.json5" + overlay.write_text( + "{\n" + " // operator overlay\n" + " agents: { defaults: {\n" + " imageMaxDimensionPx: 800,\n" + " heartbeat: { every: '30m' },\n" + " workspace: '__CODER_WORKSPACE__',\n" + " } },\n" + "}\n", + encoding="utf-8", + ) + merged_path = scratch / "openclaw.json" + approvals_path = scratch / "exec-approvals.json" + rendered = feature_host.clawops( + *common, + "--overlay", + str(overlay), + "--output", + str(merged_path), + "--exec-approvals-output", + str(approvals_path), + env=isolation, + ).require_ok() + assert f"Rendered {merged_path.resolve()}" in rendered.stdout, rendered.describe() + assert f"Rendered {approvals_path.resolve()}" in rendered.stdout, rendered.describe() + + expected = copy.deepcopy(base) + defaults = _as_object(_as_object(expected["agents"])["defaults"]) + admin_workspace = Path(str(defaults["workspace"])) + defaults["imageMaxDimensionPx"] = 800 + _as_object(defaults["heartbeat"])["every"] = "30m" + defaults["workspace"] = (admin_workspace.parent / "coder").as_posix() + merged_text = merged_path.read_text(encoding="utf-8") + assert json.loads(merged_text) == expected, "overlay must change only the keys it sets" + assert PLACEHOLDER_PATTERN.search(merged_text) is None, merged_text + + approvals_text = approvals_path.read_text(encoding="utf-8") + approvals = _as_object(json.loads(approvals_text)) + rules = _as_objects(approvals["rules"]) + assert approvals["version"] == 1 and [rule["id"] for rule in rules] == ["workspace-tests"] + prefixes = cast(list[str], _as_object(rules[0]["match"])["cwdPrefixes"]) + assert prefixes[0] == feature_host.repo_root.resolve().as_posix(), prefixes + assert all(Path(prefix).is_absolute() for prefix in prefixes), prefixes + assert PLACEHOLDER_PATTERN.search(approvals_text) is None, approvals_text + for name in ("hypermemory.yaml", "hypermemory.sqlite.yaml"): + assert (config_dir / "memory" / name).is_file(), f"{name} was not materialized" + + +def test_services_render_every_template(feature_host: FeatureHost) -> None: + home = feature_host.scratch("services-home") + state_dir = feature_host.scratch("services-state") + managers: dict[str, tuple[Path, list[str]]] = { + "systemd": ( + home / ".config" / "systemd" / "user", + sorted( + path.name + for pattern in ("*.service", "*.timer") + for path in (feature_host.repo_root / "platform" / "systemd").glob(pattern) + ), + ), + "launchd": ( + home / "Library" / "LaunchAgents", + sorted( + path.stem + for path in (feature_host.repo_root / "platform" / "launchd").glob("*.template") + ), + ), + } + for manager, (output_dir, expected_names) in managers.items(): + payload = _ok_object( + feature_host.clawops( + "services", + "--asset-root", + ".", + "--state-dir", + str(state_dir), + "--service-manager", + manager, + "render", + env={"HOME": str(home)}, + ) + ) + assert payload["serviceManager"] == manager, payload + assert Path(str(payload["outputDir"])).resolve() == output_dir.resolve(), payload + assert Path(str(payload["stateDir"])).resolve() == state_dir.resolve(), payload + rendered = [Path(item) for item in cast(list[str], payload["renderedFiles"])] + assert sorted(path.name for path in rendered) == expected_names, payload + corpus = "" + for path in rendered: + text = path.read_text(encoding="utf-8") + leftover = PLACEHOLDER_PATTERN.findall(text) + assert not leftover, f"{path} kept placeholders {sorted(set(leftover))}" + if manager == "launchd": + label = _as_object(plistlib.loads(text.encode("utf-8")))["Label"] + assert f"{label}.plist" == path.name, path + else: + section = "[Timer]" if path.suffix == ".timer" else "[Service]" + assert section in text, f"{path} has no {section} section" + corpus += text + assert feature_host.repo_root.resolve().as_posix() in corpus, manager + assert state_dir.resolve().as_posix() in corpus, manager diff --git a/tests/suites/e2e/feature_verification/test_recovery.py b/tests/suites/e2e/feature_verification/test_recovery.py new file mode 100644 index 00000000..1665bbf3 --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_recovery.py @@ -0,0 +1,117 @@ +"""Recovery features: OpenClaw-backed backups, verification, restore, retention, scheduled jobs.""" + +from __future__ import annotations + +import os +import tarfile +import time +from pathlib import Path +from typing import cast + +from clawops.strongclaw_recovery import backups_dir +from tests.utils.helpers.feature_verification import CommandOutcome, FeatureHost + +SCHEDULED_JOB_TIMEOUT_SECONDS = 600.0 +_POLL_SECONDS = 3.0 + + +def _recovery(feature_host: FeatureHost, *arguments: str, timeout: float = 900) -> CommandOutcome: + return feature_host.clawops( + "recovery", "--home-dir", str(feature_host.home), *arguments, timeout=timeout + ) + + +def _create_backup(feature_host: FeatureHost) -> dict[str, object]: + return cast(dict[str, object], _recovery(feature_host, "backup-create").require_ok().json()) + + +def test_backup_create_uses_the_openclaw_backend(feature_host: FeatureHost) -> None: + payload = _create_backup(feature_host) + assert payload["mode"] == "openclaw-cli", f"backup fell back to tar: {payload}" + assert payload["fallback_used"] is False, payload + archive = Path(str(payload["archive"])) + assert archive.is_file() and archive.stat().st_size > 0, payload + assert archive.parent == backups_dir(home_dir=feature_host.home), archive + + +def test_backup_verify_accepts_latest_archive(feature_host: FeatureHost) -> None: + created = _create_backup(feature_host) + verified = cast(dict[str, object], _recovery(feature_host, "backup-verify").require_ok().json()) + assert verified["ok"] is True and verified["archive"] == created["archive"], verified + + +def test_restore_extracts_openclaw_state(feature_host: FeatureHost) -> None: + archive = Path(str(_create_backup(feature_host)["archive"])) + destination = feature_host.scratch("restore") + restored = cast( + dict[str, object], + _recovery(feature_host, "restore", str(archive), str(destination)).require_ok().json(), + ) + assert Path(str(restored["destination"])) == destination, restored + with tarfile.open(archive, "r:gz") as bundle: + member_names = {member.name for member in bundle.getmembers() if member.isfile()} + assert member_names, "the archive holds no files" + restored_files = { + path.relative_to(destination).as_posix() + for path in destination.rglob("*") + if path.is_file() + } + assert member_names <= restored_files, sorted(member_names - restored_files)[:20] + assert any(name.endswith("openclaw.json") for name in restored_files), sorted(restored_files)[ + :40 + ] + + +def test_prune_retention_removes_only_stale_archives(feature_host: FeatureHost) -> None: + archive_root = backups_dir(home_dir=feature_host.home) + archive_root.mkdir(parents=True, exist_ok=True) + stale = archive_root / "featver-stale.tar.gz" + stale.write_bytes(b"stale") + old = time.time() - 30 * 24 * 3600 + os.utime(stale, (old, old)) + fresh = Path(str(_create_backup(feature_host)["archive"])) + payload = cast( + dict[str, object], _recovery(feature_host, "prune-retention").require_ok().json() + ) + assert stale.as_posix() in cast(list[str], payload["deleted"]), payload + assert not stale.exists() and fresh.is_file(), payload + + +def test_rotate_secrets_guidance_lists_rotation_targets(feature_host: FeatureHost) -> None: + payload = cast(dict[str, object], _recovery(feature_host, "rotate-secrets").require_ok().json()) + steps = cast(list[str], payload["steps"]) + assert payload["ok"] is True and steps, payload + assert any("Restart the gateway" in step for step in steps), steps + + +def test_scheduled_backup_job_produces_an_archive(feature_host: FeatureHost) -> None: + archive_root = backups_dir(home_dir=feature_host.home) + before: set[Path] = set(archive_root.glob("*")) if archive_root.is_dir() else set() + if feature_host.lane.platform == "linux": + timer = feature_host.run( + ["systemctl", "--user", "is-active", "openclaw-backup-create.timer"], timeout=30 + ) + assert timer.stdout.strip() == "active", timer.describe() + feature_host.run( + ["systemctl", "--user", "start", "openclaw-backup-create.service"], + timeout=SCHEDULED_JOB_TIMEOUT_SECONDS, + ).require_ok() + result = feature_host.run( + ["systemctl", "--user", "show", "--property=Result", "openclaw-backup-create.service"], + timeout=30, + ).require_ok() + assert "Result=success" in result.stdout, result.describe() + else: + label = f"gui/{os.getuid()}/ai.openclaw.backup-create" + feature_host.run(["launchctl", "kickstart", "-k", label], timeout=60).require_ok() + deadline = time.monotonic() + SCHEDULED_JOB_TIMEOUT_SECONDS + state = feature_host.run(["launchctl", "print", label], timeout=30) + while "last exit code = 0" not in state.stdout and time.monotonic() < deadline: + assert "last exit code = (never exited)" in state.stdout or "state = running" in ( + state.stdout + ), state.describe() + time.sleep(_POLL_SECONDS) + state = feature_host.run(["launchctl", "print", label], timeout=30) + assert "last exit code = 0" in state.stdout, state.describe() + created = set(archive_root.glob("*")) - before + assert created, f"the scheduled backup job wrote no archive under {archive_root}" diff --git a/tests/suites/e2e/feature_verification/test_setup_profiles.py b/tests/suites/e2e/feature_verification/test_setup_profiles.py new file mode 100644 index 00000000..edaa683d --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_setup_profiles.py @@ -0,0 +1,157 @@ +"""Setup-profile features: each profile renders, validates, and boots a real gateway. + +Each test activates its own profile through the operator path (``clawops config memory +--set-profile`` for managed profiles, ``clawops setup --profile`` for legacy ones), re-applies the +model chain as the setup guidance instructs, and restarts the harness gateway on it. +""" + +from __future__ import annotations + +import json +import uuid +from pathlib import Path +from typing import Final, cast + +from tests.utils.helpers.feature_verification import ( + ACP_STUB_REPLY_PREFIX, + FeatureHost, + GatewayController, + plugin_statuses, +) + +ACP_WORKER_AGENTS: Final[tuple[str, ...]] = ("coder-acp-codex", "reviewer-acp-claude") +STUB_AGENT: Final[Path] = Path("tests/utils/helpers/_feature_verification/acp_stub_agent.py") + + +def _activate(feature_host: FeatureHost, gateway: GatewayController, profile: str) -> None: + if profile in {"acp", "browser-lab"}: + feature_host.clawops( + "setup", + "--asset-root", + ".", + "--home-dir", + str(feature_host.home), + "--profile", + profile, + "--skip-bootstrap", + "--no-activate-services", + "--no-verify", + "--non-interactive", + timeout=1800, + ).require_ok() + else: + payload = ( + feature_host.clawops( + "config", + "--asset-root", + ".", + "--home-dir", + str(feature_host.home), + "memory", + "--set-profile", + profile, + "--json", + timeout=1800, + ) + .require_ok() + .json() + ) + assert cast(dict[str, object], payload).get("ok", True) is not False, payload + feature_host.clawops("model-auth", "--asset-root", ".", "ensure", timeout=900).require_ok() + feature_host.openclaw("config", "validate", timeout=120).require_ok() + gateway.restart() + + +def _agent_ids(feature_host: FeatureHost) -> list[str]: + payload = feature_host.openclaw("agents", "list", "--json", timeout=120).require_ok().json() + return [str(cast(dict[str, object], agent)["id"]) for agent in cast(list[object], payload)] + + +def test_openclaw_default_profile_boots_with_memory_core( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + _activate(feature_host, gateway, "openclaw-default") + plugins = cast(dict[str, object], feature_host.rendered_config().get("plugins", {})) + assert "slots" not in plugins or "memory" not in cast(dict[str, object], plugins["slots"]) + assert plugin_statuses(feature_host).get("memory-core") == "loaded" + feature_host.openclaw("memory", "status", "--deep", timeout=600).require_ok() + + +def test_openclaw_qmd_profile_installs_and_uses_qmd( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + _activate(feature_host, gateway, "openclaw-qmd") + qmd = feature_host.home / ".bun" / "bin" / "qmd" + feature_host.run([str(qmd), "status"], timeout=120).require_ok() + memory = cast(dict[str, object], feature_host.rendered_config()["memory"]) + assert memory["backend"] == "qmd", memory + status = feature_host.openclaw("memory", "status", "--deep", timeout=600).require_ok() + assert "qmd" in status.output.lower(), status.describe() + + +def test_acp_profile_registers_acp_worker_agents( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + _activate(feature_host, gateway, "acp") + listed = _agent_ids(feature_host) + for agent_id in ACP_WORKER_AGENTS: + assert agent_id in listed, listed + agents = cast(dict[str, object], feature_host.rendered_config()["agents"]) + runtimes = { + str(cast(dict[str, object], agent)["id"]): cast(dict[str, object], agent).get("runtime") + for agent in cast(list[object], agents["list"]) + } + for agent_id in ACP_WORKER_AGENTS: + runtime = cast(dict[str, object], runtimes[agent_id]) + assert runtime["type"] == "acp", runtime + assert cast(dict[str, object], runtime["acp"])["backend"] == "acpx", runtime + + +def test_acp_worker_agent_runs_a_turn_through_acpx( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + _activate(feature_host, gateway, "acp") + agents = cast(dict[str, object], feature_host.rendered_config()["agents"]) + coder = next( + cast(dict[str, object], agent) + for agent in cast(list[object], agents["list"]) + if cast(dict[str, object], agent)["id"] == "coder-acp-codex" + ) + acp_cwd = Path( + str(cast(dict[str, object], cast(dict[str, object], coder["runtime"])["acp"])["cwd"]) + ) + acp_cwd.mkdir(parents=True, exist_ok=True) + stub = feature_host.repo_root / STUB_AGENT + (acp_cwd / ".acpxrc.json").write_text( + json.dumps( + {"agents": {"codex": {"command": f"{feature_host.python} {stub} --persona codex"}}} + ), + encoding="utf-8", + ) + token = f"FV-worker-{uuid.uuid4().hex[:10]}" + outcome = feature_host.openclaw( + "agent", + "--agent", + "coder-acp-codex", + "--session-id", + f"featver-{token}", + "--message", + f"Reply {token}", + "--json", + timeout=300, + ) + assert ( + outcome.ok and ACP_STUB_REPLY_PREFIX in outcome.stdout and token in outcome.stdout + ), outcome.describe() + + +def test_browser_lab_profile_points_at_the_cdp_endpoint( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + _activate(feature_host, gateway, "browser-lab") + browser = cast(dict[str, object], feature_host.rendered_config()["browser"]) + profiles = cast(dict[str, object], browser["profiles"]) + lab = cast(dict[str, object], profiles[str(browser["defaultProfile"])]) + assert browser["enabled"] is True and lab["cdpUrl"] == "http://127.0.0.1:9222", browser + ssrf = cast(dict[str, object], browser["ssrfPolicy"]) + assert ssrf["dangerouslyAllowPrivateNetwork"] is False, ssrf diff --git a/tests/suites/e2e/feature_verification/test_sidecars.py b/tests/suites/e2e/feature_verification/test_sidecars.py new file mode 100644 index 00000000..74b33fe9 --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_sidecars.py @@ -0,0 +1,249 @@ +"""Sidecar, observability and browser-lab features against the live compose stacks.""" + +from __future__ import annotations + +import time +import uuid +from typing import cast + +from tests.utils.helpers.feature_verification import ( + FeatureHost, + basic_auth, + http_request, + tcp_open, +) + +QDRANT = "http://127.0.0.1:6333" +LITELLM = "http://127.0.0.1:4000" +NEO4J_HTTP = "http://127.0.0.1:7474" +OTLP_HTTP = "http://127.0.0.1:4318" +OTEL_METRICS = "http://127.0.0.1:9464/metrics" +QDRANT_TEST_PREFIX = "memory-v2-int-" + + +def _container(feature_host: FeatureHost, fragment: str) -> str: + listing = feature_host.run(["docker", "ps", "--format", "{{.Names}}"], timeout=60).require_ok() + names = [name for name in listing.stdout.split() if fragment in name] + assert names, f"no running container matches {fragment!r}: {listing.stdout}" + return names[0] + + +def test_postgres_sidecar_accepts_queries(feature_host: FeatureHost) -> None: + container = _container(feature_host, "postgres") + outcome = feature_host.run( + [ + "docker", + "exec", + container, + "psql", + "-U", + "litellm", + "-d", + "litellm", + "-tAc", + "select count(*) from information_schema.tables", + ], + timeout=60, + ).require_ok() + assert int(outcome.stdout.strip()) > 0, outcome.describe() + + +def test_litellm_routes_embeddings_to_the_upstream(feature_host: FeatureHost) -> None: + assert http_request("GET", f"{LITELLM}/health/liveliness").status == 200 + token = f"FV-litellm-{uuid.uuid4().hex[:10]}" + started = time.time() + response = http_request( + "POST", + f"{LITELLM}/v1/embeddings", + body={"model": "hypermemory-embedding", "input": [token]}, + headers={"Authorization": f"Bearer {feature_host.env_values()['LITELLM_MASTER_KEY']}"}, + timeout=120, + ) + assert response.status == 200, response.body[:2000] + data = cast(list[dict[str, object]], cast(dict[str, object], response.json())["data"]) + assert data and len(cast(list[float], data[0]["embedding"])) > 0, response.body[:500] + upstream = [ + request + for request in feature_host.fake_model_requests() + if float(cast(float, request.get("at", 0.0))) >= started + and str(request.get("path", "")).startswith("/api/embed") + ] + assert upstream, "litellm answered without calling the configured embedding upstream" + + +def test_qdrant_sidecar_round_trips_vectors(feature_host: FeatureHost) -> None: + del feature_host + collection = f"featver-{uuid.uuid4().hex[:8]}" + assert http_request("GET", f"{QDRANT}/healthz").status == 200 + created = http_request( + "PUT", + f"{QDRANT}/collections/{collection}", + body={"vectors": {"size": 4, "distance": "Cosine"}}, + ) + assert created.status == 200, created.body + try: + upsert = http_request( + "PUT", + f"{QDRANT}/collections/{collection}/points?wait=true", + body={"points": [{"id": 1, "vector": [0.1, 0.2, 0.3, 0.4], "payload": {"k": "fv"}}]}, + ) + assert upsert.status == 200, upsert.body + search = http_request( + "POST", + f"{QDRANT}/collections/{collection}/points/search", + body={"vector": [0.1, 0.2, 0.3, 0.4], "limit": 1, "with_payload": True}, + ) + hits = cast(list[dict[str, object]], cast(dict[str, object], search.json())["result"]) + assert hits and cast(dict[str, object], hits[0]["payload"])["k"] == "fv", search.body + finally: + http_request("DELETE", f"{QDRANT}/collections/{collection}") + + +def test_neo4j_sidecar_round_trips_cypher(feature_host: FeatureHost) -> None: + values = feature_host.env_values() + marker = f"fv-{uuid.uuid4().hex[:8]}" + response = http_request( + "POST", + f"{NEO4J_HTTP}/db/neo4j/tx/commit", + body={ + "statements": [ + { + "statement": "CREATE (n:FeatVer {id: $id}) RETURN n.id", + "parameters": {"id": marker}, + }, + { + "statement": "MATCH (n:FeatVer {id: $id}) DETACH DELETE n RETURN count(*)", + "parameters": {"id": marker}, + }, + ] + }, + headers=basic_auth(values.get("NEO4J_USERNAME", "neo4j"), values["NEO4J_PASSWORD"]), + timeout=60, + ) + payload = cast(dict[str, object], response.json()) + assert response.status == 200 and not payload.get("errors"), response.body[:2000] + results = cast(list[dict[str, object]], payload["results"]) + assert len(results) == 2, payload + + +def test_observability_pipeline_accepts_telemetry(feature_host: FeatureHost) -> None: + feature_host.clawops("verify-platform", "observability", "--asset-root", ".").require_ok() + trace = { + "resourceSpans": [ + { + "resource": { + "attributes": [{"key": "service.name", "value": {"stringValue": "featver"}}] + }, + "scopeSpans": [ + { + "spans": [ + { + "traceId": uuid.uuid4().hex, + "spanId": uuid.uuid4().hex[:16], + "name": "featver-span", + "kind": 1, + "startTimeUnixNano": str(time.time_ns() - 1_000_000), + "endTimeUnixNano": str(time.time_ns()), + } + ] + } + ], + } + ] + } + accepted = http_request("POST", f"{OTLP_HTTP}/v1/traces", body=trace) + assert accepted.status == 200, accepted.body + metrics = http_request("GET", OTEL_METRICS) + assert metrics.status == 200 and metrics.body.strip(), "otel prometheus exporter is silent" + + +def test_verify_platform_sidecars_runtime(feature_host: FeatureHost) -> None: + outcome = feature_host.clawops("verify-platform", "sidecars", "--asset-root", ".", timeout=300) + payload = cast(dict[str, object], outcome.json()) + assert outcome.ok and payload["ok"] is True, outcome.describe() + + +def test_ops_status_reports_required_sidecars_ready(feature_host: FeatureHost) -> None: + payload = cast( + dict[str, object], + feature_host.clawops("ops", "--asset-root", ".", "status", timeout=300).require_ok().json(), + ) + readiness = cast(dict[str, object], payload["readiness"]) + assert payload["ok"] is True and readiness["requiredReady"] is True, payload + services = cast(dict[str, object], payload["services"]) + for name in ("postgres", "litellm", "qdrant", "neo4j"): + assert cast(dict[str, object], services[name])["state"] == "running", services + + +def test_prune_qdrant_test_collections(feature_host: FeatureHost) -> None: + collection = f"{QDRANT_TEST_PREFIX}featver-{uuid.uuid4().hex[:8]}" + created = http_request( + "PUT", + f"{QDRANT}/collections/{collection}", + body={"vectors": {"size": 4, "distance": "Dot"}}, + ) + assert created.status == 200, created.body + dry_run = feature_host.clawops( + "ops", "--asset-root", ".", "prune-qdrant-test-collections", "--dry-run" + ).require_ok() + assert collection in dry_run.stdout, dry_run.describe() + assert http_request("GET", f"{QDRANT}/collections/{collection}").status == 200 + feature_host.clawops("ops", "--asset-root", ".", "prune-qdrant-test-collections").require_ok() + assert http_request("GET", f"{QDRANT}/collections/{collection}").status == 404 + + +def test_browser_lab_starts_and_exposes_cdp(feature_host: FeatureHost) -> None: + feature_host.clawops("ops", "--asset-root", ".", "browser-lab", "up", timeout=900).require_ok() + try: + verify = feature_host.clawops( + "verify-platform", "browser-lab", "--asset-root", ".", timeout=300 + ) + assert verify.ok and cast(dict[str, object], verify.json())["ok"] is True, verify.describe() + deadline = time.monotonic() + 120 + version = http_request("GET", "http://127.0.0.1:9222/json/version", timeout=10) + while version.status != 200 and time.monotonic() < deadline: + time.sleep(3) + version = http_request("GET", "http://127.0.0.1:9222/json/version", timeout=10) + assert version.status == 200, version.body + assert "webSocketDebuggerUrl" in version.body or "Browser" in version.body, version.body + assert tcp_open(3128), "browser-lab egress proxy is not listening on 3128" + finally: + feature_host.clawops("ops", "--asset-root", ".", "browser-lab", "down", timeout=600) + + +def test_reset_compose_state_clears_one_component(feature_host: FeatureHost) -> None: + state_root = feature_host.scratch("compose-state") + component_dir = state_root / "browser-lab" + component_dir.mkdir() + (component_dir / "profile.db").write_text("stale", encoding="utf-8") + outcome = feature_host.clawops( + "ops", + "--asset-root", + ".", + "reset-compose-state", + "--component", + "browser-lab", + "--state-dir", + str(state_root), + timeout=180, + ).require_ok() + payload = cast(dict[str, object], outcome.json()) + assert payload["ok"] is True and payload["stateDir"] == str(component_dir), payload + assert component_dir.is_dir() and not any(component_dir.iterdir()), list( + component_dir.iterdir() + ) + + +def test_optional_ollama_compose_serves_the_api(feature_host: FeatureHost) -> None: + compose = ["docker", "compose", "-f", "platform/compose/docker-compose.ollama.optional.yaml"] + env = {"STRONGCLAW_COMPOSE_STATE_DIR": str(feature_host.scratch("ollama-state"))} + feature_host.run([*compose, "up", "-d"], timeout=900, env=env).require_ok() + try: + deadline = time.monotonic() + 180 + version = http_request("GET", "http://127.0.0.1:11434/api/version", timeout=10) + while version.status != 200 and time.monotonic() < deadline: + time.sleep(3) + version = http_request("GET", "http://127.0.0.1:11434/api/version", timeout=10) + assert version.status == 200 and "version" in version.body, version.body + finally: + feature_host.run([*compose, "down", "-v"], timeout=300, env=env) diff --git a/tests/suites/unit/ci/test_feature_verification.py b/tests/suites/unit/ci/test_feature_verification.py new file mode 100644 index 00000000..dfe6783f --- /dev/null +++ b/tests/suites/unit/ci/test_feature_verification.py @@ -0,0 +1,335 @@ +"""Unit coverage for the feature-verification harness and its fresh-host scenarios.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from typing import cast + +import pytest + +from tests.plugins.infrastructure.context import TestContext +from tests.utils.helpers import fresh_host +from tests.utils.helpers._fresh_host import linux as fresh_host_linux +from tests.utils.helpers._fresh_host import shell as fresh_host_shell +from tests.utils.helpers.feature_verification import ( + ACP_STUB_REPLY_PREFIX, + REPLY_PREFIX, + FeatureResult, + FeatureVerificationError, + LaneResults, + build_report, + canned_reply, + deterministic_embedding, + extract_json, + load_lane_results, + load_manifest, + render_markdown, +) +from tests.utils.helpers.repo import REPO_ROOT + +_STUB_AGENT = ( + REPO_ROOT / "tests" / "utils" / "helpers" / "_feature_verification" / "acp_stub_agent.py" +) +_MANIFEST_HEADER = """\ +schema_version: 1 +lanes: + lane-a: {platform: linux, profile: hypermemory, scenario: linux-features, activate_services: true, description: a} + lane-b: {platform: macos, profile: openclaw-default, scenario: macos-features, activate_services: true, description: b} +features: +""" +_VALID_FEATURES = """\ + - id: alpha.one + category: cat-a + title: Alpha one + call_sites: [src/clawops/cli.py:1] + lanes: [lane-a, lane-b] + verification: live + test: tests/suites/e2e/feature_verification/test_x.py::test_alpha + - id: beta.two + category: cat-b + title: Beta two + call_sites: [src/clawops/cli.py:2] + status: unverifiable + reason: Needs a third-party account that CI does not have. +""" + + +def _write_manifest(tmp_path: Path, features: str = _VALID_FEATURES) -> Path: + path = tmp_path / "feature_matrix.yaml" + path.write_text(_MANIFEST_HEADER + features, encoding="utf-8") + return path + + +def test_canned_reply_echoes_tokens_from_the_latest_user_turn() -> None: + assert canned_reply("ctx", "please answer FV-abc123 now") == f"{REPLY_PREFIX} FV-abc123" + assert canned_reply("ctx", "no token here") == f"{REPLY_PREFIX} acknowledged" + + +def test_canned_reply_answers_reflection_and_extraction_prompts() -> None: + reflection = canned_reply("## Invariants\n## Derived\nMEMORY REFLECTION", "") + assert "FEATVER_REFLECTION_INVARIANT" in reflection and "## Derived" in reflection + extraction = canned_reply("Analyze ... extract memories worth long-term preservation.", "") + assert json.loads(extraction) == {"memories": []} + + +def test_deterministic_embedding_is_stable_and_unit_length() -> None: + first = deterministic_embedding("alpha", 64) + assert first == deterministic_embedding("alpha", 64) + assert first != deterministic_embedding("beta", 64) + assert len(first) == 64 + assert abs(sum(value * value for value in first) - 1.0) < 1e-3 + + +@pytest.mark.network_local +def test_fake_model_server_serves_openai_and_ollama_shapes() -> None: + import urllib.request + + from tests.utils.helpers.feature_verification import FakeModelServer + + server = FakeModelServer(host="127.0.0.1", ports=[0]) + server.start() + base = f"http://127.0.0.1:{server.ports[0]}" + try: + + def _post(path: str, body: object) -> bytes: + request = urllib.request.Request( + f"{base}{path}", data=json.dumps(body).encode(), method="POST" + ) + with urllib.request.urlopen(request, timeout=5) as response: # noqa: S310 + return cast(bytes, response.read()) + + chat = json.loads( + _post("/v1/chat/completions", {"messages": [{"role": "user", "content": "FV-1234"}]}) + ) + assert chat["choices"][0]["message"]["content"] == f"{REPLY_PREFIX} FV-1234" + stream = _post( + "/v1/chat/completions", + {"stream": True, "messages": [{"role": "user", "content": "FV-5678"}]}, + ).decode() + assert f"{REPLY_PREFIX} FV-5678" in stream and stream.rstrip().endswith("data: [DONE]") + embeddings = json.loads(_post("/v1/embeddings", {"input": ["a", "b"], "dimensions": 8})) + assert [len(item["embedding"]) for item in embeddings["data"]] == [8, 8] + ollama = json.loads(_post("/api/embed", {"input": ["a"], "dimensions": 4})) + assert len(ollama["embeddings"][0]) == 4 + with urllib.request.urlopen( + f"{base}/__featver/requests", timeout=5 + ) as response: # noqa: S310 + journal = json.loads(response.read())["requests"] + assert [entry["path"] for entry in journal] == [ + "/v1/chat/completions", + "/v1/chat/completions", + "/v1/embeddings", + "/api/embed", + ] + finally: + server.stop() + + +def test_acp_stub_agent_speaks_acp_and_reports_env_presence_only() -> None: + messages: list[dict[str, object]] = [ + {"jsonrpc": "2.0", "id": 0, "method": "initialize", "params": {"protocolVersion": 1}}, + { + "jsonrpc": "2.0", + "id": 1, + "method": "authenticate", + "params": {"methodId": "openai-api-key"}, + }, + { + "jsonrpc": "2.0", + "id": 2, + "method": "session/new", + "params": {"cwd": "/", "mcpServers": []}, + }, + { + "jsonrpc": "2.0", + "id": 3, + "method": "session/prompt", + "params": {"sessionId": "s", "prompt": [{"type": "text", "text": "FV-stub"}]}, + }, + ] + completed = subprocess.run( + [sys.executable, str(_STUB_AGENT), "--persona", "codex"], + input="\n".join(json.dumps(message) for message in messages) + "\n", + capture_output=True, + text=True, + timeout=30, + check=True, + env={"PATH": "/usr/bin:/bin", "OPENAI_API_KEY": "secret-value-never-echoed"}, + ) + replies = [json.loads(line) for line in completed.stdout.splitlines()] + methods = [method["id"] for method in replies[0]["result"]["authMethods"]] + assert methods == ["chatgpt", "codex-api-key", "openai-api-key"] + update = next(reply for reply in replies if reply.get("method") == "session/update") + text = update["params"]["update"]["content"]["text"] + assert text.startswith(ACP_STUB_REPLY_PREFIX) + assert "auth=openai-api-key" in text and "OPENAI_API_KEY:present" in text + assert "secret-value-never-echoed" not in completed.stdout + assert replies[-1] == {"jsonrpc": "2.0", "id": 3, "result": {"stopReason": "end_turn"}} + + +def test_extract_json_skips_leading_noise() -> None: + assert extract_json('warning: banner\n{"ok": true}\ntrailer') == {"ok": True} + with pytest.raises(AssertionError): + extract_json("no json at all") + + +def test_load_manifest_accepts_valid_matrix_and_derives_platforms(tmp_path: Path) -> None: + manifest = load_manifest(_write_manifest(tmp_path)) + alpha = manifest.feature("alpha.one") + assert manifest.platforms(alpha) == ("linux", "macos") + assert manifest.features_for_lane("lane-b") == (alpha,) + assert manifest.feature("beta.two").status == "unverifiable" + + +@pytest.mark.parametrize( + ("features", "message"), + [ + (_VALID_FEATURES.replace("lanes: [lane-a, lane-b]", "lanes: [lane-z]"), "unknown lanes"), + ( + _VALID_FEATURES.replace( + " reason: Needs", + " test: tests/suites/e2e/feature_verification/test_x.py::test_beta\n reason: Needs", + ), + "unverifiable", + ), + (_VALID_FEATURES.replace("verification: live", "verification: maybe"), "verification"), + (_VALID_FEATURES.replace("src/clawops/cli.py:1", "src/clawops/cli.py"), "file:line"), + (_VALID_FEATURES.replace("id: beta.two", "id: alpha.one"), "duplicate"), + ], +) +def test_load_manifest_rejects_invalid_entries(tmp_path: Path, features: str, message: str) -> None: + with pytest.raises(FeatureVerificationError, match=message): + load_manifest(_write_manifest(tmp_path, features)) + + +def _lane(lane: str, platform: str, outcome: str, message: str = "") -> LaneResults: + return LaneResults( + lane=lane, + platform=platform, + profile="p", + started_at="t0", + versions={"openclaw": "OpenClaw 2026.3.13", "acpx": "0.3.0"}, + results=[ + FeatureResult( + feature="alpha.one", + test="tests/suites/e2e/feature_verification/test_x.py::test_alpha", + outcome=outcome, # type: ignore[arg-type] + duration_seconds=1.0, + message=message, + ) + ], + ) + + +def test_report_requires_every_lane_to_pass(tmp_path: Path) -> None: + manifest = load_manifest(_write_manifest(tmp_path)) + passing = build_report( + manifest, + { + "lane-a": _lane("lane-a", "linux", "passed"), + "lane-b": _lane("lane-b", "macos", "passed"), + }, + ) + assert passing.counts == {"PASS": 1, "FAIL": 0, "UNVERIFIABLE": 1} + one_lane_missing = build_report(manifest, {"lane-a": _lane("lane-a", "linux", "passed")}) + assert one_lane_missing.counts == {"PASS": 0, "FAIL": 1, "UNVERIFIABLE": 1} + failing = build_report( + manifest, + { + "lane-a": _lane("lane-a", "linux", "failed", "boom | pipe"), + "lane-b": _lane("lane-b", "macos", "passed"), + }, + ) + markdown = render_markdown(failing) + assert "| `alpha.one` | cat-a | FAIL |" in markdown and "boom \\| pipe" in markdown + assert "OpenClaw 2026.3.13" in markdown + + +def test_report_maps_unverifiable_skips_and_plain_skips(tmp_path: Path) -> None: + manifest = load_manifest(_write_manifest(tmp_path)) + lanes = { + "lane-a": _lane("lane-a", "linux", "skipped", "unverifiable: provider outage"), + "lane-b": _lane("lane-b", "macos", "skipped", "unverifiable: provider outage"), + } + assert build_report(manifest, lanes).counts == {"PASS": 0, "FAIL": 0, "UNVERIFIABLE": 2} + lanes["lane-b"] = _lane("lane-b", "macos", "skipped", "some other skip") + assert build_report(manifest, lanes).counts["FAIL"] == 1 + + +def test_lane_results_round_trip(tmp_path: Path) -> None: + path = tmp_path / "feature-results-lane-a.json" + _lane("lane-a", "linux", "error", "teardown").write(path) + loaded = load_lane_results(path) + assert loaded.results[0].verdict == "FAIL" and loaded.versions["acpx"] == "0.3.0" + + +def test_feature_scenarios_install_into_the_runner_home( + tmp_path: Path, test_context: TestContext +) -> None: + test_context.apply_profiles("fresh_host_push") + workspace = tmp_path / "workspace" + workspace.mkdir() + context = fresh_host.prepare_context( + scenario_id="linux-features", + profile="hypermemory", + repo_root=workspace, + runner_temp=tmp_path / "runner-temp", + workspace=workspace, + github_env_file=None, + ) + assert Path(context.app_home) == Path.home() + assert context.xdg_runtime_dir is not None and context.xdg_runtime_dir.startswith("/run/user/") + assert fresh_host_shell.phase_env(context)["OPENCLAW_MODEL_SETUP_MODE"] == "auto" + reloaded = fresh_host.load_context(Path(context.context_path)) + assert reloaded.use_host_home is True and reloaded.model_setup_mode == "auto" + + +def test_legacy_context_files_default_the_new_fields( + tmp_path: Path, test_context: TestContext +) -> None: + test_context.apply_profiles("fresh_host_push") + workspace = tmp_path / "workspace" + workspace.mkdir() + context = fresh_host.prepare_context( + scenario_id="linux", + repo_root=workspace, + runner_temp=tmp_path / "runner-temp", + workspace=workspace, + github_env_file=None, + ) + payload = json.loads(Path(context.context_path).read_text(encoding="utf-8")) + del payload["model_setup_mode"], payload["use_host_home"] + Path(context.context_path).write_text(json.dumps(payload), encoding="utf-8") + reloaded = fresh_host.load_context(Path(context.context_path)) + assert reloaded.model_setup_mode == "skip" and reloaded.use_host_home is False + assert fresh_host_shell.phase_env(reloaded)["OPENCLAW_MODEL_SETUP_MODE"] == "skip" + + +@pytest.mark.parametrize( + ("scenario_id", "expects_flag"), [("linux", True), ("linux-features", False)] +) +def test_linux_setup_only_skips_activation_when_the_scenario_says_so( + tmp_path: Path, test_context: TestContext, scenario_id: str, expects_flag: bool +) -> None: + test_context.apply_profiles("fresh_host_push") + workspace = tmp_path / "workspace" + workspace.mkdir() + context = fresh_host.prepare_context( + scenario_id=scenario_id, + repo_root=workspace, + runner_temp=tmp_path / "runner-temp", + workspace=workspace, + github_env_file=None, + ) + calls: list[list[str]] = [] + + def _record(command: list[str], **_: object) -> None: + calls.append(command) + + test_context.patch.patch_object(fresh_host_linux, "run_command", new=_record) + command = fresh_host_linux.linux_setup(context) + assert calls == [command] + assert ("--no-activate-services" in command) is expects_flag diff --git a/tests/utils/helpers/_feature_verification/__init__.py b/tests/utils/helpers/_feature_verification/__init__.py new file mode 100644 index 00000000..a395d01d --- /dev/null +++ b/tests/utils/helpers/_feature_verification/__init__.py @@ -0,0 +1 @@ +"""Internal feature-verification helper package.""" diff --git a/tests/utils/helpers/_feature_verification/acp_stub_agent.py b/tests/utils/helpers/_feature_verification/acp_stub_agent.py new file mode 100644 index 00000000..db974e25 --- /dev/null +++ b/tests/utils/helpers/_feature_verification/acp_stub_agent.py @@ -0,0 +1,163 @@ +"""Minimal ACP (Agent Client Protocol) agent that stands in for vendor coding agents. + +Real ``acpx`` launches this over stdio in place of a vendor adapter (codex-acp, +claude-agent-acp, ``gemini --acp`` ...), so StrongClaw's auth-mode routing, env sanitization, +and acpx invocation are exercised end to end without vendor accounts. The reply reports which +auth-relevant env keys reached the agent process (presence only, never values) and which auth +method acpx selected, so tests can assert routing without the stub ever handling a secret. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import uuid +from collections.abc import Mapping +from typing import Final, TextIO, cast + +PROTOCOL_VERSION: Final[int] = 1 +REPLY_PREFIX: Final[str] = "FEATVER_ACP_STUB" +# Env keys whose presence (never value) the stub reports back. +OBSERVED_ENV_KEYS: Final[tuple[str, ...]] = ( + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "AWS_PROFILE", + "AWS_REGION", + "GOOGLE_APPLICATION_CREDENTIALS", + "VERTEXAI_PROJECT", + "ACPX_AUTH_OPENAI_API_KEY", +) +# Auth methods advertised by the real adapters each stub persona imitates. +AUTH_METHODS: Final[Mapping[str, tuple[str, ...]]] = { + "codex": ("chatgpt", "codex-api-key", "openai-api-key"), + "claude": ("claude-ai-login", "console-login"), +} + +type JsonObject = dict[str, object] + + +class StubAgent: + """Serve one ACP connection on stdio.""" + + def __init__(self, persona: str, *, out: TextIO) -> None: + self._persona = persona + self._out = out + self._authenticated: list[str] = [] + + def _send(self, payload: JsonObject) -> None: + self._out.write(json.dumps(payload) + "\n") + self._out.flush() + + def _result(self, request_id: object, result: JsonObject) -> None: + self._send({"jsonrpc": "2.0", "id": request_id, "result": result}) + + def _error(self, request_id: object, code: int, message: str) -> None: + self._send( + {"jsonrpc": "2.0", "id": request_id, "error": {"code": code, "message": message}} + ) + + def _env_report(self) -> str: + return ",".join( + f"{key}:{'present' if os.environ.get(key) else 'absent'}" for key in OBSERVED_ENV_KEYS + ) + + def _prompt_text(self, params: JsonObject) -> str: + prompt = params.get("prompt") + if not isinstance(prompt, list): + return "" + parts: list[str] = [] + for block in cast(list[object], prompt): + if isinstance(block, dict): + text = cast(Mapping[str, object], block).get("text") + if isinstance(text, str): + parts.append(text) + return " ".join(parts) + + def handle(self, message: JsonObject) -> None: + """Dispatch one JSON-RPC message.""" + method = message.get("method") + request_id = message.get("id") + params = message.get("params") + params_mapping = cast(JsonObject, params) if isinstance(params, dict) else {} + if method == "initialize": + self._result( + request_id, + { + "protocolVersion": PROTOCOL_VERSION, + "agentCapabilities": { + "loadSession": False, + "promptCapabilities": { + "image": False, + "audio": False, + "embeddedContext": False, + }, + }, + "authMethods": [ + {"id": method_id, "name": method_id, "description": None} + for method_id in AUTH_METHODS.get(self._persona, ()) + ], + "agentInfo": {"name": f"featver-{self._persona}", "version": "1.0.0"}, + }, + ) + elif method == "authenticate": + method_id = params_mapping.get("methodId") + self._authenticated.append(str(method_id)) + self._result(request_id, {}) + elif method == "session/new": + self._result(request_id, {"sessionId": f"featver-{uuid.uuid4().hex[:12]}"}) + elif method == "session/prompt": + session_id = params_mapping.get("sessionId") + reply = ( + f"{REPLY_PREFIX} persona={self._persona} " + f"auth={'+'.join(self._authenticated) or 'none'} " + f"env={self._env_report()} prompt={self._prompt_text(params_mapping)}" + ) + self._send( + { + "jsonrpc": "2.0", + "method": "session/update", + "params": { + "sessionId": session_id, + "update": { + "sessionUpdate": "agent_message_chunk", + "content": {"type": "text", "text": reply}, + }, + }, + } + ) + self._result(request_id, {"stopReason": "end_turn"}) + elif method in {"session/cancel"} or request_id is None: + return + elif str(method).startswith("session/"): + # Optional session controls (modes, config options): accept as no-ops. + self._result(request_id, {}) + else: + self._error(request_id, -32601, f"method not found: {method}") + + def serve(self, stream: TextIO) -> None: + """Read newline-delimited JSON-RPC messages until stdin closes.""" + for raw_line in stream: + line = raw_line.strip() + if not line: + continue + try: + message = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(message, dict): + self.handle(cast(JsonObject, message)) + + +def main(argv: list[str] | None = None) -> int: + """Run the stub agent on stdio.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--persona", required=True) + args = parser.parse_args(argv) + StubAgent(str(args.persona), out=sys.stdout).serve(sys.stdin) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/utils/helpers/_feature_verification/diagnostics.py b/tests/utils/helpers/_feature_verification/diagnostics.py new file mode 100644 index 00000000..150f57a7 --- /dev/null +++ b/tests/utils/helpers/_feature_verification/diagnostics.py @@ -0,0 +1,97 @@ +"""Best-effort lane diagnostics: service-manager state and gateway logs, secrets redacted.""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path +from typing import Final + +from clawops.strongclaw_runtime import load_env_assignments, varlock_local_env_file +from tests.utils.helpers._fresh_host.shell import phase_env +from tests.utils.helpers._fresh_host.storage import load_context + +LAUNCHD_LABELS: Final[tuple[str, ...]] = ( + "ai.openclaw.gateway", + "ai.openclaw.sidecars", + "ai.openclaw.backup-create", + "ai.openclaw.backup-verify", + "ai.openclaw.maintenance", +) +_SECRET_MARKERS: Final[tuple[str, ...]] = ("TOKEN", "KEY", "PASSWORD", "SECRET") +_MIN_SECRET_LENGTH: Final[int] = 8 +_LOG_TAIL_CHARS: Final[int] = 400_000 +_COMMAND_TIMEOUT_SECONDS: Final[int] = 60 + + +def _diagnostic_commands(platform: str) -> dict[str, list[str]]: + if platform == "linux": + return { + "systemd-units.txt": ["systemctl", "--user", "list-units", "openclaw*", "--all"], + "systemd-timers.txt": ["systemctl", "--user", "list-timers", "--all"], + "systemd-status.txt": ["systemctl", "--user", "status", "openclaw*", "--no-pager"], + "systemd-journal.txt": ["journalctl", "--user", "--no-pager", "-n", "2000"], + "docker-ps.txt": ["docker", "ps", "-a"], + } + commands = { + f"launchctl-{label}.txt": ["launchctl", "print", f"gui/{os.getuid()}/{label}"] + for label in LAUNCHD_LABELS + } + commands["docker-ps.txt"] = ["docker", "ps", "-a"] + return commands + + +def _secret_values(env_file: Path) -> list[str]: + values = load_env_assignments(env_file) if env_file.is_file() else {} + return sorted( + ( + value + for key, value in values.items() + if any(marker in key for marker in _SECRET_MARKERS) and len(value) >= _MIN_SECRET_LENGTH + ), + key=len, + reverse=True, + ) + + +def collect_lane_diagnostics(*, context_file: Path, output_dir: Path) -> None: + """Capture service-manager state and gateway logs into *output_dir*.""" + context = load_context(context_file) + env = phase_env(context) + secrets = _secret_values( + varlock_local_env_file( + Path(context.repo_root), home_dir=Path(context.app_home), environ=env + ) + ) + + def _redact(text: str) -> str: + for secret in secrets: + text = text.replace(secret, "[REDACTED]") + return text + + output_dir.mkdir(parents=True, exist_ok=True) + for name, command in _diagnostic_commands(context.platform).items(): + try: + completed = subprocess.run( # noqa: S603 - fixed diagnostic argv + command, + env=env, + capture_output=True, + text=True, + timeout=_COMMAND_TIMEOUT_SECONDS, + check=False, + ) + text = ( + f"$ {' '.join(command)}\nexit={completed.returncode}\n" + f"{completed.stdout}\n{completed.stderr}" + ) + except (OSError, subprocess.TimeoutExpired) as exc: + text = f"$ {' '.join(command)}\nfailed: {exc}" + (output_dir / name).write_text(_redact(text), encoding="utf-8") + for log_root in (Path("/tmp/openclaw"), Path(context.app_home) / ".openclaw" / "logs"): + if not log_root.is_dir(): + continue + for log_file in sorted(log_root.glob("*.log"))[-5:]: + tail = log_file.read_text(encoding="utf-8", errors="replace")[-_LOG_TAIL_CHARS:] + (output_dir / f"{log_root.name}-{log_file.name}").write_text( + _redact(tail), encoding="utf-8" + ) diff --git a/tests/utils/helpers/_feature_verification/envvars.py b/tests/utils/helpers/_feature_verification/envvars.py new file mode 100644 index 00000000..92bb270d --- /dev/null +++ b/tests/utils/helpers/_feature_verification/envvars.py @@ -0,0 +1,10 @@ +"""Environment variables that carry lane state from ``run-lane`` into the pytest session.""" + +from typing import Final + +LANE_ENV: Final[str] = "FEATURE_VERIFICATION_LANE" +CONTEXT_ENV: Final[str] = "FEATURE_VERIFICATION_CONTEXT" +RESULTS_ENV: Final[str] = "FEATURE_VERIFICATION_RESULTS" +WORK_DIR_ENV: Final[str] = "FEATURE_VERIFICATION_WORK_DIR" +FAKE_MODEL_DIR_ENV: Final[str] = "FEATURE_VERIFICATION_FAKE_MODEL_DIR" +MANIFEST_ENV: Final[str] = "FEATURE_VERIFICATION_MANIFEST" diff --git a/tests/utils/helpers/_feature_verification/fake_model.py b/tests/utils/helpers/_feature_verification/fake_model.py new file mode 100644 index 00000000..d7abce0b --- /dev/null +++ b/tests/utils/helpers/_feature_verification/fake_model.py @@ -0,0 +1,460 @@ +"""Deterministic OpenAI- and Ollama-compatible model server for feature verification. + +The server stands in for every model provider the product talks to on a runner: the OpenClaw +agent chain (``vllm/`` through the OpenAI-compatible ``/v1`` API), the memory plugins' +embedding and extraction endpoints, and the LiteLLM sidecar's Ollama upstream. Replies are +canned, so assertions can rely on exact content, and every request is journaled so a test can +prove the product actually reached the model rather than short-circuiting. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +import threading +import time +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final, cast + +CHAT_MODEL_ID: Final[str] = "featver-chat" +EMBEDDING_MODEL_ID: Final[str] = "featver-embed" +DEFAULT_EMBEDDING_DIMENSIONS: Final[int] = 768 +DEFAULT_CONTEXT_WINDOW: Final[int] = 131_072 +REPLY_PREFIX: Final[str] = "FEATVER_REPLY" +ECHO_TOKEN_PATTERN: Final[re.Pattern[str]] = re.compile(r"\bFV-[A-Za-z0-9][A-Za-z0-9_-]{3,63}\b") +JOURNAL_PATH: Final[str] = "/__featver/requests" +HEALTH_PATH: Final[str] = "/__featver/health" +_MAX_JOURNAL_ENTRIES: Final[int] = 2000 +_PROMPT_EXCERPT_CHARS: Final[int] = 4000 +# Models advertised by /v1/models and /api/tags; requests naming any other model still succeed. +ADVERTISED_MODELS: Final[tuple[str, ...]] = ( + CHAT_MODEL_ID, + EMBEDDING_MODEL_ID, + "nomic-embed-text", + "qwen2.5:7b", +) + +type JsonObject = dict[str, object] +type ReplyRule = tuple[Callable[[str], bool], Callable[[str], str]] + + +def _as_object(value: object) -> JsonObject: + """Return *value* as a string-keyed mapping, or an empty mapping.""" + if not isinstance(value, dict): + return {} + return {str(key): item for key, item in cast(Mapping[object, object], value).items()} + + +def _content_text(content: object) -> str: + """Flatten OpenAI/Ollama message content (string or content parts) into text.""" + if isinstance(content, str): + return content + if isinstance(content, list): + parts: list[str] = [] + for part in cast(Sequence[object], content): + part_mapping = _as_object(part) + text = part_mapping.get("text") + if isinstance(text, str): + parts.append(text) + return "\n".join(parts) + return "" + + +def _messages_text(payload: JsonObject) -> tuple[str, str]: + """Return (all message text, latest user message text) for a chat payload.""" + messages = payload.get("messages") + if not isinstance(messages, list): + prompt = payload.get("prompt") + text = prompt if isinstance(prompt, str) else "" + return text, text + all_parts: list[str] = [] + latest_user = "" + for raw_message in cast(Sequence[object], messages): + message = _as_object(raw_message) + text = _content_text(message.get("content")) + all_parts.append(text) + if message.get("role") == "user": + latest_user = text + return "\n".join(all_parts), latest_user + + +def _is_reflection_prompt(text: str) -> bool: + """Return whether *text* is the memory-lancedb-pro reflection prompt.""" + return "## Invariants" in text and "## Derived" in text + + +def _reflection_reply(_: str) -> str: + """Return a reflection document the lancedb-pro parser accepts.""" + return ( + "## Context (session background)\n" + "- Feature verification session for the StrongClaw memory plugin.\n\n" + "## Decisions (durable)\n" + "- Keep feature verification replies deterministic.\n\n" + "## User model deltas (about the human)\n" + "- (none captured)\n\n" + "## Agent model deltas (about the assistant/system)\n" + "- (none captured)\n\n" + "## Lessons & pitfalls (symptom / cause / fix / prevention)\n" + "- (none captured)\n\n" + "## Learning governance candidates (.learnings / promotion / skill extraction)\n" + "- (none captured)\n\n" + "## Open loops / next actions\n" + "- (none captured)\n\n" + "## Retrieval tags / keywords\n" + "- featver, reflection\n\n" + "## Invariants\n" + "- Always verify features against the real runtime (FEATVER_REFLECTION_INVARIANT).\n\n" + "## Derived\n" + "- Prefer the embedded runner for reflection runs (FEATVER_REFLECTION_DERIVED).\n" + ) + + +def _is_extraction_prompt(text: str) -> bool: + """Return whether *text* is the memory-lancedb-pro smart-extraction prompt.""" + return "extract memories worth long-term preservation" in text + + +def _extraction_reply(_: str) -> str: + """Return an empty structured-extraction result.""" + return json.dumps({"memories": []}) + + +def _default_reply(latest_user: str) -> str: + """Return the canned agent reply, echoing any FV- token from the latest user turn.""" + tokens = ECHO_TOKEN_PATTERN.findall(latest_user) + suffix = " ".join(dict.fromkeys(tokens)) if tokens else "acknowledged" + return f"{REPLY_PREFIX} {suffix}" + + +REPLY_RULES: Final[tuple[ReplyRule, ...]] = ( + (_is_reflection_prompt, _reflection_reply), + (_is_extraction_prompt, _extraction_reply), +) + + +def canned_reply(all_text: str, latest_user: str) -> str: + """Return the deterministic reply for one chat request.""" + for matches, respond in REPLY_RULES: + if matches(all_text): + return respond(all_text) + return _default_reply(latest_user) + + +def deterministic_embedding( + text: str, dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS +) -> list[float]: + """Return a unit-length, content-derived embedding so equal texts embed identically.""" + values: list[float] = [] + counter = 0 + while len(values) < dimensions: + digest = hashlib.sha256(f"{counter}:{text}".encode()).digest() + values.extend((byte - 127.5) / 127.5 for byte in digest) + counter += 1 + vector = values[:dimensions] + norm = math.sqrt(sum(component * component for component in vector)) or 1.0 + return [round(component / norm, 6) for component in vector] + + +@dataclass(slots=True) +class RequestJournal: + """Thread-safe bounded record of the requests the server answered.""" + + entries: list[JsonObject] = field(default_factory=lambda: cast(list[JsonObject], [])) + lock: threading.Lock = field(default_factory=threading.Lock) + + def record(self, entry: JsonObject) -> None: + """Append one entry, dropping the oldest beyond the bound.""" + with self.lock: + self.entries.append(entry) + del self.entries[:-_MAX_JOURNAL_ENTRIES] + + def snapshot(self) -> list[JsonObject]: + """Return a copy of the recorded entries.""" + with self.lock: + return list(self.entries) + + +class FakeModelHandler(BaseHTTPRequestHandler): + """Route OpenAI-compatible and Ollama-native requests to canned responders.""" + + server_version = "featver-fake-model/1" + journal: RequestJournal = RequestJournal() + + def log_message(self, format: str, *args: object) -> None: # noqa: A002 + """Silence per-request stderr logging; the journal is the audit trail.""" + del format, args + + def _read_json(self) -> JsonObject: + length = int(self.headers.get("Content-Length") or 0) + if length <= 0: + return {} + raw = self.rfile.read(length) + try: + return _as_object(json.loads(raw.decode("utf-8"))) + except (UnicodeDecodeError, json.JSONDecodeError): + return {} + + def _send_json(self, payload: object, status: HTTPStatus = HTTPStatus.OK) -> None: + body = json.dumps(payload).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _record(self, payload: JsonObject, *, reply: str | None = None) -> None: + all_text, latest_user = _messages_text(payload) + self.journal.record( + { + "at": round(time.time(), 3), + "method": self.command, + "path": self.path.split("?", 1)[0], + "port": cast(tuple[str, int], self.server.server_address)[1], + "model": payload.get("model"), + "stream": bool(payload.get("stream")), + "latestUser": latest_user[:_PROMPT_EXCERPT_CHARS], + "prompt": all_text[:_PROMPT_EXCERPT_CHARS], + "reply": reply, + } + ) + + def do_GET(self) -> None: # noqa: N802 + """Serve discovery, health, and journal endpoints.""" + path = self.path.split("?", 1)[0] + if path == HEALTH_PATH: + self._send_json({"ok": True}) + elif path == JOURNAL_PATH: + self._send_json({"requests": self.journal.snapshot()}) + elif path in {"/v1/models", "/models"}: + self._send_json( + { + "object": "list", + "data": [ + {"id": model, "object": "model", "created": 0, "owned_by": "featver"} + for model in ADVERTISED_MODELS + ], + } + ) + elif path == "/api/tags": + self._send_json( + {"models": [{"name": model, "model": model} for model in ADVERTISED_MODELS]} + ) + elif path == "/api/version": + self._send_json({"version": "0.0.0-featver"}) + elif path in {"/", "/health", "/health/liveliness"}: + self._send_json({"status": "ok"}) + else: + self._send_json({"error": f"unknown path {path}"}, HTTPStatus.NOT_FOUND) + + def do_POST(self) -> None: # noqa: N802 + """Serve chat, completion, and embedding endpoints.""" + path = self.path.split("?", 1)[0] + payload = self._read_json() + if path in {"/v1/chat/completions", "/chat/completions"}: + self._openai_chat(payload) + elif path in {"/v1/completions", "/completions"}: + self._openai_completion(payload) + elif path in {"/v1/embeddings", "/embeddings"}: + self._openai_embeddings(payload) + elif path == "/api/chat": + self._ollama_chat(payload) + elif path in {"/api/embed", "/api/embeddings"}: + self._ollama_embeddings(payload, legacy=path == "/api/embeddings") + elif path == "/api/show": + self._send_json( + { + "details": {"family": "featver"}, + "model_info": {"featver.context_length": DEFAULT_CONTEXT_WINDOW}, + "capabilities": ["completion", "embedding"], + } + ) + else: + self._record(payload) + self._send_json({"error": f"unknown path {path}"}, HTTPStatus.NOT_FOUND) + + def _openai_chat(self, payload: JsonObject) -> None: + all_text, latest_user = _messages_text(payload) + reply = canned_reply(all_text, latest_user) + self._record(payload, reply=reply) + model = str(payload.get("model") or CHAT_MODEL_ID) + completion_id = f"chatcmpl-featver-{int(time.time() * 1000)}" + usage = {"prompt_tokens": 16, "completion_tokens": 8, "total_tokens": 24} + if not payload.get("stream"): + self._send_json( + { + "id": completion_id, + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": reply}, + "finish_reason": "stop", + } + ], + "usage": usage, + } + ) + return + chunks: list[JsonObject] = [ + {"index": 0, "delta": {"role": "assistant", "content": ""}, "finish_reason": None}, + {"index": 0, "delta": {"content": reply}, "finish_reason": None}, + {"index": 0, "delta": {}, "finish_reason": "stop"}, + ] + events: list[JsonObject] = [ + { + "id": completion_id, + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model, + "choices": [choice], + } + for choice in chunks + ] + events.append( + { + "id": completion_id, + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model, + "choices": [], + "usage": usage, + } + ) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Connection", "close") + self.end_headers() + for event in events: + self.wfile.write(f"data: {json.dumps(event)}\n\n".encode()) + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + self.close_connection = True + + def _openai_completion(self, payload: JsonObject) -> None: + prompt = payload.get("prompt") + text = prompt if isinstance(prompt, str) else "" + reply = canned_reply(text, text) + self._record(payload, reply=reply) + self._send_json( + { + "id": "cmpl-featver", + "object": "text_completion", + "created": int(time.time()), + "model": str(payload.get("model") or CHAT_MODEL_ID), + "choices": [{"index": 0, "text": reply, "finish_reason": "stop"}], + } + ) + + @staticmethod + def _embedding_inputs(raw: object) -> list[str]: + if isinstance(raw, str): + return [raw] + if isinstance(raw, list): + return [str(item) for item in cast(Sequence[object], raw)] + return [""] + + @staticmethod + def _dimensions(payload: JsonObject) -> int: + raw = payload.get("dimensions") + return raw if isinstance(raw, int) and raw > 0 else DEFAULT_EMBEDDING_DIMENSIONS + + def _openai_embeddings(self, payload: JsonObject) -> None: + self._record(payload) + inputs = self._embedding_inputs(payload.get("input")) + dimensions = self._dimensions(payload) + self._send_json( + { + "object": "list", + "model": str(payload.get("model") or EMBEDDING_MODEL_ID), + "data": [ + { + "object": "embedding", + "index": index, + "embedding": deterministic_embedding(text, dimensions), + } + for index, text in enumerate(inputs) + ], + "usage": {"prompt_tokens": len(inputs), "total_tokens": len(inputs)}, + } + ) + + def _ollama_embeddings(self, payload: JsonObject, *, legacy: bool) -> None: + self._record(payload) + dimensions = self._dimensions(payload) + if legacy: + prompt = payload.get("prompt") + text = prompt if isinstance(prompt, str) else "" + self._send_json({"embedding": deterministic_embedding(text, dimensions)}) + return + inputs = self._embedding_inputs(payload.get("input")) + self._send_json( + { + "model": str(payload.get("model") or EMBEDDING_MODEL_ID), + "embeddings": [deterministic_embedding(text, dimensions) for text in inputs], + } + ) + + def _ollama_chat(self, payload: JsonObject) -> None: + all_text, latest_user = _messages_text(payload) + reply = canned_reply(all_text, latest_user) + self._record(payload, reply=reply) + model = str(payload.get("model") or CHAT_MODEL_ID) + final: JsonObject = { + "model": model, + "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "message": {"role": "assistant", "content": reply}, + "done": True, + "done_reason": "stop", + "prompt_eval_count": 16, + "eval_count": 8, + } + if payload.get("stream") is False: + self._send_json(final) + return + body = json.dumps(final).encode("utf-8") + b"\n" + self.send_response(HTTPStatus.OK) + self.send_header("Content-Type", "application/x-ndjson") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +class FakeModelServer: + """Serve the canned model API on one or more ports from a shared journal.""" + + def __init__(self, *, host: str, ports: Sequence[int]) -> None: + if not ports: + raise ValueError("FakeModelServer needs at least one port") + self._servers = [ThreadingHTTPServer((host, port), FakeModelHandler) for port in ports] + for server in self._servers: + server.daemon_threads = True + self._threads: list[threading.Thread] = [] + + @property + def ports(self) -> list[int]: + """Return the bound ports (resolves ephemeral port 0 requests).""" + return [int(server.server_address[1]) for server in self._servers] + + def start(self) -> None: + """Serve every port on a daemon thread.""" + for server in self._servers: + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self._threads.append(thread) + + def stop(self) -> None: + """Shut down every listener.""" + for server in self._servers: + server.shutdown() + server.server_close() + for thread in self._threads: + thread.join(timeout=5) + self._threads.clear() diff --git a/tests/utils/helpers/_feature_verification/host.py b/tests/utils/helpers/_feature_verification/host.py new file mode 100644 index 00000000..f41f8f3e --- /dev/null +++ b/tests/utils/helpers/_feature_verification/host.py @@ -0,0 +1,288 @@ +"""Black-box access to a provisioned feature-verification host. + +Tests drive the product only through its public entrypoints (``clawops``, ``openclaw`` via +Varlock, ``acpx``) as an operator would, with an environment scrubbed of harness variables so a +nested product run (for example ``clawops baseline verify`` running pytest) never re-enters +the feature suite. +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import time +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Final, cast + +from clawops.strongclaw_runtime import ( + load_env_assignments, + load_openclaw_config, + resolve_openclaw_config_path, + varlock_env_dir, + varlock_local_env_file, + wrap_command_with_varlock, +) +from tests.utils.helpers._feature_verification.envvars import ( + CONTEXT_ENV, + FAKE_MODEL_DIR_ENV, + LANE_ENV, + MANIFEST_ENV, + WORK_DIR_ENV, +) +from tests.utils.helpers._feature_verification.manifest import ( + FeatureManifest, + FeatureVerificationError, + Lane, + load_manifest, +) +from tests.utils.helpers._feature_verification.processes import fetch_journal, load_state +from tests.utils.helpers._fresh_host.models import FreshHostContext +from tests.utils.helpers._fresh_host.storage import load_context + +_HARNESS_ENV_PREFIXES: Final[tuple[str, ...]] = ("FEATURE_VERIFICATION_", "PYTEST_") +_FRAMEWORK_ENV_KEYS: Final[frozenset[str]] = frozenset({"TEST_ID", "RESOURCE_PREFIX", "WORKER_ID"}) +_SECRET_KEY_PATTERN: Final[re.Pattern[str]] = re.compile(r"(TOKEN|KEY|PASSWORD|SECRET)") +_MIN_REDACTED_LENGTH: Final[int] = 8 +_JSON_START: Final[re.Pattern[str]] = re.compile(r"(?m)^[ \t]*[\[{]") +_OUTPUT_TAIL_CHARS: Final[int] = 3000 + + +def default_manifest_path(repo_root: Path) -> Path: + """Return the checked-in feature matrix path.""" + return repo_root / "tests" / "feature_matrix.yaml" + + +def extract_json(text: str) -> object: + """Return the first JSON document embedded in mixed command output.""" + decoder = json.JSONDecoder() + for match in _JSON_START.finditer(text): + try: + payload, _ = decoder.raw_decode(text[match.start() :].lstrip()) + except json.JSONDecodeError: + continue + return cast(object, payload) + raise AssertionError(f"no JSON document in command output: {text[-_OUTPUT_TAIL_CHARS:]!r}") + + +@dataclass(frozen=True, slots=True) +class CommandOutcome: + """Captured result of one product command.""" + + argv: tuple[str, ...] + returncode: int + stdout: str + stderr: str + duration_seconds: float + + @property + def ok(self) -> bool: + """Return whether the command exited 0.""" + return self.returncode == 0 + + @property + def output(self) -> str: + """Return stdout and stderr combined.""" + return f"{self.stdout}\n{self.stderr}" + + def describe(self) -> str: + """Return a bounded, already-redacted description for assertion messages.""" + return ( + f"`{' '.join(self.argv)}` exited {self.returncode} after {self.duration_seconds:.1f}s\n" + f"stdout tail:\n{self.stdout[-_OUTPUT_TAIL_CHARS:]}\n" + f"stderr tail:\n{self.stderr[-_OUTPUT_TAIL_CHARS:]}" + ) + + def require_ok(self) -> CommandOutcome: + """Assert a zero exit status.""" + assert self.ok, self.describe() + return self + + def json(self) -> object: + """Parse the first JSON document from stdout.""" + return extract_json(self.stdout) + + +@dataclass(slots=True) +class FeatureHost: + """The provisioned host one feature-verification lane runs against.""" + + lane: Lane + manifest: FeatureManifest + context: FreshHostContext + work_dir: Path + fake_model_dir: Path | None + _secrets: tuple[str, ...] = field(default_factory=tuple) + + @classmethod + def from_environment(cls, environ: Mapping[str, str] | None = None) -> FeatureHost: + """Build the host from the lane environment exported by ``run-lane``.""" + env = os.environ if environ is None else environ + lane_id = env.get(LANE_ENV, "").strip() + context_file = env.get(CONTEXT_ENV, "").strip() + if not lane_id or not context_file: + raise FeatureVerificationError(f"{LANE_ENV} and {CONTEXT_ENV} must both be set") + context = load_context(Path(context_file)) + manifest_path = Path( + env.get(MANIFEST_ENV, "").strip() + or default_manifest_path(Path(context.repo_root)).as_posix() + ) + manifest = load_manifest(manifest_path) + if lane_id not in manifest.lanes: + raise FeatureVerificationError(f"lane {lane_id!r} is not in {manifest_path}") + work_dir = Path(env.get(WORK_DIR_ENV, "").strip() or Path(context.tmp_root) / "featver") + work_dir.mkdir(parents=True, exist_ok=True) + fake_model_raw = env.get(FAKE_MODEL_DIR_ENV, "").strip() + return cls( + lane=manifest.lanes[lane_id], + manifest=manifest, + context=context, + work_dir=work_dir, + fake_model_dir=Path(fake_model_raw) if fake_model_raw else None, + ) + + @property + def repo_root(self) -> Path: + """Return the StrongClaw checkout under test.""" + return Path(self.context.repo_root) + + @property + def home(self) -> Path: + """Return the HOME the product was installed into.""" + return Path(self.context.app_home) + + @property + def python(self) -> Path: + """Return the managed-environment interpreter created by bootstrap.""" + return self.repo_root / ".venv" / "bin" / "python" + + def scratch(self, name: str) -> Path: + """Return a fresh per-test scratch directory.""" + path = self.work_dir / f"{name}-{time.time_ns()}" + path.mkdir(parents=True) + return path + + def env_file(self) -> Path: + """Return the Varlock local env file the operator edits.""" + return varlock_local_env_file(self.repo_root) + + def env_dir(self) -> Path: + """Return the Varlock env directory.""" + return varlock_env_dir(self.repo_root) + + def env_values(self) -> dict[str, str]: + """Return the operator env contract assignments.""" + return load_env_assignments(self.env_file()) + + def _refresh_secrets(self) -> None: + values = self.env_values() if self.env_file().is_file() else {} + self._secrets = tuple( + sorted( + { + value + for key, value in values.items() + if _SECRET_KEY_PATTERN.search(key) and len(value) >= _MIN_REDACTED_LENGTH + }, + key=len, + reverse=True, + ) + ) + + def redact(self, text: str) -> str: + """Replace known secret values in *text*.""" + for secret in self._secrets: + text = text.replace(secret, "[REDACTED]") + return text + + def product_env(self, overrides: Mapping[str, str] | None = None) -> dict[str, str]: + """Return the operator environment with harness variables removed.""" + env = { + key: value + for key, value in os.environ.items() + if key not in _FRAMEWORK_ENV_KEYS and not key.startswith(_HARNESS_ENV_PREFIXES) + } + if overrides: + env.update(overrides) + return env + + def run( + self, + argv: Sequence[str], + *, + cwd: Path | None = None, + timeout: float = 300, + env: Mapping[str, str] | None = None, + input_text: str | None = None, + ) -> CommandOutcome: + """Run one command and capture its redacted output.""" + self._refresh_secrets() + started = time.monotonic() + command = tuple(str(part) for part in argv) + try: + completed = subprocess.run( # noqa: S603 - argv built by the harness + command, + cwd=cwd or self.repo_root, + env=self.product_env(env), + input=input_text, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout if isinstance(exc.stdout, str) else "" + stderr = exc.stderr if isinstance(exc.stderr, str) else "" + return CommandOutcome( + argv=command, + returncode=-9, + stdout=self.redact(stdout), + stderr=self.redact(f"{stderr}\n[harness] timed out after {timeout}s"), + duration_seconds=time.monotonic() - started, + ) + return CommandOutcome( + argv=command, + returncode=completed.returncode, + stdout=self.redact(completed.stdout), + stderr=self.redact(completed.stderr), + duration_seconds=time.monotonic() - started, + ) + + def clawops( + self, + *arguments: str, + cwd: Path | None = None, + timeout: float = 300, + env: Mapping[str, str] | None = None, + ) -> CommandOutcome: + """Run the managed ``clawops`` CLI.""" + return self.run( + [str(self.python), "-m", "clawops", *arguments], cwd=cwd, timeout=timeout, env=env + ) + + def openclaw( + self, + *arguments: str, + cwd: Path | None = None, + timeout: float = 300, + env: Mapping[str, str] | None = None, + ) -> CommandOutcome: + """Run ``openclaw`` exactly as StrongClaw does: inside ``varlock run``.""" + argv = wrap_command_with_varlock(self.repo_root, ["openclaw", *arguments]) + return self.run(argv, cwd=cwd, timeout=timeout, env=env) + + def fake_model_requests(self) -> list[dict[str, object]]: + """Return every request the fake model server has answered so far.""" + if self.fake_model_dir is None: + raise FeatureVerificationError(f"{FAKE_MODEL_DIR_ENV} is not set for this lane") + state = load_state(self.fake_model_dir) + ports = state.get("ports") + if not isinstance(ports, list) or not ports: + raise FeatureVerificationError("fake model state has no ports") + return fetch_journal(str(state.get("host", "127.0.0.1")), int(cast(int, ports[0]))) + + def rendered_config(self) -> dict[str, object]: + """Return the rendered OpenClaw config.""" + return dict(load_openclaw_config(resolve_openclaw_config_path(self.repo_root))) diff --git a/tests/utils/helpers/_feature_verification/http.py b/tests/utils/helpers/_feature_verification/http.py new file mode 100644 index 00000000..22f4b45f --- /dev/null +++ b/tests/utils/helpers/_feature_verification/http.py @@ -0,0 +1,53 @@ +"""Loopback HTTP helpers for probing sidecars from feature tests.""" + +from __future__ import annotations + +import base64 +import json +import urllib.error +import urllib.request +from collections.abc import Mapping +from dataclasses import dataclass +from typing import cast + + +@dataclass(frozen=True, slots=True) +class HttpResponse: + """One HTTP response.""" + + status: int + body: str + + def json(self) -> object: + """Decode the body as JSON.""" + return cast(object, json.loads(self.body)) if self.body else {} + + +def basic_auth(user: str, password: str) -> dict[str, str]: + """Return a basic-auth header.""" + token = base64.b64encode(f"{user}:{password}".encode()).decode("ascii") + return {"Authorization": f"Basic {token}"} + + +def http_request( + method: str, + url: str, + *, + body: object = None, + headers: Mapping[str, str] | None = None, + timeout: float = 30.0, +) -> HttpResponse: + """Issue one loopback HTTP request and return status plus body, even for HTTP errors.""" + data = None if body is None else json.dumps(body).encode("utf-8") + request = urllib.request.Request(url, data=data, method=method) + request.add_header("Content-Type", "application/json") + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 + return HttpResponse(int(response.status), response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return HttpResponse(int(exc.code), exc.read().decode("utf-8", errors="replace")) + except (urllib.error.URLError, OSError) as exc: + # Status 0 marks "no HTTP response" so callers can poll and assert uniformly. + return HttpResponse(0, f"unreachable: {exc}") diff --git a/tests/utils/helpers/_feature_verification/lane.py b/tests/utils/helpers/_feature_verification/lane.py new file mode 100644 index 00000000..f04e4328 --- /dev/null +++ b/tests/utils/helpers/_feature_verification/lane.py @@ -0,0 +1,139 @@ +"""Lane provisioning and execution steps for the feature-verification workflow.""" + +from __future__ import annotations + +import os +import subprocess +import time +from pathlib import Path +from typing import Final + +from clawops.strongclaw_runtime import set_env_assignment, varlock_local_env_file +from tests.utils.helpers._feature_verification.envvars import ( + CONTEXT_ENV, + FAKE_MODEL_DIR_ENV, + LANE_ENV, + MANIFEST_ENV, + RESULTS_ENV, + WORK_DIR_ENV, +) +from tests.utils.helpers._fresh_host.budget import ( + SYSTEM_CLOCK, + JobBudget, + scenario_phase_deadline, +) +from tests.utils.helpers._fresh_host.shell import phase_env +from tests.utils.helpers._fresh_host.storage import load_context + +FEATURE_SUITE_PATH: Final[str] = "tests/suites/e2e/feature_verification" +FAKE_CHAT_MODEL_REF: Final[str] = "vllm/featver-chat" +# vLLM is a keyless local provider; OpenClaw only needs a non-empty key to register it. +FAKE_PROVIDER_ENV: Final[dict[str, str]] = { + "OPENCLAW_DEFAULT_MODEL": FAKE_CHAT_MODEL_REF, + "VLLM_API_KEY": "featver-local", + "HYPERMEMORY_EMBEDDING_MODEL": "ollama/nomic-embed-text", + "HYPERMEMORY_EMBEDDING_API_BASE": "http://host.docker.internal:11434", +} +DEFAULT_LANE_TIMEOUT_SECONDS: Final[int] = 3600 + + +class LaneError(RuntimeError): + """Raised when a lane step cannot run.""" + + +LINGER_READY_TIMEOUT_SECONDS: Final[float] = 60.0 +_LINGER_POLL_SECONDS: Final[float] = 1.0 + + +def configure_model_provider(context_file: Path) -> Path: + """Point the operator env contract at the fake model server, as an operator edits it.""" + context = load_context(context_file) + env = phase_env(context) + env_file = varlock_local_env_file( + Path(context.repo_root), home_dir=Path(context.app_home), environ=env + ) + env_file.parent.mkdir(parents=True, exist_ok=True) + for key, value in FAKE_PROVIDER_ENV.items(): + set_env_assignment(env_file, key, value) + env_file.chmod(0o600) + return env_file + + +def enable_systemd_user_manager() -> Path: + """Start the runner's systemd user manager (lingering) and wait for its bus.""" + user = ( + os.environ.get("USER") + or subprocess.run(["id", "-un"], check=True, capture_output=True, text=True).stdout.strip() + ) + subprocess.run(["sudo", "loginctl", "enable-linger", user], check=True) + runtime_dir = Path(f"/run/user/{os.getuid()}") + bus = runtime_dir / "bus" + deadline = time.monotonic() + LINGER_READY_TIMEOUT_SECONDS + while time.monotonic() < deadline: + if bus.exists(): + probe = subprocess.run( + ["systemctl", "--user", "is-system-running"], + env={**os.environ, "XDG_RUNTIME_DIR": str(runtime_dir)}, + capture_output=True, + text=True, + check=False, + ) + if probe.stdout.strip() in {"running", "degraded", "starting"}: + return runtime_dir + time.sleep(_LINGER_POLL_SECONDS) + raise LaneError(f"systemd user manager bus {bus} did not come up") + + +def _lane_timeout_seconds() -> int: + budget = JobBudget.from_env(os.environ) + if budget is None: + return DEFAULT_LANE_TIMEOUT_SECONDS + return max(60, int(scenario_phase_deadline(budget).remaining(SYSTEM_CLOCK))) + + +def run_lane( + *, + context_file: Path, + lane_id: str, + results_path: Path, + fake_model_dir: Path | None, + manifest_path: Path | None = None, +) -> int: + """Run the lane's feature tests against the provisioned host; return pytest's exit code.""" + context = load_context(context_file) + repo_root = Path(context.repo_root) + # The pytest session validates the lane against the matrix (this step runs without PyYAML). + manifest_file = manifest_path or repo_root / "tests" / "feature_matrix.yaml" + env = phase_env(context) + env.update( + { + LANE_ENV: lane_id, + CONTEXT_ENV: str(context_file), + RESULTS_ENV: str(results_path), + WORK_DIR_ENV: str(Path(context.tmp_root) / "featver"), + MANIFEST_ENV: str(manifest_file), + } + ) + if fake_model_dir is not None: + env[FAKE_MODEL_DIR_ENV] = str(fake_model_dir) + command = [ + str(repo_root / ".venv" / "bin" / "python"), + "-m", + "pytest", + FEATURE_SUITE_PATH, + "-m", + "feature_verification", + "-p", + "no:cacheprovider", + "-rA", + "--durations=15", + ] + timeout = _lane_timeout_seconds() + print(f"[feature-verification] lane={lane_id} timeout={timeout}s: {' '.join(command)}") + try: + completed = subprocess.run(command, cwd=repo_root, env=env, check=False, timeout=timeout) + except subprocess.TimeoutExpired as exc: + raise LaneError(f"lane {lane_id} exceeded its {timeout}s budget") from exc + if not results_path.is_file(): + raise LaneError(f"lane {lane_id} wrote no results to {results_path}") + return completed.returncode diff --git a/tests/utils/helpers/_feature_verification/manifest.py b/tests/utils/helpers/_feature_verification/manifest.py new file mode 100644 index 00000000..0aed6afd --- /dev/null +++ b/tests/utils/helpers/_feature_verification/manifest.py @@ -0,0 +1,235 @@ +"""Typed loader and validator for the checked-in feature matrix (``tests/feature_matrix.yaml``).""" + +from __future__ import annotations + +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal, cast + +import yaml + +type PlatformName = Literal["linux", "macos"] +type Verification = Literal["live", "fake-model", "stub"] +type FeatureStatus = Literal["verifiable", "unverifiable"] + +MANIFEST_SCHEMA_VERSION: Final[int] = 1 +PLATFORMS: Final[frozenset[str]] = frozenset({"linux", "macos"}) +VERIFICATIONS: Final[frozenset[str]] = frozenset({"live", "fake-model", "stub"}) +STATUSES: Final[frozenset[str]] = frozenset({"verifiable", "unverifiable"}) +FEATURE_ID_PATTERN: Final[re.Pattern[str]] = re.compile(r"^[a-z0-9]+(?:[.-][a-z0-9]+)*$") +CALL_SITE_PATTERN: Final[re.Pattern[str]] = re.compile(r"^(?P[^:\s]+):(?P[1-9]\d*)$") +TEST_NODE_PATTERN: Final[re.Pattern[str]] = re.compile( + r"^(?Ptests/suites/e2e/feature_verification/test_[a-z0-9_]+\.py)::(?Ptest_[a-z0-9_]+)$" +) + + +class FeatureVerificationError(RuntimeError): + """Raised when the feature matrix or a lane result is invalid.""" + + +@dataclass(frozen=True, slots=True) +class Lane: + """One provisioned CI host the suite runs on.""" + + lane_id: str + platform: PlatformName + profile: str + scenario: str + activate_services: bool + description: str + + +@dataclass(frozen=True, slots=True) +class Feature: + """One product feature and the test that verifies it.""" + + feature_id: str + category: str + title: str + call_sites: tuple[str, ...] + status: FeatureStatus + verification: Verification | None + lanes: tuple[str, ...] + test: str | None + reason: str | None + limitations: str | None + + @property + def test_path(self) -> str | None: + """Return the repo-relative test module path.""" + return None if self.test is None else self.test.split("::", 1)[0] + + @property + def test_name(self) -> str | None: + """Return the test function name.""" + return None if self.test is None else self.test.split("::", 1)[1] + + +@dataclass(frozen=True, slots=True) +class FeatureManifest: + """Validated feature matrix.""" + + lanes: Mapping[str, Lane] + features: tuple[Feature, ...] + + def feature(self, feature_id: str) -> Feature: + """Return one feature by id.""" + for feature in self.features: + if feature.feature_id == feature_id: + return feature + raise FeatureVerificationError(f"unknown feature {feature_id!r}") + + def features_for_lane(self, lane_id: str) -> tuple[Feature, ...]: + """Return the verifiable features scheduled on one lane.""" + return tuple(feature for feature in self.features if lane_id in feature.lanes) + + def feature_by_test(self) -> dict[str, Feature]: + """Map each test node id to the feature it verifies.""" + return {feature.test: feature for feature in self.features if feature.test is not None} + + def platforms(self, feature: Feature) -> tuple[PlatformName, ...]: + """Return the platforms one feature runs on, derived from its lanes.""" + return tuple(sorted({self.lanes[lane_id].platform for lane_id in feature.lanes})) + + +def _mapping(value: object, *, path: str) -> Mapping[str, object]: + if not isinstance(value, Mapping): + raise FeatureVerificationError(f"{path} must be a mapping") + return cast(Mapping[str, object], value) + + +def _string(value: object, *, path: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise FeatureVerificationError(f"{path} must be a non-empty string") + return value.strip() + + +def _optional_string(value: object, *, path: str) -> str | None: + return None if value is None else _string(value, path=path) + + +def _string_tuple(value: object, *, path: str) -> tuple[str, ...]: + if value is None: + return () + if not isinstance(value, Sequence) or isinstance(value, str): + raise FeatureVerificationError(f"{path} must be a list of strings") + return tuple( + _string(item, path=f"{path}[{index}]") + for index, item in enumerate(cast(Sequence[object], value)) + ) + + +def _choice(value: object, *, allowed: frozenset[str], path: str) -> str: + text = _string(value, path=path) + if text not in allowed: + raise FeatureVerificationError(f"{path} must be one of {sorted(allowed)}, got {text!r}") + return text + + +def _load_lane(lane_id: str, raw: object) -> Lane: + path = f"lanes.{lane_id}" + payload = _mapping(raw, path=path) + activate = payload.get("activate_services") + if not isinstance(activate, bool): + raise FeatureVerificationError(f"{path}.activate_services must be a boolean") + return Lane( + lane_id=lane_id, + platform=cast( + PlatformName, + _choice(payload.get("platform"), allowed=PLATFORMS, path=f"{path}.platform"), + ), + profile=_string(payload.get("profile"), path=f"{path}.profile"), + scenario=_string(payload.get("scenario"), path=f"{path}.scenario"), + activate_services=activate, + description=_string(payload.get("description"), path=f"{path}.description"), + ) + + +def _load_feature(index: int, raw: object, *, lanes: Mapping[str, Lane]) -> Feature: + path = f"features[{index}]" + payload = _mapping(raw, path=path) + feature_id = _string(payload.get("id"), path=f"{path}.id") + if FEATURE_ID_PATTERN.fullmatch(feature_id) is None: + raise FeatureVerificationError(f"{path}.id {feature_id!r} must be dotted kebab-case") + path = f"feature {feature_id}" + status = cast( + FeatureStatus, + _choice(payload.get("status", "verifiable"), allowed=STATUSES, path=f"{path}.status"), + ) + call_sites = _string_tuple(payload.get("call_sites"), path=f"{path}.call_sites") + for call_site in call_sites: + if CALL_SITE_PATTERN.fullmatch(call_site) is None: + raise FeatureVerificationError( + f"{path}.call_sites entry {call_site!r} is not file:line" + ) + feature_lanes = _string_tuple(payload.get("lanes"), path=f"{path}.lanes") + unknown_lanes = sorted(set(feature_lanes) - set(lanes)) + if unknown_lanes: + raise FeatureVerificationError(f"{path} references unknown lanes {unknown_lanes}") + test = _optional_string(payload.get("test"), path=f"{path}.test") + if test is not None and TEST_NODE_PATTERN.fullmatch(test) is None: + raise FeatureVerificationError(f"{path}.test {test!r} is not a feature-suite node id") + raw_verification = payload.get("verification") + verification = ( + None + if raw_verification is None + else cast( + Verification, + _choice(raw_verification, allowed=VERIFICATIONS, path=f"{path}.verification"), + ) + ) + reason = _optional_string(payload.get("reason"), path=f"{path}.reason") + if status == "verifiable": + if test is None or not feature_lanes or verification is None: + raise FeatureVerificationError( + f"{path} is verifiable, so it needs a test, at least one lane, and a verification" + ) + elif reason is None or test is not None or feature_lanes: + raise FeatureVerificationError( + f"{path} is unverifiable, so it needs a reason and must not schedule a test" + ) + if not call_sites: + raise FeatureVerificationError(f"{path} must cite at least one call site") + return Feature( + feature_id=feature_id, + category=_string(payload.get("category"), path=f"{path}.category"), + title=_string(payload.get("title"), path=f"{path}.title"), + call_sites=call_sites, + status=status, + verification=verification, + lanes=feature_lanes, + test=test, + reason=reason, + limitations=_optional_string(payload.get("limitations"), path=f"{path}.limitations"), + ) + + +def load_manifest(path: Path) -> FeatureManifest: + """Load and validate the feature matrix.""" + if not path.is_file(): + raise FeatureVerificationError(f"feature matrix not found at {path}") + document = _mapping(yaml.safe_load(path.read_text(encoding="utf-8")), path="manifest") + if document.get("schema_version") != MANIFEST_SCHEMA_VERSION: + raise FeatureVerificationError(f"manifest schema_version must be {MANIFEST_SCHEMA_VERSION}") + raw_lanes = _mapping(document.get("lanes"), path="lanes") + lanes = {str(lane_id): _load_lane(str(lane_id), raw) for lane_id, raw in raw_lanes.items()} + raw_features = document.get("features") + if not isinstance(raw_features, list) or not raw_features: + raise FeatureVerificationError("features must be a non-empty list") + features = tuple( + _load_feature(index, raw, lanes=lanes) + for index, raw in enumerate(cast(list[object], raw_features)) + ) + seen_ids: set[str] = set() + seen_tests: set[str] = set() + for feature in features: + if feature.feature_id in seen_ids: + raise FeatureVerificationError(f"duplicate feature id {feature.feature_id!r}") + seen_ids.add(feature.feature_id) + if feature.test is not None: + if feature.test in seen_tests: + raise FeatureVerificationError(f"test {feature.test!r} maps to several features") + seen_tests.add(feature.test) + return FeatureManifest(lanes=lanes, features=features) diff --git a/tests/utils/helpers/_feature_verification/processes.py b/tests/utils/helpers/_feature_verification/processes.py new file mode 100644 index 00000000..83adab50 --- /dev/null +++ b/tests/utils/helpers/_feature_verification/processes.py @@ -0,0 +1,133 @@ +"""Detached lifecycle for the fake model server used by feature-verification lanes. + +Workflow steps are one-shot shell commands, so the server runs as a detached child that +outlives the step that started it; its pid and bound ports live in a small state file. +""" + +from __future__ import annotations + +import json +import os +import signal +import subprocess +import sys +import time +import urllib.error +import urllib.request +from collections.abc import Sequence +from pathlib import Path +from typing import Final, cast + +from tests.utils.helpers._feature_verification.fake_model import HEALTH_PATH, JOURNAL_PATH + +STATE_FILE_NAME: Final[str] = "fake-model.json" +LOG_FILE_NAME: Final[str] = "fake-model.log" +READY_TIMEOUT_SECONDS: Final[float] = 20.0 +STOP_TIMEOUT_SECONDS: Final[float] = 10.0 +_POLL_SECONDS: Final[float] = 0.2 + + +class FakeModelProcessError(RuntimeError): + """Raised when the detached fake model server cannot be managed.""" + + +def _probe_host(host: str) -> str: + """Return the address a local client should use for *host*.""" + return "127.0.0.1" if host in {"0.0.0.0", "", "::"} else host + + +def _healthy(host: str, port: int) -> bool: + try: + with urllib.request.urlopen( # noqa: S310 - loopback probe + f"http://{_probe_host(host)}:{port}{HEALTH_PATH}", timeout=2 + ) as response: + return bool(response.status == 200) + except (urllib.error.URLError, OSError): + return False + + +def start_detached( + *, + script_path: Path, + host: str, + ports: Sequence[int], + state_dir: Path, +) -> dict[str, object]: + """Start ``serve-fake-model`` detached and block until every port answers.""" + state_dir.mkdir(parents=True, exist_ok=True) + log_path = state_dir / LOG_FILE_NAME + command = [sys.executable, str(script_path), "serve-fake-model", "--host", host] + for port in ports: + command.extend(["--port", str(port)]) + with log_path.open("ab") as log_handle: + process = subprocess.Popen( # noqa: S603 - fixed argv built above + command, + stdin=subprocess.DEVNULL, + stdout=log_handle, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + deadline = time.monotonic() + READY_TIMEOUT_SECONDS + while time.monotonic() < deadline: + if process.poll() is not None: + raise FakeModelProcessError( + f"fake model server exited with {process.returncode}; see {log_path}" + ) + if all(_healthy(host, port) for port in ports): + state: dict[str, object] = { + "pid": process.pid, + "host": host, + "ports": list(ports), + "log": str(log_path), + } + (state_dir / STATE_FILE_NAME).write_text(json.dumps(state), encoding="utf-8") + return state + time.sleep(_POLL_SECONDS) + process.terminate() + raise FakeModelProcessError(f"fake model server not ready in {READY_TIMEOUT_SECONDS}s") + + +def load_state(state_dir: Path) -> dict[str, object]: + """Return the persisted server state.""" + state_path = state_dir / STATE_FILE_NAME + if not state_path.is_file(): + raise FakeModelProcessError(f"no fake model state at {state_path}") + payload: object = json.loads(state_path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise FakeModelProcessError(f"invalid fake model state at {state_path}") + return cast(dict[str, object], payload) + + +def fetch_journal(host: str, port: int) -> list[dict[str, object]]: + """Return the request journal recorded by a running server.""" + with urllib.request.urlopen( # noqa: S310 - loopback probe + f"http://{_probe_host(host)}:{port}{JOURNAL_PATH}", timeout=5 + ) as response: + payload: object = json.loads(response.read().decode("utf-8")) + if not isinstance(payload, dict): + raise FakeModelProcessError("fake model journal is not a JSON object") + requests = cast(dict[str, object], payload).get("requests") + if not isinstance(requests, list): + raise FakeModelProcessError("fake model journal has no request list") + return [cast(dict[str, object], entry) for entry in cast(list[object], requests)] + + +def stop_detached(state_dir: Path) -> bool: + """Terminate the detached server; return whether a live process was stopped.""" + state = load_state(state_dir) + pid = state.get("pid") + if not isinstance(pid, int): + raise FakeModelProcessError("fake model state has no pid") + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + return False + deadline = time.monotonic() + STOP_TIMEOUT_SECONDS + while time.monotonic() < deadline: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + time.sleep(_POLL_SECONDS) + os.kill(pid, signal.SIGKILL) + return True diff --git a/tests/utils/helpers/_feature_verification/recorder.py b/tests/utils/helpers/_feature_verification/recorder.py new file mode 100644 index 00000000..ac8e7c1a --- /dev/null +++ b/tests/utils/helpers/_feature_verification/recorder.py @@ -0,0 +1,153 @@ +"""Collect pytest outcomes for one lane and map them onto manifest features.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import Final + +import pytest + +from tests.utils.helpers._feature_verification.envvars import ( + CONTEXT_ENV, + LANE_ENV, + MANIFEST_ENV, + RESULTS_ENV, +) +from tests.utils.helpers._feature_verification.host import default_manifest_path +from tests.utils.helpers._feature_verification.manifest import ( + Feature, + FeatureManifest, + FeatureVerificationError, + load_manifest, +) +from tests.utils.helpers._feature_verification.results import ( + MAX_MESSAGE_CHARS, + FeatureResult, + LaneResults, + TestOutcome, +) +from tests.utils.helpers.repo import REPO_ROOT + +FEATURE_SUITE_PARTS: Final[tuple[str, ...]] = ("suites", "e2e", "feature_verification") + + +def is_feature_suite_item(item: pytest.Item) -> bool: + """Return whether *item* belongs to the feature-verification suite.""" + parts = Path(str(item.fspath)).resolve().parts + return any( + parts[index : index + len(FEATURE_SUITE_PARTS)] == FEATURE_SUITE_PARTS + for index in range(len(parts)) + ) + + +def _message(report: pytest.TestReport) -> str: + if report.outcome == "skipped" and isinstance(report.longrepr, tuple): + return str(report.longrepr[2]).removeprefix("Skipped: ") + text = report.longreprtext or "" + return text[-MAX_MESSAGE_CHARS:] + + +@dataclass(slots=True) +class LaneRecorder: + """Accumulate one lane's feature outcomes and persist them at session end.""" + + manifest: FeatureManifest + lane_id: str + results_path: Path + scheduled: dict[str, Feature] = field(default_factory=lambda: dict[str, Feature]()) + results: dict[str, FeatureResult] = field(default_factory=lambda: dict[str, FeatureResult]()) + versions: dict[str, str] = field(default_factory=lambda: dict[str, str]()) + started_at: str = field(default_factory=lambda: datetime.now(tz=UTC).isoformat()) + + @classmethod + def from_environment(cls) -> LaneRecorder | None: + """Return a recorder when the lane environment is present, else ``None``.""" + lane_id = os.environ.get(LANE_ENV, "").strip() + if not lane_id: + return None + results_raw = os.environ.get(RESULTS_ENV, "").strip() + if not results_raw or not os.environ.get(CONTEXT_ENV, "").strip(): + raise FeatureVerificationError( + f"{LANE_ENV} is set, so {RESULTS_ENV} and {CONTEXT_ENV} are required" + ) + manifest_path = Path( + os.environ.get(MANIFEST_ENV, "").strip() or default_manifest_path(REPO_ROOT).as_posix() + ) + manifest = load_manifest(manifest_path) + if lane_id not in manifest.lanes: + raise FeatureVerificationError(f"unknown lane {lane_id!r}") + return cls(manifest=manifest, lane_id=lane_id, results_path=Path(results_raw)) + + def select(self, items: list[pytest.Item]) -> tuple[list[pytest.Item], list[pytest.Item]]: + """Split suite items into (scheduled on this lane, not scheduled).""" + by_test = self.manifest.feature_by_test() + keep: list[pytest.Item] = [] + drop: list[pytest.Item] = [] + for item in items: + feature = by_test.get(item.nodeid) + if feature is None: + raise pytest.UsageError(f"{item.nodeid} is not mapped in the feature matrix") + if self.lane_id in feature.lanes: + self.scheduled[item.nodeid] = feature + keep.append(item) + else: + drop.append(item) + return keep, drop + + def record(self, report: pytest.TestReport) -> None: + """Fold one phase report into the feature's outcome.""" + feature = self.scheduled.get(report.nodeid) + if feature is None: + return + existing = self.results.get(report.nodeid) + outcome: TestOutcome | None = None + if report.when == "call": + outcome = "passed" if report.passed else "skipped" if report.skipped else "failed" + elif report.failed: + outcome = "error" + elif report.when == "setup" and report.skipped: + outcome = "skipped" + if outcome is None: + if existing is not None: + existing.duration_seconds += report.duration + return + if existing is not None and existing.outcome in {"failed", "error"}: + existing.duration_seconds += report.duration + return + self.results[report.nodeid] = FeatureResult( + feature=feature.feature_id, + test=report.nodeid, + outcome=outcome, + duration_seconds=report.duration + (existing.duration_seconds if existing else 0.0), + message=_message(report), + ) + + def write(self) -> LaneResults: + """Persist lane results, marking scheduled-but-unreported features as not run.""" + lane = self.manifest.lanes[self.lane_id] + results = list(self.results.values()) + for feature in self.manifest.features_for_lane(self.lane_id): + if feature.test is not None and feature.test not in self.results: + results.append( + FeatureResult( + feature=feature.feature_id, + test=feature.test, + outcome="not-run", + duration_seconds=0.0, + message="test was scheduled on this lane but did not run", + ) + ) + lane_results = LaneResults( + lane=self.lane_id, + platform=lane.platform, + profile=lane.profile, + started_at=self.started_at, + finished_at=datetime.now(tz=UTC).isoformat(), + versions=dict(self.versions), + results=sorted(results, key=lambda result: result.feature), + ) + lane_results.write(self.results_path) + return lane_results diff --git a/tests/utils/helpers/_feature_verification/reporting.py b/tests/utils/helpers/_feature_verification/reporting.py new file mode 100644 index 00000000..9bbd1a69 --- /dev/null +++ b/tests/utils/helpers/_feature_verification/reporting.py @@ -0,0 +1,237 @@ +"""Join lane results with the feature matrix into a per-feature verdict report.""" + +from __future__ import annotations + +import json +from collections import Counter +from dataclasses import asdict, dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Final + +from tests.utils.helpers._feature_verification.manifest import ( + Feature, + FeatureManifest, + load_manifest, +) +from tests.utils.helpers._feature_verification.results import ( + FeatureVerdict, + LaneResults, + load_lane_results, +) + +LANE_RESULTS_GLOB: Final[str] = "feature-results-*.json" +VERDICTS: Final[tuple[FeatureVerdict, ...]] = ("PASS", "FAIL", "UNVERIFIABLE") +_DETAIL_CHARS: Final[int] = 180 + + +@dataclass(frozen=True, slots=True) +class LaneVerdict: + """One feature's outcome on one lane.""" + + verdict: FeatureVerdict + outcome: str + message: str + duration_seconds: float = 0.0 + + +@dataclass(frozen=True, slots=True) +class FeatureRow: + """One feature's overall verdict across its lanes.""" + + id: str + category: str + title: str + verdict: FeatureVerdict + verification: str | None + platforms: list[str] + lanes: dict[str, LaneVerdict] + reason: str | None + limitations: str | None + test: str | None + call_sites: list[str] + + +@dataclass(frozen=True, slots=True) +class LaneSummary: + """Where one lane ran and which tool versions it verified.""" + + platform: str + profile: str + reported: bool + versions: dict[str, str] + + +@dataclass(frozen=True, slots=True) +class VerificationReport: + """The full per-feature report.""" + + generated_at: str + lanes: dict[str, LaneSummary] + counts: dict[str, int] + categories: dict[str, dict[str, int]] + features: list[FeatureRow] + schema_version: int = 1 + + @property + def failures(self) -> int: + """Return the number of FAIL features.""" + return self.counts["FAIL"] + + def to_json(self) -> str: + """Serialize the report.""" + return json.dumps(asdict(self), indent=2, sort_keys=True) + "\n" + + +def _lane_results(results_dir: Path) -> dict[str, LaneResults]: + lanes: dict[str, LaneResults] = {} + for path in sorted(results_dir.rglob(LANE_RESULTS_GLOB)): + lane = load_lane_results(path) + lanes[lane.lane] = lane + return lanes + + +def _lane_verdict(feature: Feature, lane: LaneResults | None) -> LaneVerdict: + if lane is None: + return LaneVerdict("FAIL", "not-run", "lane reported no results") + match = next((result for result in lane.results if result.feature == feature.feature_id), None) + if match is None: + return LaneVerdict("FAIL", "not-run", "test did not run on lane") + return LaneVerdict( + verdict=match.verdict, + outcome=match.outcome, + message=match.message, + duration_seconds=round(match.duration_seconds, 3), + ) + + +def _feature_verdict(feature: Feature, lanes: dict[str, LaneVerdict]) -> FeatureVerdict: + if feature.status == "unverifiable": + return "UNVERIFIABLE" + verdicts = {entry.verdict for entry in lanes.values()} + if "FAIL" in verdicts: + return "FAIL" + return "PASS" if verdicts == {"PASS"} else "UNVERIFIABLE" + + +def build_report(manifest: FeatureManifest, lanes: dict[str, LaneResults]) -> VerificationReport: + """Return the report for every manifest feature.""" + rows: list[FeatureRow] = [] + counts: Counter[str] = Counter() + by_category: dict[str, Counter[str]] = {} + for feature in manifest.features: + lane_verdicts = { + lane_id: _lane_verdict(feature, lanes.get(lane_id)) for lane_id in feature.lanes + } + verdict = _feature_verdict(feature, lane_verdicts) + counts[verdict] += 1 + by_category.setdefault(feature.category, Counter())[verdict] += 1 + rows.append( + FeatureRow( + id=feature.feature_id, + category=feature.category, + title=feature.title, + verdict=verdict, + verification=feature.verification, + platforms=list(manifest.platforms(feature)), + lanes=lane_verdicts, + reason=feature.reason, + limitations=feature.limitations, + test=feature.test, + call_sites=list(feature.call_sites), + ) + ) + return VerificationReport( + generated_at=datetime.now(tz=UTC).isoformat(), + lanes={ + lane_id: LaneSummary( + platform=lane.platform, + profile=lane.profile, + reported=lane_id in lanes, + versions=dict(lanes[lane_id].versions) if lane_id in lanes else {}, + ) + for lane_id, lane in manifest.lanes.items() + }, + counts={verdict: counts.get(verdict, 0) for verdict in VERDICTS}, + categories={ + category: {verdict: tally.get(verdict, 0) for verdict in VERDICTS} + for category, tally in sorted(by_category.items()) + }, + features=rows, + ) + + +def _cell(text: str) -> str: + flattened = " ".join(text.split()) + if len(flattened) > _DETAIL_CHARS: + flattened = flattened[: _DETAIL_CHARS - 3] + "..." + return flattened.replace("|", "\\|") + + +def _detail(row: FeatureRow) -> str: + if row.verdict == "UNVERIFIABLE" and row.reason: + return _cell(row.reason) + for lane_id, entry in row.lanes.items(): + if entry.verdict != "PASS": + return _cell(f"{lane_id}: {entry.message}") + return _cell(row.limitations or "") + + +def render_markdown(report: VerificationReport) -> str: + """Render the report as a GitHub step-summary section.""" + lines = [ + "## Feature Verification", + "", + "| Lane | Platform | Profile | Versions |", + "| --- | --- | --- | --- |", + ] + for lane_id, lane in report.lanes.items(): + versions = ", ".join(f"{name} {value}" for name, value in lane.versions.items()) + lines.append( + f"| {lane_id} | {lane.platform} | {lane.profile} | {_cell(versions or 'not reported')} |" + ) + lines.extend( + [ + "", + " · ".join(f"**{verdict}** {report.counts[verdict]}" for verdict in VERDICTS), + "", + "| Category | PASS | FAIL | UNVERIFIABLE |", + "| --- | --- | --- | --- |", + ] + ) + for category, tally in report.categories.items(): + lines.append( + f"| {category} | {tally['PASS']} | {tally['FAIL']} | {tally['UNVERIFIABLE']} |" + ) + lines.extend( + [ + "", + "| Feature | Category | Verdict | Verification | Platforms | Detail |", + "| --- | --- | --- | --- | --- | --- |", + ] + ) + for row in report.features: + lines.append( + f"| `{row.id}` | {row.category} | {row.verdict} | {row.verification or '-'} " + f"| {', '.join(row.platforms) or '-'} | {_detail(row)} |" + ) + lines.append("") + return "\n".join(lines) + + +def write_lane_report( + *, + results_dir: Path, + manifest_path: Path, + output_path: Path, + summary_file: Path | None, +) -> int: + """Write the JSON report (and optional step summary); return the FAIL count.""" + report = build_report(load_manifest(manifest_path), _lane_results(results_dir)) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(report.to_json(), encoding="utf-8") + if summary_file is not None: + summary_file.parent.mkdir(parents=True, exist_ok=True) + with summary_file.open("a", encoding="utf-8") as handle: + handle.write(render_markdown(report)) + return report.failures diff --git a/tests/utils/helpers/_feature_verification/results.py b/tests/utils/helpers/_feature_verification/results.py new file mode 100644 index 00000000..03575667 --- /dev/null +++ b/tests/utils/helpers/_feature_verification/results.py @@ -0,0 +1,121 @@ +"""Per-lane feature results written by the pytest plugin and read by the reporter.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Final, Literal, cast + +from tests.utils.helpers._feature_verification.manifest import FeatureVerificationError + +type TestOutcome = Literal["passed", "failed", "error", "skipped", "not-run"] +type FeatureVerdict = Literal["PASS", "FAIL", "UNVERIFIABLE"] + +RESULT_SCHEMA_VERSION: Final[int] = 1 +TEST_OUTCOMES: Final[frozenset[str]] = frozenset( + {"passed", "failed", "error", "skipped", "not-run"} +) +# A test that discovers at runtime that its feature cannot be exercised skips with this prefix. +UNVERIFIABLE_SKIP_PREFIX: Final[str] = "unverifiable:" +MAX_MESSAGE_CHARS: Final[int] = 4000 + + +@dataclass(slots=True) +class FeatureResult: + """Outcome of one feature test on one lane.""" + + feature: str + test: str + outcome: TestOutcome + duration_seconds: float + message: str = "" + + @property + def verdict(self) -> FeatureVerdict: + """Map the pytest outcome onto the report vocabulary.""" + if self.outcome == "passed": + return "PASS" + if self.outcome == "skipped" and self.message.startswith(UNVERIFIABLE_SKIP_PREFIX): + return "UNVERIFIABLE" + return "FAIL" + + +def _empty_results() -> list[FeatureResult]: + return [] + + +def _empty_versions() -> dict[str, str]: + return {} + + +@dataclass(slots=True) +class LaneResults: + """All feature outcomes recorded on one lane.""" + + lane: str + platform: str + profile: str + started_at: str + finished_at: str = "" + versions: dict[str, str] = field(default_factory=_empty_versions) + results: list[FeatureResult] = field(default_factory=_empty_results) + schema_version: int = RESULT_SCHEMA_VERSION + + def write(self, path: Path) -> None: + """Persist the lane results as JSON.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(asdict(self), indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _str(value: object, *, path: str) -> str: + if not isinstance(value, str): + raise FeatureVerificationError(f"{path} must be a string") + return value + + +def load_lane_results(path: Path) -> LaneResults: + """Load one lane result file.""" + payload: object = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, Mapping): + raise FeatureVerificationError(f"{path} must hold a JSON object") + document = cast(Mapping[str, object], payload) + if document.get("schema_version") != RESULT_SCHEMA_VERSION: + raise FeatureVerificationError(f"{path} has an unsupported schema_version") + raw_results = document.get("results") + if not isinstance(raw_results, Sequence): + raise FeatureVerificationError(f"{path}.results must be a list") + results: list[FeatureResult] = [] + for index, raw in enumerate(cast(Sequence[object], raw_results)): + if not isinstance(raw, Mapping): + raise FeatureVerificationError(f"{path}.results[{index}] must be an object") + entry = cast(Mapping[str, object], raw) + outcome = _str(entry.get("outcome"), path=f"{path}.results[{index}].outcome") + if outcome not in TEST_OUTCOMES: + raise FeatureVerificationError(f"{path}.results[{index}] has outcome {outcome!r}") + duration = entry.get("duration_seconds", 0.0) + results.append( + FeatureResult( + feature=_str(entry.get("feature"), path=f"{path}.results[{index}].feature"), + test=_str(entry.get("test"), path=f"{path}.results[{index}].test"), + outcome=cast(TestOutcome, outcome), + duration_seconds=float(duration) if isinstance(duration, int | float) else 0.0, + message=str(entry.get("message", "")), + ) + ) + raw_versions = document.get("versions", {}) + versions = ( + {str(key): str(value) for key, value in cast(Mapping[object, object], raw_versions).items()} + if isinstance(raw_versions, Mapping) + else {} + ) + return LaneResults( + lane=_str(document.get("lane"), path=f"{path}.lane"), + platform=_str(document.get("platform"), path=f"{path}.platform"), + profile=_str(document.get("profile"), path=f"{path}.profile"), + started_at=_str(document.get("started_at"), path=f"{path}.started_at"), + finished_at=str(document.get("finished_at", "")), + versions=versions, + results=results, + ) diff --git a/tests/utils/helpers/_feature_verification/runtime.py b/tests/utils/helpers/_feature_verification/runtime.py new file mode 100644 index 00000000..bb6d2380 --- /dev/null +++ b/tests/utils/helpers/_feature_verification/runtime.py @@ -0,0 +1,218 @@ +"""Gateway and sidecar runtime control for feature-verification lanes. + +On lanes that activate host services the gateway belongs to systemd/launchd and the harness +only restarts it through the service manager. When activation is off (or failed, which the +setup feature reports on its own) the harness starts ``clawops ops gateway start`` itself, so +the remaining features still get a real gateway rather than cascading into failures. +""" + +from __future__ import annotations + +import os +import signal +import socket +import subprocess +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal, cast + +from tests.utils.helpers._feature_verification.host import FeatureHost + +type GatewayMode = Literal["systemd", "launchd", "harness"] + +SYSTEMD_GATEWAY_UNIT: Final[str] = "openclaw-gateway.service" +LAUNCHD_GATEWAY_LABEL: Final[str] = "ai.openclaw.gateway" +DEFAULT_GATEWAY_PORT: Final[int] = 18789 +GATEWAY_READY_TIMEOUT_SECONDS: Final[float] = 150.0 +_POLL_SECONDS: Final[float] = 2.0 +_STOP_TIMEOUT_SECONDS: Final[float] = 20.0 +_LOG_TAIL_CHARS: Final[int] = 2_000_000 +# OpenClaw writes its rolling file log here regardless of TMPDIR. +OPENCLAW_TMP_LOG_DIR: Final[str] = "/tmp/openclaw" + + +def tcp_open(port: int, host: str = "127.0.0.1", timeout: float = 2.0) -> bool: + """Return whether *host*:*port* accepts TCP connections.""" + try: + with socket.create_connection((host, port), timeout=timeout): + return True + except OSError: + return False + + +def gateway_port(host: FeatureHost) -> int: + """Return the gateway port from the rendered config.""" + gateway = host.rendered_config().get("gateway") + if isinstance(gateway, dict): + port = cast(dict[str, object], gateway).get("port") + if isinstance(port, int): + return port + return DEFAULT_GATEWAY_PORT + + +def gateway_status(host: FeatureHost) -> dict[str, object]: + """Return ``openclaw gateway status --json``.""" + payload = host.openclaw("gateway", "status", "--json", timeout=120).require_ok().json() + assert isinstance(payload, dict), payload + return cast(dict[str, object], payload) + + +def gateway_rpc_ok(host: FeatureHost) -> bool: + """Return whether the gateway answers RPC.""" + outcome = host.openclaw("gateway", "status", "--json", timeout=120) + if not outcome.ok: + return False + try: + payload = outcome.json() + except AssertionError: + return False + rpc = cast(dict[str, object], payload).get("rpc") if isinstance(payload, dict) else None + return isinstance(rpc, dict) and cast(dict[str, object], rpc).get("ok") is True + + +def plugin_statuses(host: FeatureHost) -> dict[str, str]: + """Return ``{plugin id: status}`` from ``openclaw plugins list --json``.""" + payload = host.openclaw("plugins", "list", "--json", timeout=300).require_ok().json() + assert isinstance(payload, dict), payload + plugins = cast(dict[str, object], payload).get("plugins") + assert isinstance(plugins, list), payload + statuses: dict[str, str] = {} + for raw in cast(list[object], plugins): + if isinstance(raw, dict): + entry = cast(dict[str, object], raw) + statuses[str(entry.get("id"))] = str(entry.get("status")) + return statuses + + +def service_active(host: FeatureHost) -> bool: + """Return whether the service manager reports the gateway as running.""" + if host.lane.platform == "linux": + return host.run( + ["systemctl", "--user", "is-active", "--quiet", SYSTEMD_GATEWAY_UNIT], timeout=30 + ).ok + outcome = host.run( + ["launchctl", "print", f"gui/{os.getuid()}/{LAUNCHD_GATEWAY_LABEL}"], timeout=30 + ) + return outcome.ok and "state = running" in outcome.stdout + + +@dataclass(slots=True) +class GatewayController: + """Own the gateway lifecycle for one lane session.""" + + host: FeatureHost + _process: subprocess.Popen[bytes] | None = None + _log_path: Path | None = None + + @property + def mode(self) -> GatewayMode: + """Return who supervises the gateway.""" + if self._process is not None or not self.host.lane.activate_services: + return "harness" + return "systemd" if self.host.lane.platform == "linux" else "launchd" + + @property + def log_path(self) -> Path | None: + """Return the harness-managed gateway log, when the harness started it.""" + return self._log_path + + def log_text(self) -> str: + """Return recent gateway output from every place the active supervisor writes it.""" + chunks: list[str] = [] + if self._log_path is not None and self._log_path.is_file(): + chunks.append(self._log_path.read_text(encoding="utf-8", errors="replace")) + if self.host.lane.platform == "linux": + journal = self.host.run( + ["journalctl", "--user", "-u", SYSTEMD_GATEWAY_UNIT, "--no-pager", "-n", "5000"], + timeout=60, + ) + chunks.append(journal.stdout) + state_logs = self.host.home / ".openclaw" / "logs" + file_logs = [ + *sorted(state_logs.glob("*gateway*.log")), + *sorted(Path(OPENCLAW_TMP_LOG_DIR).glob("openclaw-*.log")), + ] + for log_file in file_logs: + chunks.append(log_file.read_text(encoding="utf-8", errors="replace")[-_LOG_TAIL_CHARS:]) + return self.host.redact("\n".join(chunks)) + + def wait_ready(self, timeout: float = GATEWAY_READY_TIMEOUT_SECONDS) -> None: + """Block until the gateway port is open and RPC answers.""" + port = gateway_port(self.host) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if self._process is not None and self._process.poll() is not None: + raise AssertionError( + f"harness gateway exited with {self._process.returncode}; see {self._log_path}" + ) + if tcp_open(port) and gateway_rpc_ok(self.host): + return + time.sleep(_POLL_SECONDS) + raise AssertionError(f"gateway on port {port} not ready within {timeout:.0f}s") + + def _start_harness_process(self) -> None: + self._log_path = self.host.work_dir / f"gateway-{time.time_ns()}.log" + with self._log_path.open("ab") as log_handle: + self._process = subprocess.Popen( # noqa: S603 - fixed clawops argv + [ + str(self.host.python), + "-m", + "clawops", + "ops", + "--asset-root", + ".", + "gateway", + "start", + ], + cwd=self.host.repo_root, + env=self.host.product_env(), + stdin=subprocess.DEVNULL, + stdout=log_handle, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + + def ensure_running(self) -> GatewayMode: + """Make sure a gateway answers; start a harness gateway when none is supervised.""" + if gateway_rpc_ok(self.host): + return self.mode + if self.mode != "harness" and service_active(self.host): + self.wait_ready() + return self.mode + if self._process is None: + self._start_harness_process() + self.wait_ready() + return self.mode + + def restart(self) -> None: + """Restart the gateway so it reloads the rendered config.""" + if self.mode == "systemd": + self.host.run( + ["systemctl", "--user", "restart", SYSTEMD_GATEWAY_UNIT], timeout=60 + ).require_ok() + elif self.mode == "launchd": + self.host.run( + ["launchctl", "kickstart", "-k", f"gui/{os.getuid()}/{LAUNCHD_GATEWAY_LABEL}"], + timeout=60, + ).require_ok() + else: + self.stop() + self._start_harness_process() + time.sleep(_POLL_SECONDS) + self.wait_ready() + + def stop(self) -> None: + """Stop a harness-started gateway (its whole process group).""" + process = self._process + if process is None: + return + self._process = None + if process.poll() is not None: + return + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=_STOP_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=_STOP_TIMEOUT_SECONDS) diff --git a/tests/utils/helpers/_fresh_host/context.py b/tests/utils/helpers/_fresh_host/context.py index 9dc6d3a0..e9c90c3d 100644 --- a/tests/utils/helpers/_fresh_host/context.py +++ b/tests/utils/helpers/_fresh_host/context.py @@ -11,6 +11,7 @@ DEFAULT_DOCKER_PULL_PARALLELISM, FreshHostContext, FreshHostReport, + ScenarioSpec, ) from tests.utils.helpers._fresh_host.storage import ( load_context, @@ -70,6 +71,15 @@ def _ensure_cache_dirs() -> None: Path(raw_path).expanduser().resolve().mkdir(parents=True, exist_ok=True) +def _xdg_runtime_dir(spec: ScenarioSpec, *, tmp_root: Path) -> Path | None: + """Return the XDG runtime dir; host-home scenarios use the real user manager's.""" + if spec.platform != "linux": + return None + if spec.use_host_home: + return Path(f"/run/user/{os.getuid()}") + return tmp_root / "xdg-runtime" + + def prepare_context( *, scenario_id: str, @@ -84,8 +94,8 @@ def prepare_context( tmp_root = runner_temp / f"strongclaw-{spec.platform}-host" report_dir = runner_temp / "fresh-host-reports" / scenario_id context_dir = runner_temp / "fresh-host" / scenario_id - app_home = tmp_root / "home" - xdg_runtime_dir = tmp_root / "xdg-runtime" if spec.platform == "linux" else None + app_home = Path.home() if spec.use_host_home else tmp_root / "home" + xdg_runtime_dir = _xdg_runtime_dir(spec, tmp_root=tmp_root) diagnostics_dir = report_dir / "diagnostics" context_file = context_dir / "context.json" report_file = report_dir / "report.json" @@ -127,6 +137,8 @@ def prepare_context( compose_files=[ str((repo_root / relative_path).resolve()) for relative_path in spec.compose_files ], + model_setup_mode=spec.model_setup_mode, + use_host_home=spec.use_host_home, ) report = FreshHostReport( scenario_id=context.scenario_id, diff --git a/tests/utils/helpers/_fresh_host/linux.py b/tests/utils/helpers/_fresh_host/linux.py index 95ff4714..725bb763 100644 --- a/tests/utils/helpers/_fresh_host/linux.py +++ b/tests/utils/helpers/_fresh_host/linux.py @@ -73,8 +73,9 @@ def linux_setup(context: FreshHostContext) -> list[str]: context.profile, "--non-interactive", "--no-verify", - "--no-activate-services", ) + if not context.activate_services: + command.append("--no-activate-services") run_command(command, cwd=repo_root, env=phase_env(context)) return command diff --git a/tests/utils/helpers/_fresh_host/models.py b/tests/utils/helpers/_fresh_host/models.py index 88f967b8..e4664200 100644 --- a/tests/utils/helpers/_fresh_host/models.py +++ b/tests/utils/helpers/_fresh_host/models.py @@ -5,7 +5,14 @@ from dataclasses import dataclass, field from typing import Final, Literal -ScenarioId = Literal["linux", "macos-sidecars", "macos-browser-lab"] +ScenarioId = Literal[ + "linux", + "macos-sidecars", + "macos-browser-lab", + "linux-features", + "linux-features-local", + "macos-features", +] PlatformName = Literal["linux", "macos"] PhaseStatus = Literal["success", "failure", "skipped"] @@ -40,6 +47,10 @@ class ScenarioSpec: verify_rendered_files: bool exercise_sidecars: bool exercise_browser_lab: bool + # Feature-verification lanes run model auth for real and, on Linux, install into the + # runner's own HOME because the systemd user manager only reads units from there. + model_setup_mode: str = "skip" + use_host_home: bool = False @dataclass(slots=True) @@ -90,6 +101,8 @@ class FreshHostContext: exercise_browser_lab: bool phase_names: list[str] compose_files: list[str] + model_setup_mode: str = "skip" + use_host_home: bool = False @dataclass(slots=True) @@ -176,4 +189,60 @@ class FreshHostReport: exercise_sidecars=False, exercise_browser_lab=True, ), + "linux-features": ScenarioSpec( + scenario_id="linux-features", + platform="linux", + job_name="Linux Feature Verification", + compose_files=( + "platform/compose/docker-compose.aux-stack.yaml", + "platform/compose/docker-compose.browser-lab.yaml", + ), + compose_variant=None, + phase_names=("bootstrap", "setup"), + activate_services=True, + ensure_images=True, + normalize_machine_name=False, + verify_launchd=False, + verify_rendered_files=False, + exercise_sidecars=False, + exercise_browser_lab=False, + model_setup_mode="auto", + use_host_home=True, + ), + "linux-features-local": ScenarioSpec( + scenario_id="linux-features-local", + platform="linux", + job_name="Linux Feature Verification (no host services)", + compose_files=( + "platform/compose/docker-compose.aux-stack.yaml", + "platform/compose/docker-compose.browser-lab.yaml", + ), + compose_variant=None, + phase_names=("bootstrap", "setup"), + activate_services=False, + ensure_images=False, + normalize_machine_name=False, + verify_launchd=False, + verify_rendered_files=False, + exercise_sidecars=False, + exercise_browser_lab=False, + model_setup_mode="auto", + use_host_home=True, + ), + "macos-features": ScenarioSpec( + scenario_id="macos-features", + platform="macos", + job_name="macOS Feature Verification", + compose_files=("platform/compose/docker-compose.aux-stack.ci-hosted-macos.yaml",), + compose_variant="ci-hosted-macos", + phase_names=("normalize-machine-name", "bootstrap", "setup", "verify-launchd"), + activate_services=True, + ensure_images=True, + normalize_machine_name=True, + verify_launchd=True, + verify_rendered_files=False, + exercise_sidecars=False, + exercise_browser_lab=False, + model_setup_mode="auto", + ), } diff --git a/tests/utils/helpers/_fresh_host/shell.py b/tests/utils/helpers/_fresh_host/shell.py index 604ab2be..7d8a10ba 100644 --- a/tests/utils/helpers/_fresh_host/shell.py +++ b/tests/utils/helpers/_fresh_host/shell.py @@ -56,7 +56,7 @@ def phase_env(context: FreshHostContext) -> dict[str, str]: { "HOME": context.app_home, "OPENCLAW_CONFIG_PROFILE": context.profile, - "OPENCLAW_MODEL_SETUP_MODE": "skip", + "OPENCLAW_MODEL_SETUP_MODE": context.model_setup_mode, "PYTHONPATH": "src", "XDG_CONFIG_HOME": f"{context.app_home}/.config", } diff --git a/tests/utils/helpers/_fresh_host/storage.py b/tests/utils/helpers/_fresh_host/storage.py index 9fe2e906..d66d4025 100644 --- a/tests/utils/helpers/_fresh_host/storage.py +++ b/tests/utils/helpers/_fresh_host/storage.py @@ -190,6 +190,10 @@ def load_context(path: Path) -> FreshHostContext: compose_files=list( as_string_list(payload.get("compose_files"), path="context.compose_files") ), + model_setup_mode=as_string( + payload.get("model_setup_mode", "skip"), path="context.model_setup_mode" + ), + use_host_home=as_bool(payload.get("use_host_home", False), path="context.use_host_home"), ) diff --git a/tests/utils/helpers/feature_verification.py b/tests/utils/helpers/feature_verification.py new file mode 100644 index 00000000..16b51a95 --- /dev/null +++ b/tests/utils/helpers/feature_verification.py @@ -0,0 +1,106 @@ +"""Public feature-verification helper facade.""" + +from tests.utils.helpers._feature_verification.acp_stub_agent import ( + REPLY_PREFIX as ACP_STUB_REPLY_PREFIX, +) +from tests.utils.helpers._feature_verification.diagnostics import collect_lane_diagnostics +from tests.utils.helpers._feature_verification.fake_model import ( + CHAT_MODEL_ID, + REPLY_PREFIX, + FakeModelServer, + canned_reply, + deterministic_embedding, +) +from tests.utils.helpers._feature_verification.host import ( + CommandOutcome, + FeatureHost, + extract_json, +) +from tests.utils.helpers._feature_verification.http import ( + HttpResponse, + basic_auth, + http_request, +) +from tests.utils.helpers._feature_verification.lane import ( + FAKE_CHAT_MODEL_REF, + LaneError, + configure_model_provider, + enable_systemd_user_manager, + run_lane, +) +from tests.utils.helpers._feature_verification.manifest import ( + Feature, + FeatureManifest, + FeatureVerificationError, + Lane, + load_manifest, +) +from tests.utils.helpers._feature_verification.processes import ( + FakeModelProcessError, + start_detached, + stop_detached, +) +from tests.utils.helpers._feature_verification.recorder import LaneRecorder, is_feature_suite_item +from tests.utils.helpers._feature_verification.reporting import ( + build_report, + render_markdown, + write_lane_report, +) +from tests.utils.helpers._feature_verification.results import ( + UNVERIFIABLE_SKIP_PREFIX, + FeatureResult, + LaneResults, + load_lane_results, +) +from tests.utils.helpers._feature_verification.runtime import ( + GatewayController, + gateway_rpc_ok, + gateway_status, + plugin_statuses, + service_active, + tcp_open, +) + +__all__ = [ + "ACP_STUB_REPLY_PREFIX", + "CHAT_MODEL_ID", + "FAKE_CHAT_MODEL_REF", + "REPLY_PREFIX", + "UNVERIFIABLE_SKIP_PREFIX", + "CommandOutcome", + "FakeModelProcessError", + "FakeModelServer", + "Feature", + "FeatureHost", + "FeatureManifest", + "FeatureResult", + "FeatureVerificationError", + "GatewayController", + "HttpResponse", + "Lane", + "LaneError", + "LaneRecorder", + "LaneResults", + "basic_auth", + "build_report", + "canned_reply", + "collect_lane_diagnostics", + "configure_model_provider", + "deterministic_embedding", + "enable_systemd_user_manager", + "extract_json", + "gateway_rpc_ok", + "gateway_status", + "http_request", + "is_feature_suite_item", + "load_lane_results", + "load_manifest", + "plugin_statuses", + "render_markdown", + "run_lane", + "service_active", + "start_detached", + "stop_detached", + "tcp_open", + "write_lane_report", +] diff --git a/tests/utils/helpers/fresh_host.py b/tests/utils/helpers/fresh_host.py index 2f261840..7873380b 100644 --- a/tests/utils/helpers/fresh_host.py +++ b/tests/utils/helpers/fresh_host.py @@ -8,6 +8,7 @@ from tests.utils.helpers._fresh_host.models import ( DEFAULT_DOCKER_PULL_MAX_ATTEMPTS, DEFAULT_DOCKER_PULL_PARALLELISM, + SCENARIO_SPECS, FreshHostContext, FreshHostError, FreshHostReport, @@ -33,6 +34,7 @@ "PhaseResult", "PlatformName", "PhaseStatus", + "SCENARIO_SPECS", "ScenarioId", "ScenarioSpec", "_deactivate_macos_host_services", From 0b13ca8f6e4cf29a9e6a3f00ec21d1ba6dc525ad Mon Sep 17 00:00:00 2001 From: Juan Sugg Date: Wed, 11 Nov 2026 13:07:52 -0300 Subject: [PATCH 2/5] test(e2e): harden feature lanes against setup cascades Declare the fake model provider as an explicit models.providers entry after setup, bring sidecars up through clawops when the service manager did not, run hypermemory/context through varlock as the runtime does, redact env-contract secrets from recorded results, and probe the fake server without proxies. --- .github/workflows/feature-verification.yml | 28 ++-- tests/feature_matrix.yaml | 20 ++- tests/fixtures/feature_verification/lane.py | 7 + tests/scripts/feature_verification.py | 29 +++- .../repo/test_feature_matrix_contract.py | 2 +- .../test_context_workflows.py | 26 ++-- .../test_host_lifecycle.py | 12 +- .../test_memory_hypermemory.py | 48 +++++-- .../test_memory_lancedb.py | 3 +- .../test_model_gateway.py | 10 ++ .../e2e/feature_verification/test_recovery.py | 9 +- .../test_setup_profiles.py | 17 ++- .../e2e/feature_verification/test_sidecars.py | 28 ++-- .../unit/ci/test_feature_verification.py | 10 +- .../_feature_verification/diagnostics.py | 29 +--- .../_feature_verification/fake_model.py | 2 +- .../helpers/_feature_verification/host.py | 59 +++++--- .../helpers/_feature_verification/http.py | 6 +- .../helpers/_feature_verification/lane.py | 25 +--- .../_feature_verification/processes.py | 37 +++-- .../helpers/_feature_verification/provider.py | 133 ++++++++++++++++++ .../helpers/_feature_verification/recorder.py | 5 + .../_feature_verification/redaction.py | 36 +++++ .../helpers/_feature_verification/runtime.py | 24 ++++ tests/utils/helpers/_fresh_host/context.py | 1 - tests/utils/helpers/_fresh_host/models.py | 9 +- tests/utils/helpers/_fresh_host/shell.py | 2 +- tests/utils/helpers/_fresh_host/storage.py | 3 - tests/utils/helpers/feature_verification.py | 18 ++- 29 files changed, 463 insertions(+), 175 deletions(-) create mode 100644 tests/utils/helpers/_feature_verification/provider.py create mode 100644 tests/utils/helpers/_feature_verification/redaction.py diff --git a/.github/workflows/feature-verification.yml b/.github/workflows/feature-verification.yml index bac6b7a3..a38d93ea 100644 --- a/.github/workflows/feature-verification.yml +++ b/.github/workflows/feature-verification.yml @@ -53,18 +53,18 @@ jobs: scenario: linux-features # LiteLLM reaches the fake Ollama upstream from its container via host-gateway. fake_model_host: 0.0.0.0 - fake_model_ports: --port 8000 --port 11434 + fake_model_ports: --port 18000 --port 11434 - lane: linux-lancedb profile: memory-lancedb-pro scenario: linux-features fake_model_host: 0.0.0.0 - fake_model_ports: --port 8000 --port 11434 + fake_model_ports: --port 18000 --port 11434 - lane: linux-profiles profile: openclaw-default scenario: linux-features-local fake_model_host: 127.0.0.1 # 11434 stays free for the optional Ollama compose stack this lane starts. - fake_model_ports: --port 8000 + fake_model_ports: --port 18000 env: FRESH_HOST_SELECTED_DOCKER_PULL_PARALLELISM: "2" FRESH_HOST_SELECTED_DOCKER_PULL_MAX_ATTEMPTS: "5" @@ -102,14 +102,20 @@ jobs: - name: Start the fake model provider timeout-minutes: 2 run: ./tests/scripts/feature_verification.py start-fake-model --host "${{ matrix.fake_model_host }}" ${{ matrix.fake_model_ports }} --state-dir "${RUNNER_TEMP}/fake-model" - - name: Point the env contract at the fake model provider - run: ./tests/scripts/feature_verification.py configure-model-provider --context "${FRESH_HOST_CONTEXT}" + - name: Select the fake model provider in the env contract + run: ./tests/scripts/feature_verification.py seed-env-contract --context "${FRESH_HOST_CONTEXT}" # Bootstrap + setup are themselves verified features; a failure here is reported per # feature by the lane, which falls back to a harness gateway for the remaining features. - name: Bootstrap and set up the host continue-on-error: true timeout-minutes: 45 run: ./tests/scripts/fresh_host.py run-scenario --context "${FRESH_HOST_CONTEXT}" + # The operator step after setup: declare the provider, then `clawops model-auth ensure`. + - name: Register the fake model provider + if: always() + continue-on-error: true + timeout-minutes: 16 + run: ./tests/scripts/feature_verification.py apply-model-provider --context "${FRESH_HOST_CONTEXT}" - name: Run lane features timeout-minutes: 60 run: ./tests/scripts/feature_verification.py run-lane --context "${FRESH_HOST_CONTEXT}" --lane "${{ matrix.lane }}" --results "${FEATURE_RESULTS_DIR}/feature-results-${{ matrix.lane }}.json" --fake-model-dir "${RUNNER_TEMP}/fake-model" @@ -209,14 +215,20 @@ jobs: run: ./tests/scripts/hosted_docker.py ensure-images --context "${FRESH_HOST_CONTEXT}" - name: Start the fake model provider timeout-minutes: 2 - run: ./tests/scripts/feature_verification.py start-fake-model --host 127.0.0.1 --port 8000 --port 11434 --state-dir "${RUNNER_TEMP}/fake-model" - - name: Point the env contract at the fake model provider - run: ./tests/scripts/feature_verification.py configure-model-provider --context "${FRESH_HOST_CONTEXT}" + run: ./tests/scripts/feature_verification.py start-fake-model --host 127.0.0.1 --port 18000 --port 11434 --state-dir "${RUNNER_TEMP}/fake-model" + - name: Select the fake model provider in the env contract + run: ./tests/scripts/feature_verification.py seed-env-contract --context "${FRESH_HOST_CONTEXT}" # The scenario recovers a Docker runtime lost mid-run inside its own budget. - name: Bootstrap and set up the host continue-on-error: true timeout-minutes: 40 run: ./tests/scripts/fresh_host.py run-scenario --context "${FRESH_HOST_CONTEXT}" + # The operator step after setup: declare the provider, then `clawops model-auth ensure`. + - name: Register the fake model provider + if: always() + continue-on-error: true + timeout-minutes: 16 + run: ./tests/scripts/feature_verification.py apply-model-provider --context "${FRESH_HOST_CONTEXT}" - name: Run lane features timeout-minutes: 35 run: ./tests/scripts/feature_verification.py run-lane --context "${FRESH_HOST_CONTEXT}" --lane macos-launchd --results "${FEATURE_RESULTS_DIR}/feature-results-macos-launchd.json" --fake-model-dir "${RUNNER_TEMP}/fake-model" diff --git a/tests/feature_matrix.yaml b/tests/feature_matrix.yaml index e52fdff7..cd450cbc 100644 --- a/tests/feature_matrix.yaml +++ b/tests/feature_matrix.yaml @@ -50,12 +50,12 @@ features: test: tests/suites/e2e/feature_verification/test_host_lifecycle.py::test_bootstrap_installs_the_locked_toolchain - id: setup.guided category: host-lifecycle - title: clawops setup (non-interactive) renders config, applies model auth, activates services + title: clawops setup (non-interactive) renders the profile and activates the host services call_sites: - src/clawops/setup_cli.py:310 - src/clawops/setup_cli.py:139 lanes: [linux-core, macos-launchd] - verification: fake-model + verification: live test: tests/suites/e2e/feature_verification/test_host_lifecycle.py::test_guided_setup_completes_with_service_activation - id: services.systemd category: host-lifecycle @@ -872,3 +872,19 @@ features: lanes: [linux-profiles] verification: live test: tests/suites/e2e/feature_verification/test_devflow_planning.py::test_devflow_plan_status_audit + - id: devflow.run + category: devflow + title: devflow run dispatches the architect-to-lead stage workers and audits the run + call_sites: [src/clawops/devflow.py:657] + status: unverifiable + reason: Every stage dispatches a subscription-authenticated vendor coding agent (claude/codex) that must author the stage artifacts (design, tasks, tests, QA report); CI has no vendor accounts and a stub cannot author meaningful artifacts. Dispatch routing is covered by the acp.* features and planning by devflow.plan-status-audit. + - id: profile.bundled-provider-plugins + category: profiles + title: Memory-profile plugin allowlists keep OpenClaw's bundled ollama provider loadable + call_sites: + - platform/configs/openclaw/77-hypermemory.example.json5:3 + - platform/configs/openclaw/75-memory-lancedb-pro.local.json5:3 + - src/clawops/strongclaw_model_auth.py:232 + lanes: [linux-core, linux-lancedb] + verification: live + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_profile_allowlist_keeps_bundled_provider_plugins diff --git a/tests/fixtures/feature_verification/lane.py b/tests/fixtures/feature_verification/lane.py index a436d656..d683699d 100644 --- a/tests/fixtures/feature_verification/lane.py +++ b/tests/fixtures/feature_verification/lane.py @@ -17,6 +17,7 @@ FeatureHost, GatewayController, LaneRecorder, + ensure_sidecars, is_feature_suite_item, ) @@ -82,6 +83,12 @@ def feature_host(request: pytest.FixtureRequest) -> FeatureHost: return host +@pytest.fixture(scope="session") +def sidecars(feature_host: FeatureHost) -> str: + """Return how the sidecar stack came up: by its service, or by the harness fallback.""" + return ensure_sidecars(feature_host) + + @pytest.fixture(scope="session") def gateway(feature_host: FeatureHost) -> Iterator[GatewayController]: """Return a running gateway, started by the harness when no service supervises one.""" diff --git a/tests/scripts/feature_verification.py b/tests/scripts/feature_verification.py index b252f8b2..8758f928 100755 --- a/tests/scripts/feature_verification.py +++ b/tests/scripts/feature_verification.py @@ -28,7 +28,6 @@ from tests.utils.helpers._feature_verification.fake_model import FakeModelServer # noqa: E402 from tests.utils.helpers._feature_verification.lane import ( # noqa: E402 LaneError, - configure_model_provider, enable_systemd_user_manager, run_lane, ) @@ -37,6 +36,11 @@ start_detached, stop_detached, ) +from tests.utils.helpers._feature_verification.provider import ( # noqa: E402 + ProviderError, + apply_model_provider, + seed_env_contract, +) def _resolved(path: Path) -> Path: @@ -60,9 +64,15 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: stop = subparsers.add_parser("stop-fake-model", help="Stop the detached fake model API.") stop.add_argument("--state-dir", type=Path, required=True) + seed = subparsers.add_parser( + "seed-env-contract", + help="Select the fake model provider in the operator env contract (before bootstrap).", + ) + seed.add_argument("--context", type=Path, required=True) + provider = subparsers.add_parser( - "configure-model-provider", - help="Point the operator env contract at the fake model server.", + "apply-model-provider", + help="Register the fake provider in the rendered config and run model-auth ensure.", ) provider.add_argument("--context", type=Path, required=True) @@ -87,6 +97,7 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: def _serve(host: str, ports: list[int]) -> int: """Serve until SIGTERM or SIGINT.""" + print(f"starting fake model server on {host}:{ports}", flush=True) server = FakeModelServer(host=host, ports=ports) stopped = threading.Event() @@ -121,9 +132,13 @@ def main(argv: list[str] | None = None) -> int: if args.command == "stop-fake-model": print(json.dumps({"stopped": stop_detached(_resolved(args.state_dir))})) return 0 - if args.command == "configure-model-provider": - env_file = configure_model_provider(_resolved(args.context)) - print(f"configured the fake model provider in {env_file}") + if args.command == "seed-env-contract": + print( + f"selected the fake model provider in {seed_env_contract(_resolved(args.context))}" + ) + return 0 + if args.command == "apply-model-provider": + print(json.dumps(apply_model_provider(_resolved(args.context)), indent=2)) return 0 if args.command == "enable-systemd-user": print(f"systemd user manager ready at {enable_systemd_user_manager()}") @@ -145,7 +160,7 @@ def main(argv: list[str] | None = None) -> int: context_file=_resolved(args.context), output_dir=_resolved(args.output_dir) ) return 0 - except (FakeModelProcessError, LaneError) as exc: + except (FakeModelProcessError, LaneError, ProviderError) as exc: print(f"feature-verification error: {exc}", file=sys.stderr) return 1 raise AssertionError(f"Unhandled command: {args.command}") diff --git a/tests/suites/contracts/repo/test_feature_matrix_contract.py b/tests/suites/contracts/repo/test_feature_matrix_contract.py index 41ad669b..37103f38 100644 --- a/tests/suites/contracts/repo/test_feature_matrix_contract.py +++ b/tests/suites/contracts/repo/test_feature_matrix_contract.py @@ -123,7 +123,7 @@ def test_lanes_match_fresh_host_scenarios() -> None: assert spec is not None, f"lane {lane.lane_id} uses unknown scenario {lane.scenario}" assert spec.platform == lane.platform, lane assert spec.activate_services == lane.activate_services, lane - assert spec.model_setup_mode == "auto", lane + assert spec.use_host_home == (lane.platform == "linux"), lane def test_workflow_runs_every_lane_and_nothing_else() -> None: diff --git a/tests/suites/e2e/feature_verification/test_context_workflows.py b/tests/suites/e2e/feature_verification/test_context_workflows.py index 0a95a45e..b161dc50 100644 --- a/tests/suites/e2e/feature_verification/test_context_workflows.py +++ b/tests/suites/e2e/feature_verification/test_context_workflows.py @@ -35,17 +35,6 @@ def _sample_repo(feature_host: FeatureHost, marker: str) -> Path: return repo -def _context_env(feature_host: FeatureHost) -> dict[str, str]: - values = feature_host.env_values() - keys = ( - "NEO4J_USERNAME", - "NEO4J_PASSWORD", - "HYPERMEMORY_QDRANT_URL", - "HYPERMEMORY_EMBEDDING_MODEL", - ) - return {key: values[key] for key in keys if values.get(key)} - - def _neo4j_node_count(feature_host: FeatureHost) -> int: values = feature_host.env_values() response = http_request( @@ -61,21 +50,23 @@ def _neo4j_node_count(feature_host: FeatureHost) -> int: return int(cast(list[int], data[0]["row"])[0]) -def test_codebase_context_indexes_into_the_neo4j_graph(feature_host: FeatureHost) -> None: +def test_codebase_context_indexes_into_the_neo4j_graph( + feature_host: FeatureHost, sidecars: str +) -> None: + del sidecars marker = f"fv{uuid.uuid4().hex[:8]}" repo = _sample_repo(feature_host, marker) - env = _context_env(feature_host) common = ("--config", CONTEXT_CONFIG, "--repo", str(repo), "--scale", "large") - index = feature_host.clawops("context", "codebase", "index", *common, "--json", env=env) + index = feature_host.clawops_managed("context", "codebase", "index", *common, "--json") stats = cast(dict[str, object], index.require_ok().json()) assert int(cast(int, stats["total_files"])) >= 2, stats assert _neo4j_node_count(feature_host) > 0, "large-scale indexing wrote nothing to neo4j" - query = feature_host.clawops( - "context", "codebase", "query", *common, "--query", "LedgerReconciler reconcile", env=env + query = feature_host.clawops_managed( + "context", "codebase", "query", *common, "--query", "LedgerReconciler reconcile" ).require_ok() assert "pkg/ledger.py" in query.stdout, query.describe() pack_path = feature_host.scratch("context-pack") / "pack.md" - feature_host.clawops( + feature_host.clawops_managed( "context", "codebase", "pack", @@ -84,7 +75,6 @@ def test_codebase_context_indexes_into_the_neo4j_graph(feature_host: FeatureHost "reconcile", "--output", str(pack_path), - env=env, ).require_ok() assert "LedgerReconciler" in pack_path.read_text(encoding="utf-8") diff --git a/tests/suites/e2e/feature_verification/test_host_lifecycle.py b/tests/suites/e2e/feature_verification/test_host_lifecycle.py index 39428635..63322174 100644 --- a/tests/suites/e2e/feature_verification/test_host_lifecycle.py +++ b/tests/suites/e2e/feature_verification/test_host_lifecycle.py @@ -9,7 +9,7 @@ from clawops.platform_compat import DEFAULT_ACPX_VERSION, DEFAULT_OPENCLAW_VERSION from clawops.strongclaw_runtime import load_bootstrap_state -from tests.utils.helpers.feature_verification import FeatureHost +from tests.utils.helpers.feature_verification import FAKE_CHAT_MODEL_REF, FeatureHost from tests.utils.helpers.fresh_host import load_report SYSTEMD_PERSISTENT_UNITS = ("openclaw-gateway.service",) @@ -54,7 +54,7 @@ def test_guided_setup_completes_with_service_activation(feature_host: FeatureHos agents = cast(dict[str, object], config["agents"]) defaults = cast(dict[str, object], agents["defaults"]) model = cast(dict[str, object], defaults["model"]) - assert str(model["primary"]).startswith("vllm/"), model + assert model["primary"] == FAKE_CHAT_MODEL_REF, model feature_host.openclaw("config", "validate", timeout=120).require_ok() @@ -122,7 +122,8 @@ def test_doctor_host_reports_toolchain(feature_host: FeatureHost) -> None: assert Path(str(payload["config"])).is_file(), payload -def test_deep_doctor_passes_every_check(feature_host: FeatureHost) -> None: +def test_deep_doctor_passes_every_check(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars outcome = feature_host.clawops("doctor", "--asset-root", ".", timeout=1800) payload = cast(dict[str, object], outcome.json()) checks = cast(list[dict[str, object]], payload["checks"]) @@ -132,7 +133,10 @@ def test_deep_doctor_passes_every_check(feature_host: FeatureHost) -> None: assert payload["status"] == "pass" and outcome.ok, json.dumps(failed, indent=2) -def test_baseline_verify_passes_with_runtime_probes(feature_host: FeatureHost) -> None: +def test_baseline_verify_passes_with_runtime_probes( + feature_host: FeatureHost, sidecars: str +) -> None: + del sidecars outcome = feature_host.clawops("baseline", "verify", timeout=3000) assert outcome.ok, outcome.describe() payload = cast(dict[str, object], outcome.json()) diff --git a/tests/suites/e2e/feature_verification/test_memory_hypermemory.py b/tests/suites/e2e/feature_verification/test_memory_hypermemory.py index a9fa086d..6c57afe9 100644 --- a/tests/suites/e2e/feature_verification/test_memory_hypermemory.py +++ b/tests/suites/e2e/feature_verification/test_memory_hypermemory.py @@ -36,7 +36,7 @@ def _config_path(feature_host: FeatureHost) -> Path: def _hypermemory( feature_host: FeatureHost, *arguments: str, timeout: float = 600 ) -> CommandOutcome: - return feature_host.clawops( + return feature_host.clawops_managed( "hypermemory", "--config", str(_config_path(feature_host)), *arguments, timeout=timeout ) @@ -56,7 +56,8 @@ def _search(feature_host: FeatureHost, query: str) -> list[dict[str, object]]: return cast(list[dict[str, object]], cast(dict[str, object], payload)["results"]) -def test_hypermemory_profile_plugins_are_loaded(feature_host: FeatureHost) -> None: +def test_hypermemory_profile_plugins_are_loaded(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars statuses = plugin_statuses(feature_host) for plugin_id in ("strongclaw-hypermemory", "lossless-claw"): assert statuses.get(plugin_id) == "loaded", f"{plugin_id}: {statuses.get(plugin_id)}" @@ -65,21 +66,25 @@ def test_hypermemory_profile_plugins_are_loaded(feature_host: FeatureHost) -> No assert slots == {"contextEngine": "lossless-claw", "memory": "strongclaw-hypermemory"}, slots -def test_openclaw_memory_status_is_healthy(feature_host: FeatureHost) -> None: +def test_openclaw_memory_status_is_healthy(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars outcome = feature_host.openclaw("memory", "status", "--deep", timeout=600).require_ok() - assert ( - "strongclaw-hypermemory" in outcome.output or "hypermemory" in outcome.output - ), outcome.describe() + # StrongClaw gates on a clean exit (setup_cli.py:501); the report must not be empty. + assert outcome.stdout.strip(), outcome.describe() -def test_openclaw_memory_search_returns_corpus_hits(feature_host: FeatureHost) -> None: +def test_openclaw_memory_search_returns_corpus_hits( + feature_host: FeatureHost, sidecars: str +) -> None: + del sidecars outcome = feature_host.openclaw( "memory", "search", "--query", "ClawOps", "--max-results", "1", timeout=600 ).require_ok() assert ".md" in outcome.stdout, f"memory search returned no corpus hit\n{outcome.describe()}" -def test_hypermemory_index_and_hybrid_search(feature_host: FeatureHost) -> None: +def test_hypermemory_index_and_hybrid_search(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars index = cast( dict[str, object], _hypermemory(feature_host, "index", "--json").require_ok().json() ) @@ -93,14 +98,18 @@ def test_hypermemory_index_and_hybrid_search(feature_host: FeatureHost) -> None: assert any("BACKUP" in str(hit["path"]).upper() for hit in hits), hits -def test_hypermemory_verify_reports_consistent_index(feature_host: FeatureHost) -> None: +def test_hypermemory_verify_reports_consistent_index( + feature_host: FeatureHost, sidecars: str +) -> None: + del sidecars payload = cast( dict[str, object], _hypermemory(feature_host, "verify", "--json").require_ok().json() ) assert payload["ok"] is True, payload -def test_hypermemory_store_search_and_forget(feature_host: FeatureHost) -> None: +def test_hypermemory_store_search_and_forget(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars token = f"FV-store-{uuid.uuid4().hex[:10]}" text = f"The feature verification vault code is {token}." stored = _hypermemory(feature_host, "store", "--json", "--type", "fact", "--text", text) @@ -111,7 +120,8 @@ def test_hypermemory_store_search_and_forget(feature_host: FeatureHost) -> None: assert not any(token in str(hit.get("snippet")) for hit in _search(feature_host, token)) -def test_hypermemory_maintenance_operations(feature_host: FeatureHost) -> None: +def test_hypermemory_maintenance_operations(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars for arguments in ( ("reflect", "--json", "--mode", "safe"), ("lifecycle", "--json"), @@ -123,15 +133,17 @@ def test_hypermemory_maintenance_operations(feature_host: FeatureHost) -> None: assert isinstance(outcome.json(), dict | list), outcome.describe() -def test_hypermemory_exports_memory_pro_snapshot(feature_host: FeatureHost) -> None: +def test_hypermemory_exports_memory_pro_snapshot(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars output = feature_host.scratch("hypermemory-export") / "memory-pro.json" _hypermemory(feature_host, "export-memory-pro", "--json", "--output", str(output)).require_ok() assert output.is_file() and output.stat().st_size > 2, f"no export written to {output}" def test_hypermemory_plugin_recalls_memory_into_turns( - feature_host: FeatureHost, gateway: GatewayController + feature_host: FeatureHost, gateway: GatewayController, sidecars: str ) -> None: + del sidecars del gateway token = f"FV-recall-{uuid.uuid4().hex[:10]}" _hypermemory( @@ -165,7 +177,10 @@ def test_hypermemory_plugin_recalls_memory_into_turns( ), "strongclaw-hypermemory autoRecall did not inject the stored fact into the model prompt" -def test_qdrant_storage_stamp_tracks_pinned_release(feature_host: FeatureHost) -> None: +def test_qdrant_storage_stamp_tracks_pinned_release( + feature_host: FeatureHost, sidecars: str +) -> None: + del sidecars image_match = _QDRANT_IMAGE.search((feature_host.repo_root / AUX_COMPOSE).read_text("utf-8")) assert image_match is not None target = QdrantRelease.from_image(image_match.group(1)) @@ -174,7 +189,10 @@ def test_qdrant_storage_stamp_tracks_pinned_release(feature_host: FeatureHost) - assert stamp.release_line == target.line and stamp.reindex_pending is False, stamp -def test_qdrant_release_change_resets_storage_and_reindexes(feature_host: FeatureHost) -> None: +def test_qdrant_release_change_resets_storage_and_reindexes( + feature_host: FeatureHost, sidecars: str +) -> None: + del sidecars stamp_file = stamp_path(resolve_compose_state_dir()) current = read_stamp(stamp_file) assert current is not None, f"no qdrant stamp at {stamp_file}" diff --git a/tests/suites/e2e/feature_verification/test_memory_lancedb.py b/tests/suites/e2e/feature_verification/test_memory_lancedb.py index c23f3ec7..753aa7ba 100644 --- a/tests/suites/e2e/feature_verification/test_memory_lancedb.py +++ b/tests/suites/e2e/feature_verification/test_memory_lancedb.py @@ -95,7 +95,8 @@ def test_import_pro_snapshot_lands_in_lancedb(feature_host: FeatureHost) -> None assert token in search.stdout, f"memory-pro search cannot find the import\n{search.describe()}" -def test_verify_pro_parity_against_live_plugin(feature_host: FeatureHost) -> None: +def test_verify_pro_parity_against_live_plugin(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars token = f"FV-parity-{uuid.uuid4().hex[:10]}" snapshot, _ = _migrate(feature_host, token) feature_host.clawops( diff --git a/tests/suites/e2e/feature_verification/test_model_gateway.py b/tests/suites/e2e/feature_verification/test_model_gateway.py index 0a33401d..04062efa 100644 --- a/tests/suites/e2e/feature_verification/test_model_gateway.py +++ b/tests/suites/e2e/feature_verification/test_model_gateway.py @@ -12,6 +12,7 @@ FeatureHost, GatewayController, gateway_status, + plugin_statuses, ) TURN_TIMEOUT_SECONDS = 300 @@ -148,3 +149,12 @@ def test_security_audit_passes(feature_host: FeatureHost, gateway: GatewayContro def test_secrets_audit_passes(feature_host: FeatureHost) -> None: feature_host.openclaw("secrets", "audit", "--check", timeout=600).require_ok() + + +def test_profile_allowlist_keeps_bundled_provider_plugins(feature_host: FeatureHost) -> None: + allow = cast(dict[str, object], feature_host.rendered_config()["plugins"]).get("allow") + statuses = plugin_statuses(feature_host) + # The documented local model path (OLLAMA_API_KEY) needs OpenClaw's bundled ollama provider. + assert ( + statuses.get("ollama") == "loaded" + ), f"bundled provider plugin 'ollama' is {statuses.get('ollama')!r} under plugins.allow={allow}" diff --git a/tests/suites/e2e/feature_verification/test_recovery.py b/tests/suites/e2e/feature_verification/test_recovery.py index 1665bbf3..0c0aa239 100644 --- a/tests/suites/e2e/feature_verification/test_recovery.py +++ b/tests/suites/e2e/feature_verification/test_recovery.py @@ -21,12 +21,15 @@ def _recovery(feature_host: FeatureHost, *arguments: str, timeout: float = 900) ) -def _create_backup(feature_host: FeatureHost) -> dict[str, object]: - return cast(dict[str, object], _recovery(feature_host, "backup-create").require_ok().json()) +def _create_backup(feature_host: FeatureHost, *, allow_fallback: bool = True) -> dict[str, object]: + """Create an archive; dependent features accept the operator tar fallback so they verify + their own behaviour even when the OpenClaw-backed path is broken (recovery.backup-create).""" + arguments = ("backup-create", "--allow-fallback") if allow_fallback else ("backup-create",) + return cast(dict[str, object], _recovery(feature_host, *arguments).require_ok().json()) def test_backup_create_uses_the_openclaw_backend(feature_host: FeatureHost) -> None: - payload = _create_backup(feature_host) + payload = _create_backup(feature_host, allow_fallback=False) assert payload["mode"] == "openclaw-cli", f"backup fell back to tar: {payload}" assert payload["fallback_used"] is False, payload archive = Path(str(payload["archive"])) diff --git a/tests/suites/e2e/feature_verification/test_setup_profiles.py b/tests/suites/e2e/feature_verification/test_setup_profiles.py index edaa683d..a7a23e90 100644 --- a/tests/suites/e2e/feature_verification/test_setup_profiles.py +++ b/tests/suites/e2e/feature_verification/test_setup_profiles.py @@ -12,11 +12,14 @@ from pathlib import Path from typing import Final, cast +from clawops.strongclaw_runtime import resolve_openclaw_config_path from tests.utils.helpers.feature_verification import ( ACP_STUB_REPLY_PREFIX, FeatureHost, GatewayController, plugin_statuses, + register_provider, + reply_texts, ) ACP_WORKER_AGENTS: Final[tuple[str, ...]] = ("coder-acp-codex", "reviewer-acp-claude") @@ -57,6 +60,8 @@ def _activate(feature_host: FeatureHost, gateway: GatewayController, profile: st .json() ) assert cast(dict[str, object], payload).get("ok", True) is not False, payload + # Setup re-renders openclaw.json, so the operator re-declares the provider first. + register_provider(resolve_openclaw_config_path(feature_host.repo_root)) feature_host.clawops("model-auth", "--asset-root", ".", "ensure", timeout=900).require_ok() feature_host.openclaw("config", "validate", timeout=120).require_ok() gateway.restart() @@ -71,8 +76,9 @@ def test_openclaw_default_profile_boots_with_memory_core( feature_host: FeatureHost, gateway: GatewayController ) -> None: _activate(feature_host, gateway, "openclaw-default") - plugins = cast(dict[str, object], feature_host.rendered_config().get("plugins", {})) - assert "slots" not in plugins or "memory" not in cast(dict[str, object], plugins["slots"]) + plugins = cast(dict[str, object], feature_host.rendered_config()["plugins"]) + slots = cast(dict[str, object], plugins["slots"]) + assert slots == {"contextEngine": "legacy", "memory": "memory-core"}, slots assert plugin_statuses(feature_host).get("memory-core") == "loaded" feature_host.openclaw("memory", "status", "--deep", timeout=600).require_ok() @@ -140,9 +146,10 @@ def test_acp_worker_agent_runs_a_turn_through_acpx( "--json", timeout=300, ) - assert ( - outcome.ok and ACP_STUB_REPLY_PREFIX in outcome.stdout and token in outcome.stdout - ), outcome.describe() + replies = reply_texts(outcome.require_ok().json()) + assert any( + ACP_STUB_REPLY_PREFIX in reply and token in reply for reply in replies + ), f"the ACP worker did not answer through acpx; agent replies: {replies}" def test_browser_lab_profile_points_at_the_cdp_endpoint( diff --git a/tests/suites/e2e/feature_verification/test_sidecars.py b/tests/suites/e2e/feature_verification/test_sidecars.py index 74b33fe9..fed994cc 100644 --- a/tests/suites/e2e/feature_verification/test_sidecars.py +++ b/tests/suites/e2e/feature_verification/test_sidecars.py @@ -28,7 +28,8 @@ def _container(feature_host: FeatureHost, fragment: str) -> str: return names[0] -def test_postgres_sidecar_accepts_queries(feature_host: FeatureHost) -> None: +def test_postgres_sidecar_accepts_queries(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars container = _container(feature_host, "postgres") outcome = feature_host.run( [ @@ -48,7 +49,10 @@ def test_postgres_sidecar_accepts_queries(feature_host: FeatureHost) -> None: assert int(outcome.stdout.strip()) > 0, outcome.describe() -def test_litellm_routes_embeddings_to_the_upstream(feature_host: FeatureHost) -> None: +def test_litellm_routes_embeddings_to_the_upstream( + feature_host: FeatureHost, sidecars: str +) -> None: + del sidecars assert http_request("GET", f"{LITELLM}/health/liveliness").status == 200 token = f"FV-litellm-{uuid.uuid4().hex[:10]}" started = time.time() @@ -71,7 +75,8 @@ def test_litellm_routes_embeddings_to_the_upstream(feature_host: FeatureHost) -> assert upstream, "litellm answered without calling the configured embedding upstream" -def test_qdrant_sidecar_round_trips_vectors(feature_host: FeatureHost) -> None: +def test_qdrant_sidecar_round_trips_vectors(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars del feature_host collection = f"featver-{uuid.uuid4().hex[:8]}" assert http_request("GET", f"{QDRANT}/healthz").status == 200 @@ -99,7 +104,8 @@ def test_qdrant_sidecar_round_trips_vectors(feature_host: FeatureHost) -> None: http_request("DELETE", f"{QDRANT}/collections/{collection}") -def test_neo4j_sidecar_round_trips_cypher(feature_host: FeatureHost) -> None: +def test_neo4j_sidecar_round_trips_cypher(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars values = feature_host.env_values() marker = f"fv-{uuid.uuid4().hex[:8]}" response = http_request( @@ -126,7 +132,8 @@ def test_neo4j_sidecar_round_trips_cypher(feature_host: FeatureHost) -> None: assert len(results) == 2, payload -def test_observability_pipeline_accepts_telemetry(feature_host: FeatureHost) -> None: +def test_observability_pipeline_accepts_telemetry(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars feature_host.clawops("verify-platform", "observability", "--asset-root", ".").require_ok() trace = { "resourceSpans": [ @@ -157,13 +164,17 @@ def test_observability_pipeline_accepts_telemetry(feature_host: FeatureHost) -> assert metrics.status == 200 and metrics.body.strip(), "otel prometheus exporter is silent" -def test_verify_platform_sidecars_runtime(feature_host: FeatureHost) -> None: +def test_verify_platform_sidecars_runtime(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars outcome = feature_host.clawops("verify-platform", "sidecars", "--asset-root", ".", timeout=300) payload = cast(dict[str, object], outcome.json()) assert outcome.ok and payload["ok"] is True, outcome.describe() -def test_ops_status_reports_required_sidecars_ready(feature_host: FeatureHost) -> None: +def test_ops_status_reports_required_sidecars_ready( + feature_host: FeatureHost, sidecars: str +) -> None: + del sidecars payload = cast( dict[str, object], feature_host.clawops("ops", "--asset-root", ".", "status", timeout=300).require_ok().json(), @@ -175,7 +186,8 @@ def test_ops_status_reports_required_sidecars_ready(feature_host: FeatureHost) - assert cast(dict[str, object], services[name])["state"] == "running", services -def test_prune_qdrant_test_collections(feature_host: FeatureHost) -> None: +def test_prune_qdrant_test_collections(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars collection = f"{QDRANT_TEST_PREFIX}featver-{uuid.uuid4().hex[:8]}" created = http_request( "PUT", diff --git a/tests/suites/unit/ci/test_feature_verification.py b/tests/suites/unit/ci/test_feature_verification.py index dfe6783f..3913e155 100644 --- a/tests/suites/unit/ci/test_feature_verification.py +++ b/tests/suites/unit/ci/test_feature_verification.py @@ -13,7 +13,6 @@ from tests.plugins.infrastructure.context import TestContext from tests.utils.helpers import fresh_host from tests.utils.helpers._fresh_host import linux as fresh_host_linux -from tests.utils.helpers._fresh_host import shell as fresh_host_shell from tests.utils.helpers.feature_verification import ( ACP_STUB_REPLY_PREFIX, REPLY_PREFIX, @@ -282,9 +281,8 @@ def test_feature_scenarios_install_into_the_runner_home( ) assert Path(context.app_home) == Path.home() assert context.xdg_runtime_dir is not None and context.xdg_runtime_dir.startswith("/run/user/") - assert fresh_host_shell.phase_env(context)["OPENCLAW_MODEL_SETUP_MODE"] == "auto" reloaded = fresh_host.load_context(Path(context.context_path)) - assert reloaded.use_host_home is True and reloaded.model_setup_mode == "auto" + assert reloaded.use_host_home is True def test_legacy_context_files_default_the_new_fields( @@ -301,11 +299,11 @@ def test_legacy_context_files_default_the_new_fields( github_env_file=None, ) payload = json.loads(Path(context.context_path).read_text(encoding="utf-8")) - del payload["model_setup_mode"], payload["use_host_home"] + del payload["use_host_home"] Path(context.context_path).write_text(json.dumps(payload), encoding="utf-8") reloaded = fresh_host.load_context(Path(context.context_path)) - assert reloaded.model_setup_mode == "skip" and reloaded.use_host_home is False - assert fresh_host_shell.phase_env(reloaded)["OPENCLAW_MODEL_SETUP_MODE"] == "skip" + assert reloaded.use_host_home is False + assert Path(reloaded.app_home) == Path(context.tmp_root) / "home" @pytest.mark.parametrize( diff --git a/tests/utils/helpers/_feature_verification/diagnostics.py b/tests/utils/helpers/_feature_verification/diagnostics.py index 150f57a7..c61ce88a 100644 --- a/tests/utils/helpers/_feature_verification/diagnostics.py +++ b/tests/utils/helpers/_feature_verification/diagnostics.py @@ -7,7 +7,8 @@ from pathlib import Path from typing import Final -from clawops.strongclaw_runtime import load_env_assignments, varlock_local_env_file +from clawops.strongclaw_runtime import varlock_local_env_file +from tests.utils.helpers._feature_verification.redaction import redact, secret_values from tests.utils.helpers._fresh_host.shell import phase_env from tests.utils.helpers._fresh_host.storage import load_context @@ -18,8 +19,6 @@ "ai.openclaw.backup-verify", "ai.openclaw.maintenance", ) -_SECRET_MARKERS: Final[tuple[str, ...]] = ("TOKEN", "KEY", "PASSWORD", "SECRET") -_MIN_SECRET_LENGTH: Final[int] = 8 _LOG_TAIL_CHARS: Final[int] = 400_000 _COMMAND_TIMEOUT_SECONDS: Final[int] = 60 @@ -41,34 +40,16 @@ def _diagnostic_commands(platform: str) -> dict[str, list[str]]: return commands -def _secret_values(env_file: Path) -> list[str]: - values = load_env_assignments(env_file) if env_file.is_file() else {} - return sorted( - ( - value - for key, value in values.items() - if any(marker in key for marker in _SECRET_MARKERS) and len(value) >= _MIN_SECRET_LENGTH - ), - key=len, - reverse=True, - ) - - def collect_lane_diagnostics(*, context_file: Path, output_dir: Path) -> None: """Capture service-manager state and gateway logs into *output_dir*.""" context = load_context(context_file) env = phase_env(context) - secrets = _secret_values( + secrets = secret_values( varlock_local_env_file( Path(context.repo_root), home_dir=Path(context.app_home), environ=env ) ) - def _redact(text: str) -> str: - for secret in secrets: - text = text.replace(secret, "[REDACTED]") - return text - output_dir.mkdir(parents=True, exist_ok=True) for name, command in _diagnostic_commands(context.platform).items(): try: @@ -86,12 +67,12 @@ def _redact(text: str) -> str: ) except (OSError, subprocess.TimeoutExpired) as exc: text = f"$ {' '.join(command)}\nfailed: {exc}" - (output_dir / name).write_text(_redact(text), encoding="utf-8") + (output_dir / name).write_text(redact(text, secrets), encoding="utf-8") for log_root in (Path("/tmp/openclaw"), Path(context.app_home) / ".openclaw" / "logs"): if not log_root.is_dir(): continue for log_file in sorted(log_root.glob("*.log"))[-5:]: tail = log_file.read_text(encoding="utf-8", errors="replace")[-_LOG_TAIL_CHARS:] (output_dir / f"{log_root.name}-{log_file.name}").write_text( - _redact(tail), encoding="utf-8" + redact(tail, secrets), encoding="utf-8" ) diff --git a/tests/utils/helpers/_feature_verification/fake_model.py b/tests/utils/helpers/_feature_verification/fake_model.py index d7abce0b..37e1010b 100644 --- a/tests/utils/helpers/_feature_verification/fake_model.py +++ b/tests/utils/helpers/_feature_verification/fake_model.py @@ -1,7 +1,7 @@ """Deterministic OpenAI- and Ollama-compatible model server for feature verification. The server stands in for every model provider the product talks to on a runner: the OpenClaw -agent chain (``vllm/`` through the OpenAI-compatible ``/v1`` API), the memory plugins' +agent chain (an explicit OpenAI-compatible ``models.providers`` entry), the memory plugins' embedding and extraction endpoints, and the LiteLLM sidecar's Ollama upstream. Replies are canned, so assertions can rely on exact content, and every request is journaled so a test can prove the product actually reached the model rather than short-circuiting. diff --git a/tests/utils/helpers/_feature_verification/host.py b/tests/utils/helpers/_feature_verification/host.py index f41f8f3e..0b063f57 100644 --- a/tests/utils/helpers/_feature_verification/host.py +++ b/tests/utils/helpers/_feature_verification/host.py @@ -40,13 +40,12 @@ load_manifest, ) from tests.utils.helpers._feature_verification.processes import fetch_journal, load_state +from tests.utils.helpers._feature_verification.redaction import redact, secret_values from tests.utils.helpers._fresh_host.models import FreshHostContext from tests.utils.helpers._fresh_host.storage import load_context _HARNESS_ENV_PREFIXES: Final[tuple[str, ...]] = ("FEATURE_VERIFICATION_", "PYTEST_") _FRAMEWORK_ENV_KEYS: Final[frozenset[str]] = frozenset({"TEST_ID", "RESOURCE_PREFIX", "WORKER_ID"}) -_SECRET_KEY_PATTERN: Final[re.Pattern[str]] = re.compile(r"(TOKEN|KEY|PASSWORD|SECRET)") -_MIN_REDACTED_LENGTH: Final[int] = 8 _JSON_START: Final[re.Pattern[str]] = re.compile(r"(?m)^[ \t]*[\[{]") _OUTPUT_TAIL_CHARS: Final[int] = 3000 @@ -68,6 +67,26 @@ def extract_json(text: str) -> object: raise AssertionError(f"no JSON document in command output: {text[-_OUTPUT_TAIL_CHARS:]!r}") +def reply_texts(payload: object) -> list[str]: + """Collect every ``payloads[].text`` from an ``openclaw agent --json`` envelope.""" + texts: list[str] = [] + if isinstance(payload, dict): + mapping = cast(dict[str, object], payload) + for key, value in mapping.items(): + if key == "payloads" and isinstance(value, list): + texts.extend( + str(cast(dict[str, object], item).get("text")) + for item in cast(list[object], value) + if isinstance(item, dict) and cast(dict[str, object], item).get("text") + ) + else: + texts.extend(reply_texts(value)) + elif isinstance(payload, list): + for item in cast(list[object], payload): + texts.extend(reply_texts(item)) + return texts + + @dataclass(frozen=True, slots=True) class CommandOutcome: """Captured result of one product command.""" @@ -111,11 +130,11 @@ class FeatureHost: """The provisioned host one feature-verification lane runs against.""" lane: Lane - manifest: FeatureManifest - context: FreshHostContext + manifest: FeatureManifest = field(repr=False) + context: FreshHostContext = field(repr=False) work_dir: Path fake_model_dir: Path | None - _secrets: tuple[str, ...] = field(default_factory=tuple) + _secrets: tuple[str, ...] = field(default_factory=tuple, repr=False) @classmethod def from_environment(cls, environ: Mapping[str, str] | None = None) -> FeatureHost: @@ -178,24 +197,11 @@ def env_values(self) -> dict[str, str]: return load_env_assignments(self.env_file()) def _refresh_secrets(self) -> None: - values = self.env_values() if self.env_file().is_file() else {} - self._secrets = tuple( - sorted( - { - value - for key, value in values.items() - if _SECRET_KEY_PATTERN.search(key) and len(value) >= _MIN_REDACTED_LENGTH - }, - key=len, - reverse=True, - ) - ) + self._secrets = secret_values(self.env_file()) def redact(self, text: str) -> str: """Replace known secret values in *text*.""" - for secret in self._secrets: - text = text.replace(secret, "[REDACTED]") - return text + return redact(text, self._secrets) def product_env(self, overrides: Mapping[str, str] | None = None) -> dict[str, str]: """Return the operator environment with harness variables removed.""" @@ -273,6 +279,19 @@ def openclaw( argv = wrap_command_with_varlock(self.repo_root, ["openclaw", *arguments]) return self.run(argv, cwd=cwd, timeout=timeout, env=env) + def clawops_managed( + self, + *arguments: str, + cwd: Path | None = None, + timeout: float = 300, + env: Mapping[str, str] | None = None, + ) -> CommandOutcome: + """Run ``clawops`` inside ``varlock run``, as StrongClaw's own runtime callers do.""" + argv = wrap_command_with_varlock( + self.repo_root, [str(self.python), "-m", "clawops", *arguments] + ) + return self.run(argv, cwd=cwd, timeout=timeout, env=env) + def fake_model_requests(self) -> list[dict[str, object]]: """Return every request the fake model server has answered so far.""" if self.fake_model_dir is None: diff --git a/tests/utils/helpers/_feature_verification/http.py b/tests/utils/helpers/_feature_verification/http.py index 22f4b45f..82c542d9 100644 --- a/tests/utils/helpers/_feature_verification/http.py +++ b/tests/utils/helpers/_feature_verification/http.py @@ -23,6 +23,10 @@ def json(self) -> object: return cast(object, json.loads(self.body)) if self.body else {} +# Loopback probes must never route through system or environment proxies. +_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + def basic_auth(user: str, password: str) -> dict[str, str]: """Return a basic-auth header.""" token = base64.b64encode(f"{user}:{password}".encode()).decode("ascii") @@ -44,7 +48,7 @@ def http_request( for key, value in (headers or {}).items(): request.add_header(key, value) try: - with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 + with _OPENER.open(request, timeout=timeout) as response: return HttpResponse(int(response.status), response.read().decode("utf-8")) except urllib.error.HTTPError as exc: return HttpResponse(int(exc.code), exc.read().decode("utf-8", errors="replace")) diff --git a/tests/utils/helpers/_feature_verification/lane.py b/tests/utils/helpers/_feature_verification/lane.py index f04e4328..fe8d40e8 100644 --- a/tests/utils/helpers/_feature_verification/lane.py +++ b/tests/utils/helpers/_feature_verification/lane.py @@ -8,7 +8,6 @@ from pathlib import Path from typing import Final -from clawops.strongclaw_runtime import set_env_assignment, varlock_local_env_file from tests.utils.helpers._feature_verification.envvars import ( CONTEXT_ENV, FAKE_MODEL_DIR_ENV, @@ -26,14 +25,6 @@ from tests.utils.helpers._fresh_host.storage import load_context FEATURE_SUITE_PATH: Final[str] = "tests/suites/e2e/feature_verification" -FAKE_CHAT_MODEL_REF: Final[str] = "vllm/featver-chat" -# vLLM is a keyless local provider; OpenClaw only needs a non-empty key to register it. -FAKE_PROVIDER_ENV: Final[dict[str, str]] = { - "OPENCLAW_DEFAULT_MODEL": FAKE_CHAT_MODEL_REF, - "VLLM_API_KEY": "featver-local", - "HYPERMEMORY_EMBEDDING_MODEL": "ollama/nomic-embed-text", - "HYPERMEMORY_EMBEDDING_API_BASE": "http://host.docker.internal:11434", -} DEFAULT_LANE_TIMEOUT_SECONDS: Final[int] = 3600 @@ -45,20 +36,6 @@ class LaneError(RuntimeError): _LINGER_POLL_SECONDS: Final[float] = 1.0 -def configure_model_provider(context_file: Path) -> Path: - """Point the operator env contract at the fake model server, as an operator edits it.""" - context = load_context(context_file) - env = phase_env(context) - env_file = varlock_local_env_file( - Path(context.repo_root), home_dir=Path(context.app_home), environ=env - ) - env_file.parent.mkdir(parents=True, exist_ok=True) - for key, value in FAKE_PROVIDER_ENV.items(): - set_env_assignment(env_file, key, value) - env_file.chmod(0o600) - return env_file - - def enable_systemd_user_manager() -> Path: """Start the runner's systemd user manager (lingering) and wait for its bus.""" user = ( @@ -107,6 +84,8 @@ def run_lane( env = phase_env(context) env.update( { + # Fresh-host setup skips model auth; the lane's own commands run it for real. + "OPENCLAW_MODEL_SETUP_MODE": "auto", LANE_ENV: lane_id, CONTEXT_ENV: str(context_file), RESULTS_ENV: str(results_path), diff --git a/tests/utils/helpers/_feature_verification/processes.py b/tests/utils/helpers/_feature_verification/processes.py index 83adab50..fb9d4d71 100644 --- a/tests/utils/helpers/_feature_verification/processes.py +++ b/tests/utils/helpers/_feature_verification/processes.py @@ -6,14 +6,13 @@ from __future__ import annotations +import http.client import json import os import signal import subprocess import sys import time -import urllib.error -import urllib.request from collections.abc import Sequence from pathlib import Path from typing import Final, cast @@ -22,7 +21,7 @@ STATE_FILE_NAME: Final[str] = "fake-model.json" LOG_FILE_NAME: Final[str] = "fake-model.log" -READY_TIMEOUT_SECONDS: Final[float] = 20.0 +READY_TIMEOUT_SECONDS: Final[float] = 45.0 STOP_TIMEOUT_SECONDS: Final[float] = 10.0 _POLL_SECONDS: Final[float] = 0.2 @@ -36,14 +35,23 @@ def _probe_host(host: str) -> str: return "127.0.0.1" if host in {"0.0.0.0", "", "::"} else host +def _get(host: str, port: int, path: str, *, timeout: float) -> tuple[int, bytes]: + """GET a loopback path directly; http.client never consults system or env proxies.""" + connection = http.client.HTTPConnection(_probe_host(host), port, timeout=timeout) + try: + connection.request("GET", path) + response = connection.getresponse() + return response.status, response.read() + finally: + connection.close() + + def _healthy(host: str, port: int) -> bool: try: - with urllib.request.urlopen( # noqa: S310 - loopback probe - f"http://{_probe_host(host)}:{port}{HEALTH_PATH}", timeout=2 - ) as response: - return bool(response.status == 200) - except (urllib.error.URLError, OSError): + status, _ = _get(host, port, HEALTH_PATH, timeout=2) + except OSError: return False + return status == 200 def start_detached( @@ -84,7 +92,10 @@ def start_detached( return state time.sleep(_POLL_SECONDS) process.terminate() - raise FakeModelProcessError(f"fake model server not ready in {READY_TIMEOUT_SECONDS}s") + log_tail = log_path.read_text(encoding="utf-8", errors="replace")[-2000:] + raise FakeModelProcessError( + f"fake model server not ready in {READY_TIMEOUT_SECONDS}s; log tail:\n{log_tail}" + ) def load_state(state_dir: Path) -> dict[str, object]: @@ -100,10 +111,10 @@ def load_state(state_dir: Path) -> dict[str, object]: def fetch_journal(host: str, port: int) -> list[dict[str, object]]: """Return the request journal recorded by a running server.""" - with urllib.request.urlopen( # noqa: S310 - loopback probe - f"http://{_probe_host(host)}:{port}{JOURNAL_PATH}", timeout=5 - ) as response: - payload: object = json.loads(response.read().decode("utf-8")) + status, body = _get(host, port, JOURNAL_PATH, timeout=10) + if status != 200: + raise FakeModelProcessError(f"fake model journal returned HTTP {status}") + payload: object = json.loads(body.decode("utf-8")) if not isinstance(payload, dict): raise FakeModelProcessError("fake model journal is not a JSON object") requests = cast(dict[str, object], payload).get("requests") diff --git a/tests/utils/helpers/_feature_verification/provider.py b/tests/utils/helpers/_feature_verification/provider.py new file mode 100644 index 00000000..a27e5cec --- /dev/null +++ b/tests/utils/helpers/_feature_verification/provider.py @@ -0,0 +1,133 @@ +"""Wire StrongClaw's model chain to the fake model server, the way an operator would. + +The provider is declared as an explicit OpenAI-compatible entry under ``models.providers`` +(core OpenClaw config, independent of bundled provider plugins), and the operator env contract +selects it through StrongClaw's env-driven chain (``OPENCLAW_DEFAULT_MODEL``). ``clawops setup`` +re-renders ``openclaw.json``, so the provider is re-registered after every render. +""" + +from __future__ import annotations + +import json +import os +import subprocess +from collections.abc import Mapping +from pathlib import Path +from typing import Final, cast + +from clawops.strongclaw_runtime import ( + resolve_openclaw_config_path, + set_env_assignment, + varlock_local_env_file, +) +from tests.utils.helpers._fresh_host.shell import phase_env +from tests.utils.helpers._fresh_host.storage import load_context + +PROVIDER_ID: Final[str] = "featver" +CHAT_MODEL: Final[str] = "featver-chat" +FAKE_CHAT_MODEL_REF: Final[str] = f"{PROVIDER_ID}/{CHAT_MODEL}" +PROVIDER_BASE_URL: Final[str] = "http://127.0.0.1:18000/v1" +# Not a credential: the fake server accepts any key; OpenClaw only needs one to be present. +PROVIDER_API_KEY: Final[str] = "featver-local" +ENV_CONTRACT: Final[dict[str, str]] = { + "OPENCLAW_DEFAULT_MODEL": FAKE_CHAT_MODEL_REF, + "HYPERMEMORY_EMBEDDING_MODEL": "ollama/nomic-embed-text", + "HYPERMEMORY_EMBEDDING_API_BASE": "http://host.docker.internal:11434", +} +MODEL_AUTH_TIMEOUT_SECONDS: Final[int] = 900 +SYSTEMD_GATEWAY_UNIT: Final[str] = "openclaw-gateway.service" +LAUNCHD_GATEWAY_LABEL: Final[str] = "ai.openclaw.gateway" + + +class ProviderError(RuntimeError): + """Raised when the fake provider cannot be wired into the host.""" + + +def provider_entry() -> dict[str, object]: + """Return the ``models.providers`` entry for the fake model server.""" + return { + "baseUrl": PROVIDER_BASE_URL, + "apiKey": PROVIDER_API_KEY, + "api": "openai-completions", + "models": [ + { + "id": CHAT_MODEL, + "name": "Feature verification fake model", + "reasoning": False, + "input": ["text"], + "cost": {"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0}, + "contextWindow": 131072, + "maxTokens": 8192, + } + ], + } + + +def seed_env_contract(context_file: Path) -> Path: + """Select the fake provider in the operator env contract (before bootstrap).""" + context = load_context(context_file) + env_file = varlock_local_env_file( + Path(context.repo_root), home_dir=Path(context.app_home), environ=phase_env(context) + ) + env_file.parent.mkdir(parents=True, exist_ok=True) + for key, value in ENV_CONTRACT.items(): + set_env_assignment(env_file, key, value) + env_file.chmod(0o600) + return env_file + + +def register_provider(config_path: Path) -> None: + """Add the fake provider to a rendered ``openclaw.json``.""" + if not config_path.is_file(): + raise ProviderError(f"rendered OpenClaw config not found at {config_path}") + payload = cast(dict[str, object], json.loads(config_path.read_text(encoding="utf-8"))) + models = payload.setdefault("models", {}) + if not isinstance(models, dict): + raise ProviderError(f"{config_path}: `models` is not a mapping") + providers = cast(dict[str, object], models).setdefault("providers", {}) + if not isinstance(providers, dict): + raise ProviderError(f"{config_path}: `models.providers` is not a mapping") + cast(dict[str, object], providers)[PROVIDER_ID] = provider_entry() + config_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + +def _restart_supervised_gateway(platform: str, env: Mapping[str, str]) -> str: + if platform == "linux": + active = subprocess.run( + ["systemctl", "--user", "is-active", "--quiet", SYSTEMD_GATEWAY_UNIT], + env=dict(env), + check=False, + ) + if active.returncode != 0: + return "no supervised gateway" + command = ["systemctl", "--user", "restart", SYSTEMD_GATEWAY_UNIT] + else: + command = ["launchctl", "kickstart", "-k", f"gui/{os.getuid()}/{LAUNCHD_GATEWAY_LABEL}"] + restarted = subprocess.run(command, env=dict(env), capture_output=True, text=True, check=False) + return "restarted" if restarted.returncode == 0 else f"restart failed: {restarted.stderr}" + + +def apply_model_provider(context_file: Path) -> dict[str, object]: + """Register the provider, run ``clawops model-auth ensure``, and restart the gateway.""" + context = load_context(context_file) + env = {**phase_env(context), "OPENCLAW_MODEL_SETUP_MODE": "auto"} + repo_root = Path(context.repo_root) + config_path = resolve_openclaw_config_path( + repo_root, home_dir=Path(context.app_home), environ=env + ) + register_provider(config_path) + ensure = subprocess.run( + [str(repo_root / ".venv" / "bin" / "python"), "-m", "clawops", "model-auth", "ensure"], + cwd=repo_root, + env=env, + capture_output=True, + text=True, + timeout=MODEL_AUTH_TIMEOUT_SECONDS, + check=False, + ) + return { + "config": str(config_path), + "modelAuthExitCode": ensure.returncode, + "modelAuthOutput": ensure.stdout[-2000:], + "gateway": _restart_supervised_gateway(context.platform, env), + } diff --git a/tests/utils/helpers/_feature_verification/recorder.py b/tests/utils/helpers/_feature_verification/recorder.py index ac8e7c1a..75077bd3 100644 --- a/tests/utils/helpers/_feature_verification/recorder.py +++ b/tests/utils/helpers/_feature_verification/recorder.py @@ -10,6 +10,7 @@ import pytest +from clawops.strongclaw_runtime import varlock_local_env_file from tests.utils.helpers._feature_verification.envvars import ( CONTEXT_ENV, LANE_ENV, @@ -23,6 +24,7 @@ FeatureVerificationError, load_manifest, ) +from tests.utils.helpers._feature_verification.redaction import redact, secret_values from tests.utils.helpers._feature_verification.results import ( MAX_MESSAGE_CHARS, FeatureResult, @@ -140,6 +142,9 @@ def write(self) -> LaneResults: message="test was scheduled on this lane but did not run", ) ) + secrets = secret_values(varlock_local_env_file(REPO_ROOT)) + for result in results: + result.message = redact(result.message, secrets) lane_results = LaneResults( lane=self.lane_id, platform=lane.platform, diff --git a/tests/utils/helpers/_feature_verification/redaction.py b/tests/utils/helpers/_feature_verification/redaction.py new file mode 100644 index 00000000..ce9ebc80 --- /dev/null +++ b/tests/utils/helpers/_feature_verification/redaction.py @@ -0,0 +1,36 @@ +"""Redact generated env-contract secrets from anything the harness records or uploads.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Final + +from clawops.strongclaw_runtime import load_env_assignments + +REDACTED: Final[str] = "[REDACTED]" +_SECRET_KEY_MARKERS: Final[tuple[str, ...]] = ("TOKEN", "KEY", "PASSWORD", "SECRET") +_MIN_SECRET_LENGTH: Final[int] = 8 + + +def secret_values(env_file: Path) -> tuple[str, ...]: + """Return secret-looking values from an env contract, longest first.""" + values = load_env_assignments(env_file) if env_file.is_file() else {} + return tuple( + sorted( + { + value + for key, value in values.items() + if any(marker in key for marker in _SECRET_KEY_MARKERS) + and len(value) >= _MIN_SECRET_LENGTH + }, + key=len, + reverse=True, + ) + ) + + +def redact(text: str, secrets: tuple[str, ...]) -> str: + """Replace every known secret value in *text*.""" + for secret in secrets: + text = text.replace(secret, REDACTED) + return text diff --git a/tests/utils/helpers/_feature_verification/runtime.py b/tests/utils/helpers/_feature_verification/runtime.py index bb6d2380..1d67279c 100644 --- a/tests/utils/helpers/_feature_verification/runtime.py +++ b/tests/utils/helpers/_feature_verification/runtime.py @@ -216,3 +216,27 @@ def stop(self) -> None: except subprocess.TimeoutExpired: os.killpg(process.pid, signal.SIGKILL) process.wait(timeout=_STOP_TIMEOUT_SECONDS) + + +def sidecars_ready(host: FeatureHost) -> bool: + """Return whether ``clawops ops status`` reports every required sidecar ready.""" + outcome = host.clawops("ops", "--asset-root", ".", "status", timeout=300) + try: + payload = outcome.json() + except AssertionError: + return False + if not isinstance(payload, dict): + return False + readiness = cast(dict[str, object], payload).get("readiness") + return ( + isinstance(readiness, dict) + and cast(dict[str, object], readiness).get("requiredReady") is True + ) + + +def ensure_sidecars(host: FeatureHost) -> str: + """Bring the sidecar stack up with ``clawops ops sidecars up`` when no service did.""" + if sidecars_ready(host): + return "ready" + host.clawops("ops", "--asset-root", ".", "sidecars", "up", timeout=1800).require_ok() + return "started-by-harness" diff --git a/tests/utils/helpers/_fresh_host/context.py b/tests/utils/helpers/_fresh_host/context.py index e9c90c3d..5cd496a7 100644 --- a/tests/utils/helpers/_fresh_host/context.py +++ b/tests/utils/helpers/_fresh_host/context.py @@ -137,7 +137,6 @@ def prepare_context( compose_files=[ str((repo_root / relative_path).resolve()) for relative_path in spec.compose_files ], - model_setup_mode=spec.model_setup_mode, use_host_home=spec.use_host_home, ) report = FreshHostReport( diff --git a/tests/utils/helpers/_fresh_host/models.py b/tests/utils/helpers/_fresh_host/models.py index e4664200..e10c8965 100644 --- a/tests/utils/helpers/_fresh_host/models.py +++ b/tests/utils/helpers/_fresh_host/models.py @@ -47,9 +47,8 @@ class ScenarioSpec: verify_rendered_files: bool exercise_sidecars: bool exercise_browser_lab: bool - # Feature-verification lanes run model auth for real and, on Linux, install into the - # runner's own HOME because the systemd user manager only reads units from there. - model_setup_mode: str = "skip" + # Feature-verification lanes install into the runner's own HOME on Linux because the + # systemd user manager only reads user units from there. use_host_home: bool = False @@ -101,7 +100,6 @@ class FreshHostContext: exercise_browser_lab: bool phase_names: list[str] compose_files: list[str] - model_setup_mode: str = "skip" use_host_home: bool = False @@ -206,7 +204,6 @@ class FreshHostReport: verify_rendered_files=False, exercise_sidecars=False, exercise_browser_lab=False, - model_setup_mode="auto", use_host_home=True, ), "linux-features-local": ScenarioSpec( @@ -226,7 +223,6 @@ class FreshHostReport: verify_rendered_files=False, exercise_sidecars=False, exercise_browser_lab=False, - model_setup_mode="auto", use_host_home=True, ), "macos-features": ScenarioSpec( @@ -243,6 +239,5 @@ class FreshHostReport: verify_rendered_files=False, exercise_sidecars=False, exercise_browser_lab=False, - model_setup_mode="auto", ), } diff --git a/tests/utils/helpers/_fresh_host/shell.py b/tests/utils/helpers/_fresh_host/shell.py index 7d8a10ba..604ab2be 100644 --- a/tests/utils/helpers/_fresh_host/shell.py +++ b/tests/utils/helpers/_fresh_host/shell.py @@ -56,7 +56,7 @@ def phase_env(context: FreshHostContext) -> dict[str, str]: { "HOME": context.app_home, "OPENCLAW_CONFIG_PROFILE": context.profile, - "OPENCLAW_MODEL_SETUP_MODE": context.model_setup_mode, + "OPENCLAW_MODEL_SETUP_MODE": "skip", "PYTHONPATH": "src", "XDG_CONFIG_HOME": f"{context.app_home}/.config", } diff --git a/tests/utils/helpers/_fresh_host/storage.py b/tests/utils/helpers/_fresh_host/storage.py index d66d4025..378b2b19 100644 --- a/tests/utils/helpers/_fresh_host/storage.py +++ b/tests/utils/helpers/_fresh_host/storage.py @@ -190,9 +190,6 @@ def load_context(path: Path) -> FreshHostContext: compose_files=list( as_string_list(payload.get("compose_files"), path="context.compose_files") ), - model_setup_mode=as_string( - payload.get("model_setup_mode", "skip"), path="context.model_setup_mode" - ), use_host_home=as_bool(payload.get("use_host_home", False), path="context.use_host_home"), ) diff --git a/tests/utils/helpers/feature_verification.py b/tests/utils/helpers/feature_verification.py index 16b51a95..14941fd9 100644 --- a/tests/utils/helpers/feature_verification.py +++ b/tests/utils/helpers/feature_verification.py @@ -15,6 +15,7 @@ CommandOutcome, FeatureHost, extract_json, + reply_texts, ) from tests.utils.helpers._feature_verification.http import ( HttpResponse, @@ -22,9 +23,7 @@ http_request, ) from tests.utils.helpers._feature_verification.lane import ( - FAKE_CHAT_MODEL_REF, LaneError, - configure_model_provider, enable_systemd_user_manager, run_lane, ) @@ -40,6 +39,13 @@ start_detached, stop_detached, ) +from tests.utils.helpers._feature_verification.provider import ( + FAKE_CHAT_MODEL_REF, + ProviderError, + apply_model_provider, + register_provider, + seed_env_contract, +) from tests.utils.helpers._feature_verification.recorder import LaneRecorder, is_feature_suite_item from tests.utils.helpers._feature_verification.reporting import ( build_report, @@ -54,6 +60,7 @@ ) from tests.utils.helpers._feature_verification.runtime import ( GatewayController, + ensure_sidecars, gateway_rpc_ok, gateway_status, plugin_statuses, @@ -80,24 +87,29 @@ "Lane", "LaneError", "LaneRecorder", + "ProviderError", "LaneResults", "basic_auth", "build_report", "canned_reply", "collect_lane_diagnostics", - "configure_model_provider", + "apply_model_provider", "deterministic_embedding", "enable_systemd_user_manager", "extract_json", "gateway_rpc_ok", + "ensure_sidecars", "gateway_status", "http_request", "is_feature_suite_item", "load_lane_results", "load_manifest", "plugin_statuses", + "register_provider", "render_markdown", + "reply_texts", "run_lane", + "seed_env_contract", "service_active", "start_detached", "stop_detached", From b86d6248d33fcfaea62dfe16e8c5529c83dd67c9 Mon Sep 17 00:00:00 2001 From: Juan Sugg Date: Wed, 11 Nov 2026 13:58:49 -0300 Subject: [PATCH 3/5] test(e2e): sharpen feature assertions and isolate shared causes Use OpenClaw's custom-local marker for the fake provider key, prove the OTLP metrics pipeline end to end, read the Qdrant stamp from the stack's own compose state dir, drive smart extraction past its message floor, and split the default Ollama embedding lane into its own feature. --- tests/feature_matrix.yaml | 11 ++- .../test_host_lifecycle.py | 7 +- .../test_memory_hypermemory.py | 13 ++- .../test_memory_lancedb.py | 50 +++++------ .../test_setup_profiles.py | 7 +- .../e2e/feature_verification/test_sidecars.py | 90 ++++++++++++++++--- .../helpers/_feature_verification/provider.py | 14 ++- tests/utils/helpers/feature_verification.py | 4 + 8 files changed, 138 insertions(+), 58 deletions(-) diff --git a/tests/feature_matrix.yaml b/tests/feature_matrix.yaml index cd450cbc..b71c634c 100644 --- a/tests/feature_matrix.yaml +++ b/tests/feature_matrix.yaml @@ -132,7 +132,7 @@ features: call_sites: - src/clawops/strongclaw_model_auth.py:343 - src/clawops/strongclaw_model_auth.py:239 - lanes: [linux-core] + lanes: [linux-core, macos-launchd] verification: fake-model test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_model_auth_ensure_applies_env_driven_chain - id: model-auth.probe @@ -528,6 +528,15 @@ features: lanes: [linux-core] verification: fake-model test: tests/suites/e2e/feature_verification/test_sidecars.py::test_litellm_routes_embeddings_to_the_upstream + - id: sidecars.litellm-ollama-embeddings + category: sidecars + title: LiteLLM serves the default Ollama embedding lane that varlock-env configure selects + call_sites: + - src/clawops/strongclaw_varlock_env.py:571 + - platform/configs/litellm/config.yaml:22 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_litellm_serves_the_default_ollama_embedding_lane - id: sidecars.qdrant category: sidecars title: Qdrant sidecar creates collections, upserts and searches vectors diff --git a/tests/suites/e2e/feature_verification/test_host_lifecycle.py b/tests/suites/e2e/feature_verification/test_host_lifecycle.py index 63322174..1a4be0ad 100644 --- a/tests/suites/e2e/feature_verification/test_host_lifecycle.py +++ b/tests/suites/e2e/feature_verification/test_host_lifecycle.py @@ -9,7 +9,7 @@ from clawops.platform_compat import DEFAULT_ACPX_VERSION, DEFAULT_OPENCLAW_VERSION from clawops.strongclaw_runtime import load_bootstrap_state -from tests.utils.helpers.feature_verification import FAKE_CHAT_MODEL_REF, FeatureHost +from tests.utils.helpers.feature_verification import FeatureHost from tests.utils.helpers.fresh_host import load_report SYSTEMD_PERSISTENT_UNITS = ("openclaw-gateway.service",) @@ -50,11 +50,6 @@ def test_bootstrap_installs_the_locked_toolchain(feature_host: FeatureHost) -> N def test_guided_setup_completes_with_service_activation(feature_host: FeatureHost) -> None: status, reason = _phase_status(feature_host, "setup") assert status == "success", f"clawops setup (with service activation) failed: {reason}" - config = feature_host.rendered_config() - agents = cast(dict[str, object], config["agents"]) - defaults = cast(dict[str, object], agents["defaults"]) - model = cast(dict[str, object], defaults["model"]) - assert model["primary"] == FAKE_CHAT_MODEL_REF, model feature_host.openclaw("config", "validate", timeout=120).require_ok() diff --git a/tests/suites/e2e/feature_verification/test_memory_hypermemory.py b/tests/suites/e2e/feature_verification/test_memory_hypermemory.py index 6c57afe9..97b8ade9 100644 --- a/tests/suites/e2e/feature_verification/test_memory_hypermemory.py +++ b/tests/suites/e2e/feature_verification/test_memory_hypermemory.py @@ -11,7 +11,6 @@ from clawops.qdrant_storage import QdrantRelease, StorageStamp, read_stamp, stamp_path, write_stamp from clawops.strongclaw_runtime import ( rendered_openclaw_hypermemory_config_path, - resolve_compose_state_dir, resolve_openclaw_config_path, ) from tests.utils.helpers.feature_verification import ( @@ -41,6 +40,11 @@ def _hypermemory( ) +def _compose_state_dir(feature_host: FeatureHost) -> Path: + payload = feature_host.clawops("ops", "--asset-root", ".", "status").require_ok().json() + return Path(str(cast(dict[str, object], payload)["composeStateDir"])) + + def _status(feature_host: FeatureHost) -> dict[str, object]: return cast( dict[str, object], _hypermemory(feature_host, "status", "--json").require_ok().json() @@ -184,8 +188,9 @@ def test_qdrant_storage_stamp_tracks_pinned_release( image_match = _QDRANT_IMAGE.search((feature_host.repo_root / AUX_COMPOSE).read_text("utf-8")) assert image_match is not None target = QdrantRelease.from_image(image_match.group(1)) - stamp = read_stamp(stamp_path(resolve_compose_state_dir())) - assert stamp is not None, f"no qdrant stamp under {resolve_compose_state_dir()}" + state_dir = _compose_state_dir(feature_host) + stamp = read_stamp(stamp_path(state_dir)) + assert stamp is not None, f"no qdrant stamp under {state_dir}" assert stamp.release_line == target.line and stamp.reindex_pending is False, stamp @@ -193,7 +198,7 @@ def test_qdrant_release_change_resets_storage_and_reindexes( feature_host: FeatureHost, sidecars: str ) -> None: del sidecars - stamp_file = stamp_path(resolve_compose_state_dir()) + stamp_file = stamp_path(_compose_state_dir(feature_host)) current = read_stamp(stamp_file) assert current is not None, f"no qdrant stamp at {stamp_file}" write_stamp( diff --git a/tests/suites/e2e/feature_verification/test_memory_lancedb.py b/tests/suites/e2e/feature_verification/test_memory_lancedb.py index 753aa7ba..87d040a4 100644 --- a/tests/suites/e2e/feature_verification/test_memory_lancedb.py +++ b/tests/suites/e2e/feature_verification/test_memory_lancedb.py @@ -20,6 +20,13 @@ PLUGIN_ID = "memory-lancedb-pro" SCOPE = "project:featver" STRATEGY_KEY = f"plugins.entries.{PLUGIN_ID}.config.sessionStrategy" +MAIN_SESSION_KEY = "agent:admin:main" +EXTRACTION_TURNS = ( + "Remember that I prefer feature verification reports rendered as tables.", + "I always run the lancedb lane on Ubuntu runners.", + "My preferred review cadence for this project is weekly on Mondays.", + "Our team name for the verification effort is Project Lighthouse.", +) REFLECTION_TIMEOUT_SECONDS = 240.0 _POLL_SECONDS = 3.0 @@ -124,17 +131,19 @@ def test_smart_extraction_calls_the_extraction_model( ) -> None: del gateway started = time.time() - feature_host.openclaw( - "agent", - "--agent", - "admin", - "--session-id", - f"featver-extract-{uuid.uuid4().hex[:8]}", - "--message", - "Remember that I prefer feature verification reports in tables.", - "--json", - timeout=300, - ).require_ok() + session_id = f"featver-extract-{uuid.uuid4().hex[:8]}" + for message in EXTRACTION_TURNS: + feature_host.openclaw( + "agent", + "--agent", + "admin", + "--session-id", + session_id, + "--message", + message, + "--json", + timeout=300, + ).require_ok() deadline = time.monotonic() + 120 while time.monotonic() < deadline: requests = [ @@ -152,23 +161,6 @@ def test_smart_extraction_calls_the_extraction_model( raise AssertionError("autoCapture smart extraction never called the extraction model") -def _session_key(feature_host: FeatureHost, session_id: str) -> str: - payload = ( - feature_host.openclaw("sessions", "--agent", "admin", "--json", timeout=120) - .require_ok() - .json() - ) - raw_sessions = cast( - list[object], - payload if isinstance(payload, list) else cast(dict[str, object], payload).get("sessions"), - ) - for raw in raw_sessions or []: - entry = cast(dict[str, object], raw) - if entry.get("sessionId") == session_id or session_id in str(entry.get("key", "")): - return str(entry["key"]) - raise AssertionError(f"session {session_id} not found in {json.dumps(payload)[:2000]}") - - def _reflection_files(feature_host: FeatureHost) -> list[Path]: agents = cast(dict[str, object], feature_host.rendered_config()["agents"]) admin = next( @@ -199,7 +191,7 @@ def test_reflection_runs_through_the_embedded_runner( timeout=300, ).require_ok() started = time.time() - params = json.dumps({"key": _session_key(feature_host, session_id), "reason": "new"}) + params = json.dumps({"key": MAIN_SESSION_KEY, "reason": "new"}) feature_host.openclaw( "gateway", "call", "sessions.reset", "--params", params, "--json", timeout=300 ).require_ok() diff --git a/tests/suites/e2e/feature_verification/test_setup_profiles.py b/tests/suites/e2e/feature_verification/test_setup_profiles.py index a7a23e90..034f8fde 100644 --- a/tests/suites/e2e/feature_verification/test_setup_profiles.py +++ b/tests/suites/e2e/feature_verification/test_setup_profiles.py @@ -147,9 +147,10 @@ def test_acp_worker_agent_runs_a_turn_through_acpx( timeout=300, ) replies = reply_texts(outcome.require_ok().json()) - assert any( - ACP_STUB_REPLY_PREFIX in reply and token in reply for reply in replies - ), f"the ACP worker did not answer through acpx; agent replies: {replies}" + assert any(ACP_STUB_REPLY_PREFIX in reply and token in reply for reply in replies), ( + f"the ACP worker did not answer through acpx (bundled acpx plugin: " + f"{plugin_statuses(feature_host).get('acpx')!r}); agent replies: {replies}" + ) def test_browser_lab_profile_points_at_the_cdp_endpoint( diff --git a/tests/suites/e2e/feature_verification/test_sidecars.py b/tests/suites/e2e/feature_verification/test_sidecars.py index fed994cc..856a1134 100644 --- a/tests/suites/e2e/feature_verification/test_sidecars.py +++ b/tests/suites/e2e/feature_verification/test_sidecars.py @@ -6,7 +6,10 @@ import uuid from typing import cast +from clawops.strongclaw_runtime import set_env_assignment from tests.utils.helpers.feature_verification import ( + OLLAMA_EMBEDDING_API_BASE, + OLLAMA_EMBEDDING_MODEL, FeatureHost, basic_auth, http_request, @@ -49,30 +52,64 @@ def test_postgres_sidecar_accepts_queries(feature_host: FeatureHost, sidecars: s assert int(outcome.stdout.strip()) > 0, outcome.describe() -def test_litellm_routes_embeddings_to_the_upstream( - feature_host: FeatureHost, sidecars: str -) -> None: - del sidecars - assert http_request("GET", f"{LITELLM}/health/liveliness").status == 200 - token = f"FV-litellm-{uuid.uuid4().hex[:10]}" +def _embed_through_litellm(feature_host: FeatureHost, *, upstream_path: str) -> None: + """Embed through the hypermemory-embedding alias and prove LiteLLM called its upstream.""" + assert http_request("GET", f"{LITELLM}/health/liveliness", timeout=10).status == 200 started = time.time() response = http_request( "POST", f"{LITELLM}/v1/embeddings", - body={"model": "hypermemory-embedding", "input": [token]}, + body={"model": "hypermemory-embedding", "input": [f"FV-litellm-{uuid.uuid4().hex[:10]}"]}, headers={"Authorization": f"Bearer {feature_host.env_values()['LITELLM_MASTER_KEY']}"}, timeout=120, ) - assert response.status == 200, response.body[:2000] + assert response.status == 200, feature_host.redact(response.body[:2000]) data = cast(list[dict[str, object]], cast(dict[str, object], response.json())["data"]) assert data and len(cast(list[float], data[0]["embedding"])) > 0, response.body[:500] upstream = [ request for request in feature_host.fake_model_requests() if float(cast(float, request.get("at", 0.0))) >= started - and str(request.get("path", "")).startswith("/api/embed") + and str(request.get("path", "")).startswith(upstream_path) ] - assert upstream, "litellm answered without calling the configured embedding upstream" + assert upstream, f"litellm answered without calling its {upstream_path} upstream" + + +def _wait_for_litellm(timeout: float = 240.0) -> None: + deadline = time.monotonic() + timeout + while http_request("GET", f"{LITELLM}/health/liveliness", timeout=10).status != 200: + assert time.monotonic() < deadline, "litellm did not come back after recreation" + time.sleep(3) + + +def test_litellm_routes_embeddings_to_the_upstream( + feature_host: FeatureHost, sidecars: str +) -> None: + del sidecars + _embed_through_litellm(feature_host, upstream_path="/v1/embeddings") + + +def test_litellm_serves_the_default_ollama_embedding_lane( + feature_host: FeatureHost, sidecars: str +) -> None: + del sidecars + env_file = feature_host.env_file() + configured = feature_host.env_values() + keys = ("HYPERMEMORY_EMBEDDING_MODEL", "HYPERMEMORY_EMBEDDING_API_BASE") + # The lane `clawops varlock-env configure` selects when a local Ollama has the model. + for key, value in zip(keys, (OLLAMA_EMBEDDING_MODEL, OLLAMA_EMBEDDING_API_BASE), strict=True): + set_env_assignment(env_file, key, value) + try: + feature_host.clawops( + "ops", "--asset-root", ".", "sidecars", "up", timeout=1500 + ).require_ok() + _wait_for_litellm() + _embed_through_litellm(feature_host, upstream_path="/api/embed") + finally: + for key in keys: + set_env_assignment(env_file, key, configured.get(key, "")) + feature_host.clawops("ops", "--asset-root", ".", "sidecars", "up", timeout=1500) + _wait_for_litellm() def test_qdrant_sidecar_round_trips_vectors(feature_host: FeatureHost, sidecars: str) -> None: @@ -160,8 +197,39 @@ def test_observability_pipeline_accepts_telemetry(feature_host: FeatureHost, sid } accepted = http_request("POST", f"{OTLP_HTTP}/v1/traces", body=trace) assert accepted.status == 200, accepted.body + metric_name = f"featver_probe_{uuid.uuid4().hex[:8]}" + gauge = { + "resourceMetrics": [ + { + "resource": { + "attributes": [{"key": "service.name", "value": {"stringValue": "featver"}}] + }, + "scopeMetrics": [ + { + "metrics": [ + { + "name": metric_name, + "gauge": { + "dataPoints": [ + {"asDouble": 1.0, "timeUnixNano": str(time.time_ns())} + ] + }, + } + ] + } + ], + } + ] + } + assert http_request("POST", f"{OTLP_HTTP}/v1/metrics", body=gauge).status == 200 + deadline = time.monotonic() + 60 metrics = http_request("GET", OTEL_METRICS) - assert metrics.status == 200 and metrics.body.strip(), "otel prometheus exporter is silent" + while metric_name not in metrics.body and time.monotonic() < deadline: + time.sleep(2) + metrics = http_request("GET", OTEL_METRICS) + assert ( + metric_name in metrics.body + ), f"OTLP metric never reached the exporter: {metrics.body[:500]}" def test_verify_platform_sidecars_runtime(feature_host: FeatureHost, sidecars: str) -> None: diff --git a/tests/utils/helpers/_feature_verification/provider.py b/tests/utils/helpers/_feature_verification/provider.py index a27e5cec..0cb3aa08 100644 --- a/tests/utils/helpers/_feature_verification/provider.py +++ b/tests/utils/helpers/_feature_verification/provider.py @@ -27,12 +27,17 @@ CHAT_MODEL: Final[str] = "featver-chat" FAKE_CHAT_MODEL_REF: Final[str] = f"{PROVIDER_ID}/{CHAT_MODEL}" PROVIDER_BASE_URL: Final[str] = "http://127.0.0.1:18000/v1" -# Not a credential: the fake server accepts any key; OpenClaw only needs one to be present. -PROVIDER_API_KEY: Final[str] = "featver-local" +# OpenClaw's non-secret marker for keyless custom local providers (not flagged by audits). +PROVIDER_API_KEY: Final[str] = "custom-local" +# LiteLLM reaches the fake server from its container through host-gateway. +OPENAI_COMPATIBLE_EMBEDDING_MODEL: Final[str] = "openai/featver-embed" +OPENAI_COMPATIBLE_EMBEDDING_API_BASE: Final[str] = "http://host.docker.internal:11434/v1" +OLLAMA_EMBEDDING_MODEL: Final[str] = "ollama/nomic-embed-text" +OLLAMA_EMBEDDING_API_BASE: Final[str] = "http://host.docker.internal:11434" ENV_CONTRACT: Final[dict[str, str]] = { "OPENCLAW_DEFAULT_MODEL": FAKE_CHAT_MODEL_REF, - "HYPERMEMORY_EMBEDDING_MODEL": "ollama/nomic-embed-text", - "HYPERMEMORY_EMBEDDING_API_BASE": "http://host.docker.internal:11434", + "HYPERMEMORY_EMBEDDING_MODEL": OPENAI_COMPATIBLE_EMBEDDING_MODEL, + "HYPERMEMORY_EMBEDDING_API_BASE": OPENAI_COMPATIBLE_EMBEDDING_API_BASE, } MODEL_AUTH_TIMEOUT_SECONDS: Final[int] = 900 SYSTEMD_GATEWAY_UNIT: Final[str] = "openclaw-gateway.service" @@ -129,5 +134,6 @@ def apply_model_provider(context_file: Path) -> dict[str, object]: "config": str(config_path), "modelAuthExitCode": ensure.returncode, "modelAuthOutput": ensure.stdout[-2000:], + "modelAuthErrors": ensure.stderr[-4000:], "gateway": _restart_supervised_gateway(context.platform, env), } diff --git a/tests/utils/helpers/feature_verification.py b/tests/utils/helpers/feature_verification.py index 14941fd9..b69a477c 100644 --- a/tests/utils/helpers/feature_verification.py +++ b/tests/utils/helpers/feature_verification.py @@ -41,6 +41,8 @@ ) from tests.utils.helpers._feature_verification.provider import ( FAKE_CHAT_MODEL_REF, + OLLAMA_EMBEDDING_API_BASE, + OLLAMA_EMBEDDING_MODEL, ProviderError, apply_model_provider, register_provider, @@ -72,6 +74,8 @@ "ACP_STUB_REPLY_PREFIX", "CHAT_MODEL_ID", "FAKE_CHAT_MODEL_REF", + "OLLAMA_EMBEDDING_API_BASE", + "OLLAMA_EMBEDDING_MODEL", "REPLY_PREFIX", "UNVERIFIABLE_SKIP_PREFIX", "CommandOutcome", From 6f8b8a56a1441e6670339fb1cbed7aa32bb81b2d Mon Sep 17 00:00:00 2001 From: Juan Sugg Date: Wed, 11 Nov 2026 14:49:10 -0300 Subject: [PATCH 4/5] test(e2e): resolve the fake provider key from the env contract Reference the key through OpenClaw's provider env marker so it is never stored or audited as plaintext, keep fresh-host's embedding default out of lane commands, restore Qdrant when the reset feature fails, and let bootstrap provision Node on lane hosts. --- .github/workflows/feature-verification.yml | 10 +--- .../test_memory_hypermemory.py | 53 +++++++++++-------- .../helpers/_feature_verification/lane.py | 2 + .../helpers/_feature_verification/provider.py | 9 ++-- 4 files changed, 41 insertions(+), 33 deletions(-) diff --git a/.github/workflows/feature-verification.yml b/.github/workflows/feature-verification.yml index a38d93ea..eee44467 100644 --- a/.github/workflows/feature-verification.yml +++ b/.github/workflows/feature-verification.yml @@ -2,6 +2,8 @@ name: Feature Verification # Verifies every feature in tests/feature_matrix.yaml against the OpenClaw/ACPX versions that # `clawops bootstrap` installs from platform/node-tools, on provisioned Linux and macOS hosts. +# No setup-node: bootstrap provisions Node itself and installs the CLIs where the rendered +# systemd/launchd units look for them, as on an operator host. on: workflow_dispatch: inputs: @@ -77,10 +79,6 @@ jobs: - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: "3.12" - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 - with: - node-version: "22.22.1" - package-manager-cache: false - name: Restore package download caches timeout-minutes: 5 uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -182,10 +180,6 @@ jobs: - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: "3.12" - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 - with: - node-version: "22.22.1" - package-manager-cache: false - name: Restore package download caches timeout-minutes: 5 uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/tests/suites/e2e/feature_verification/test_memory_hypermemory.py b/tests/suites/e2e/feature_verification/test_memory_hypermemory.py index 97b8ade9..caa450ad 100644 --- a/tests/suites/e2e/feature_verification/test_memory_hypermemory.py +++ b/tests/suites/e2e/feature_verification/test_memory_hypermemory.py @@ -201,25 +201,34 @@ def test_qdrant_release_change_resets_storage_and_reindexes( stamp_file = stamp_path(_compose_state_dir(feature_host)) current = read_stamp(stamp_file) assert current is not None, f"no qdrant stamp at {stamp_file}" - write_stamp( - stamp_file, - StorageStamp( - release_line="0.0", - image="qdrant/qdrant:v0.0.0", - reindex_pending=False, - updated_at=current.updated_at, - ), - ) - outcome = feature_host.clawops( - "ops", "--asset-root", ".", "sidecars", "up", "--json", timeout=1500 - ).require_ok() - assert "Resetting qdrant storage" in outcome.stderr, outcome.describe() - command = cast(dict[str, object], cast(dict[str, object], outcome.json())["command"]) - steps = { - str(cast(dict[str, object], step)["step"]) for step in cast(list[object], command["steps"]) - } - assert {"reset-qdrant-storage", "reindex-hypermemory"} <= steps, steps - restored = read_stamp(stamp_file) - assert restored is not None and restored.release_line == current.release_line, restored - assert restored.reindex_pending is False, restored - assert int(cast(int, _status(feature_host)["vectorItems"])) > 0 + reset_verified = False + try: + write_stamp( + stamp_file, + StorageStamp( + release_line="0.0", + image="qdrant/qdrant:v0.0.0", + reindex_pending=False, + updated_at=current.updated_at, + ), + ) + outcome = feature_host.clawops( + "ops", "--asset-root", ".", "sidecars", "up", "--json", timeout=1500 + ).require_ok() + assert "Resetting qdrant storage" in outcome.stderr, outcome.describe() + command = cast(dict[str, object], cast(dict[str, object], outcome.json())["command"]) + steps = { + str(cast(dict[str, object], step)["step"]) + for step in cast(list[object], command["steps"]) + } + assert {"reset-qdrant-storage", "reindex-hypermemory"} <= steps, steps + restored = read_stamp(stamp_file) + assert restored is not None and restored.release_line == current.release_line, restored + assert restored.reindex_pending is False, restored + assert int(cast(int, _status(feature_host)["vectorItems"])) > 0 + reset_verified = True + finally: + # A failed reset must not take Qdrant down for the features after this one. + if not reset_verified: + write_stamp(stamp_file, current) + feature_host.clawops("ops", "--asset-root", ".", "sidecars", "up", timeout=1500) diff --git a/tests/utils/helpers/_feature_verification/lane.py b/tests/utils/helpers/_feature_verification/lane.py index fe8d40e8..4180fe43 100644 --- a/tests/utils/helpers/_feature_verification/lane.py +++ b/tests/utils/helpers/_feature_verification/lane.py @@ -93,6 +93,8 @@ def run_lane( MANIFEST_ENV: str(manifest_file), } ) + # fresh-host injects an embedding-model default; the lane reads it from the env contract. + env.pop("HYPERMEMORY_EMBEDDING_MODEL", None) if fake_model_dir is not None: env[FAKE_MODEL_DIR_ENV] = str(fake_model_dir) command = [ diff --git a/tests/utils/helpers/_feature_verification/provider.py b/tests/utils/helpers/_feature_verification/provider.py index 0cb3aa08..235bb3e9 100644 --- a/tests/utils/helpers/_feature_verification/provider.py +++ b/tests/utils/helpers/_feature_verification/provider.py @@ -27,8 +27,10 @@ CHAT_MODEL: Final[str] = "featver-chat" FAKE_CHAT_MODEL_REF: Final[str] = f"{PROVIDER_ID}/{CHAT_MODEL}" PROVIDER_BASE_URL: Final[str] = "http://127.0.0.1:18000/v1" -# OpenClaw's non-secret marker for keyless custom local providers (not flagged by audits). -PROVIDER_API_KEY: Final[str] = "custom-local" +# A known provider env-var name is OpenClaw's env marker: resolved from the environment at +# runtime and never stored (or audited) as plaintext. The fake server accepts any value. +PROVIDER_API_KEY_ENV: Final[str] = "VLLM_API_KEY" +PROVIDER_API_KEY_VALUE: Final[str] = "featver-local" # LiteLLM reaches the fake server from its container through host-gateway. OPENAI_COMPATIBLE_EMBEDDING_MODEL: Final[str] = "openai/featver-embed" OPENAI_COMPATIBLE_EMBEDDING_API_BASE: Final[str] = "http://host.docker.internal:11434/v1" @@ -36,6 +38,7 @@ OLLAMA_EMBEDDING_API_BASE: Final[str] = "http://host.docker.internal:11434" ENV_CONTRACT: Final[dict[str, str]] = { "OPENCLAW_DEFAULT_MODEL": FAKE_CHAT_MODEL_REF, + PROVIDER_API_KEY_ENV: PROVIDER_API_KEY_VALUE, "HYPERMEMORY_EMBEDDING_MODEL": OPENAI_COMPATIBLE_EMBEDDING_MODEL, "HYPERMEMORY_EMBEDDING_API_BASE": OPENAI_COMPATIBLE_EMBEDDING_API_BASE, } @@ -52,7 +55,7 @@ def provider_entry() -> dict[str, object]: """Return the ``models.providers`` entry for the fake model server.""" return { "baseUrl": PROVIDER_BASE_URL, - "apiKey": PROVIDER_API_KEY, + "apiKey": PROVIDER_API_KEY_ENV, "api": "openai-completions", "models": [ { From b6e14cbd370c0dc2efa3dcc95d8439c55694b37b Mon Sep 17 00:00:00 2001 From: Juan Sugg Date: Wed, 11 Nov 2026 15:16:30 -0300 Subject: [PATCH 5/5] ci(feature-verification): run on dispatch and weekly schedule only --- .github/workflows/feature-verification.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/feature-verification.yml b/.github/workflows/feature-verification.yml index eee44467..912d35b9 100644 --- a/.github/workflows/feature-verification.yml +++ b/.github/workflows/feature-verification.yml @@ -23,9 +23,6 @@ on: default: true schedule: - cron: "23 5 * * 1" - push: - branches: - - test/feature-verification-suite concurrency: group: feature-verification-${{ github.ref }}