diff --git a/.github/workflows/feature-verification.yml b/.github/workflows/feature-verification.yml new file mode 100644 index 00000000..912d35b9 --- /dev/null +++ b/.github/workflows/feature-verification.yml @@ -0,0 +1,290 @@ +name: Feature Verification + +# Verifies every feature in tests/feature_matrix.yaml against the OpenClaw/ACPX versions that +# `clawops bootstrap` installs from platform/node-tools, on provisioned Linux and macOS hosts. +# No setup-node: bootstrap provisions Node itself and installs the CLIs where the rendered +# systemd/launchd units look for them, as on an operator host. +on: + workflow_dispatch: + inputs: + platforms: + description: Which lanes to run. + required: true + type: choice + default: all + options: + - all + - linux + - macos + fail_on_feature_failures: + description: Fail the run when any feature is FAIL. + required: true + type: boolean + default: true + schedule: + - cron: "23 5 * * 1" + +concurrency: + group: feature-verification-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + FRESH_HOST_CACHE_ROOT: ${{ github.workspace }}/.github-cache + UV_CACHE_DIR: ${{ github.workspace }}/.github-cache/uv + npm_config_cache: ${{ github.workspace }}/.github-cache/npm + +jobs: + linux-lanes: + name: Linux Features (${{ matrix.lane }}) + if: ${{ github.event_name == 'schedule' || inputs.platforms != 'macos' }} + runs-on: ubuntu-latest + timeout-minutes: 150 + strategy: + fail-fast: false + matrix: + include: + - lane: linux-core + profile: hypermemory + scenario: linux-features + # LiteLLM reaches the fake Ollama upstream from its container via host-gateway. + fake_model_host: 0.0.0.0 + fake_model_ports: --port 18000 --port 11434 + - lane: linux-lancedb + profile: memory-lancedb-pro + scenario: linux-features + fake_model_host: 0.0.0.0 + fake_model_ports: --port 18000 --port 11434 + - lane: linux-profiles + profile: openclaw-default + scenario: linux-features-local + fake_model_host: 127.0.0.1 + # 11434 stays free for the optional Ollama compose stack this lane starts. + fake_model_ports: --port 18000 + env: + FRESH_HOST_SELECTED_DOCKER_PULL_PARALLELISM: "2" + FRESH_HOST_SELECTED_DOCKER_PULL_MAX_ATTEMPTS: "5" + FEATURE_RESULTS_DIR: ${{ github.workspace }}/.feature-results + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + # Reserves equal the scenario+lane step timeouts and the summed observer step timeouts. + - name: Initialize Linux feature job budget + run: ./tests/scripts/fresh_host.py initialize-budget --job-timeout-minutes 150 --scenario-reserve-minutes 105 --observer-reserve-minutes 10 --github-env-file "${GITHUB_ENV}" + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + - name: Restore package download caches + timeout-minutes: 5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + .github-cache/npm + .github-cache/uv + key: fresh-host-packages-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('uv.lock', 'platform/node-tools/package-lock.json', 'platform/plugins/memory-lancedb-pro/package-lock.json') }} + - name: Start the systemd user manager + if: ${{ matrix.scenario == 'linux-features' }} + timeout-minutes: 3 + run: ./tests/scripts/feature_verification.py enable-systemd-user + - name: Prepare Linux feature context + run: ./tests/scripts/fresh_host.py prepare-context --scenario "${{ matrix.scenario }}" --profile "${{ matrix.profile }}" --repo-root . --runner-temp "${RUNNER_TEMP}" --workspace "${GITHUB_WORKSPACE}" --github-env-file "${GITHUB_ENV}" + - name: Ensure Linux sidecar images + if: ${{ matrix.scenario == 'linux-features' }} + timeout-minutes: 32 + run: ./tests/scripts/hosted_docker.py ensure-images --context "${FRESH_HOST_CONTEXT}" + - name: Start the fake model provider + timeout-minutes: 2 + run: ./tests/scripts/feature_verification.py start-fake-model --host "${{ matrix.fake_model_host }}" ${{ matrix.fake_model_ports }} --state-dir "${RUNNER_TEMP}/fake-model" + - name: Select the fake model provider in the env contract + run: ./tests/scripts/feature_verification.py seed-env-contract --context "${FRESH_HOST_CONTEXT}" + # Bootstrap + setup are themselves verified features; a failure here is reported per + # feature by the lane, which falls back to a harness gateway for the remaining features. + - name: Bootstrap and set up the host + continue-on-error: true + timeout-minutes: 45 + run: ./tests/scripts/fresh_host.py run-scenario --context "${FRESH_HOST_CONTEXT}" + # The operator step after setup: declare the provider, then `clawops model-auth ensure`. + - name: Register the fake model provider + if: always() + continue-on-error: true + timeout-minutes: 16 + run: ./tests/scripts/feature_verification.py apply-model-provider --context "${FRESH_HOST_CONTEXT}" + - name: Run lane features + timeout-minutes: 60 + run: ./tests/scripts/feature_verification.py run-lane --context "${FRESH_HOST_CONTEXT}" --lane "${{ matrix.lane }}" --results "${FEATURE_RESULTS_DIR}/feature-results-${{ matrix.lane }}.json" --fake-model-dir "${RUNNER_TEMP}/fake-model" + - name: Collect lane diagnostics + if: ${{ always() && env.FRESH_HOST_CONTEXT != '' }} + continue-on-error: true + timeout-minutes: 3 + run: ./tests/scripts/feature_verification.py collect-diagnostics --context "${FRESH_HOST_CONTEXT}" --output-dir "${FEATURE_RESULTS_DIR}/diagnostics" + - name: Collect fresh-host diagnostics + if: ${{ always() && env.FRESH_HOST_CONTEXT != '' }} + continue-on-error: true + timeout-minutes: 3 + run: ./tests/scripts/fresh_host.py collect-diagnostics --context "${FRESH_HOST_CONTEXT}" + - name: Stop the fake model provider + if: always() + continue-on-error: true + timeout-minutes: 1 + run: ./tests/scripts/feature_verification.py stop-fake-model --state-dir "${RUNNER_TEMP}/fake-model" + - name: Upload lane results + if: always() + continue-on-error: true + timeout-minutes: 3 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: feature-results-${{ matrix.lane }} + path: | + ${{ github.workspace }}/.feature-results + ${{ runner.temp }}/fresh-host-reports + ${{ runner.temp }}/fake-model/fake-model.log + include-hidden-files: true + + macos-lane: + name: macOS Features (macos-launchd) + if: ${{ github.event_name == 'schedule' || inputs.platforms != 'linux' }} + runs-on: macos-15-intel + timeout-minutes: 150 + env: + FRESH_HOST_SELECTED_DOCKER_PULL_PARALLELISM: "2" + FRESH_HOST_SELECTED_DOCKER_PULL_MAX_ATTEMPTS: "5" + FEATURE_RESULTS_DIR: ${{ github.workspace }}/.feature-results + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + - name: Initialize macOS feature job budget + run: ./tests/scripts/fresh_host.py initialize-budget --job-timeout-minutes 150 --scenario-reserve-minutes 75 --observer-reserve-minutes 12 --github-env-file "${GITHUB_ENV}" + - name: Restore OrbStack DMG cache + id: orbstack_dmg_cache + timeout-minutes: 5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: /tmp/orbstack.dmg + key: orbstack-v1.5.1-16857-amd64 + - name: Set up Docker via OrbStack + timeout-minutes: 10 + # v1.6.0+ panics on GHA macos-15-intel (Skylake CPU check); v1.5.1 is safe. + env: + ORBSTACK_DMG_PATH: /tmp/orbstack.dmg + ORBSTACK_DMG_SHA256: fb95108ded54a27603b68184a13f7e666e0e758167652c0b65cd4dc5eff94617 + ORBSTACK_DMG_URL: https://cdn-updates.orbstack.dev/amd64/OrbStack_v1.5.1_16857_amd64.dmg + run: >- + ./tests/scripts/hosted_docker.py setup-orbstack + --dmg-path "${ORBSTACK_DMG_PATH}" + --url "${ORBSTACK_DMG_URL}" + --sha256 "${ORBSTACK_DMG_SHA256}" + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + - name: Restore package download caches + timeout-minutes: 5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + .github-cache/npm + .github-cache/uv + key: fresh-host-packages-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('uv.lock', 'platform/node-tools/package-lock.json', 'platform/plugins/memory-lancedb-pro/package-lock.json') }} + - name: Prepare macOS feature context + run: >- + ./tests/scripts/fresh_host.py prepare-context + --scenario macos-features + --profile openclaw-default + --repo-root . + --runner-temp "${RUNNER_TEMP}" + --workspace "${GITHUB_WORKSPACE}" + --github-env-file "${GITHUB_ENV}" + # Backstop only: the helper ends at its own deadline with recovery reserved inside it. + - name: Wait for hosted macOS Docker runtime + timeout-minutes: 12 + run: >- + ./tests/scripts/hosted_docker.py wait-runtime-ready + --context "${FRESH_HOST_CONTEXT}" + --github-env-file "${GITHUB_ENV}" + - name: Ensure hosted macOS Docker images + timeout-minutes: 32 + run: ./tests/scripts/hosted_docker.py ensure-images --context "${FRESH_HOST_CONTEXT}" + - name: Start the fake model provider + timeout-minutes: 2 + run: ./tests/scripts/feature_verification.py start-fake-model --host 127.0.0.1 --port 18000 --port 11434 --state-dir "${RUNNER_TEMP}/fake-model" + - name: Select the fake model provider in the env contract + run: ./tests/scripts/feature_verification.py seed-env-contract --context "${FRESH_HOST_CONTEXT}" + # The scenario recovers a Docker runtime lost mid-run inside its own budget. + - name: Bootstrap and set up the host + continue-on-error: true + timeout-minutes: 40 + run: ./tests/scripts/fresh_host.py run-scenario --context "${FRESH_HOST_CONTEXT}" + # The operator step after setup: declare the provider, then `clawops model-auth ensure`. + - name: Register the fake model provider + if: always() + continue-on-error: true + timeout-minutes: 16 + run: ./tests/scripts/feature_verification.py apply-model-provider --context "${FRESH_HOST_CONTEXT}" + - name: Run lane features + timeout-minutes: 35 + run: ./tests/scripts/feature_verification.py run-lane --context "${FRESH_HOST_CONTEXT}" --lane macos-launchd --results "${FEATURE_RESULTS_DIR}/feature-results-macos-launchd.json" --fake-model-dir "${RUNNER_TEMP}/fake-model" + - name: Collect lane diagnostics + if: ${{ always() && env.FRESH_HOST_CONTEXT != '' }} + continue-on-error: true + timeout-minutes: 3 + run: ./tests/scripts/feature_verification.py collect-diagnostics --context "${FRESH_HOST_CONTEXT}" --output-dir "${FEATURE_RESULTS_DIR}/diagnostics" + - name: Collect hosted macOS Docker diagnostics + if: ${{ always() && env.FRESH_HOST_CONTEXT != '' }} + continue-on-error: true + timeout-minutes: 4 + run: ./tests/scripts/hosted_docker.py collect-diagnostics --context "${FRESH_HOST_CONTEXT}" + - name: Stop the fake model provider + if: always() + continue-on-error: true + timeout-minutes: 1 + run: ./tests/scripts/feature_verification.py stop-fake-model --state-dir "${RUNNER_TEMP}/fake-model" + - name: Upload lane results + if: always() + continue-on-error: true + timeout-minutes: 3 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: feature-results-macos-launchd + path: | + ${{ github.workspace }}/.feature-results + ${{ runner.temp }}/fresh-host-reports + ${{ runner.temp }}/fake-model/fake-model.log + include-hidden-files: true + - name: Clean up macOS feature context + if: ${{ always() && env.FRESH_HOST_CONTEXT != '' }} + continue-on-error: true + timeout-minutes: 2 + run: ./tests/scripts/fresh_host.py cleanup --context "${FRESH_HOST_CONTEXT}" + + report: + name: Feature Verification Report + needs: + - linux-lanes + - macos-lane + if: always() + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version: "0.10.9" + enable-cache: true + - run: uv sync --locked + - name: Download lane results + continue-on-error: true + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + pattern: feature-results-* + path: lane-results + - name: Write the per-feature report + run: uv run python3 ./tests/scripts/feature_report.py --results-dir lane-results --output feature-report/feature-verification.json --summary-file "${GITHUB_STEP_SUMMARY}" ${{ (github.event_name == 'schedule' || inputs.fail_on_feature_failures) && '--fail-on-failures' || '' }} + - name: Upload the per-feature report + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: feature-verification-report + path: feature-report/feature-verification.json diff --git a/pyproject.toml b/pyproject.toml index 28949de0..81c2c020 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -167,6 +167,7 @@ markers = [ "integration: cross-module and service-shaped tests", "contract: repository and artifact contract checks", "e2e: black-box end-to-end orchestration tests", + "feature_verification: per-feature verification against a provisioned host (lane-only)", "devflow: production devflow coverage", "framework: explicit validation of the pytest framework itself", "test_profile: named infrastructure runtime profile declarations", diff --git a/tests/conftest.py b/tests/conftest.py index d5588629..0cfcfcfb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,6 +56,8 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item item.add_marker(pytest.mark.e2e) if "framework" in parts: item.add_marker(pytest.mark.framework) + if "feature_verification" in parts: + item.add_marker(pytest.mark.feature_verification) if "hypermemory" in parts: item.add_marker(pytest.mark.hypermemory) if rel_path_posix in _DEVFLOW_MARKED_FILES: diff --git a/tests/feature_matrix.yaml b/tests/feature_matrix.yaml new file mode 100644 index 00000000..b71c634c --- /dev/null +++ b/tests/feature_matrix.yaml @@ -0,0 +1,899 @@ +# StrongClaw end-to-end feature matrix. +# +# One entry per product feature: where it lives (call sites), the single test that verifies it, +# and the lanes (provisioned CI hosts) it runs on; platforms are derived from the lanes. +# `verification` says how deep the check goes: +# live - real StrongClaw + real OpenClaw/acpx + real sidecars, no stand-ins +# fake-model - live, with the local deterministic model server as the model provider +# stub - a third-party endpoint (vendor CLI/adapter, messaging API) is stood in for +# `status: unverifiable` features cannot be exercised without third-party live services; they +# carry the reason and are reported UNVERIFIABLE, never PASS. +# Contract: tests/suites/contracts/repo/test_feature_matrix_contract.py. +schema_version: 1 + +lanes: + linux-core: + platform: linux + profile: hypermemory + scenario: linux-features + activate_services: true + description: Default product profile with systemd user services, sidecars, and the gateway. + linux-lancedb: + platform: linux + profile: memory-lancedb-pro + scenario: linux-features + activate_services: true + description: Vendored memory-lancedb-pro profile with systemd services and sidecars. + linux-profiles: + platform: linux + profile: openclaw-default + scenario: linux-features-local + activate_services: false + description: Profile matrix plus CLI-only operator tools on a harness-managed gateway. + macos-launchd: + platform: macos + profile: openclaw-default + scenario: macos-features + activate_services: true + description: macOS bootstrap and launchd-supervised runtime. + +features: + # --- host lifecycle ------------------------------------------------------------------------- + - id: bootstrap.host + category: host-lifecycle + title: clawops bootstrap installs the locked OpenClaw/ACPX toolchain, varlock and the venv + call_sites: + - src/clawops/strongclaw_bootstrap.py:576 + - src/clawops/strongclaw_node_tools.py:715 + lanes: [linux-core, macos-launchd] + verification: live + test: tests/suites/e2e/feature_verification/test_host_lifecycle.py::test_bootstrap_installs_the_locked_toolchain + - id: setup.guided + category: host-lifecycle + title: clawops setup (non-interactive) renders the profile and activates the host services + call_sites: + - src/clawops/setup_cli.py:310 + - src/clawops/setup_cli.py:139 + lanes: [linux-core, macos-launchd] + verification: live + test: tests/suites/e2e/feature_verification/test_host_lifecycle.py::test_guided_setup_completes_with_service_activation + - id: services.systemd + category: host-lifecycle + title: systemd user units and timers are installed, enabled and active on Linux + call_sites: + - src/clawops/strongclaw_services.py:51 + - src/clawops/strongclaw_services.py:398 + - platform/systemd/openclaw-gateway.service:16 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_host_lifecycle.py::test_systemd_user_units_are_installed_and_active + - id: services.launchd + category: host-lifecycle + title: launchd agents are bootstrapped and running on macOS + call_sites: + - src/clawops/strongclaw_services.py:27 + - src/clawops/strongclaw_services.py:233 + - platform/launchd/ai.openclaw.gateway.plist.template:7 + lanes: [macos-launchd] + verification: live + test: tests/suites/e2e/feature_verification/test_host_lifecycle.py::test_launchd_agents_are_loaded_and_running + - id: varlock.env-contract + category: host-lifecycle + title: varlock-env configure/check generates and validates the operator env contract + call_sites: + - src/clawops/strongclaw_varlock_env.py:641 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_host_lifecycle.py::test_varlock_env_contract_is_valid + - id: doctor.host + category: host-lifecycle + title: clawops doctor-host validates the toolchain and the rendered config + call_sites: + - src/clawops/setup_cli.py:127 + lanes: [linux-core, macos-launchd] + verification: live + test: tests/suites/e2e/feature_verification/test_host_lifecycle.py::test_doctor_host_reports_toolchain + - id: doctor.deep + category: host-lifecycle + title: clawops doctor runs every readiness check (model probe, audits, memory, sidecars) + call_sites: + - src/clawops/setup_cli.py:392 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_host_lifecycle.py::test_deep_doctor_passes_every_check + - id: baseline.verify + category: host-lifecycle + title: clawops baseline verify passes with runtime probes + call_sites: + - src/clawops/strongclaw_baseline.py:138 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_host_lifecycle.py::test_baseline_verify_passes_with_runtime_probes + - id: host.runtime-user-provisioning + category: host-lifecycle + title: Dedicated runtime-user provisioning per the setup runbooks + call_sites: + - SETUP_GUIDE.md:12 + status: unverifiable + reason: Manual operator runbook (OS account creation and service-user SSH setup); StrongClaw ships no code path that performs it, so there is nothing to execute. + + # --- model auth ----------------------------------------------------------------------------- + - id: model-auth.agents-list + category: model-auth + title: openclaw agents list --json reports every configured agent + call_sites: + - src/clawops/strongclaw_model_auth.py:114 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_agents_list_matches_rendered_config + - id: model-auth.ensure + category: model-auth + title: clawops model-auth ensure applies the env-driven model chain to every agent + call_sites: + - src/clawops/strongclaw_model_auth.py:343 + - src/clawops/strongclaw_model_auth.py:239 + lanes: [linux-core, macos-launchd] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_model_auth_ensure_applies_env_driven_chain + - id: model-auth.probe + category: model-auth + title: models status --check --probe reaches the provider for every agent + call_sites: + - src/clawops/strongclaw_model_auth.py:170 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_model_probe_reaches_the_provider + - id: model-auth.models-list + category: model-auth + title: openclaw models --agent X list --json reports an available model (fallback path) + call_sites: + - src/clawops/strongclaw_model_auth.py:141 + lanes: [linux-profiles] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_models_list_reports_available_models + - id: model-auth.configure-wizard + category: model-auth + title: Interactive model wizard (openclaw configure --section model) + call_sites: + - src/clawops/strongclaw_model_auth.py:393 + status: unverifiable + reason: Runs only on an interactive TTY and walks a vendor sign-in/provider picker; the non-interactive env-driven chain (model-auth.ensure) covers the same outcome. + + # --- gateway -------------------------------------------------------------------------------- + - id: gateway.status + category: gateway + title: openclaw gateway status --json reports a loopback RPC endpoint + call_sites: + - src/clawops/setup_cli.py:493 + - src/clawops/strongclaw_ops.py:977 + lanes: [linux-core, macos-launchd] + verification: live + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_gateway_status_reports_loopback_rpc + - id: gateway.agent-turn + category: gateway + title: The supervised gateway runs a full agent turn through the configured provider + call_sites: + - src/clawops/strongclaw_ops.py:977 + - platform/systemd/openclaw-gateway.service:16 + lanes: [linux-core, macos-launchd] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_gateway_runs_an_agent_turn_through_the_provider + - id: gateway.loopback-bind + category: gateway + title: The gateway listens on loopback only + call_sites: + - platform/configs/openclaw/00-baseline.json5:116 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_gateway_binds_only_loopback + - id: openclaw.doctor + category: gateway + title: openclaw doctor --non-interactive is clean on the rendered profile + call_sites: + - src/clawops/strongclaw_baseline.py:156 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_openclaw_doctor_is_clean + - id: security.audit + category: security + title: openclaw security audit --deep passes + call_sites: + - src/clawops/strongclaw_baseline.py:157 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_security_audit_passes + - id: secrets.audit + category: security + title: openclaw secrets audit --check passes + call_sites: + - src/clawops/strongclaw_baseline.py:158 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_secrets_audit_passes + - id: secrets.external-backends + category: security + title: Varlock secret backends other than local (1Password, AWS, Azure, Bitwarden, GCP, Infisical) + call_sites: + - platform/configs/varlock/.env.schema:15 + - platform/docs/SECRETS_AND_ENV.md:22 + status: unverifiable + reason: Each backend resolves secrets from a third-party vault account; no account or credentials exist in CI. + + # --- memory: hypermemory -------------------------------------------------------------------- + - id: profile.hypermemory + category: memory + title: hypermemory profile loads strongclaw-hypermemory and lossless-claw into their slots + call_sites: + - src/clawops/memory_profiles.py:23 + - src/clawops/strongclaw_bootstrap.py:425 + - platform/configs/openclaw/77-hypermemory.example.json5:12 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_hypermemory_profile_plugins_are_loaded + - id: memory.openclaw-status + category: memory + title: openclaw memory status --deep reports the active memory backend + call_sites: + - src/clawops/setup_cli.py:501 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_openclaw_memory_status_is_healthy + - id: memory.openclaw-search + category: memory + title: openclaw memory search returns corpus hits through the memory plugin + call_sites: + - src/clawops/setup_cli.py:510 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_openclaw_memory_search_returns_corpus_hits + - id: hypermemory.index-search + category: memory + title: hypermemory index + sparse/dense hybrid search over Qdrant with real embeddings route + call_sites: + - src/clawops/hypermemory/cli.py:37 + - src/clawops/hypermemory/cli.py:40 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_hypermemory_index_and_hybrid_search + - id: hypermemory.verify + category: memory + title: hypermemory verify reports a consistent Qdrant index + call_sites: + - src/clawops/hypermemory/cli.py:31 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_hypermemory_verify_reports_consistent_index + - id: hypermemory.store-forget + category: memory + title: hypermemory store makes a fact searchable and forget removes it + call_sites: + - src/clawops/hypermemory/cli.py:85 + - src/clawops/hypermemory/cli.py:125 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_hypermemory_store_search_and_forget + - id: hypermemory.maintenance + category: memory + title: hypermemory reflect/lifecycle/list-facts/flush-metadata run against the live index + call_sites: + - src/clawops/hypermemory/cli.py:108 + - src/clawops/hypermemory/cli.py:174 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_hypermemory_maintenance_operations + - id: hypermemory.export-memory-pro + category: memory + title: hypermemory export-memory-pro writes a memory-pro import snapshot + call_sites: + - src/clawops/hypermemory/cli.py:195 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_hypermemory_exports_memory_pro_snapshot + - id: hypermemory.plugin-recall + category: memory + title: strongclaw-hypermemory autoRecall injects stored memory into gateway agent turns + call_sites: + - platform/plugins/strongclaw-hypermemory/index.js:884 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_hypermemory_plugin_recalls_memory_into_turns + - id: qdrant.storage-stamp + category: memory + title: sidecars up stamps Qdrant storage with the pinned release line (#237) + call_sites: + - src/clawops/qdrant_storage.py:278 + - src/clawops/strongclaw_ops.py:677 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_qdrant_storage_stamp_tracks_pinned_release + - id: qdrant.reset-reindex + category: memory + title: A Qdrant release-line change resets storage and reindexes hypermemory (#237) + call_sites: + - src/clawops/qdrant_storage.py:318 + - src/clawops/strongclaw_ops.py:710 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_hypermemory.py::test_qdrant_release_change_resets_storage_and_reindexes + + # --- memory: memory-lancedb-pro ------------------------------------------------------------- + - id: profile.memory-lancedb-pro + category: memory + title: memory-lancedb-pro profile installs the vendored plugin and loads it into the memory slot + call_sites: + - src/clawops/memory_profiles.py:41 + - src/clawops/strongclaw_bootstrap.py:391 + - platform/configs/openclaw/75-memory-lancedb-pro.local.json5:11 + lanes: [linux-lancedb] + verification: live + test: tests/suites/e2e/feature_verification/test_memory_lancedb.py::test_lancedb_plugin_is_loaded_in_the_memory_slot + - id: memory.migrate-hypermemory-to-pro + category: memory + title: clawops memory migrate-hypermemory-to-pro exports a scope into memory-pro import JSON + call_sites: + - src/clawops/memory_tools.py:147 + lanes: [linux-lancedb] + verification: live + test: tests/suites/e2e/feature_verification/test_memory_lancedb.py::test_migrate_hypermemory_exports_memory_pro_snapshot + - id: memory.import-pro-snapshot + category: memory + title: clawops memory import-pro-snapshot imports through openclaw memory-pro import + call_sites: + - src/clawops/memory_tools.py:174 + lanes: [linux-lancedb] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_lancedb.py::test_import_pro_snapshot_lands_in_lancedb + - id: memory.verify-pro-parity + category: memory + title: clawops memory verify-pro-parity compares hypermemory hits with live memory-pro search + call_sites: + - src/clawops/memory_tools.py:476 + lanes: [linux-lancedb] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_lancedb.py::test_verify_pro_parity_against_live_plugin + - id: lancedb.smart-extraction + category: memory + title: memory-lancedb-pro autoCapture runs smart extraction and embeddings after a turn + call_sites: + - platform/plugins/memory-lancedb-pro/index.ts:2725 + lanes: [linux-lancedb] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_lancedb.py::test_smart_extraction_calls_the_extraction_model + - id: lancedb.reflection-embedded-runner + category: memory + title: memory reflection runs through extensionAPI.js runEmbeddedPiAgent on session reset + call_sites: + - platform/plugins/memory-lancedb-pro/index.ts:399 + - platform/plugins/memory-lancedb-pro/index.ts:1212 + - platform/plugins/memory-lancedb-pro/index.ts:3252 + lanes: [linux-lancedb] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_memory_lancedb.py::test_reflection_runs_through_the_embedded_runner + + # --- setup profiles ------------------------------------------------------------------------- + - id: profile.openclaw-default + category: profiles + title: openclaw-default profile renders, validates and boots the gateway with memory-core + call_sites: + - src/clawops/memory_profiles.py:30 + - src/clawops/config_cli.py:86 + lanes: [linux-profiles] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_setup_profiles.py::test_openclaw_default_profile_boots_with_memory_core + - id: profile.openclaw-qmd + category: profiles + title: openclaw-qmd profile installs QMD and boots the gateway on the qmd memory backend + call_sites: + - src/clawops/memory_profiles.py:35 + - src/clawops/strongclaw_bootstrap.py:342 + - platform/configs/openclaw/40-qmd-context.json5:3 + lanes: [linux-profiles] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_setup_profiles.py::test_openclaw_qmd_profile_installs_and_uses_qmd + - id: profile.acp + category: profiles + title: acp profile registers the ACP worker agents and boots the gateway + call_sites: + - src/clawops/memory_profiles.py:48 + - platform/configs/openclaw/20-acp-workers.json5:11 + lanes: [linux-profiles] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_setup_profiles.py::test_acp_profile_registers_acp_worker_agents + - id: profile.browser-lab + category: profiles + title: browser-lab profile renders the CDP browser profile and boots the gateway + call_sites: + - src/clawops/memory_profiles.py:55 + - platform/configs/openclaw/60-browser-lab.json5:8 + lanes: [linux-profiles] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_setup_profiles.py::test_browser_lab_profile_points_at_the_cdp_endpoint + + # --- ACP workers ---------------------------------------------------------------------------- + - id: acp.worker-agent-turn + category: acp + title: An OpenClaw ACP worker agent (acp profile) runs a turn through OpenClaw's acpx backend + call_sites: + - platform/configs/openclaw/20-acp-workers.json5:11 + lanes: [linux-profiles] + verification: stub + limitations: The codex adapter behind acpx is the ACP stub agent; a live vendor session is acp.vendor-live-sessions. + test: tests/suites/e2e/feature_verification/test_setup_profiles.py::test_acp_worker_agent_runs_a_turn_through_acpx + - id: acp.openclaw-local + category: acp + title: acp-runner --backend openclaw --auth-mode local runs a live turn (acpx -> openclaw acp) + call_sites: + - src/clawops/acp_runner.py:394 + - src/clawops/acpx_adapter.py:49 + - src/clawops/backend_registry.py:109 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_acp.py::test_openclaw_backend_runs_a_live_acp_turn + - id: acp.codex-api + category: acp + title: codex backend in api mode keeps OPENAI_API_KEY and acpx authenticates with it + call_sites: + - src/clawops/backend_registry.py:69 + - src/clawops/credential_broker.py:160 + lanes: [linux-core] + verification: stub + limitations: Real acpx; the codex-acp adapter is the ACP stub agent (same advertised auth methods). + test: tests/suites/e2e/feature_verification/test_acp.py::test_codex_api_mode_passes_the_key_and_authenticates + - id: acp.codex-subscription + category: acp + title: codex backend in subscription mode strips API keys before dispatch + call_sites: + - src/clawops/backend_registry.py:69 + lanes: [linux-core] + verification: stub + limitations: Vendor `codex login status` and the adapter are stubs. + test: tests/suites/e2e/feature_verification/test_acp.py::test_codex_subscription_mode_strips_api_keys + - id: acp.claude-api + category: acp + title: claude backend in api mode passes ANTHROPIC_API_KEY to the agent + call_sites: + - src/clawops/backend_registry.py:82 + lanes: [linux-core] + verification: stub + limitations: Real acpx; the claude-agent-acp adapter is the ACP stub agent. + test: tests/suites/e2e/feature_verification/test_acp.py::test_claude_api_mode_passes_the_key + - id: acp.claude-subscription + category: acp + title: claude backend in subscription mode strips API keys before dispatch + call_sites: + - src/clawops/backend_registry.py:82 + lanes: [linux-core] + verification: stub + limitations: Vendor `claude auth status` and the adapter are stubs. + test: tests/suites/e2e/feature_verification/test_acp.py::test_claude_subscription_mode_strips_api_keys + - id: acp.claude-cloud-provider + category: acp + title: claude backend in cloud-provider mode keeps cloud credentials and strips the API key + call_sites: + - src/clawops/backend_registry.py:82 + lanes: [linux-core] + verification: stub + limitations: Cloud credentials are presence-only markers; no cloud account is used. + test: tests/suites/e2e/feature_verification/test_acp.py::test_claude_cloud_provider_mode_keeps_cloud_credentials + - id: acp.local-backends + category: acp + title: gemini/qwen/copilot/cursor local backends dispatch through acpx + call_sites: + - src/clawops/backend_registry.py:105 + lanes: [linux-core] + verification: stub + limitations: The vendor CLIs' ACP servers are the ACP stub agent. + test: tests/suites/e2e/feature_verification/test_acp.py::test_local_backends_dispatch_through_acpx + - id: acp.auth-policy + category: acp + title: acp-runner refuses unsupported and unready auth modes at preflight + call_sites: + - src/clawops/credential_broker.py:236 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_acp.py::test_auth_policy_blocks_unsupported_and_unready_modes + - id: acp.credential-status + category: acp + title: clawops credential-status reports machine-readable readiness per auth mode + call_sites: + - src/clawops/credential_broker.py:292 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_acp.py::test_credential_status_reports_each_mode + - id: acp.vendor-live-sessions + category: acp + title: Live Codex/Claude/Gemini/Qwen/Copilot/Cursor coding sessions through acpx + call_sites: + - src/clawops/backend_registry.py:69 + - src/clawops/backend_registry.py:82 + status: unverifiable + reason: Requires signed-in vendor accounts or paid API keys for each coding agent; CI has none. Routing is covered at stub level by the acp.* features. + + # --- sidecars and platform ------------------------------------------------------------------ + - id: sidecars.postgres + category: sidecars + title: Postgres sidecar is up and serves the LiteLLM database + call_sites: + - platform/compose/docker-compose.aux-stack.yaml:2 + - src/clawops/strongclaw_ops.py:495 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_postgres_sidecar_accepts_queries + - id: sidecars.litellm + category: sidecars + title: LiteLLM routes the hypermemory-embedding alias to its upstream + call_sites: + - platform/compose/docker-compose.aux-stack.yaml:25 + - platform/configs/litellm/config.yaml:22 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_litellm_routes_embeddings_to_the_upstream + - id: sidecars.litellm-ollama-embeddings + category: sidecars + title: LiteLLM serves the default Ollama embedding lane that varlock-env configure selects + call_sites: + - src/clawops/strongclaw_varlock_env.py:571 + - platform/configs/litellm/config.yaml:22 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_litellm_serves_the_default_ollama_embedding_lane + - id: sidecars.qdrant + category: sidecars + title: Qdrant sidecar creates collections, upserts and searches vectors + call_sites: + - platform/compose/docker-compose.aux-stack.yaml:90 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_qdrant_sidecar_round_trips_vectors + - id: sidecars.neo4j + category: sidecars + title: Neo4j sidecar accepts authenticated Cypher writes and reads + call_sites: + - platform/compose/docker-compose.aux-stack.yaml:111 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_neo4j_sidecar_round_trips_cypher + - id: observability.otel + category: observability + title: OTel collector accepts OTLP traces and exports Prometheus metrics; verify-platform observability + call_sites: + - platform/compose/docker-compose.aux-stack.yaml:69 + - src/clawops/platform_verify.py:383 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_observability_pipeline_accepts_telemetry + - id: verify-platform.sidecars + category: sidecars + title: clawops verify-platform sidecars passes with runtime probes + call_sites: + - src/clawops/platform_verify.py:261 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_verify_platform_sidecars_runtime + - id: ops.status + category: sidecars + title: clawops ops status reports every required sidecar ready + call_sites: + - src/clawops/strongclaw_ops.py:1024 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_ops_status_reports_required_sidecars_ready + - id: ops.prune-qdrant-test-collections + category: sidecars + title: clawops ops prune-qdrant-test-collections removes prefixed collections (dry-run first) + call_sites: + - src/clawops/strongclaw_ops.py:1174 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_prune_qdrant_test_collections + - id: browser-lab.runtime + category: sidecars + title: Browser lab starts, exposes CDP on 9222 and the egress proxy on 3128, then stops + call_sites: + - src/clawops/strongclaw_ops.py:1004 + - src/clawops/platform_verify.py:678 + - platform/compose/docker-compose.browser-lab.yaml:12 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_browser_lab_starts_and_exposes_cdp + - id: ops.reset-compose-state + category: sidecars + title: clawops ops reset-compose-state clears one component's state directory + call_sites: + - src/clawops/strongclaw_ops.py:1114 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_reset_compose_state_clears_one_component + - id: sidecars.ollama-optional + category: sidecars + title: Optional Ollama compose stack serves the Ollama API on loopback + call_sites: + - platform/compose/docker-compose.ollama.optional.yaml:2 + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_sidecars.py::test_optional_ollama_compose_serves_the_api + - id: observability.langfuse-optional + category: observability + title: Optional Langfuse OTLP exporter stack + call_sites: + - platform/compose/docker-compose.langfuse.optional.yaml:2 + - platform/docs/OBSERVABILITY.md:14 + status: unverifiable + reason: The optional compose file only runs langfuse-web/worker and expects an operator-provided Postgres, Redis and ClickHouse (LANGFUSE_* URLs); StrongClaw ships none of them. + + # --- recovery ------------------------------------------------------------------------------- + - id: recovery.backup-create + category: recovery + title: recovery backup-create writes an archive through openclaw backup create + call_sites: + - src/clawops/strongclaw_recovery.py:144 + - src/clawops/recovery/backends.py:49 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_recovery.py::test_backup_create_uses_the_openclaw_backend + - id: recovery.backup-verify + category: recovery + title: recovery backup-verify validates the latest archive with openclaw backup verify + call_sites: + - src/clawops/strongclaw_recovery.py:199 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_recovery.py::test_backup_verify_accepts_latest_archive + - id: recovery.restore + category: recovery + title: recovery restore extracts every archived file safely into a destination + call_sites: + - src/clawops/strongclaw_recovery.py:259 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_recovery.py::test_restore_extracts_openclaw_state + - id: recovery.prune-retention + category: recovery + title: recovery prune-retention deletes only archives past retention + call_sites: + - src/clawops/strongclaw_recovery.py:274 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_recovery.py::test_prune_retention_removes_only_stale_archives + - id: recovery.rotate-secrets + category: recovery + title: recovery rotate-secrets prints the rotation runbook steps + call_sites: + - src/clawops/strongclaw_recovery.py:302 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_recovery.py::test_rotate_secrets_guidance_lists_rotation_targets + - id: recovery.scheduled-jobs + category: recovery + title: The scheduled backup job (systemd timer / launchd agent) produces an archive + call_sites: + - platform/systemd/openclaw-backup-create.timer:5 + - platform/launchd/ai.openclaw.backup-create.plist.template:7 + lanes: [linux-core, macos-launchd] + verification: live + test: tests/suites/e2e/feature_verification/test_recovery.py::test_scheduled_backup_job_produces_an_archive + + # --- context, workflows, channels ----------------------------------------------------------- + - id: context.codebase-graph + category: context + title: context codebase index/query/pack at large scale writes the Neo4j graph + call_sites: + - src/clawops/context/codebase/service.py:1686 + - src/clawops/context/codebase/service.py:3504 + lanes: [linux-core] + verification: fake-model + test: tests/suites/e2e/feature_verification/test_context_workflows.py::test_codebase_context_indexes_into_the_neo4j_graph + - id: workflow.daily-healthcheck + category: operator-tools + title: The shipped daily-healthcheck workflow runs openclaw doctor and security audit + call_sites: + - src/clawops/workflow_runner.py:1168 + - platform/configs/workflows/daily_healthcheck.yaml:1 + lanes: [linux-core] + verification: live + test: tests/suites/e2e/feature_verification/test_context_workflows.py::test_daily_healthcheck_workflow_runs_openclaw_checks + - id: channels.openclaw-config + category: channels + title: The channels overlay renders and validates under OpenClaw with Telegram and WhatsApp configured + call_sites: + - platform/configs/openclaw/30-channels.json5:12 + - src/clawops/openclaw_config.py:338 + lanes: [linux-profiles] + verification: stub + limitations: Telegram uses a placeholder bot token; no message reaches a messaging network. + test: tests/suites/e2e/feature_verification/test_context_workflows.py::test_channels_overlay_validates_under_openclaw + - id: channels.telegram-live + category: channels + title: Live Telegram bot messaging through the gateway + call_sites: + - platform/docs/CHANNELS.md:3 + status: unverifiable + reason: Needs a real Telegram bot token and a Telegram account to message it; neither exists in CI. + - id: channels.whatsapp-live + category: channels + title: Live WhatsApp messaging through the gateway + call_sites: + - platform/docs/channels/whatsapp.md:1 + status: unverifiable + reason: Needs a WhatsApp account paired by QR code on a real phone. + - id: ops.remote-operator-access + category: operator-tools + title: Remote operator access to the gateway over an SSH tunnel + call_sites: + - USAGE_GUIDE.md:65 + status: unverifiable + reason: Operator procedure between two hosts; StrongClaw ships no code that opens the tunnel. + + # --- operator tools and governance (CLI-only) ----------------------------------------------- + - id: policy.evaluate + category: governance + title: Policy engine evaluates payloads against the shipped YAML rules + call_sites: [src/clawops/policy_engine.py:261, src/clawops/policy_engine.py:267] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_policy_evaluates_shipped_rules + - id: op-journal.lifecycle + category: governance + title: Operation journal records begin, transition, approval, and stuck listings + call_sites: [src/clawops/op_journal.py:1012, src/clawops/op_journal.py:1048] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_op_journal_tracks_operation_lifecycle + - id: approvals.review-queue + category: governance + title: Approval queue lists, approves, rejects, delegates, and ingests reviews + call_sites: [src/clawops/approvals.py:32, src/clawops/approvals.py:82] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_approvals_review_queue_resolves_operations + - id: skills.intake + category: governance + title: Skill intake scans, quarantines, promotes, and demotes bundles + call_sites: [src/clawops/skill_scanner.py:236, src/clawops/skill_scanner.py:298] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_skills_intake_moves_bundle_through_stages + - id: wrapper.webhook + category: governance + title: Webhook wrapper posts exactly once after journaled approval + call_sites: [src/clawops/wrappers/webhook.py:104, src/clawops/wrappers/webhook.py:119] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_webhook_wrapper_posts_once_after_approval + - id: wrapper.github + category: governance + title: GitHub wrapper dry run journals planned comment and label operations + call_sites: [src/clawops/wrappers/github.py:327, src/clawops/wrappers/github.py:359] + lanes: [linux-profiles] + verification: stub + limitations: Dry run only; executing against the live GitHub API needs a repo token. + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_github_wrapper_dry_run_plans_without_executing + - id: harness.regression-suites + category: operator-tools + title: YAML regression harness runs the shipped policy and security suites + call_sites: [src/clawops/harness.py:102, src/clawops/harness.py:110] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_harness_regression_suites_pass + - id: charts.render + category: operator-tools + title: Charts render harness results to a PNG + call_sites: [src/clawops/charts.py:26, src/clawops/charts.py:34] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_charts_render_harness_results + - id: merge-json.overlays + category: operator-tools + title: merge-json deep-merges JSON5 overlays onto a base document + call_sites: [src/clawops/json_merge.py:20, src/clawops/json_merge.py:29] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_merge_json_deep_merges_overlays + - id: allowlists.render + category: channels + title: Channel allowlists render a normalized channels fragment + call_sites: [src/clawops/allowlist_sync.py:59, src/clawops/allowlist_sync.py:67] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_allowlists_render_channel_fragment + - id: worktree.lifecycle + category: operator-tools + title: Managed git worktrees are created, listed, and pruned + call_sites: [src/clawops/repo_tools.py:272, src/clawops/repo_tools.py:298] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_worktree_lifecycle_creates_lists_and_prunes + - id: repo.doctor + category: operator-tools + title: Repo doctor validates the repo/upstream and repo/worktrees contract + call_sites: [src/clawops/repo_tools.py:267, src/clawops/repo_tools.py:286] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_repo_doctor_accepts_valid_layout + - id: workflow.code-review + category: operator-tools + title: Workflow runner executes the shipped code-review workflow and its dry run + call_sites: [src/clawops/workflow_runner.py:1158, src/clawops/workflow_runner.py:1168] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_workflow_code_review_records_step_results + - id: supply-chain.inventory + category: operator-tools + title: Supply-chain inventory lists pinned workflow actions and compose digests + call_sites: [src/clawops/supply_chain.py:550, src/clawops/supply_chain.py:592] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_supply_chain_inventory_lists_pins + - id: supply-chain.propose-refresh + category: operator-tools + title: Supply-chain propose-refresh dry run plans pin updates without side effects + call_sites: [src/clawops/supply_chain.py:565, src/clawops/supply_chain.py:423] + lanes: [linux-profiles] + verification: stub + limitations: Runs against a scratch repo with a docker shim; opening a real refresh PR needs repo write access. + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_supply_chain_propose_refresh_dry_run_plans_updates + - id: observability.structured-logs + category: observability + title: CLAWOPS_STRUCTURED_LOGS emits JSON event lines on stderr + call_sites: [src/clawops/observability.py:37, src/clawops/observability.py:61] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_structured_logs_emit_json_events + - id: baseline.harness-smoke + category: host-lifecycle + title: Baseline harness-smoke runs both regression suites into a runs dir + call_sites: [src/clawops/strongclaw_baseline.py:347, src/clawops/strongclaw_baseline.py:34] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_baseline_harness_smoke_writes_results + - id: channels.verify-platform + category: channels + title: verify-platform channels checks overlays, docs, and allowlist parity + call_sites: [src/clawops/platform_verify.py:751, src/clawops/platform_verify.py:528] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_verify_platform_channels_passes + - id: config.memory-profiles + category: profiles + title: config memory lists every managed memory profile + call_sites: [src/clawops/config_cli.py:84, src/clawops/config_cli.py:110] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_config_lists_managed_memory_profiles + - id: render.overlays + category: profiles + title: render-openclaw-config merges operator overlays and renders exec approvals + call_sites: [src/clawops/openclaw_config.py:386, src/clawops/openclaw_config.py:431] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_render_openclaw_config_merges_overlay + - id: services.render + category: host-lifecycle + title: services render materializes every systemd and launchd template + call_sites: [src/clawops/strongclaw_services.py:421, src/clawops/strongclaw_services.py:113] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_operator_tools.py::test_services_render_every_template + - id: devflow.plan-status-audit + category: devflow + title: devflow plans stages and roles; status/audit reject unknown run ids + call_sites: [src/clawops/devflow.py:651, src/clawops/devflow.py:671, src/clawops/devflow.py:705] + lanes: [linux-profiles] + verification: live + test: tests/suites/e2e/feature_verification/test_devflow_planning.py::test_devflow_plan_status_audit + - id: devflow.run + category: devflow + title: devflow run dispatches the architect-to-lead stage workers and audits the run + call_sites: [src/clawops/devflow.py:657] + status: unverifiable + reason: Every stage dispatches a subscription-authenticated vendor coding agent (claude/codex) that must author the stage artifacts (design, tasks, tests, QA report); CI has no vendor accounts and a stub cannot author meaningful artifacts. Dispatch routing is covered by the acp.* features and planning by devflow.plan-status-audit. + - id: profile.bundled-provider-plugins + category: profiles + title: Memory-profile plugin allowlists keep OpenClaw's bundled ollama provider loadable + call_sites: + - platform/configs/openclaw/77-hypermemory.example.json5:3 + - platform/configs/openclaw/75-memory-lancedb-pro.local.json5:3 + - src/clawops/strongclaw_model_auth.py:232 + lanes: [linux-core, linux-lancedb] + verification: live + test: tests/suites/e2e/feature_verification/test_model_gateway.py::test_profile_allowlist_keeps_bundled_provider_plugins diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py index 1b5f1748..6500a789 100644 --- a/tests/fixtures/__init__.py +++ b/tests/fixtures/__init__.py @@ -2,6 +2,7 @@ pytest_plugins = ( "tests.fixtures.core", + "tests.fixtures.feature_verification", "tests.fixtures.hypermemory", "tests.fixtures.platform", ) diff --git a/tests/fixtures/feature_verification/__init__.py b/tests/fixtures/feature_verification/__init__.py new file mode 100644 index 00000000..d6fdca23 --- /dev/null +++ b/tests/fixtures/feature_verification/__init__.py @@ -0,0 +1,3 @@ +"""Feature-verification suite fixtures and lane hooks.""" + +pytest_plugins = ("tests.fixtures.feature_verification.lane",) diff --git a/tests/fixtures/feature_verification/lane.py b/tests/fixtures/feature_verification/lane.py new file mode 100644 index 00000000..d683699d --- /dev/null +++ b/tests/fixtures/feature_verification/lane.py @@ -0,0 +1,100 @@ +"""Pytest hooks and fixtures for the end-to-end feature-verification suite. + +Outside a provisioned lane (``FEATURE_VERIFICATION_LANE`` unset) the suite is deselected, so +``make test`` and nested product runs such as ``clawops baseline verify`` never touch it. +Inside a lane, only the features the matrix schedules on that lane run, and every outcome is +written to the lane results file the reporter joins with the matrix. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Final + +import pytest + +from tests.utils.helpers.feature_verification import ( + FeatureHost, + GatewayController, + LaneRecorder, + ensure_sidecars, + is_feature_suite_item, +) + +_RECORDER_KEY: Final[pytest.StashKey[LaneRecorder]] = pytest.StashKey[LaneRecorder]() +# The logreport hook receives no config, so the active recorder is also held here. +_ACTIVE_RECORDERS: Final[dict[str, LaneRecorder]] = {} + + +def pytest_configure(config: pytest.Config) -> None: + """Attach the lane recorder when running inside a provisioned lane.""" + recorder = LaneRecorder.from_environment() + if recorder is not None: + config.stash[_RECORDER_KEY] = recorder + _ACTIVE_RECORDERS["lane"] = recorder + + +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + """Deselect the suite outside a lane, and features scheduled on other lanes inside one.""" + suite_items = [item for item in items if is_feature_suite_item(item)] + if not suite_items: + return + recorder = config.stash.get(_RECORDER_KEY, None) + if recorder is None: + deselected = suite_items + else: + _, deselected = recorder.select(suite_items) + if deselected: + dropped = {id(item) for item in deselected} + config.hook.pytest_deselected(items=deselected) + items[:] = [item for item in items if id(item) not in dropped] + + +def pytest_runtest_logreport(report: pytest.TestReport) -> None: + """Record feature outcomes as phases finish.""" + recorder = _ACTIVE_RECORDERS.get("lane") + if recorder is not None: + recorder.record(report) + + +def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: + """Persist the lane results file.""" + del exitstatus + recorder = session.config.stash.get(_RECORDER_KEY, None) + if recorder is not None: + recorder.write() + _ACTIVE_RECORDERS.pop("lane", None) + + +@pytest.fixture(scope="session") +def feature_host(request: pytest.FixtureRequest) -> FeatureHost: + """Return the provisioned host this lane verifies, recording tool versions once.""" + host = FeatureHost.from_environment() + recorder = request.config.stash.get(_RECORDER_KEY, None) + if recorder is not None and not recorder.versions: + for tool in ("openclaw", "acpx"): + outcome = host.run([tool, "--version"], timeout=60) + recorder.versions[tool] = ( + outcome.stdout.strip().splitlines()[-1] + if outcome.ok and outcome.stdout.strip() + else f"unavailable (exit {outcome.returncode})" + ) + return host + + +@pytest.fixture(scope="session") +def sidecars(feature_host: FeatureHost) -> str: + """Return how the sidecar stack came up: by its service, or by the harness fallback.""" + return ensure_sidecars(feature_host) + + +@pytest.fixture(scope="session") +def gateway(feature_host: FeatureHost) -> Iterator[GatewayController]: + """Return a running gateway, started by the harness when no service supervises one.""" + controller = GatewayController(feature_host) + controller.ensure_running() + try: + yield controller + finally: + controller.stop() diff --git a/tests/scripts/feature_report.py b/tests/scripts/feature_report.py new file mode 100755 index 00000000..0c0bc8b2 --- /dev/null +++ b/tests/scripts/feature_report.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Join feature-verification lane results with the feature matrix into a verdict report.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +SRC_ROOT = REPO_ROOT / "src" +for import_root in (SRC_ROOT, REPO_ROOT): + if str(import_root) not in sys.path: + sys.path.insert(0, str(import_root)) + +from tests.utils.helpers.feature_verification import ( # noqa: E402 + FeatureVerificationError, + write_lane_report, +) + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--results-dir", type=Path, required=True) + parser.add_argument( + "--manifest", type=Path, default=REPO_ROOT / "tests" / "feature_matrix.yaml" + ) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--summary-file", type=Path) + parser.add_argument( + "--fail-on-failures", + action="store_true", + help="Exit non-zero when any feature is FAIL.", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + """Write the report and optionally fail on FAIL verdicts.""" + args = _parse_args(argv) + try: + failures = write_lane_report( + results_dir=Path(args.results_dir).expanduser().resolve(), + manifest_path=Path(args.manifest).expanduser().resolve(), + output_path=Path(args.output).expanduser().resolve(), + summary_file=( + Path(args.summary_file).expanduser().resolve() + if args.summary_file is not None + else None + ), + ) + except FeatureVerificationError as exc: + print(f"feature-report error: {exc}", file=sys.stderr) + return 1 + print(f"feature verification: {failures} feature(s) FAIL") + return 1 if failures and bool(args.fail_on_failures) else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/scripts/feature_verification.py b/tests/scripts/feature_verification.py new file mode 100755 index 00000000..8758f928 --- /dev/null +++ b/tests/scripts/feature_verification.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Semantic CLI for the host-side steps of the feature-verification workflow. + +These steps run on a fresh runner's bare Python (before ``clawops bootstrap`` creates the +managed environment), so this CLI only imports stdlib-backed helpers; the matrix report lives +in ``feature_report.py``. +""" + +from __future__ import annotations + +import argparse +import json +import signal +import sys +import threading +from pathlib import Path +from types import FrameType + +REPO_ROOT = Path(__file__).resolve().parents[2] +SRC_ROOT = REPO_ROOT / "src" +for import_root in (SRC_ROOT, REPO_ROOT): + if str(import_root) not in sys.path: + sys.path.insert(0, str(import_root)) + +from tests.utils.helpers._feature_verification.diagnostics import ( # noqa: E402 + collect_lane_diagnostics, +) +from tests.utils.helpers._feature_verification.fake_model import FakeModelServer # noqa: E402 +from tests.utils.helpers._feature_verification.lane import ( # noqa: E402 + LaneError, + enable_systemd_user_manager, + run_lane, +) +from tests.utils.helpers._feature_verification.processes import ( # noqa: E402 + FakeModelProcessError, + start_detached, + stop_detached, +) +from tests.utils.helpers._feature_verification.provider import ( # noqa: E402 + ProviderError, + apply_model_provider, + seed_env_contract, +) + + +def _resolved(path: Path) -> Path: + return path.expanduser().resolve() + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + serve = subparsers.add_parser("serve-fake-model", help="Serve the fake model API (blocks).") + serve.add_argument("--host", default="127.0.0.1") + serve.add_argument("--port", type=int, action="append", required=True) + + start = subparsers.add_parser("start-fake-model", help="Start the fake model API detached.") + start.add_argument("--host", default="127.0.0.1") + start.add_argument("--port", type=int, action="append", required=True) + start.add_argument("--state-dir", type=Path, required=True) + + stop = subparsers.add_parser("stop-fake-model", help="Stop the detached fake model API.") + stop.add_argument("--state-dir", type=Path, required=True) + + seed = subparsers.add_parser( + "seed-env-contract", + help="Select the fake model provider in the operator env contract (before bootstrap).", + ) + seed.add_argument("--context", type=Path, required=True) + + provider = subparsers.add_parser( + "apply-model-provider", + help="Register the fake provider in the rendered config and run model-auth ensure.", + ) + provider.add_argument("--context", type=Path, required=True) + + subparsers.add_parser( + "enable-systemd-user", + help="Start the runner's systemd user manager so user units can be activated.", + ) + + lane = subparsers.add_parser("run-lane", help="Run one lane's feature tests.") + lane.add_argument("--context", type=Path, required=True) + lane.add_argument("--lane", required=True) + lane.add_argument("--results", type=Path, required=True) + lane.add_argument("--fake-model-dir", type=Path) + + diagnostics = subparsers.add_parser( + "collect-diagnostics", help="Capture service-manager and gateway diagnostics." + ) + diagnostics.add_argument("--context", type=Path, required=True) + diagnostics.add_argument("--output-dir", type=Path, required=True) + return parser.parse_args(argv) + + +def _serve(host: str, ports: list[int]) -> int: + """Serve until SIGTERM or SIGINT.""" + print(f"starting fake model server on {host}:{ports}", flush=True) + server = FakeModelServer(host=host, ports=ports) + stopped = threading.Event() + + def _stop(signum: int, frame: FrameType | None) -> None: + del signum, frame + stopped.set() + + signal.signal(signal.SIGTERM, _stop) + signal.signal(signal.SIGINT, _stop) + server.start() + print(f"fake model server listening on {host}:{server.ports}", flush=True) + stopped.wait() + server.stop() + return 0 + + +def main(argv: list[str] | None = None) -> int: + """Run the requested feature-verification subcommand.""" + args = _parse_args(argv) + try: + if args.command == "serve-fake-model": + return _serve(str(args.host), [int(port) for port in args.port]) + if args.command == "start-fake-model": + state = start_detached( + script_path=Path(__file__).resolve(), + host=str(args.host), + ports=[int(port) for port in args.port], + state_dir=_resolved(args.state_dir), + ) + print(json.dumps(state, sort_keys=True)) + return 0 + if args.command == "stop-fake-model": + print(json.dumps({"stopped": stop_detached(_resolved(args.state_dir))})) + return 0 + if args.command == "seed-env-contract": + print( + f"selected the fake model provider in {seed_env_contract(_resolved(args.context))}" + ) + return 0 + if args.command == "apply-model-provider": + print(json.dumps(apply_model_provider(_resolved(args.context)), indent=2)) + return 0 + if args.command == "enable-systemd-user": + print(f"systemd user manager ready at {enable_systemd_user_manager()}") + return 0 + if args.command == "run-lane": + exit_code = run_lane( + context_file=_resolved(args.context), + lane_id=str(args.lane), + results_path=_resolved(args.results), + fake_model_dir=( + _resolved(args.fake_model_dir) if args.fake_model_dir is not None else None + ), + ) + # Feature failures are report data; only a missing results file fails this step. + print(f"lane pytest exit code: {exit_code}") + return 0 + if args.command == "collect-diagnostics": + collect_lane_diagnostics( + context_file=_resolved(args.context), output_dir=_resolved(args.output_dir) + ) + return 0 + except (FakeModelProcessError, LaneError, ProviderError) as exc: + print(f"feature-verification error: {exc}", file=sys.stderr) + return 1 + raise AssertionError(f"Unhandled command: {args.command}") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/scripts/fresh_host.py b/tests/scripts/fresh_host.py index 42094e55..d3f10717 100755 --- a/tests/scripts/fresh_host.py +++ b/tests/scripts/fresh_host.py @@ -14,7 +14,9 @@ if str(import_root) not in sys.path: sys.path.insert(0, str(import_root)) +from clawops.memory_profiles import MANAGED_MEMORY_PROFILE_IDS # noqa: E402 from tests.utils.helpers.fresh_host import ( # noqa: E402 + SCENARIO_SPECS, FreshHostError, cleanup, collect_diagnostics, @@ -44,13 +46,13 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: prepare_parser.add_argument( "--scenario", required=True, - choices=("linux", "macos-sidecars", "macos-browser-lab"), + choices=tuple(SCENARIO_SPECS), ) prepare_parser.add_argument("--repo-root", type=Path, default=REPO_ROOT) prepare_parser.add_argument( "--profile", default="openclaw-default", - choices=("openclaw-default", "hypermemory"), + choices=MANAGED_MEMORY_PROFILE_IDS, ) prepare_parser.add_argument("--runner-temp", type=Path, required=True) prepare_parser.add_argument("--workspace", type=Path, default=REPO_ROOT) diff --git a/tests/suites/contracts/repo/test_feature_matrix_contract.py b/tests/suites/contracts/repo/test_feature_matrix_contract.py new file mode 100644 index 00000000..37103f38 --- /dev/null +++ b/tests/suites/contracts/repo/test_feature_matrix_contract.py @@ -0,0 +1,154 @@ +"""Contracts binding the feature matrix to the product surface, the suite, and the workflow.""" + +from __future__ import annotations + +import ast +from pathlib import Path +from typing import cast + +import yaml + +from clawops.backend_registry import BACKEND_REGISTRY +from clawops.cli import COMMANDS, WRAPPER_COMMANDS +from clawops.memory_profiles import MEMORY_PROFILES +from tests.utils.helpers.feature_verification import FeatureManifest, load_manifest +from tests.utils.helpers.fresh_host import SCENARIO_SPECS +from tests.utils.helpers.repo import REPO_ROOT + +_MANIFEST = REPO_ROOT / "tests" / "feature_matrix.yaml" +_SUITE_DIR = REPO_ROOT / "tests" / "suites" / "e2e" / "feature_verification" +_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "feature-verification.yml" +_COMPOSE_DIR = REPO_ROOT / "platform" / "compose" + + +def _manifest() -> FeatureManifest: + return load_manifest(_MANIFEST) + + +def _suite_tests() -> set[str]: + node_ids: set[str] = set() + for path in sorted(_SUITE_DIR.glob("test_*.py")): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=path.as_posix()) + rel_path = path.relative_to(REPO_ROOT).as_posix() + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name.startswith("test_"): + node_ids.add(f"{rel_path}::{node.name}") + return node_ids + + +def _cited_files(manifest: FeatureManifest) -> set[str]: + return { + call_site.rsplit(":", 1)[0] + for feature in manifest.features + for call_site in feature.call_sites + } + + +def _module_path(import_path: str) -> str: + candidate = Path("src", *import_path.split(".")) + return candidate.as_posix() if (REPO_ROOT / candidate).is_dir() else f"{candidate}.py" + + +def test_every_feature_test_exists_in_the_suite() -> None: + referenced = set(_manifest().feature_by_test()) + missing = sorted(referenced - _suite_tests()) + assert not missing, f"matrix references tests that do not exist: {missing}" + + +def test_every_suite_test_maps_to_exactly_one_feature() -> None: + unmapped = sorted(_suite_tests() - set(_manifest().feature_by_test())) + assert not unmapped, f"suite tests missing from the matrix: {unmapped}" + + +def test_every_call_site_resolves_to_a_real_line() -> None: + for feature in _manifest().features: + for call_site in feature.call_sites: + rel_path, line_text = call_site.rsplit(":", 1) + path = REPO_ROOT / rel_path + assert path.is_file(), f"{feature.feature_id}: {rel_path} does not exist" + line_count = len(path.read_text(encoding="utf-8").splitlines()) + assert int(line_text) <= line_count, f"{feature.feature_id}: {call_site} is past EOF" + + +def test_every_clawops_command_is_covered() -> None: + cited = _cited_files(_manifest()) + modules = {spec.name: _module_path(spec.import_path) for spec in COMMANDS} + modules.update( + {f"wrapper {name}": _module_path(module) for name, (module, _) in WRAPPER_COMMANDS.items()} + ) + uncovered = sorted( + command + for command, module in modules.items() + if not any(path == module or path.startswith(f"{module}/") for path in cited) + ) + assert not uncovered, f"clawops commands without a feature: {uncovered}" + + +def test_every_profile_has_a_profile_feature() -> None: + feature_ids = {feature.feature_id for feature in _manifest().features} + missing = sorted( + profile for profile in MEMORY_PROFILES if f"profile.{profile}" not in feature_ids + ) + assert not missing, f"profiles without a profile. feature: {missing}" + + +def test_every_backend_auth_mode_is_covered() -> None: + feature_ids = {feature.feature_id for feature in _manifest().features} + missing: list[str] = [] + for name, backend in BACKEND_REGISTRY.items(): + for mode in backend.supported_auth_modes: + if mode == "local": + expected = "acp.openclaw-local" if name == "openclaw" else "acp.local-backends" + else: + expected = f"acp.{name}-{mode}" + if expected not in feature_ids: + missing.append(f"{name}/{mode} -> {expected}") + assert not missing, f"backend auth modes without a feature: {missing}" + + +def test_every_shipped_compose_stack_is_covered() -> None: + cited = _cited_files(_manifest()) + stacks = sorted( + path.relative_to(REPO_ROOT).as_posix() + for path in _COMPOSE_DIR.glob("docker-compose.*.yaml") + if ".ci-hosted-" not in path.name + ) + uncovered = [stack for stack in stacks if stack not in cited] + assert not uncovered, f"compose stacks without a feature: {uncovered}" + + +def test_lanes_match_fresh_host_scenarios() -> None: + for lane in _manifest().lanes.values(): + spec = SCENARIO_SPECS.get(lane.scenario) # type: ignore[call-overload] + assert spec is not None, f"lane {lane.lane_id} uses unknown scenario {lane.scenario}" + assert spec.platform == lane.platform, lane + assert spec.activate_services == lane.activate_services, lane + assert spec.use_host_home == (lane.platform == "linux"), lane + + +def test_workflow_runs_every_lane_and_nothing_else() -> None: + workflow = cast(dict[str, object], yaml.safe_load(_WORKFLOW.read_text(encoding="utf-8"))) + jobs = cast(dict[str, dict[str, object]], workflow["jobs"]) + linux_matrix = cast(dict[str, object], jobs["linux-lanes"]["strategy"])["matrix"] + entries = cast(list[dict[str, str]], cast(dict[str, object], linux_matrix)["include"]) + workflow_lanes = {entry["lane"]: entry for entry in entries} + workflow_text = _WORKFLOW.read_text(encoding="utf-8") + manifest = _manifest() + for lane_id, lane in manifest.lanes.items(): + if lane.platform == "linux": + assert lane_id in workflow_lanes, f"workflow does not run lane {lane_id}" + entry = workflow_lanes[lane_id] + assert (entry["profile"], entry["scenario"]) == (lane.profile, lane.scenario), entry + else: + assert f"--lane {lane_id}" in workflow_text, f"workflow does not run lane {lane_id}" + assert f"--scenario {lane.scenario}" in workflow_text + assert f"--profile {lane.profile}" in workflow_text + assert set(workflow_lanes) <= set(manifest.lanes), sorted( + set(workflow_lanes) - set(manifest.lanes) + ) + + +def test_unverifiable_features_explain_why() -> None: + for feature in _manifest().features: + if feature.status == "unverifiable": + assert feature.reason is not None and len(feature.reason) >= 40, feature.feature_id diff --git a/tests/suites/contracts/taf/framework/test_fixture_plugin_registration.py b/tests/suites/contracts/taf/framework/test_fixture_plugin_registration.py index a013372b..bca595cd 100644 --- a/tests/suites/contracts/taf/framework/test_fixture_plugin_registration.py +++ b/tests/suites/contracts/taf/framework/test_fixture_plugin_registration.py @@ -41,10 +41,14 @@ def test_recursive_fixture_plugin_registries_are_explicit() -> None: fixture_root / "__init__.py": ( "tests.fixtures.core", + "tests.fixtures.feature_verification", "tests.fixtures.hypermemory", "tests.fixtures.platform", ), fixture_root + / "feature_verification" + / "__init__.py": ("tests.fixtures.feature_verification.lane",), + fixture_root / "core" / "__init__.py": ( "tests.fixtures.core.cli", diff --git a/tests/suites/contracts/taf/test_framework_health.py b/tests/suites/contracts/taf/test_framework_health.py index 94436a57..602542f1 100644 --- a/tests/suites/contracts/taf/test_framework_health.py +++ b/tests/suites/contracts/taf/test_framework_health.py @@ -15,6 +15,7 @@ "contract", "devflow", "e2e", + "feature_verification", "framework", "hypermemory", "integration", diff --git a/tests/suites/e2e/feature_verification/test_acp.py b/tests/suites/e2e/feature_verification/test_acp.py new file mode 100644 index 00000000..b1481a25 --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_acp.py @@ -0,0 +1,255 @@ +"""ACP worker features: acp-runner through the real acpx in every backend auth mode. + +The ``openclaw`` backend runs fully live (acpx -> ``openclaw acp`` -> gateway -> provider). +Vendor backends replace only the vendor adapter with an ACP stub agent through acpx's own +agent-override config, so StrongClaw's credential broker, env sanitization and the real acpx +client are exercised without vendor accounts. +""" + +from __future__ import annotations + +import json +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Final, cast + +from tests.utils.helpers.feature_verification import ( + ACP_STUB_REPLY_PREFIX, + REPLY_PREFIX, + FeatureHost, + GatewayController, +) + +STUB_AGENT: Final[Path] = Path("tests/utils/helpers/_feature_verification/acp_stub_agent.py") +STUBBED_AGENTS: Final[tuple[str, ...]] = ("codex", "claude", "gemini", "qwen", "copilot", "cursor") +FAKE_KEY: Final[str] = "featver-not-a-real-key" + + +@dataclass(frozen=True, slots=True) +class AcpSession: + """One finished acp-runner session.""" + + summary: dict[str, object] + stdout: str + + def stub_reply(self) -> str: + """Return the stub agent's reply line from the session transcript.""" + for line in self.stdout.splitlines(): + if ACP_STUB_REPLY_PREFIX in line: + return line + raise AssertionError(f"no stub reply in acpx output: {self.stdout[-2000:]}") + + +def _workspace(feature_host: FeatureHost, *, stubbed: bool) -> tuple[Path, Path]: + root = feature_host.scratch("acp") + project = root / "project" + workspace = project / "workspace" + workspace.mkdir(parents=True) + if stubbed: + stub = feature_host.repo_root / STUB_AGENT + agents = { + name: {"command": f"{feature_host.python} {stub} --persona {name}"} + for name in STUBBED_AGENTS + } + (workspace / ".acpxrc.json").write_text(json.dumps({"agents": agents}), encoding="utf-8") + return project, workspace + + +def _vendor_cli_stubs(feature_host: FeatureHost) -> Path: + """Stand-in vendor CLIs that report a logged-in subscription, as the real ones do.""" + bin_dir = feature_host.scratch("vendor-bin") + scripts = { + "codex": "#!/bin/sh\necho 'Logged in using ChatGPT'\n", + "claude": '#!/bin/sh\necho \'{"loggedIn": true, "authMethod": "claude.ai"}\'\n', + } + for name, body in scripts.items(): + path = bin_dir / name + path.write_text(body, encoding="utf-8") + path.chmod(0o755) + return bin_dir + + +def _run_session( + feature_host: FeatureHost, + *, + backend: str, + auth_mode: str, + prompt: str, + stubbed: bool = True, + env: dict[str, str] | None = None, +) -> AcpSession: + project, workspace = _workspace(feature_host, stubbed=stubbed) + outcome = feature_host.clawops( + "acp-runner", + "--backend", + backend, + "--auth-mode", + auth_mode, + "--prompt", + prompt, + "--project-root", + str(project), + "--workspace", + str(workspace), + "--allowed-workspace-root", + str(project), + "--state-dir", + str(project / "state"), + "--permissions-mode", + "approve-reads", + "--output-format", + "json", + "--timeout-seconds", + "240", + timeout=300, + env=env, + ) + summary = cast(dict[str, object], outcome.json()) + stdout_path = summary.get("stdout_path") + stdout = Path(str(stdout_path)).read_text(encoding="utf-8") if stdout_path else "" + assert summary.get("status") == "succeeded", f"{outcome.describe()}\n{stdout[-2000:]}" + return AcpSession(summary=summary, stdout=stdout) + + +def _env_with(feature_host: FeatureHost, **values: str) -> dict[str, str]: + env = {"PATH": f"{_vendor_cli_stubs(feature_host)}:{feature_host.product_env()['PATH']}"} + env.update(values) + return env + + +def test_openclaw_backend_runs_a_live_acp_turn( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + del gateway + token = f"FV-acp-{uuid.uuid4().hex[:10]}" + session = _run_session( + feature_host, backend="openclaw", auth_mode="local", prompt=f"Reply {token}", stubbed=False + ) + assert ( + f"{REPLY_PREFIX} {token}" in session.stdout + ), f"openclaw acp did not return the provider reply:\n{session.stdout[-3000:]}" + + +def test_codex_api_mode_passes_the_key_and_authenticates(feature_host: FeatureHost) -> None: + session = _run_session( + feature_host, + backend="codex", + auth_mode="api", + prompt="FV-codex-api", + env=_env_with(feature_host, OPENAI_API_KEY=FAKE_KEY), + ) + reply = session.stub_reply() + assert "OPENAI_API_KEY:present" in reply, reply + assert "auth=openai-api-key" in reply, f"acpx did not select the env-backed auth: {reply}" + assert session.summary["credential_source_class"] == "process-env", session.summary + + +def test_codex_subscription_mode_strips_api_keys(feature_host: FeatureHost) -> None: + session = _run_session( + feature_host, + backend="codex", + auth_mode="subscription", + prompt="FV-codex-sub", + env=_env_with(feature_host, OPENAI_API_KEY=FAKE_KEY), + ) + reply = session.stub_reply() + assert "OPENAI_API_KEY:absent" in reply and "auth=none" in reply, reply + assert session.summary["credential_source_class"] == "vendor-cli-state", session.summary + + +def test_claude_api_mode_passes_the_key(feature_host: FeatureHost) -> None: + session = _run_session( + feature_host, + backend="claude", + auth_mode="api", + prompt="FV-claude-api", + env=_env_with(feature_host, ANTHROPIC_API_KEY=FAKE_KEY), + ) + assert "ANTHROPIC_API_KEY:present" in session.stub_reply() + + +def test_claude_subscription_mode_strips_api_keys(feature_host: FeatureHost) -> None: + session = _run_session( + feature_host, + backend="claude", + auth_mode="subscription", + prompt="FV-claude-sub", + env=_env_with(feature_host, ANTHROPIC_API_KEY=FAKE_KEY), + ) + assert "ANTHROPIC_API_KEY:absent" in session.stub_reply() + assert session.summary["credential_source_class"] == "vendor-cli-state", session.summary + + +def test_claude_cloud_provider_mode_keeps_cloud_credentials(feature_host: FeatureHost) -> None: + session = _run_session( + feature_host, + backend="claude", + auth_mode="cloud-provider", + prompt="FV-claude-cloud", + env=_env_with(feature_host, ANTHROPIC_API_KEY=FAKE_KEY, AWS_PROFILE="featver"), + ) + reply = session.stub_reply() + assert "ANTHROPIC_API_KEY:absent" in reply and "AWS_PROFILE:present" in reply, reply + assert session.summary["credential_source_class"] == "cloud-env", session.summary + + +def test_local_backends_dispatch_through_acpx(feature_host: FeatureHost) -> None: + for backend in ("gemini", "qwen", "copilot", "cursor"): + session = _run_session( + feature_host, backend=backend, auth_mode="local", prompt=f"FV-{backend}" + ) + reply = session.stub_reply() + assert f"persona={backend}" in reply and f"prompt=FV-{backend}" in reply, reply + + +def test_auth_policy_blocks_unsupported_and_unready_modes(feature_host: FeatureHost) -> None: + project, workspace = _workspace(feature_host, stubbed=True) + common = ( + "--prompt", + "FV-policy", + "--project-root", + str(project), + "--workspace", + str(workspace), + "--allowed-workspace-root", + str(project), + "--state-dir", + str(project / "state"), + ) + unsupported = feature_host.clawops( + "acp-runner", "--backend", "codex", "--auth-mode", "cloud-provider", *common + ) + payload = cast(dict[str, object], unsupported.json()) + assert not unsupported.ok and payload["status"] == "preflight_failed", unsupported.describe() + assert payload["credential_state"] == "forbidden_by_policy", payload + env = _env_with(feature_host) + env["PATH"] = f"{feature_host.scratch('empty-bin')}:{feature_host.product_env()['PATH']}" + unready = feature_host.clawops( + "acp-runner", + "--backend", + "codex", + "--auth-mode", + "api", + *common, + env={**env, "OPENAI_API_KEY": ""}, + ) + unready_payload = cast(dict[str, object], unready.json()) + assert unready_payload["status"] == "preflight_failed", unready.describe() + + +def test_credential_status_reports_each_mode(feature_host: FeatureHost) -> None: + ready = feature_host.clawops( + "credential-status", + "--backend", + "claude", + "--auth-mode", + "cloud-provider", + env={"AWS_PROFILE": "featver"}, + ) + ready_payload = cast(dict[str, object], ready.json()) + assert ready_payload["state"] == "ready" and ready_payload["source_class"] == "cloud-env" + missing = feature_host.clawops( + "credential-status", "--backend", "codex", "--auth-mode", "api", env={"OPENAI_API_KEY": ""} + ) + assert cast(dict[str, object], missing.json())["state"] == "misconfigured", missing.describe() diff --git a/tests/suites/e2e/feature_verification/test_context_workflows.py b/tests/suites/e2e/feature_verification/test_context_workflows.py new file mode 100644 index 00000000..b161dc50 --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_context_workflows.py @@ -0,0 +1,114 @@ +"""Context-service, workflow-runner and channel-config features against the real runtime.""" + +from __future__ import annotations + +import json +import subprocess +import uuid +from pathlib import Path +from typing import cast + +from tests.utils.helpers.feature_verification import FeatureHost, basic_auth, http_request + +CONTEXT_CONFIG = "platform/configs/context/codebase.yaml" +CHANNELS_OVERLAY = "platform/configs/openclaw/30-channels.json5" +NEO4J_HTTP = "http://127.0.0.1:7474/db/neo4j/tx/commit" + + +def _sample_repo(feature_host: FeatureHost, marker: str) -> Path: + repo = feature_host.scratch("context-repo") + (repo / "pkg").mkdir() + (repo / "pkg" / "ledger.py").write_text( + f'"""Ledger module {marker}."""\n\n\n' + "class LedgerReconciler:\n" + " def reconcile(self, entries: list[int]) -> int:\n" + " return sum(entries)\n", + encoding="utf-8", + ) + (repo / "README.md").write_text(f"# Ledger\n\nReconciliation notes {marker}.\n", "utf-8") + for command in ( + ["git", "init", "-q", "-b", "main"], + ["git", "add", "."], + ["git", "-c", "user.email=fv@example.com", "-c", "user.name=fv", "commit", "-qm", "seed"], + ): + subprocess.run(command, cwd=repo, check=True, capture_output=True) + return repo + + +def _neo4j_node_count(feature_host: FeatureHost) -> int: + values = feature_host.env_values() + response = http_request( + "POST", + NEO4J_HTTP, + body={"statements": [{"statement": "MATCH (n) RETURN count(n) AS nodes"}]}, + headers=basic_auth(values.get("NEO4J_USERNAME", "neo4j"), values["NEO4J_PASSWORD"]), + timeout=60, + ) + assert response.status == 200, response.body[:1000] + results = cast(list[dict[str, object]], cast(dict[str, object], response.json())["results"]) + data = cast(list[dict[str, object]], results[0]["data"]) + return int(cast(list[int], data[0]["row"])[0]) + + +def test_codebase_context_indexes_into_the_neo4j_graph( + feature_host: FeatureHost, sidecars: str +) -> None: + del sidecars + marker = f"fv{uuid.uuid4().hex[:8]}" + repo = _sample_repo(feature_host, marker) + common = ("--config", CONTEXT_CONFIG, "--repo", str(repo), "--scale", "large") + index = feature_host.clawops_managed("context", "codebase", "index", *common, "--json") + stats = cast(dict[str, object], index.require_ok().json()) + assert int(cast(int, stats["total_files"])) >= 2, stats + assert _neo4j_node_count(feature_host) > 0, "large-scale indexing wrote nothing to neo4j" + query = feature_host.clawops_managed( + "context", "codebase", "query", *common, "--query", "LedgerReconciler reconcile" + ).require_ok() + assert "pkg/ledger.py" in query.stdout, query.describe() + pack_path = feature_host.scratch("context-pack") / "pack.md" + feature_host.clawops_managed( + "context", + "codebase", + "pack", + *common, + "--query", + "reconcile", + "--output", + str(pack_path), + ).require_ok() + assert "LedgerReconciler" in pack_path.read_text(encoding="utf-8") + + +def test_daily_healthcheck_workflow_runs_openclaw_checks(feature_host: FeatureHost) -> None: + outcome = feature_host.clawops( + "workflow", "--workflow", "platform/configs/workflows/daily_healthcheck.yaml", timeout=1200 + ) + rows = [line.split("\t") for line in outcome.stdout.splitlines() if "\t" in line] + results = {row[0]: row[1] for row in rows if len(row) >= 2} + assert results == {"openclaw doctor": "ok", "security audit": "ok"}, outcome.describe() + assert outcome.ok, outcome.describe() + + +def test_channels_overlay_validates_under_openclaw(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("channels") + rendered = scratch / "openclaw.json" + feature_host.clawops( + "render-openclaw-config", + "--asset-root", + ".", + "--home-dir", + str(feature_host.home), + "--profile", + "openclaw-default", + "--overlay", + CHANNELS_OVERLAY, + "--output", + str(rendered), + "--exec-approvals-output", + str(scratch / "exec-approvals.json"), + ).require_ok() + env = {"OPENCLAW_CONFIG_PATH": str(rendered), "TELEGRAM_BOT_TOKEN": "123456:featver-stub-token"} + feature_host.openclaw("config", "validate", env=env, timeout=120).require_ok() + listed = feature_host.openclaw("channels", "list", "--no-usage", "--json", env=env, timeout=180) + payload = json.dumps(listed.require_ok().json()) + assert "telegram" in payload and "whatsapp" in payload, payload[:2000] diff --git a/tests/suites/e2e/feature_verification/test_devflow_planning.py b/tests/suites/e2e/feature_verification/test_devflow_planning.py new file mode 100644 index 00000000..af7e2430 --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_devflow_planning.py @@ -0,0 +1,129 @@ +"""Devflow operator CLI: plan a run for a scratch project and inspect its state and audit surface. + +``devflow run`` dispatches real ACP workers, so it is out of scope here. ``devflow plan`` is a +pure planner that persists nothing, which makes ``status`` and ``audit`` for the planned run id +observable only through their unknown-run contract. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Final, cast + +import yaml + +from tests.utils.helpers.feature_verification import CommandOutcome, FeatureHost + +type JsonObject = dict[str, object] + +DEVFLOW_CONFIG_DIR: Final[Path] = Path("platform/configs/devflow") +RUN_ID_PATTERN: Final[re.Pattern[str]] = re.compile(r"^df_[0-9a-f]{12}$") +GIT_CONFIG_ARGS: Final[tuple[str, ...]] = ( + "-c", + "user.name=Feature Verification", + "-c", + "user.email=featver@example.invalid", + "-c", + "commit.gpgsign=false", + "-c", + "core.hooksPath=/dev/null", +) + + +def _as_object(value: object) -> JsonObject: + assert isinstance(value, dict), f"expected a JSON object, got {value!r}" + return cast(JsonObject, value) + + +def _as_objects(value: object) -> list[JsonObject]: + assert isinstance(value, list), f"expected a JSON array, got {value!r}" + return [_as_object(item) for item in cast(list[object], value)] + + +def _load_yaml(path: Path) -> JsonObject: + return _as_object(yaml.safe_load(path.read_text(encoding="utf-8"))) + + +def _scratch_project(feature_host: FeatureHost) -> Path: + project = feature_host.scratch("devflow-project") + (project / "pyproject.toml").write_text( + '[project]\nname = "featver-demo"\nversion = "0.1.0"\n', encoding="utf-8" + ) + (project / "app.py").write_text("def health() -> str:\n return 'ok'\n", encoding="utf-8") + for arguments in (("init", "-q"), ("add", "-A"), ("commit", "-q", "-m", "initial commit")): + feature_host.run( + ["git", *GIT_CONFIG_ARGS, "-C", str(project), *arguments], timeout=60 + ).require_ok() + return project + + +def _devflow(feature_host: FeatureHost, project: Path, *arguments: str) -> CommandOutcome: + command, *rest = arguments + return feature_host.clawops("devflow", command, "--project-root", str(project), *rest) + + +def test_devflow_plan_status_audit(feature_host: FeatureHost) -> None: + project = _scratch_project(feature_host) + goal = "Add a /health endpoint" + plan_outcome = _devflow(feature_host, project, "plan", "--goal", goal).require_ok() + plan = _as_object(plan_outcome.json()) + run_id = str(plan["run_id"]) + assert RUN_ID_PATTERN.fullmatch(run_id), plan + assert plan["goal"] == goal and plan["lane"] == "default", plan + assert plan["repo_root"] == project.resolve().as_posix(), plan + assert plan["bootstrap_profile"] == "python-basic", plan + commands = _as_object(plan["bootstrap_commands"]) + assert commands["test_unit"] == [["python", "-m", "pytest", "-q"]], commands + + workflow = _load_yaml(feature_host.repo_root / DEVFLOW_CONFIG_DIR / "workflows/production.yaml") + roles = _as_object( + _load_yaml(feature_host.repo_root / DEVFLOW_CONFIG_DIR / "roles.yaml")["roles"] + ) + assert plan["run_profile"] == workflow["run_profile"], plan + stages = _as_objects(plan["stages"]) + assert [stage["name"] for stage in stages] == workflow["stage_order"], stages + for stage in stages: + role = _as_object(roles[str(stage["role"])]) + assert stage["role"] == stage["name"], stage + assert stage["backend"] == role["default_backend"], stage + assert stage["permissions_mode"] == role["permissions_mode"], stage + assert stage["workspace_mode"] == role["workspace_mode"], stage + assert stage["required_auth_mode"] == role["required_auth_mode"], stage + assert stage["approval_required"] is bool(role.get("approval_required", False)), stage + assert stage["expected_artifacts"] == role["expected_artifacts"], stage + prompt = Path(str(stage["worker_prompt"])) + assert prompt.as_posix().endswith(str(role["worker_prompt"])), stage + assert prompt.is_file(), f"planned worker prompt {prompt} does not exist" + assert [stage["name"] for stage in stages if stage["approval_required"]] == ["lead"], stages + + replanned = _as_object( + _devflow(feature_host, project, "plan", "--goal", goal).require_ok().json() + ) + assert replanned == plan, "planning the same goal must be deterministic" + other_lane = _as_object( + _devflow(feature_host, project, "plan", "--goal", goal, "--lane", "hotfix") + .require_ok() + .json() + ) + assert other_lane["run_id"] != run_id and other_lane["lane"] == "hotfix", other_lane + run_root = project / ".clawops" / "devflow" / run_id + assert not run_root.exists(), "devflow plan must not materialize a run" + + status = _devflow(feature_host, project, "status", "--run-id", run_id) + assert status.returncode == 2, status.describe() + status_payload = _as_object(status.json()) + assert status_payload["ok"] is False and run_id in str( + status_payload["message"] + ), status_payload + stuck = _as_object( + _devflow(feature_host, project, "status", "--stuck-only").require_ok().json() + ) + assert stuck == {"stuck_runs": []}, stuck + + audit = _devflow(feature_host, project, "audit", "--run-id", run_id) + assert not audit.ok, audit.describe() + assert f"unknown devflow run: {run_id}" in audit.output, audit.describe() + assert not (run_root / "audit" / "bundle.json").exists(), "no audit bundle for an unknown run" + assert json.loads(plan_outcome.stdout) == plan, "plan output must be a single JSON document" diff --git a/tests/suites/e2e/feature_verification/test_host_lifecycle.py b/tests/suites/e2e/feature_verification/test_host_lifecycle.py new file mode 100644 index 00000000..1a4be0ad --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_host_lifecycle.py @@ -0,0 +1,138 @@ +"""Host lifecycle features: bootstrap, guided setup, service managers, env contract, doctors.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import cast + +from clawops.platform_compat import DEFAULT_ACPX_VERSION, DEFAULT_OPENCLAW_VERSION +from clawops.strongclaw_runtime import load_bootstrap_state +from tests.utils.helpers.feature_verification import FeatureHost +from tests.utils.helpers.fresh_host import load_report + +SYSTEMD_PERSISTENT_UNITS = ("openclaw-gateway.service",) +SYSTEMD_ONESHOT_UNITS = ("openclaw-sidecars.service",) +SYSTEMD_TIMERS = ( + "openclaw-backup-create.timer", + "openclaw-backup-verify.timer", + "openclaw-maintenance.timer", +) +LAUNCHD_JOBS = ( + "ai.openclaw.backup-create", + "ai.openclaw.backup-verify", + "ai.openclaw.maintenance", +) + + +def _phase_status(feature_host: FeatureHost, phase: str) -> tuple[str, str]: + report = load_report(Path(feature_host.context.report_path)) + for result in report.phases: + if result.name == phase: + return result.status, result.failure_reason or "" + return "missing", f"phase {phase!r} never ran; scenario status={report.status}" + + +def test_bootstrap_installs_the_locked_toolchain(feature_host: FeatureHost) -> None: + status, reason = _phase_status(feature_host, "bootstrap") + assert status == "success", f"clawops bootstrap failed: {reason}" + state = load_bootstrap_state() + assert state is not None, "bootstrap did not persist its completion state" + assert state.profile == feature_host.lane.profile + openclaw_version = feature_host.run(["openclaw", "--version"]).require_ok().stdout + acpx_version = feature_host.run(["acpx", "--version"]).require_ok().stdout + assert DEFAULT_OPENCLAW_VERSION in openclaw_version, openclaw_version + assert DEFAULT_ACPX_VERSION in acpx_version, acpx_version + assert feature_host.python.is_file(), "bootstrap did not create the managed venv" + + +def test_guided_setup_completes_with_service_activation(feature_host: FeatureHost) -> None: + status, reason = _phase_status(feature_host, "setup") + assert status == "success", f"clawops setup (with service activation) failed: {reason}" + feature_host.openclaw("config", "validate", timeout=120).require_ok() + + +def test_systemd_user_units_are_installed_and_active(feature_host: FeatureHost) -> None: + unit_dir = feature_host.home / ".config" / "systemd" / "user" + for unit in (*SYSTEMD_PERSISTENT_UNITS, *SYSTEMD_ONESHOT_UNITS, *SYSTEMD_TIMERS): + rendered = unit_dir / unit + assert rendered.is_file(), f"{rendered} was not rendered" + assert "__REPO_ROOT__" not in rendered.read_text(encoding="utf-8") + enabled = feature_host.run(["systemctl", "--user", "is-enabled", unit], timeout=30) + assert enabled.stdout.strip() == "enabled", enabled.describe() + for unit in (*SYSTEMD_PERSISTENT_UNITS, *SYSTEMD_TIMERS): + active = feature_host.run(["systemctl", "--user", "is-active", unit], timeout=30) + assert active.stdout.strip() == "active", active.describe() + sidecars = feature_host.run( + ["systemctl", "--user", "show", "--property=ActiveState,Result", *SYSTEMD_ONESHOT_UNITS], + timeout=30, + ).require_ok() + assert ( + "ActiveState=active" in sidecars.stdout and "Result=success" in sidecars.stdout + ), sidecars.describe() + + +def test_launchd_agents_are_loaded_and_running(feature_host: FeatureHost) -> None: + domain = f"gui/{os.getuid()}" + gateway = feature_host.run(["launchctl", "print", f"{domain}/ai.openclaw.gateway"], timeout=30) + assert gateway.ok and "state = running" in gateway.stdout, gateway.describe() + sidecars = feature_host.run( + ["launchctl", "print", f"{domain}/ai.openclaw.sidecars"], timeout=30 + ) + assert sidecars.ok and "last exit code = 0" in sidecars.stdout, sidecars.describe() + for label in LAUNCHD_JOBS: + job = feature_host.run(["launchctl", "print", f"{domain}/{label}"], timeout=30) + assert job.ok, f"{label} is not loaded:\n{job.describe()}" + plist_dir = feature_host.home / "Library" / "LaunchAgents" + for label in ("ai.openclaw.gateway", *LAUNCHD_JOBS): + plist = plist_dir / f"{label}.plist" + assert plist.is_file(), f"{plist} was not rendered" + lint = feature_host.run(["plutil", "-lint", str(plist)], timeout=30) + assert lint.ok, lint.describe() + + +def test_varlock_env_contract_is_valid(feature_host: FeatureHost) -> None: + check = feature_host.clawops("varlock-env", "--asset-root", ".", "check").require_ok() + payload = cast(dict[str, object], check.json()) + assert payload["ok"] is True and payload["checkOnly"] is True, payload + assert payload["varlockValidated"] is True, payload + values = feature_host.env_values() + for key in ("OPENCLAW_GATEWAY_TOKEN", "LITELLM_MASTER_KEY", "NEO4J_PASSWORD"): + assert len(values.get(key, "")) >= 16, f"{key} was not generated by the env contract" + mode = feature_host.env_file().stat().st_mode & 0o777 + assert mode == 0o600, f"{feature_host.env_file()} has mode {oct(mode)}" + reconfigure = feature_host.clawops( + "varlock-env", "--asset-root", ".", "configure", "--non-interactive" + ).require_ok() + assert cast(dict[str, object], reconfigure.json())["created"] is False + + +def test_doctor_host_reports_toolchain(feature_host: FeatureHost) -> None: + outcome = feature_host.clawops("doctor-host", "--asset-root", ".", timeout=300).require_ok() + payload = cast(dict[str, object], outcome.json()) + assert payload["ok"] is True, payload + assert DEFAULT_OPENCLAW_VERSION in str(payload["openclawVersion"]), payload + assert DEFAULT_ACPX_VERSION in str(payload["acpxVersion"]), payload + assert Path(str(payload["config"])).is_file(), payload + + +def test_deep_doctor_passes_every_check(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars + outcome = feature_host.clawops("doctor", "--asset-root", ".", timeout=1800) + payload = cast(dict[str, object], outcome.json()) + checks = cast(list[dict[str, object]], payload["checks"]) + failed = { + str(check["name"]): str(check["message"])[:600] for check in checks if not check["ok"] + } + assert payload["status"] == "pass" and outcome.ok, json.dumps(failed, indent=2) + + +def test_baseline_verify_passes_with_runtime_probes( + feature_host: FeatureHost, sidecars: str +) -> None: + del sidecars + outcome = feature_host.clawops("baseline", "verify", timeout=3000) + assert outcome.ok, outcome.describe() + payload = cast(dict[str, object], outcome.json()) + assert payload["ok"] is True and payload["verificationMode"] == "runtime", payload diff --git a/tests/suites/e2e/feature_verification/test_memory_hypermemory.py b/tests/suites/e2e/feature_verification/test_memory_hypermemory.py new file mode 100644 index 00000000..caa450ad --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_memory_hypermemory.py @@ -0,0 +1,234 @@ +"""Hypermemory features: OpenClaw memory surface, hypermemory engine, plugin recall, Qdrant upkeep.""" + +from __future__ import annotations + +import re +import time +import uuid +from pathlib import Path +from typing import cast + +from clawops.qdrant_storage import QdrantRelease, StorageStamp, read_stamp, stamp_path, write_stamp +from clawops.strongclaw_runtime import ( + rendered_openclaw_hypermemory_config_path, + resolve_openclaw_config_path, +) +from tests.utils.helpers.feature_verification import ( + CommandOutcome, + FeatureHost, + GatewayController, + plugin_statuses, +) + +AUX_COMPOSE = Path("platform/compose/docker-compose.aux-stack.yaml") +_QDRANT_IMAGE = re.compile(r"^\s*image:\s*(qdrant/qdrant:\S+)", re.MULTILINE) + + +def _config_path(feature_host: FeatureHost) -> Path: + path = rendered_openclaw_hypermemory_config_path( + resolve_openclaw_config_path(feature_host.repo_root) + ) + assert path is not None and path.is_file(), "the hypermemory profile has no rendered config" + return path + + +def _hypermemory( + feature_host: FeatureHost, *arguments: str, timeout: float = 600 +) -> CommandOutcome: + return feature_host.clawops_managed( + "hypermemory", "--config", str(_config_path(feature_host)), *arguments, timeout=timeout + ) + + +def _compose_state_dir(feature_host: FeatureHost) -> Path: + payload = feature_host.clawops("ops", "--asset-root", ".", "status").require_ok().json() + return Path(str(cast(dict[str, object], payload)["composeStateDir"])) + + +def _status(feature_host: FeatureHost) -> dict[str, object]: + return cast( + dict[str, object], _hypermemory(feature_host, "status", "--json").require_ok().json() + ) + + +def _search(feature_host: FeatureHost, query: str) -> list[dict[str, object]]: + payload = ( + _hypermemory(feature_host, "search", "--json", "--query", query, "--max-results", "5") + .require_ok() + .json() + ) + return cast(list[dict[str, object]], cast(dict[str, object], payload)["results"]) + + +def test_hypermemory_profile_plugins_are_loaded(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars + statuses = plugin_statuses(feature_host) + for plugin_id in ("strongclaw-hypermemory", "lossless-claw"): + assert statuses.get(plugin_id) == "loaded", f"{plugin_id}: {statuses.get(plugin_id)}" + plugins = cast(dict[str, object], feature_host.rendered_config()["plugins"]) + slots = cast(dict[str, object], plugins["slots"]) + assert slots == {"contextEngine": "lossless-claw", "memory": "strongclaw-hypermemory"}, slots + + +def test_openclaw_memory_status_is_healthy(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars + outcome = feature_host.openclaw("memory", "status", "--deep", timeout=600).require_ok() + # StrongClaw gates on a clean exit (setup_cli.py:501); the report must not be empty. + assert outcome.stdout.strip(), outcome.describe() + + +def test_openclaw_memory_search_returns_corpus_hits( + feature_host: FeatureHost, sidecars: str +) -> None: + del sidecars + outcome = feature_host.openclaw( + "memory", "search", "--query", "ClawOps", "--max-results", "1", timeout=600 + ).require_ok() + assert ".md" in outcome.stdout, f"memory search returned no corpus hit\n{outcome.describe()}" + + +def test_hypermemory_index_and_hybrid_search(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars + index = cast( + dict[str, object], _hypermemory(feature_host, "index", "--json").require_ok().json() + ) + assert int(cast(int, index["files"])) > 0 and int(cast(int, index["chunks"])) > 0, index + status = _status(feature_host) + assert status["backendActive"] == "qdrant_sparse_dense_hybrid", status + assert int(cast(int, status["vectorItems"])) > 0, status + assert int(cast(int, status["sparseVectorItems"])) > 0, status + hits = _search(feature_host, "backup and recovery") + assert hits, "hybrid search returned no hits for a shipped runbook topic" + assert any("BACKUP" in str(hit["path"]).upper() for hit in hits), hits + + +def test_hypermemory_verify_reports_consistent_index( + feature_host: FeatureHost, sidecars: str +) -> None: + del sidecars + payload = cast( + dict[str, object], _hypermemory(feature_host, "verify", "--json").require_ok().json() + ) + assert payload["ok"] is True, payload + + +def test_hypermemory_store_search_and_forget(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars + token = f"FV-store-{uuid.uuid4().hex[:10]}" + text = f"The feature verification vault code is {token}." + stored = _hypermemory(feature_host, "store", "--json", "--type", "fact", "--text", text) + assert cast(dict[str, object], stored.require_ok().json()).get("ok", True) is not False + assert any(token in str(hit.get("snippet")) for hit in _search(feature_host, token)), token + forgot = _hypermemory(feature_host, "forget", "--json", "--entry-text", text).require_ok() + assert cast(dict[str, object], forgot.json()).get("ok", True) is not False, forgot.describe() + assert not any(token in str(hit.get("snippet")) for hit in _search(feature_host, token)) + + +def test_hypermemory_maintenance_operations(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars + for arguments in ( + ("reflect", "--json", "--mode", "safe"), + ("lifecycle", "--json"), + ("list-facts", "--json"), + ("flush-metadata", "--json"), + ): + outcome = _hypermemory(feature_host, *arguments) + assert outcome.ok, outcome.describe() + assert isinstance(outcome.json(), dict | list), outcome.describe() + + +def test_hypermemory_exports_memory_pro_snapshot(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars + output = feature_host.scratch("hypermemory-export") / "memory-pro.json" + _hypermemory(feature_host, "export-memory-pro", "--json", "--output", str(output)).require_ok() + assert output.is_file() and output.stat().st_size > 2, f"no export written to {output}" + + +def test_hypermemory_plugin_recalls_memory_into_turns( + feature_host: FeatureHost, gateway: GatewayController, sidecars: str +) -> None: + del sidecars + del gateway + token = f"FV-recall-{uuid.uuid4().hex[:10]}" + _hypermemory( + feature_host, + "store", + "--json", + "--type", + "fact", + "--text", + f"The featver lighthouse passphrase is {token}.", + ).require_ok() + started = time.time() + feature_host.openclaw( + "agent", + "--agent", + "admin", + "--session-id", + f"featver-recall-{token}", + "--message", + "What is the featver lighthouse passphrase?", + "--json", + timeout=300, + ).require_ok() + prompts = [ + str(request.get("prompt", "")) + for request in feature_host.fake_model_requests() + if float(cast(float, request.get("at", 0.0))) >= started + ] + assert any( + token in prompt for prompt in prompts + ), "strongclaw-hypermemory autoRecall did not inject the stored fact into the model prompt" + + +def test_qdrant_storage_stamp_tracks_pinned_release( + feature_host: FeatureHost, sidecars: str +) -> None: + del sidecars + image_match = _QDRANT_IMAGE.search((feature_host.repo_root / AUX_COMPOSE).read_text("utf-8")) + assert image_match is not None + target = QdrantRelease.from_image(image_match.group(1)) + state_dir = _compose_state_dir(feature_host) + stamp = read_stamp(stamp_path(state_dir)) + assert stamp is not None, f"no qdrant stamp under {state_dir}" + assert stamp.release_line == target.line and stamp.reindex_pending is False, stamp + + +def test_qdrant_release_change_resets_storage_and_reindexes( + feature_host: FeatureHost, sidecars: str +) -> None: + del sidecars + stamp_file = stamp_path(_compose_state_dir(feature_host)) + current = read_stamp(stamp_file) + assert current is not None, f"no qdrant stamp at {stamp_file}" + reset_verified = False + try: + write_stamp( + stamp_file, + StorageStamp( + release_line="0.0", + image="qdrant/qdrant:v0.0.0", + reindex_pending=False, + updated_at=current.updated_at, + ), + ) + outcome = feature_host.clawops( + "ops", "--asset-root", ".", "sidecars", "up", "--json", timeout=1500 + ).require_ok() + assert "Resetting qdrant storage" in outcome.stderr, outcome.describe() + command = cast(dict[str, object], cast(dict[str, object], outcome.json())["command"]) + steps = { + str(cast(dict[str, object], step)["step"]) + for step in cast(list[object], command["steps"]) + } + assert {"reset-qdrant-storage", "reindex-hypermemory"} <= steps, steps + restored = read_stamp(stamp_file) + assert restored is not None and restored.release_line == current.release_line, restored + assert restored.reindex_pending is False, restored + assert int(cast(int, _status(feature_host)["vectorItems"])) > 0 + reset_verified = True + finally: + # A failed reset must not take Qdrant down for the features after this one. + if not reset_verified: + write_stamp(stamp_file, current) + feature_host.clawops("ops", "--asset-root", ".", "sidecars", "up", timeout=1500) diff --git a/tests/suites/e2e/feature_verification/test_memory_lancedb.py b/tests/suites/e2e/feature_verification/test_memory_lancedb.py new file mode 100644 index 00000000..87d040a4 --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_memory_lancedb.py @@ -0,0 +1,223 @@ +"""memory-lancedb-pro features: plugin load, memory-pro CLI, migration tools, extraction, reflection.""" + +from __future__ import annotations + +import json +import time +import uuid +from datetime import UTC, datetime +from pathlib import Path +from typing import cast + +from clawops.runtime_assets import resolve_runtime_layout +from tests.utils.helpers.feature_verification import ( + CommandOutcome, + FeatureHost, + GatewayController, + plugin_statuses, +) + +PLUGIN_ID = "memory-lancedb-pro" +SCOPE = "project:featver" +STRATEGY_KEY = f"plugins.entries.{PLUGIN_ID}.config.sessionStrategy" +MAIN_SESSION_KEY = "agent:admin:main" +EXTRACTION_TURNS = ( + "Remember that I prefer feature verification reports rendered as tables.", + "I always run the lancedb lane on Ubuntu runners.", + "My preferred review cadence for this project is weekly on Mondays.", + "Our team name for the verification effort is Project Lighthouse.", +) +REFLECTION_TIMEOUT_SECONDS = 240.0 +_POLL_SECONDS = 3.0 + + +def _sqlite_config(feature_host: FeatureHost) -> Path: + path = resolve_runtime_layout(repo_root=feature_host.repo_root).hypermemory_sqlite_config_path + assert path.is_file(), f"setup did not materialize {path}" + return path + + +def _seed_hypermemory_fact(feature_host: FeatureHost, token: str) -> None: + feature_host.clawops( + "hypermemory", + "--config", + str(_sqlite_config(feature_host)), + "store", + "--json", + "--type", + "fact", + "--scope", + SCOPE, + "--text", + f"The featver orchard code is {token}.", + ).require_ok() + + +def _migrate(feature_host: FeatureHost, token: str) -> tuple[Path, dict[str, object]]: + _seed_hypermemory_fact(feature_host, token) + scratch = feature_host.scratch("memory-pro-migration") + snapshot = scratch / "snapshot.json" + outcome = feature_host.clawops( + "memory", + "migrate-hypermemory-to-pro", + "--config", + str(_sqlite_config(feature_host)), + "--scope", + SCOPE, + "--output", + str(snapshot), + "--report", + str(scratch / "report.json"), + ).require_ok() + return snapshot, cast(dict[str, object], outcome.json()) + + +def _memory_pro_search(feature_host: FeatureHost, query: str) -> CommandOutcome: + return feature_host.openclaw( + "memory-pro", "search", query, "--scope", SCOPE, "--limit", "5", "--json", timeout=300 + ) + + +def test_lancedb_plugin_is_loaded_in_the_memory_slot(feature_host: FeatureHost) -> None: + assert plugin_statuses(feature_host).get(PLUGIN_ID) == "loaded" + plugins = cast(dict[str, object], feature_host.rendered_config()["plugins"]) + assert cast(dict[str, object], plugins["slots"])["memory"] == PLUGIN_ID + + +def test_migrate_hypermemory_exports_memory_pro_snapshot(feature_host: FeatureHost) -> None: + token = f"FV-migrate-{uuid.uuid4().hex[:10]}" + snapshot, summary = _migrate(feature_host, token) + assert summary["dryRun"] is False and summary["scope"] == SCOPE, summary + assert token in snapshot.read_text(encoding="utf-8"), "snapshot is missing the seeded fact" + + +def test_import_pro_snapshot_lands_in_lancedb(feature_host: FeatureHost) -> None: + token = f"FV-import-{uuid.uuid4().hex[:10]}" + snapshot, _ = _migrate(feature_host, token) + imported = feature_host.clawops( + "memory", "import-pro-snapshot", "--input", str(snapshot), "--scope", SCOPE, timeout=600 + ) + assert cast(dict[str, object], imported.json())["ok"] is True, imported.describe() + search = _memory_pro_search(feature_host, token).require_ok() + assert token in search.stdout, f"memory-pro search cannot find the import\n{search.describe()}" + + +def test_verify_pro_parity_against_live_plugin(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars + token = f"FV-parity-{uuid.uuid4().hex[:10]}" + snapshot, _ = _migrate(feature_host, token) + feature_host.clawops( + "memory", "import-pro-snapshot", "--input", str(snapshot), "--scope", SCOPE, timeout=600 + ).require_ok() + parity = feature_host.clawops( + "memory", + "verify-pro-parity", + "--config", + str(_sqlite_config(feature_host)), + "--scope", + SCOPE, + "--mode", + "openclaw", + "--query", + token, + timeout=600, + ) + payload = cast(dict[str, object], parity.json()) + assert parity.ok and payload["ok"] is True, parity.describe() + + +def test_smart_extraction_calls_the_extraction_model( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + del gateway + started = time.time() + session_id = f"featver-extract-{uuid.uuid4().hex[:8]}" + for message in EXTRACTION_TURNS: + feature_host.openclaw( + "agent", + "--agent", + "admin", + "--session-id", + session_id, + "--message", + message, + "--json", + timeout=300, + ).require_ok() + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + requests = [ + request + for request in feature_host.fake_model_requests() + if float(cast(float, request.get("at", 0.0))) >= started + ] + if any( + "extract memories worth long-term preservation" in str(r.get("prompt")) + for r in requests + ): + assert any(r.get("path") == "/v1/embeddings" for r in requests), "no embedding call" + return + time.sleep(_POLL_SECONDS) + raise AssertionError("autoCapture smart extraction never called the extraction model") + + +def _reflection_files(feature_host: FeatureHost) -> list[Path]: + agents = cast(dict[str, object], feature_host.rendered_config()["agents"]) + admin = next( + cast(dict[str, object], agent) + for agent in cast(list[object], agents["list"]) + if cast(dict[str, object], agent)["id"] == "admin" + ) + day = datetime.now(tz=UTC).strftime("%Y-%m-%d") + return sorted((Path(str(admin["workspace"])) / "memory" / "reflections" / day).glob("*.md")) + + +def test_reflection_runs_through_the_embedded_runner( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + feature_host.openclaw("config", "set", STRATEGY_KEY, "memoryReflection").require_ok() + try: + gateway.restart() + session_id = f"featver-reflect-{uuid.uuid4().hex[:8]}" + feature_host.openclaw( + "agent", + "--agent", + "admin", + "--session-id", + session_id, + "--message", + "We decided feature verification must use the real runtime.", + "--json", + timeout=300, + ).require_ok() + started = time.time() + params = json.dumps({"key": MAIN_SESSION_KEY, "reason": "new"}) + feature_host.openclaw( + "gateway", "call", "sessions.reset", "--params", params, "--json", timeout=300 + ).require_ok() + deadline = time.monotonic() + REFLECTION_TIMEOUT_SECONDS + while time.monotonic() < deadline: + reflected = [ + path + for path in _reflection_files(feature_host) + if "FEATVER_REFLECTION_INVARIANT" in path.read_text(encoding="utf-8") + ] + if reflected: + break + time.sleep(_POLL_SECONDS) + else: + raise AssertionError("no reflection file was written after sessions.reset") + prompts = [ + str(request.get("prompt")) + for request in feature_host.fake_model_requests() + if float(cast(float, request.get("at", 0.0))) >= started + ] + assert any("MEMORY REFLECTION" in prompt for prompt in prompts), "no reflection prompt" + logs = gateway.log_text() + assert "runner=embedded" in logs, ( + "reflection did not run through extensionAPI runEmbeddedPiAgent " + f"(runner line: {[line for line in logs.splitlines() if 'runner=' in line][-3:]})" + ) + finally: + feature_host.openclaw("config", "set", STRATEGY_KEY, "none") + gateway.restart() diff --git a/tests/suites/e2e/feature_verification/test_model_gateway.py b/tests/suites/e2e/feature_verification/test_model_gateway.py new file mode 100644 index 00000000..04062efa --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_model_gateway.py @@ -0,0 +1,160 @@ +"""Model auth and gateway features against the real OpenClaw runtime and the fake model server.""" + +from __future__ import annotations + +import time +import uuid +from typing import cast + +from tests.utils.helpers.feature_verification import ( + FAKE_CHAT_MODEL_REF, + REPLY_PREFIX, + FeatureHost, + GatewayController, + gateway_status, + plugin_statuses, +) + +TURN_TIMEOUT_SECONDS = 300 + + +def _agent_ids(feature_host: FeatureHost) -> list[str]: + agents = cast(dict[str, object], feature_host.rendered_config()["agents"]) + return [ + str(cast(dict[str, object], agent)["id"]) for agent in cast(list[object], agents["list"]) + ] + + +def _token(label: str) -> str: + return f"FV-{label}-{uuid.uuid4().hex[:10]}" + + +def _requests_since(feature_host: FeatureHost, started: float) -> list[dict[str, object]]: + return [ + request + for request in feature_host.fake_model_requests() + if float(cast(float, request.get("at", 0.0))) >= started + ] + + +def test_agents_list_matches_rendered_config(feature_host: FeatureHost) -> None: + payload = feature_host.openclaw("agents", "list", "--json", timeout=120).require_ok().json() + listed = [str(cast(dict[str, object], agent)["id"]) for agent in cast(list[object], payload)] + assert sorted(listed) == sorted(_agent_ids(feature_host)), (listed, _agent_ids(feature_host)) + + +def test_model_auth_ensure_applies_env_driven_chain(feature_host: FeatureHost) -> None: + outcome = feature_host.clawops("model-auth", "--asset-root", ".", "ensure", timeout=600) + payload = cast(dict[str, object], outcome.require_ok().json()) + assert payload["ok"] is True and payload["missingAgents"] == [], payload + agents = cast(dict[str, object], feature_host.rendered_config()["agents"]) + defaults = cast(dict[str, object], agents["defaults"]) + assert cast(dict[str, object], defaults["model"])["primary"] == FAKE_CHAT_MODEL_REF + for raw_agent in cast(list[object], agents["list"]): + agent = cast(dict[str, object], raw_agent) + assert cast(dict[str, object], agent["model"])["primary"] == FAKE_CHAT_MODEL_REF, agent + + +def test_model_probe_reaches_the_provider(feature_host: FeatureHost) -> None: + started = time.time() + outcome = feature_host.clawops( + "model-auth", + "--asset-root", + ".", + "check", + "--probe", + "--probe-max-tokens", + "8", + timeout=900, + ) + payload = cast(dict[str, object], outcome.require_ok().json()) + assert payload["ok"] is True and payload["checkedOnly"] is True, payload + probes = [ + request + for request in _requests_since(feature_host, started) + if request.get("path") == "/v1/chat/completions" + ] + assert len(probes) >= len( + _agent_ids(feature_host) + ), f"expected one live probe per agent, fake provider saw {len(probes)}" + + +def test_models_list_reports_available_models(feature_host: FeatureHost) -> None: + for agent_id in _agent_ids(feature_host): + payload = ( + feature_host.openclaw("models", "--agent", agent_id, "list", "--json", timeout=120) + .require_ok() + .json() + ) + models = cast(list[dict[str, object]], cast(dict[str, object], payload)["models"]) + available = [model for model in models if model.get("available") is True] + assert available, f"agent {agent_id} has no available model: {models}" + + +def test_gateway_status_reports_loopback_rpc( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + del gateway + status = gateway_status(feature_host) + rpc = cast(dict[str, object], status["rpc"]) + bind = cast(dict[str, object], status["gateway"]) + assert rpc["ok"] is True, status + assert bind["bindMode"] == "loopback" and bind["bindHost"] == "127.0.0.1", bind + + +def test_gateway_runs_an_agent_turn_through_the_provider( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + del gateway + token = _token("turn") + started = time.time() + outcome = feature_host.openclaw( + "agent", + "--agent", + _agent_ids(feature_host)[0], + "--session-id", + f"featver-{token}", + "--message", + f"Feature verification turn {token}", + "--json", + timeout=TURN_TIMEOUT_SECONDS, + ).require_ok() + turns = [ + request + for request in _requests_since(feature_host, started) + if token in str(request.get("latestUser", "")) + ] + assert turns, f"the gateway never sent the turn to the provider\n{outcome.describe()}" + assert f"{REPLY_PREFIX} {token}" in outcome.stdout, outcome.describe() + + +def test_gateway_binds_only_loopback(feature_host: FeatureHost, gateway: GatewayController) -> None: + del gateway + listeners = feature_host.run(["ss", "-H", "-ltn", "sport = :18789"], timeout=30).require_ok() + addresses = [line.split()[3] for line in listeners.stdout.splitlines() if line.strip()] + assert addresses, listeners.describe() + for address in addresses: + host = address.rsplit(":", 1)[0].strip("[]") + assert host in {"127.0.0.1", "::1"}, f"gateway listens on non-loopback {address}" + + +def test_openclaw_doctor_is_clean(feature_host: FeatureHost) -> None: + feature_host.openclaw("doctor", "--non-interactive", timeout=600).require_ok() + + +def test_security_audit_passes(feature_host: FeatureHost, gateway: GatewayController) -> None: + del gateway + feature_host.openclaw("security", "audit", "--deep", timeout=600).require_ok() + + +def test_secrets_audit_passes(feature_host: FeatureHost) -> None: + feature_host.openclaw("secrets", "audit", "--check", timeout=600).require_ok() + + +def test_profile_allowlist_keeps_bundled_provider_plugins(feature_host: FeatureHost) -> None: + allow = cast(dict[str, object], feature_host.rendered_config()["plugins"]).get("allow") + statuses = plugin_statuses(feature_host) + # The documented local model path (OLLAMA_API_KEY) needs OpenClaw's bundled ollama provider. + assert ( + statuses.get("ollama") == "loaded" + ), f"bundled provider plugin 'ollama' is {statuses.get('ollama')!r} under plugins.allow={allow}" diff --git a/tests/suites/e2e/feature_verification/test_operator_tools.py b/tests/suites/e2e/feature_verification/test_operator_tools.py new file mode 100644 index 00000000..0bddee16 --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_operator_tools.py @@ -0,0 +1,1307 @@ +"""CLI-only operator tools: governance, journaling, harness, intake, repo, and render surfaces. + +Every test drives the real ``clawops`` entrypoint against scratch inputs and asserts on the +command's own observable outputs: exit codes, JSON payloads, files it writes, journal rows read +back through the CLI, and HTTP requests received by a loopback listener. +""" + +from __future__ import annotations + +import contextlib +import copy +import json +import os +import plistlib +import re +import shutil +import threading +from collections.abc import Generator +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Final, cast + +import yaml + +from clawops.memory_profiles import MANAGED_MEMORY_PROFILE_IDS, MEMORY_PROFILES +from tests.utils.helpers.feature_verification import CommandOutcome, FeatureHost + +type JsonObject = dict[str, object] + +SHIPPED_POLICY: Final[Path] = Path("platform/configs/policy/policy.yaml") +HARNESS_DIR: Final[Path] = Path("platform/configs/harness") +HARNESS_SUITES: Final[tuple[str, ...]] = ("policy_regressions.yaml", "security_regressions.yaml") +CODE_REVIEW_WORKFLOW: Final[Path] = Path("platform/configs/workflows/code_review.yaml") +CODEBASE_CONTEXT_CONFIG: Final[Path] = Path("platform/configs/context/codebase.yaml") +ALLOWLIST_EXAMPLE: Final[Path] = Path("platform/configs/source-allowlists.example.yaml") +PLACEHOLDER_PATTERN: Final[re.Pattern[str]] = re.compile(r"__[A-Z][A-Z0-9_]*__") +SHA1_PATTERN: Final[re.Pattern[str]] = re.compile(r"^[0-9a-f]{40}$") +SHA256_DIGEST_PATTERN: Final[re.Pattern[str]] = re.compile(r"^sha256:[0-9a-f]{64}$") +PNG_SIGNATURE: Final[bytes] = b"\x89PNG\r\n\x1a\n" +LOOPBACK_ENV: Final[dict[str, str]] = {"NO_PROXY": "127.0.0.1,localhost", "no_proxy": "127.0.0.1"} +GIT_CONFIG_ARGS: Final[tuple[str, ...]] = ( + "-c", + "user.name=Feature Verification", + "-c", + "user.email=featver@example.invalid", + "-c", + "commit.gpgsign=false", + "-c", + "core.hooksPath=/dev/null", + "-c", + "init.defaultBranch=main", +) +CHANNEL_CHECKS: Final[frozenset[str]] = frozenset( + { + "telegram-dm-policy", + "telegram-group-policy", + "telegram-allowlist", + "whatsapp-dm-policy", + "whatsapp-group-policy", + "default-group-policy", + "channel-docs-pairing", + "channel-docs-allowlists", + "telegram-guidance-pairing", + "whatsapp-guidance-pairing", + "rendered-telegram-allowlist", + "rendered-whatsapp-allowlist", + } +) + + +def _as_object(value: object) -> JsonObject: + assert isinstance(value, dict), f"expected a JSON object, got {value!r}" + return cast(JsonObject, value) + + +def _as_objects(value: object) -> list[JsonObject]: + assert isinstance(value, list), f"expected a JSON array, got {value!r}" + return [_as_object(item) for item in cast(list[object], value)] + + +def _ok_object(outcome: CommandOutcome) -> JsonObject: + return _as_object(outcome.require_ok().json()) + + +def _read_json(path: Path) -> object: + assert path.is_file(), f"{path} was not written" + return cast(object, json.loads(path.read_text(encoding="utf-8"))) + + +def _read_jsonl(path: Path) -> list[JsonObject]: + assert path.is_file(), f"{path} was not written" + lines = path.read_text(encoding="utf-8").splitlines() + return [_as_object(json.loads(line)) for line in lines if line.strip()] + + +def _write_json(path: Path, payload: object) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2), encoding="utf-8") + return path + + +def _write_yaml(path: Path, payload: object) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + return path + + +def _git(feature_host: FeatureHost, repo: Path, *arguments: str) -> CommandOutcome: + return feature_host.run( + ["git", *GIT_CONFIG_ARGS, "-C", str(repo), *arguments], timeout=60 + ).require_ok() + + +def _init_git_repo(feature_host: FeatureHost, repo: Path) -> None: + repo.mkdir(parents=True, exist_ok=True) + readme = repo / "README.md" + if not readme.exists(): + readme.write_text("# feature verification fixture\n", encoding="utf-8") + _git(feature_host, repo, "init", "-q") + _git(feature_host, repo, "add", "-A") + _git(feature_host, repo, "commit", "-q", "-m", "initial commit") + + +def _evaluate_policy(feature_host: FeatureHost, policy: Path, payload: Path) -> JsonObject: + return _ok_object( + feature_host.clawops("policy", "--policy", str(policy), "--input", str(payload)) + ) + + +def _journal_begin( + feature_host: FeatureHost, db: Path, *, target: str, payload: Path +) -> JsonObject: + return _ok_object( + feature_host.clawops( + "op-journal", + "begin", + "--db", + str(db), + "--scope", + "featver", + "--kind", + "webhook_post", + "--trust-zone", + "automation", + "--target", + target, + "--payload-file", + str(payload), + ) + ) + + +def _journal_transition( + feature_host: FeatureHost, db: Path, op_id: str, status: str +) -> CommandOutcome: + return feature_host.clawops( + "op-journal", "transition", "--db", str(db), "--op-id", op_id, "--status", status + ) + + +def _list_stuck_ids(feature_host: FeatureHost, db: Path, older_than_ms: int) -> set[str]: + stuck = feature_host.clawops( + "op-journal", "list-stuck", "--db", str(db), "--older-than-ms", str(older_than_ms) + ).require_ok() + return {str(row["op_id"]) for row in _as_objects(stuck.json())} + + +def _approvals( + feature_host: FeatureHost, command: str, db: Path, *arguments: str +) -> CommandOutcome: + return feature_host.clawops("approvals", command, "--db", str(db), *arguments) + + +def _run_harness_suite(feature_host: FeatureHost, suite: str, output: Path) -> list[JsonObject]: + outcome = feature_host.clawops( + "harness", "--suite", str(HARNESS_DIR / suite), "--output", str(output), timeout=900 + ).require_ok() + results = _read_jsonl(output) + assert f"passed={len(results)} total={len(results)}" in outcome.stdout, outcome.describe() + return results + + +def _suite_case_ids(feature_host: FeatureHost, suite: str) -> list[str]: + document = _as_object( + yaml.safe_load((feature_host.repo_root / HARNESS_DIR / suite).read_text("utf-8")) + ) + return [str(case["id"]) for case in _as_objects(document["cases"])] + + +@dataclass(frozen=True, slots=True) +class _ReceivedRequest: + method: str + path: str + content_type: str + body: bytes + + +class _RecordingServer(ThreadingHTTPServer): + """Loopback HTTP endpoint on an ephemeral port that records every request it answers.""" + + def __init__(self) -> None: + super().__init__(("127.0.0.1", 0), _RecordingHandler) + self._lock = threading.Lock() + self._received: list[_ReceivedRequest] = [] + + def url(self, path: str) -> str: + return f"http://127.0.0.1:{self.server_port}{path}" + + def record(self, request: _ReceivedRequest) -> None: + with self._lock: + self._received.append(request) + + def received(self) -> list[_ReceivedRequest]: + with self._lock: + return list(self._received) + + +class _RecordingHandler(BaseHTTPRequestHandler): + def do_POST(self) -> None: + body = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + cast(_RecordingServer, self.server).record( + _ReceivedRequest( + method="POST", + path=self.path, + content_type=self.headers.get("Content-Type", ""), + body=body, + ) + ) + reply = b'{"received": true}' + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(reply))) + self.end_headers() + self.wfile.write(reply) + + def log_message(self, format: str, *args: object) -> None: + del format, args + + +@contextlib.contextmanager +def _recording_server() -> Generator[_RecordingServer, None, None]: + server = _RecordingServer() + thread = threading.Thread(target=server.serve_forever, name="featver-webhook", daemon=True) + thread.start() + try: + yield server + finally: + server.shutdown() + server.server_close() + thread.join(timeout=10) + + +def test_policy_evaluates_shipped_rules(feature_host: FeatureHost) -> None: + fixtures = HARNESS_DIR / "fixtures" + approval = _evaluate_policy( + feature_host, SHIPPED_POLICY, fixtures / "policy_github_approval.json" + ) + assert approval["decision"] == "require_approval", approval + assert approval["review_policy_id"] == "approval.actions.github.comment.create", approval + assert approval["review_mode"] == "manual", approval + assert approval["reasons"] == ["approval required by approval matrix"], approval + + webhook = _evaluate_policy(feature_host, SHIPPED_POLICY, fixtures / "policy_webhook_deny.json") + assert webhook["decision"] == "deny", webhook + assert webhook["reasons"] == ["target:webhook_url:not allowlisted"], webhook + assert "review_mode" not in webhook, webhook + + browser = _evaluate_policy(feature_host, SHIPPED_POLICY, fixtures / "policy_browser_deny.json") + assert browser["decision"] == "deny", browser + assert browser["reasons"] == ["zone:coder:not in allowlist"], browser + + scratch = feature_host.scratch("policy") + prod_webhook = _evaluate_policy( + feature_host, + SHIPPED_POLICY, + _write_json( + scratch / "prod-webhook.json", + { + "trust_zone": "automation", + "action": "webhook.post", + "category": "external_write", + "target_kind": "webhook_url", + "target": "https://example.internal/hooks/deploy", + "environment": "prod", + }, + ), + ) + assert prod_webhook["decision"] == "require_approval", prod_webhook + assert prod_webhook["matched_rules"] == ["approval-for-prod-webhook"], prod_webhook + assert prod_webhook["review_reason"] == "production webhooks always require approval" + + browser_rule = _evaluate_policy( + feature_host, + SHIPPED_POLICY, + _write_json( + scratch / "browser-main-host.json", + { + "trust_zone": "admin", + "action": "browser.navigate", + "host_boundary": "main-control-plane", + }, + ), + ) + assert browser_rule["decision"] == "deny", browser_rule + assert browser_rule["matched_rules"] == ["deny-browser-on-main-host"], browser_rule + + read_only = _write_json( + scratch / "reader-search.json", + {"trust_zone": "reader", "action": "memory.search", "category": "read_only"}, + ) + default_deny = _evaluate_policy(feature_host, SHIPPED_POLICY, read_only) + assert default_deny == {"decision": "deny", "matched_rules": [], "reasons": ["default"]} + + # The shipped policy has no allow rule, so the allow path needs an operator default. + shipped = _as_object( + yaml.safe_load((feature_host.repo_root / SHIPPED_POLICY).read_text("utf-8")) + ) + permissive = _write_yaml( + scratch / "policy.yaml", {**shipped, "defaults": {"decision": "allow"}} + ) + allowed = _evaluate_policy(feature_host, permissive, read_only) + assert allowed == {"decision": "allow", "matched_rules": [], "reasons": ["default"]} + still_gated = _evaluate_policy( + feature_host, permissive, fixtures / "policy_github_approval.json" + ) + assert still_gated["decision"] == "require_approval", still_gated + + +def test_op_journal_tracks_operation_lifecycle(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("op-journal") + db = scratch / "journal.sqlite" + feature_host.clawops("op-journal", "init", "--db", str(db)).require_ok() + assert db.is_file(), f"op-journal init did not create {db}" + payload = _write_json(scratch / "payload.json", {"message": "deploy finished"}) + target = "https://example.internal/hooks/deploy" + + proposed = _journal_begin(feature_host, db, target=target, payload=payload) + op_id = str(proposed["op_id"]) + assert proposed["status"] == "proposed" and proposed["attempt"] == 0, proposed + assert json.loads(str(proposed["inputs_json"])) == {"message": "deploy finished"}, proposed + replayed = _journal_begin(feature_host, db, target=target, payload=payload) + assert replayed["op_id"] == op_id, "begin with identical inputs must be idempotent" + + pending = _ok_object(_journal_transition(feature_host, db, op_id, "pending_approval")) + assert pending["status"] == "pending_approval", pending + approved = _ok_object( + feature_host.clawops( + "op-journal", + "approve", + "--db", + str(db), + "--op-id", + op_id, + "--approved-by", + "operator", + "--note", + "change window open", + ) + ) + assert approved["status"] == "approved" and approved["approved_by"] == "operator", approved + assert approved["approval_note"] == "change window open", approved + running = _ok_object(_journal_transition(feature_host, db, op_id, "running")) + assert running["status"] == "running" and running["attempt"] == 1, running + + assert op_id in _list_stuck_ids(feature_host, db, 0), "running op must be listed as stuck" + assert op_id not in _list_stuck_ids(feature_host, db, 3_600_000), "fresh op is not stale" + + succeeded = _ok_object(_journal_transition(feature_host, db, op_id, "succeeded")) + assert succeeded["status"] == "succeeded", succeeded + assert op_id not in _list_stuck_ids(feature_host, db, 0), "terminal ops are never stuck" + + invalid = _journal_transition(feature_host, db, op_id, "running") + assert not invalid.ok, invalid.describe() + assert ( + "invalid operation transition: succeeded -> running" in invalid.stderr + ), invalid.describe() + + +def test_approvals_review_queue_resolves_operations(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("approvals") + db = scratch / "journal.sqlite" + payload = _write_json(scratch / "payload.json", {"message": "review me"}) + op_ids: list[str] = [] + for name in ("approve", "reject", "delegate"): + op = _journal_begin( + feature_host, db, target=f"https://example.internal/hooks/{name}", payload=payload + ) + _journal_transition(feature_host, db, str(op["op_id"]), "pending_approval").require_ok() + op_ids.append(str(op["op_id"])) + approve_id, reject_id, delegate_id = op_ids + + queue_file = scratch / "queue.json" + queue_outcome = _approvals(feature_host, "queue", db, "--output", str(queue_file)).require_ok() + queue = _as_objects(queue_outcome.json()) + assert [str(row["op_id"]) for row in queue] == op_ids, queue + assert all(row["status"] == "pending_approval" for row in queue), queue + assert _read_json(queue_file) == queue, "queue --output must match the printed queue" + + shown = _ok_object(_approvals(feature_host, "show", db, "--op-id", approve_id)) + assert shown["op_id"] == approve_id and shown["status"] == "pending_approval", shown + + approved = _ok_object( + _approvals(feature_host, "approve", db, "--op-id", approve_id, "--approved-by", "alice") + ) + assert approved["status"] == "approved" and approved["review_status"] == "approved", approved + assert approved["approved_by"] == "alice" and approved["reviewed_by"] == "alice", approved + + rejected = _ok_object( + _approvals( + feature_host, + "reject", + db, + "--op-id", + reject_id, + "--reviewed-by", + "bob", + "--note", + "target out of scope", + ) + ) + assert rejected["status"] == "rejected" and rejected["review_status"] == "rejected", rejected + assert rejected["last_error"] == "target out of scope", rejected + + delegated = _ok_object( + _approvals( + feature_host, + "delegate", + db, + "--op-id", + delegate_id, + "--reviewed-by", + "carol", + "--to", + "reviewer-acp-claude", + ) + ) + assert delegated["status"] == "pending_approval", delegated + assert delegated["review_status"] == "delegated", delegated + assert delegated["review_target"] == "reviewer-acp-claude", delegated + + review_payload = _write_json(scratch / "review.json", {"summary": "change is safe"}) + review_artifact = scratch / "review.md" + review_artifact.write_text("LGTM\n", encoding="utf-8") + ingested = _ok_object( + _approvals( + feature_host, + "ingest-review", + db, + "--op-id", + delegate_id, + "--reviewed-by", + "reviewer-acp-claude", + "--decision", + "allow", + "--payload-file", + str(review_payload), + "--artifact-path", + str(review_artifact), + ) + ) + assert ingested["status"] == "approved", ingested + assert ingested["approved_by"] == "reviewer-acp-claude", ingested + assert ingested["review_artifact_path"] == str(review_artifact), ingested + assert json.loads(str(ingested["review_payload_json"])) == { + "delegate_to": "reviewer-acp-claude", + "review_decision": "allow", + "summary": "change is safe", + }, ingested + + final_queue = _approvals(feature_host, "queue", db).require_ok() + assert _as_objects(final_queue.json()) == [], final_queue.describe() + reapprove = _approvals( + feature_host, "approve", db, "--op-id", reject_id, "--approved-by", "alice" + ) + assert not reapprove.ok, "a rejected operation must not be approvable" + + +def test_harness_regression_suites_pass(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("harness") + for suite in HARNESS_SUITES: + results = _run_harness_suite(feature_host, suite, scratch / f"{Path(suite).stem}.jsonl") + assert [str(item["id"]) for item in results] == _suite_case_ids(feature_host, suite) + failed = { + str(item["id"]): item["failures"] for item in results if item["passed"] is not True + } + assert not failed, json.dumps(failed, indent=2) + assert all(item["failures"] == [] for item in results), results + + +def test_charts_render_harness_results(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("charts") + results_path = scratch / "policy.jsonl" + results = _run_harness_suite(feature_host, "policy_regressions.yaml", results_path) + chart = scratch / "nested" / "harness.png" + feature_host.clawops( + "charts", "--input", str(results_path), "--output", str(chart) + ).require_ok() + assert chart.is_file(), f"charts did not write {chart}" + data = chart.read_bytes() + assert data.startswith(PNG_SIGNATURE), "chart output is not a PNG" + assert len(data) > 4096, f"chart is suspiciously small ({len(data)} bytes)" + width, height = int.from_bytes(data[16:20], "big"), int.from_bytes(data[20:24], "big") + expected_ratio = max(6.0, len(results) * 1.2) / 4.0 + assert abs(width / height - expected_ratio) < 0.02, (width, height, expected_ratio) + + +def test_merge_json_deep_merges_overlays(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("merge-json") + base = _write_json( + scratch / "base.json", + { + "gateway": {"port": 18789, "bind": "loopback", "auth": {"mode": "token"}}, + "agents": {"list": [{"id": "admin"}, {"id": "reader"}]}, + "keep": True, + }, + ) + first = scratch / "10-gateway.json5" + first.write_text( + "{\n" + " // operator overlay: json5 comments and trailing commas are accepted\n" + " gateway: { port: 19001, auth: { token: 'from-overlay' } },\n" + " agents: { list: [{ id: 'coder' }] },\n" + "}\n", + encoding="utf-8", + ) + second = _write_json(scratch / "20-extra.json", {"gateway": {"bind": "lan"}, "extra": {"n": 1}}) + output = scratch / "merged.json" + feature_host.clawops( + "merge-json", + "--base", + str(base), + "--overlay", + str(first), + str(second), + "--output", + str(output), + ).require_ok() + assert _read_json(output) == { + "gateway": { + "port": 19001, + "bind": "lan", + "auth": {"mode": "token", "token": "from-overlay"}, + }, + "agents": {"list": [{"id": "coder"}]}, + "keep": True, + "extra": {"n": 1}, + } + + duplicate = scratch / "duplicate.json5" + duplicate.write_text('{"gateway": {"port": 1}, "gateway": {"port": 2}}\n', encoding="utf-8") + rejected_output = scratch / "rejected.json" + rejected = feature_host.clawops( + "merge-json", + "--base", + str(base), + "--overlay", + str(duplicate), + "--output", + str(rejected_output), + ) + assert not rejected.ok, "overlays with duplicate keys must be rejected" + assert not rejected_output.exists(), "a rejected merge must not write output" + + +def test_allowlists_render_channel_fragment(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("allowlists") + output = scratch / "channels.json" + outcome = feature_host.clawops( + "allowlists", "--source", str(ALLOWLIST_EXAMPLE), "--output", str(output) + ).require_ok() + expected = { + "channels": { + "telegram": {"allowFrom": ["tg:12345678"]}, + "whatsapp": {"allowFrom": ["+5511999999999"]}, + "modelByChannel": { + "telegram": {"12345678": "messaging"}, + "whatsapp": {"+5511999999999": "messaging"}, + }, + } + } + assert outcome.json() == expected, outcome.describe() + assert _read_json(output) == expected + + invalid = _write_json( + scratch / "invalid.json", {"telegram_allow": ["telegram:42"], "whatsapp_allow": ["12345"]} + ) + rejected = feature_host.clawops( + "allowlists", "--source", str(invalid), "--output", str(scratch / "rejected.json") + ) + assert ( + not rejected.ok and "invalid WhatsApp E.164: 12345" in rejected.stderr + ), rejected.describe() + + +def test_skills_intake_moves_bundle_through_stages(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("skills") + source = scratch / "incoming" / "weather-skill" + (source / "scripts").mkdir(parents=True) + (source / "SKILL.md").write_text("# Weather\nSummarize the forecast.\n", encoding="utf-8") + script = "import subprocess\nsubprocess.run(['curl', 'https://example.invalid'])\n" + (source / "scripts" / "fetch.py").write_text(script, encoding="utf-8") + skills_root = scratch / "skills" + resolved_root = skills_root.resolve() + + scan_report = scratch / "scan.json" + scanned = _ok_object( + feature_host.clawops( + "skills", "scan", "--source", str(source), "--report", str(scan_report) + ) + ) + assert scanned["status"] == "scanned", scanned + assert scanned["bundlePath"] == source.resolve().as_posix(), scanned + assert scanned["findingCounts"] == {"critical": 1, "high": 1}, scanned + findings = _as_objects(scanned["findings"]) + assert {(str(item["rule"]), item["line"]) for item in findings} == { + ("shell exfil", 2), + ("python subprocess", 2), + }, findings + assert {str(item["path"]) for item in _as_objects(scanned["files"])} == { + "SKILL.md", + "scripts/fetch.py", + }, scanned + assert _read_json(scan_report) == scanned + + manifest = scratch / "manifest.json" + quarantined = _ok_object( + feature_host.clawops( + "skills", + "quarantine", + "--source", + str(source), + "--report", + str(manifest), + "--quarantine-root", + str(skills_root / "quarantine"), + ) + ) + quarantine_copy = resolved_root / "quarantine" / "weather-skill" + assert quarantined["status"] == "quarantined", quarantined + assert quarantined["bundlePath"] == quarantine_copy.as_posix(), quarantined + assert (quarantine_copy / "scripts" / "fetch.py").read_text(encoding="utf-8") == script + assert (source / "SKILL.md").is_file(), "quarantine must copy, not move, the source" + + premature = feature_host.clawops( + "skills", + "promote", + "--manifest", + str(manifest), + "--skills-root", + str(skills_root), + "--stage", + "approved", + ) + assert not premature.ok, "quarantined bundles must be reviewed before approval" + assert "cannot transition quarantined -> approved" in premature.stderr, premature.describe() + + transitions = (("promote", "reviewed"), ("promote", "approved"), ("demote", "reviewed")) + previous = quarantine_copy + for command, stage in transitions: + staged = _ok_object( + feature_host.clawops( + "skills", + command, + "--manifest", + str(manifest), + "--skills-root", + str(skills_root), + "--stage", + stage, + ) + ) + destination = resolved_root / stage / "weather-skill" + assert staged["status"] == stage, staged + assert staged["bundlePath"] == destination.as_posix(), staged + assert (destination / "SKILL.md").is_file(), f"{destination} was not populated" + assert not previous.exists(), f"{command} left the bundle behind at {previous}" + previous = destination + + history = _as_objects(_as_object(_read_json(manifest))["stageHistory"]) + assert [(item["status"], item["reason"]) for item in history] == [ + ("quarantined", "initial_scan"), + ("reviewed", "promote"), + ("approved", "promote"), + ("reviewed", "demote"), + ], history + + +def test_worktree_lifecycle_creates_lists_and_prunes(feature_host: FeatureHost) -> None: + root = feature_host.scratch("worktree") + _init_git_repo(feature_host, root / "repo" / "upstream") + managed_path = (root / "repo" / "worktrees" / "feature-x").resolve() + + initial = _ok_object(feature_host.clawops("worktree", "--repo-root", str(root), "list")) + initial_entries = _as_objects(initial["worktrees"]) + assert len(initial_entries) == 1 and initial_entries[0]["managed"] is False, initial + + created = _ok_object( + feature_host.clawops("worktree", "--repo-root", str(root), "new", "--branch", "feature-x") + ) + entry = _as_object(created["created"]) + assert created["branch"] == "feature-x", created + assert Path(str(entry["path"])).resolve() == managed_path, entry + assert entry["branch"] == "feature-x" and entry["managed"] is True, entry + assert (managed_path / "README.md").is_file(), "worktree was not checked out" + + duplicate = feature_host.clawops( + "worktree", "--repo-root", str(root), "new", "--branch", "feature-x" + ) + assert duplicate.returncode == 1, duplicate.describe() + assert _as_object(duplicate.json())["ok"] is False, duplicate.describe() + + listed = _as_objects( + _ok_object(feature_host.clawops("worktree", "--repo-root", str(root), "list"))["worktrees"] + ) + managed = [item for item in listed if item["managed"] is True] + assert len(listed) == 2 and [item["branch"] for item in managed] == ["feature-x"], listed + + shutil.rmtree(managed_path) + pruned = _ok_object(feature_host.clawops("worktree", "--repo-root", str(root), "prune")) + assert pruned["beforeCount"] == 2 and pruned["afterCount"] == 1, pruned + remaining = _as_objects(pruned["worktrees"]) + assert all(item["managed"] is False for item in remaining), remaining + + +def test_repo_doctor_accepts_valid_layout(feature_host: FeatureHost) -> None: + root = feature_host.scratch("repo-doctor") + _init_git_repo(feature_host, root / "repo" / "upstream") + (root / "repo" / "worktrees").mkdir() + + report = _ok_object(feature_host.clawops("repo", "--repo-root", str(root), "doctor")) + checks = _as_object(report["checks"]) + assert report["ok"] is True and "errors" not in report, report + assert checks["gitAvailable"] is True and checks["upstreamExists"] is True, checks + assert checks["worktreesRootExists"] is True and checks["managedWorktrees"] == 0, checks + assert checks["repoRoot"] == root.resolve().as_posix(), checks + assert len(_as_objects(report["worktrees"])) == 1, report + + missing = feature_host.clawops( + "repo", "--repo-root", str(root), "doctor", "--branch", "missing-branch" + ) + missing_report = _as_object(missing.json()) + assert missing.returncode == 1 and missing_report["ok"] is False, missing.describe() + assert any( + "missing managed worktree for branch missing-branch" in str(error) + for error in cast(list[object], missing_report["errors"]) + ), missing_report + + +def _code_review_base(feature_host: FeatureHost, name: str) -> Path: + base = feature_host.scratch(name) + for relative in (SHIPPED_POLICY, CODEBASE_CONTEXT_CONFIG): + (base / relative).parent.mkdir(parents=True, exist_ok=True) + shutil.copy(feature_host.repo_root / relative, base / relative) + notes = base / "docs" / "auth-changes.md" + notes.parent.mkdir(parents=True) + # Verbatim query phrase: multi-term FTS hits are dropped by the contentless files_fts JOIN. + notes.write_text( + "# Auth notes\n\nPlease review recent auth changes before merge.\n", encoding="utf-8" + ) + return base + + +def _workflow_steps(outcome: CommandOutcome) -> list[tuple[str, str, str]]: + rows = [line.split("\t") for line in outcome.stdout.splitlines() if "\t" in line] + assert all(len(row) == 3 for row in rows), outcome.describe() + return [(row[0], row[1], row[2]) for row in rows] + + +def test_workflow_code_review_records_step_results(feature_host: FeatureHost) -> None: + workflow = _as_object( + yaml.safe_load((feature_host.repo_root / CODE_REVIEW_WORKFLOW).read_text(encoding="utf-8")) + ) + step_names = [str(step["name"]) for step in _as_objects(workflow["steps"])] + assert step_names == ["init journal", "policy preflight", "dry context pack"], step_names + + base = _code_review_base(feature_host, "code-review") + state_dir = feature_host.scratch("code-review-state") + run = feature_host.clawops( + "workflow", + "--workflow", + str(CODE_REVIEW_WORKFLOW), + "--base-dir", + str(base), + env={"STRONGCLAW_STATE_DIR": str(state_dir)}, + timeout=600, + ).require_ok() + steps = _workflow_steps(run) + assert [(name, status) for name, status, _ in steps] == [ + (name, "ok") for name in step_names + ], run.describe() + journal_message, decision_message, pack_message = (message for _, _, message in steps) + journal = base.resolve() / ".clawops" / "op_journal.sqlite" + assert journal_message == f"journal={journal}", run.describe() + assert journal.is_file(), f"journal_init did not create {journal}" + assert decision_message == "require_approval", run.describe() + pack_path = Path(pack_message.removeprefix("context packed -> ")) + assert pack_message.startswith("context packed -> "), run.describe() + assert pack_path.resolve().is_relative_to(state_dir.resolve()), pack_path + pack = pack_path.read_text(encoding="utf-8") + assert "- query: review recent auth changes" in pack, pack + assert "### docs/auth-changes.md" in pack, pack + + dry_base = _code_review_base(feature_host, "code-review-dry") + dry_state = feature_host.scratch("code-review-dry-state") + dry_run = feature_host.clawops( + "workflow", + "--workflow", + str(CODE_REVIEW_WORKFLOW), + "--base-dir", + str(dry_base), + "--dry-run", + env={"STRONGCLAW_STATE_DIR": str(dry_state)}, + ).require_ok() + dry_journal = dry_base.resolve() / ".clawops" / "op_journal.sqlite" + assert _workflow_steps(dry_run) == [ + ("init journal", "ok", f"journal={dry_journal}"), + ("policy preflight", "ok", "require_approval"), + ("dry context pack", "ok", "dry-run context pack"), + ], dry_run.describe() + assert not dry_journal.exists(), "--dry-run must not create the journal" + assert not any(dry_state.iterdir()), "--dry-run must not write context packs" + + +def test_webhook_wrapper_posts_once_after_approval(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("webhook") + db = scratch / "journal.sqlite" + body = {"event": "deploy.finished", "build": 42} + payload = _write_json(scratch / "payload.json", body) + with _recording_server() as listener: + url = listener.url("/hooks/deploy") + policy = _write_yaml( + scratch / "policy.yaml", + { + "defaults": {"decision": "deny"}, + "zones": { + "automation": { + "allow_actions": ["webhook.post"], + "allow_categories": ["external_write"], + } + }, + "approval": {"require_for_actions": ["webhook.post"]}, + "allowlists": {"webhook_url": [url]}, + }, + ) + + def wrapper(*arguments: str) -> CommandOutcome: + return feature_host.clawops( + "wrapper", + "webhook", + "--db", + str(db), + "--policy", + str(policy), + *arguments, + env=LOOPBACK_ENV, + timeout=120, + ) + + def propose(target: str) -> CommandOutcome: + return wrapper( + "--scope", + "featver", + "--trust-zone", + "automation", + "--url", + target, + "--payload-file", + str(payload), + ) + + proposal = _ok_object(propose(url)) + op_id = str(proposal["op_id"]) + assert proposal["status"] == "pending_approval", proposal + assert proposal["accepted"] is True and proposal["executed"] is False, proposal + assert _as_object(proposal["decision"])["decision"] == "require_approval", proposal + assert Path(str(proposal["review_artifact_path"])).is_file(), proposal + assert listener.received() == [], "nothing may be sent before approval" + + early = _ok_object(wrapper("--op-id", op_id, "--execute-approved")) + assert early["status"] == "pending_approval" and early["executed"] is False, early + assert listener.received() == [], "an unapproved op must not be sent" + + _approvals( + feature_host, "approve", db, "--op-id", op_id, "--approved-by", "ops" + ).require_ok() + executed = _ok_object(wrapper("--op-id", op_id, "--execute-approved")) + assert executed["ok"] is True and executed["executed"] is True, executed + assert executed["status"] == "succeeded" and executed["status_code"] == 200, executed + assert executed["request_method"] == "POST" and executed["request_url"] == url, executed + + replay = _ok_object(wrapper("--op-id", op_id, "--execute-approved")) + assert replay["status"] == "succeeded" and replay["status_code"] == 200, replay + + denied = propose(listener.url("/hooks/not-allowlisted")) + denied_result = _as_object(denied.json()) + assert denied.returncode == 1 and denied_result["accepted"] is False, denied.describe() + assert denied_result["status"] == "failed", denied_result + + received = listener.received() + assert len(received) == 1, f"listener received {len(received)} requests, expected exactly 1" + request = received[0] + assert (request.method, request.path) == ("POST", "/hooks/deploy"), request + assert request.content_type == "application/json", request + assert json.loads(request.body) == body, request + journal_row = _ok_object(_approvals(feature_host, "show", db, "--op-id", op_id)) + assert journal_row["status"] == "succeeded" and journal_row["attempt"] == 1, journal_row + assert journal_row["result_status_code"] == 200, journal_row + assert journal_row["result_body_excerpt"] == '{"received": true}', journal_row + + +def test_github_wrapper_dry_run_plans_without_executing(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("github") + db = scratch / "journal.sqlite" + repo = "your-org/openclaw-platform" + target = f"github://{repo}/issues/7" + common = ("--db", str(db), "--scope", "featver", "--trust-zone", "coder", "--repo", repo) + + comment = _ok_object( + feature_host.clawops( + "wrapper", + "github", + *common, + "--policy", + str(SHIPPED_POLICY), + "--operation", + "comment", + "--issue-number", + "7", + "--body", + "Feature verification comment", + "--dry-run", + ) + ) + assert comment["status"] == "pending_approval", comment + assert comment["accepted"] is True and comment["executed"] is False, comment + assert _as_object(comment["decision"])["decision"] == "require_approval", comment + + allow_policy = _write_yaml( + scratch / "policy.yaml", + { + "defaults": {"decision": "allow"}, + "zones": {"coder": {"allow_actions": ["github.issue.labels.add"]}}, + "allowlists": {"github_repo": [repo]}, + }, + ) + labels = _ok_object( + feature_host.clawops( + "wrapper", + "github", + *common, + "--policy", + str(allow_policy), + "--operation", + "labels", + "--issue-number", + "7", + "--label", + "needs-review", + "--label", + "featver", + "--dry-run", + ) + ) + assert labels["status"] == "approved" and labels["dry_run"] is True, labels + assert labels["executed"] is False and "status_code" not in labels, labels + + expected_inputs = { + str(comment["op_id"]): ("github_comment", {"body": "Feature verification comment"}), + str(labels["op_id"]): ("github_issue_labels", {"labels": ["needs-review", "featver"]}), + } + for op_id, (kind, inputs) in expected_inputs.items(): + row = _ok_object(_approvals(feature_host, "show", db, "--op-id", op_id)) + assert row["kind"] == kind and row["normalized_target"] == target, row + assert json.loads(str(row["inputs_json"])) == inputs, row + assert row["attempt"] == 0 and row["result_status_code"] is None, row + assert row["result_request_url"] is None and row["result_ok"] is None, row + queued = _as_objects(_approvals(feature_host, "queue", db).require_ok().json()) + assert [row["op_id"] for row in queued] == [comment["op_id"]], queued + + +def test_supply_chain_inventory_lists_pins(feature_host: FeatureHost) -> None: + inventory = _ok_object(feature_host.clawops("supply-chain", "inventory")) + assert inventory["ok"] is True, inventory + assert inventory["repoRoot"] == feature_host.repo_root.resolve().as_posix(), inventory + actions = _as_objects(inventory["workflowActions"]) + images = _as_objects(inventory["composeImages"]) + assert actions and images, inventory + file_lines: dict[str, list[str]] = {} + + def line_at(relative: object, line: object) -> str: + key = str(relative) + if key not in file_lines: + file_lines[key] = ( + (feature_host.repo_root / key).read_text(encoding="utf-8").splitlines() + ) + return file_lines[key][cast(int, line) - 1] + + for pin in actions: + assert str(pin["path"]).startswith(".github/workflows/"), pin + assert SHA1_PATTERN.fullmatch(str(pin["ref"])), f"action not pinned to a commit: {pin}" + assert f"{pin['action']}@{pin['ref']}" in line_at(pin["path"], pin["line"]), pin + for pin in images: + assert str(pin["path"]).startswith("platform/compose/"), pin + assert SHA256_DIGEST_PATTERN.fullmatch(str(pin["digest"])), pin + assert f"{pin['image']}@{pin['digest']}" in line_at(pin["path"], pin["line"]), pin + + +def test_supply_chain_propose_refresh_dry_run_plans_updates(feature_host: FeatureHost) -> None: + root = feature_host.scratch("supply-chain") + old_digest = f"sha256:{'1' * 64}" + new_digest = f"sha256:{'2' * 64}" + workflow = root / ".github" / "workflows" / "ci.yml" + workflow.parent.mkdir(parents=True) + # No version-tag comment, so the action refresh resolves nothing over the network. + workflow.write_text( + f"jobs:\n build:\n steps:\n - uses: actions/checkout@{'a' * 40}\n", + encoding="utf-8", + ) + compose = root / "platform" / "compose" / "docker-compose.yaml" + compose.parent.mkdir(parents=True) + compose_text = f"services:\n db:\n image: postgres:16-alpine@{old_digest}\n" + compose.write_text(compose_text, encoding="utf-8") + _init_git_repo(feature_host, root) + + shim_dir = feature_host.scratch("supply-chain-shim") + shim_log = shim_dir / "docker.log" + docker_shim = shim_dir / "docker" + docker_shim.write_text( + "#!/bin/sh\n" + f"printf '%s\\n' \"$*\" >> '{shim_log}'\n" + f"printf 'Name: %s\\nDigest: {new_digest}\\n' \"$4\"\n", + encoding="utf-8", + ) + docker_shim.chmod(0o755) + path = os.environ.get("PATH", "") + head_before = _git(feature_host, root, "rev-parse", "HEAD").stdout.strip() + + plan = _ok_object( + feature_host.clawops( + "supply-chain", + "--repo-root", + str(root), + "propose-refresh", + "--dry-run", + "--create-pr", + "--refresh-command", + "uv lock", + env={"PATH": f"{shim_dir}:{path}"}, + ) + ) + assert plan["ok"] is True and plan["dryRun"] is True, plan + assert plan["workflowActions"] == [], plan + assert plan["composeImages"] == [ + { + "path": "platform/compose/docker-compose.yaml", + "line": 3, + "image": "postgres:16-alpine", + "from": old_digest, + "to": new_digest, + } + ], plan + assert plan["refreshCommands"] == ["uv lock"], plan + assert "prUrl" not in plan and "branch" not in plan, plan + assert shim_log.read_text(encoding="utf-8").splitlines() == [ + "buildx imagetools inspect postgres:16-alpine" + ] + assert compose.read_text(encoding="utf-8") == compose_text, "dry-run must not rewrite pins" + assert _git(feature_host, root, "status", "--porcelain").stdout.strip() == "" + assert _git(feature_host, root, "rev-parse", "HEAD").stdout.strip() == head_before + branches = _git(feature_host, root, "branch", "--format=%(refname:short)").stdout.split() + assert branches == ["main"], f"dry-run must not create branches: {branches}" + + +def test_structured_logs_emit_json_events(feature_host: FeatureHost) -> None: + project = feature_host.scratch("structured-logs") + (project / "src").mkdir() + (project / "src" / "service.py").write_text("def handler() -> int:\n return 1\n") + (project / "README.md").write_text("# Service\n", encoding="utf-8") + index_args = ( + "context", + "codebase", + "index", + "--config", + str(CODEBASE_CONTEXT_CONFIG), + "--repo", + str(project), + "--scale", + "small", + "--json", + ) + + enabled = feature_host.clawops(*index_args, env={"CLAWOPS_STRUCTURED_LOGS": "1"}) + stats = _ok_object(enabled) + records = [ + _as_object(json.loads(line)) for line in enabled.stderr.splitlines() if line.startswith("{") + ] + index_events = [item for item in records if item["event"] == "clawops.context.codebase.index"] + assert len(index_events) == 1, enabled.describe() + event = index_events[0] + assert event["provider"] == "codebase" and event["scale"] == "small", event + assert event["repo"] == project.resolve().as_posix(), event + assert event["indexed_files"] == stats["indexed_files"] == 2, (event, stats) + assert isinstance(event["elapsed_ms"], int), event + assert "clawops.context.codebase.index" not in enabled.stdout, "logs belong on stderr" + + (project / "src" / "extra.py").write_text("VALUE = 2\n", encoding="utf-8") + disabled = feature_host.clawops(*index_args, env={"CLAWOPS_STRUCTURED_LOGS": "0"}) + assert _ok_object(disabled)["indexed_files"] == 1, disabled.describe() + assert '"event"' not in disabled.stderr, disabled.describe() + + +def test_baseline_harness_smoke_writes_results(feature_host: FeatureHost) -> None: + runs_dir = feature_host.scratch("baseline-runs") + payload = _ok_object( + feature_host.clawops("baseline", "--runs-dir", str(runs_dir), "harness-smoke", timeout=900) + ) + assert payload == {"ok": True, "runsDir": str(runs_dir)}, payload + for output, suite in ( + ("security.jsonl", "security_regressions.yaml"), + ("policy.jsonl", "policy_regressions.yaml"), + ): + results = _read_jsonl(runs_dir / output) + assert [str(item["id"]) for item in results] == _suite_case_ids(feature_host, suite) + assert all(item["passed"] is True for item in results), results + + +def test_verify_platform_channels_passes(feature_host: FeatureHost) -> None: + report = _ok_object(feature_host.clawops("verify-platform", "channels", "--asset-root", ".")) + checks = _as_objects(report["checks"]) + assert report["name"] == "channels" and report["ok"] is True, report + assert {str(check["name"]) for check in checks} == CHANNEL_CHECKS, report + assert all(check["ok"] is True for check in checks), report + + drifted = _write_json( + feature_host.scratch("channels") / "30-channels.json", + { + "channels": { + "telegram": {"dmPolicy": "open", "groupPolicy": "allowlist", "allowFrom": ["tg:1"]}, + "whatsapp": {"dmPolicy": "pairing", "groupPolicy": "allowlist"}, + "defaults": {"groupPolicy": "allowlist"}, + } + }, + ) + failing = feature_host.clawops( + "verify-platform", "channels", "--asset-root", ".", "--overlay", str(drifted) + ) + failing_report = _as_object(failing.json()) + failed = { + str(check["name"]) for check in _as_objects(failing_report["checks"]) if not check["ok"] + } + assert failing.returncode == 1 and failing_report["ok"] is False, failing.describe() + assert failed == {"telegram-dm-policy"}, failing_report + + +def test_config_lists_managed_memory_profiles(feature_host: FeatureHost) -> None: + home = feature_host.scratch("config-home") + listing = _ok_object( + feature_host.clawops( + "config", + "--asset-root", + ".", + "--home-dir", + str(home), + "memory", + "--list-profiles", + "--json", + ) + ) + profiles = _as_objects(listing["profiles"]) + assert [profile["id"] for profile in profiles] == list(MANAGED_MEMORY_PROFILE_IDS), listing + for profile in profiles: + spec = MEMORY_PROFILES[str(profile["id"])] + assert profile["renderProfile"] == spec.render_profile, profile + assert profile["description"] == spec.description, profile + unmanaged = {profile_id for profile_id, spec in MEMORY_PROFILES.items() if not spec.managed} + assert unmanaged.isdisjoint(str(profile["id"]) for profile in profiles), listing + assert not any(home.iterdir()), "listing profiles must not write into the home dir" + + +def test_render_openclaw_config_merges_overlay(feature_host: FeatureHost) -> None: + scratch = feature_host.scratch("render") + home = scratch / "home" + home.mkdir() + config_dir = scratch / "strongclaw-config" + isolation = { + "STRONGCLAW_CONFIG_DIR": str(config_dir), + "STRONGCLAW_DATA_DIR": str(scratch / "strongclaw-data"), + "STRONGCLAW_STATE_DIR": str(scratch / "strongclaw-state"), + } + common = ( + "render-openclaw-config", + "--asset-root", + ".", + "--profile", + "openclaw-default", + "--home-dir", + str(home), + "--user-timezone", + "UTC", + ) + base_path = scratch / "base.json" + feature_host.clawops(*common, "--output", str(base_path), env=isolation).require_ok() + base = _as_object(_read_json(base_path)) + + overlay = scratch / "90-operator.json5" + overlay.write_text( + "{\n" + " // operator overlay\n" + " agents: { defaults: {\n" + " imageMaxDimensionPx: 800,\n" + " heartbeat: { every: '30m' },\n" + " workspace: '__CODER_WORKSPACE__',\n" + " } },\n" + "}\n", + encoding="utf-8", + ) + merged_path = scratch / "openclaw.json" + approvals_path = scratch / "exec-approvals.json" + rendered = feature_host.clawops( + *common, + "--overlay", + str(overlay), + "--output", + str(merged_path), + "--exec-approvals-output", + str(approvals_path), + env=isolation, + ).require_ok() + assert f"Rendered {merged_path.resolve()}" in rendered.stdout, rendered.describe() + assert f"Rendered {approvals_path.resolve()}" in rendered.stdout, rendered.describe() + + expected = copy.deepcopy(base) + defaults = _as_object(_as_object(expected["agents"])["defaults"]) + admin_workspace = Path(str(defaults["workspace"])) + defaults["imageMaxDimensionPx"] = 800 + _as_object(defaults["heartbeat"])["every"] = "30m" + defaults["workspace"] = (admin_workspace.parent / "coder").as_posix() + merged_text = merged_path.read_text(encoding="utf-8") + assert json.loads(merged_text) == expected, "overlay must change only the keys it sets" + assert PLACEHOLDER_PATTERN.search(merged_text) is None, merged_text + + approvals_text = approvals_path.read_text(encoding="utf-8") + approvals = _as_object(json.loads(approvals_text)) + rules = _as_objects(approvals["rules"]) + assert approvals["version"] == 1 and [rule["id"] for rule in rules] == ["workspace-tests"] + prefixes = cast(list[str], _as_object(rules[0]["match"])["cwdPrefixes"]) + assert prefixes[0] == feature_host.repo_root.resolve().as_posix(), prefixes + assert all(Path(prefix).is_absolute() for prefix in prefixes), prefixes + assert PLACEHOLDER_PATTERN.search(approvals_text) is None, approvals_text + for name in ("hypermemory.yaml", "hypermemory.sqlite.yaml"): + assert (config_dir / "memory" / name).is_file(), f"{name} was not materialized" + + +def test_services_render_every_template(feature_host: FeatureHost) -> None: + home = feature_host.scratch("services-home") + state_dir = feature_host.scratch("services-state") + managers: dict[str, tuple[Path, list[str]]] = { + "systemd": ( + home / ".config" / "systemd" / "user", + sorted( + path.name + for pattern in ("*.service", "*.timer") + for path in (feature_host.repo_root / "platform" / "systemd").glob(pattern) + ), + ), + "launchd": ( + home / "Library" / "LaunchAgents", + sorted( + path.stem + for path in (feature_host.repo_root / "platform" / "launchd").glob("*.template") + ), + ), + } + for manager, (output_dir, expected_names) in managers.items(): + payload = _ok_object( + feature_host.clawops( + "services", + "--asset-root", + ".", + "--state-dir", + str(state_dir), + "--service-manager", + manager, + "render", + env={"HOME": str(home)}, + ) + ) + assert payload["serviceManager"] == manager, payload + assert Path(str(payload["outputDir"])).resolve() == output_dir.resolve(), payload + assert Path(str(payload["stateDir"])).resolve() == state_dir.resolve(), payload + rendered = [Path(item) for item in cast(list[str], payload["renderedFiles"])] + assert sorted(path.name for path in rendered) == expected_names, payload + corpus = "" + for path in rendered: + text = path.read_text(encoding="utf-8") + leftover = PLACEHOLDER_PATTERN.findall(text) + assert not leftover, f"{path} kept placeholders {sorted(set(leftover))}" + if manager == "launchd": + label = _as_object(plistlib.loads(text.encode("utf-8")))["Label"] + assert f"{label}.plist" == path.name, path + else: + section = "[Timer]" if path.suffix == ".timer" else "[Service]" + assert section in text, f"{path} has no {section} section" + corpus += text + assert feature_host.repo_root.resolve().as_posix() in corpus, manager + assert state_dir.resolve().as_posix() in corpus, manager diff --git a/tests/suites/e2e/feature_verification/test_recovery.py b/tests/suites/e2e/feature_verification/test_recovery.py new file mode 100644 index 00000000..0c0aa239 --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_recovery.py @@ -0,0 +1,120 @@ +"""Recovery features: OpenClaw-backed backups, verification, restore, retention, scheduled jobs.""" + +from __future__ import annotations + +import os +import tarfile +import time +from pathlib import Path +from typing import cast + +from clawops.strongclaw_recovery import backups_dir +from tests.utils.helpers.feature_verification import CommandOutcome, FeatureHost + +SCHEDULED_JOB_TIMEOUT_SECONDS = 600.0 +_POLL_SECONDS = 3.0 + + +def _recovery(feature_host: FeatureHost, *arguments: str, timeout: float = 900) -> CommandOutcome: + return feature_host.clawops( + "recovery", "--home-dir", str(feature_host.home), *arguments, timeout=timeout + ) + + +def _create_backup(feature_host: FeatureHost, *, allow_fallback: bool = True) -> dict[str, object]: + """Create an archive; dependent features accept the operator tar fallback so they verify + their own behaviour even when the OpenClaw-backed path is broken (recovery.backup-create).""" + arguments = ("backup-create", "--allow-fallback") if allow_fallback else ("backup-create",) + return cast(dict[str, object], _recovery(feature_host, *arguments).require_ok().json()) + + +def test_backup_create_uses_the_openclaw_backend(feature_host: FeatureHost) -> None: + payload = _create_backup(feature_host, allow_fallback=False) + assert payload["mode"] == "openclaw-cli", f"backup fell back to tar: {payload}" + assert payload["fallback_used"] is False, payload + archive = Path(str(payload["archive"])) + assert archive.is_file() and archive.stat().st_size > 0, payload + assert archive.parent == backups_dir(home_dir=feature_host.home), archive + + +def test_backup_verify_accepts_latest_archive(feature_host: FeatureHost) -> None: + created = _create_backup(feature_host) + verified = cast(dict[str, object], _recovery(feature_host, "backup-verify").require_ok().json()) + assert verified["ok"] is True and verified["archive"] == created["archive"], verified + + +def test_restore_extracts_openclaw_state(feature_host: FeatureHost) -> None: + archive = Path(str(_create_backup(feature_host)["archive"])) + destination = feature_host.scratch("restore") + restored = cast( + dict[str, object], + _recovery(feature_host, "restore", str(archive), str(destination)).require_ok().json(), + ) + assert Path(str(restored["destination"])) == destination, restored + with tarfile.open(archive, "r:gz") as bundle: + member_names = {member.name for member in bundle.getmembers() if member.isfile()} + assert member_names, "the archive holds no files" + restored_files = { + path.relative_to(destination).as_posix() + for path in destination.rglob("*") + if path.is_file() + } + assert member_names <= restored_files, sorted(member_names - restored_files)[:20] + assert any(name.endswith("openclaw.json") for name in restored_files), sorted(restored_files)[ + :40 + ] + + +def test_prune_retention_removes_only_stale_archives(feature_host: FeatureHost) -> None: + archive_root = backups_dir(home_dir=feature_host.home) + archive_root.mkdir(parents=True, exist_ok=True) + stale = archive_root / "featver-stale.tar.gz" + stale.write_bytes(b"stale") + old = time.time() - 30 * 24 * 3600 + os.utime(stale, (old, old)) + fresh = Path(str(_create_backup(feature_host)["archive"])) + payload = cast( + dict[str, object], _recovery(feature_host, "prune-retention").require_ok().json() + ) + assert stale.as_posix() in cast(list[str], payload["deleted"]), payload + assert not stale.exists() and fresh.is_file(), payload + + +def test_rotate_secrets_guidance_lists_rotation_targets(feature_host: FeatureHost) -> None: + payload = cast(dict[str, object], _recovery(feature_host, "rotate-secrets").require_ok().json()) + steps = cast(list[str], payload["steps"]) + assert payload["ok"] is True and steps, payload + assert any("Restart the gateway" in step for step in steps), steps + + +def test_scheduled_backup_job_produces_an_archive(feature_host: FeatureHost) -> None: + archive_root = backups_dir(home_dir=feature_host.home) + before: set[Path] = set(archive_root.glob("*")) if archive_root.is_dir() else set() + if feature_host.lane.platform == "linux": + timer = feature_host.run( + ["systemctl", "--user", "is-active", "openclaw-backup-create.timer"], timeout=30 + ) + assert timer.stdout.strip() == "active", timer.describe() + feature_host.run( + ["systemctl", "--user", "start", "openclaw-backup-create.service"], + timeout=SCHEDULED_JOB_TIMEOUT_SECONDS, + ).require_ok() + result = feature_host.run( + ["systemctl", "--user", "show", "--property=Result", "openclaw-backup-create.service"], + timeout=30, + ).require_ok() + assert "Result=success" in result.stdout, result.describe() + else: + label = f"gui/{os.getuid()}/ai.openclaw.backup-create" + feature_host.run(["launchctl", "kickstart", "-k", label], timeout=60).require_ok() + deadline = time.monotonic() + SCHEDULED_JOB_TIMEOUT_SECONDS + state = feature_host.run(["launchctl", "print", label], timeout=30) + while "last exit code = 0" not in state.stdout and time.monotonic() < deadline: + assert "last exit code = (never exited)" in state.stdout or "state = running" in ( + state.stdout + ), state.describe() + time.sleep(_POLL_SECONDS) + state = feature_host.run(["launchctl", "print", label], timeout=30) + assert "last exit code = 0" in state.stdout, state.describe() + created = set(archive_root.glob("*")) - before + assert created, f"the scheduled backup job wrote no archive under {archive_root}" diff --git a/tests/suites/e2e/feature_verification/test_setup_profiles.py b/tests/suites/e2e/feature_verification/test_setup_profiles.py new file mode 100644 index 00000000..034f8fde --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_setup_profiles.py @@ -0,0 +1,165 @@ +"""Setup-profile features: each profile renders, validates, and boots a real gateway. + +Each test activates its own profile through the operator path (``clawops config memory +--set-profile`` for managed profiles, ``clawops setup --profile`` for legacy ones), re-applies the +model chain as the setup guidance instructs, and restarts the harness gateway on it. +""" + +from __future__ import annotations + +import json +import uuid +from pathlib import Path +from typing import Final, cast + +from clawops.strongclaw_runtime import resolve_openclaw_config_path +from tests.utils.helpers.feature_verification import ( + ACP_STUB_REPLY_PREFIX, + FeatureHost, + GatewayController, + plugin_statuses, + register_provider, + reply_texts, +) + +ACP_WORKER_AGENTS: Final[tuple[str, ...]] = ("coder-acp-codex", "reviewer-acp-claude") +STUB_AGENT: Final[Path] = Path("tests/utils/helpers/_feature_verification/acp_stub_agent.py") + + +def _activate(feature_host: FeatureHost, gateway: GatewayController, profile: str) -> None: + if profile in {"acp", "browser-lab"}: + feature_host.clawops( + "setup", + "--asset-root", + ".", + "--home-dir", + str(feature_host.home), + "--profile", + profile, + "--skip-bootstrap", + "--no-activate-services", + "--no-verify", + "--non-interactive", + timeout=1800, + ).require_ok() + else: + payload = ( + feature_host.clawops( + "config", + "--asset-root", + ".", + "--home-dir", + str(feature_host.home), + "memory", + "--set-profile", + profile, + "--json", + timeout=1800, + ) + .require_ok() + .json() + ) + assert cast(dict[str, object], payload).get("ok", True) is not False, payload + # Setup re-renders openclaw.json, so the operator re-declares the provider first. + register_provider(resolve_openclaw_config_path(feature_host.repo_root)) + feature_host.clawops("model-auth", "--asset-root", ".", "ensure", timeout=900).require_ok() + feature_host.openclaw("config", "validate", timeout=120).require_ok() + gateway.restart() + + +def _agent_ids(feature_host: FeatureHost) -> list[str]: + payload = feature_host.openclaw("agents", "list", "--json", timeout=120).require_ok().json() + return [str(cast(dict[str, object], agent)["id"]) for agent in cast(list[object], payload)] + + +def test_openclaw_default_profile_boots_with_memory_core( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + _activate(feature_host, gateway, "openclaw-default") + plugins = cast(dict[str, object], feature_host.rendered_config()["plugins"]) + slots = cast(dict[str, object], plugins["slots"]) + assert slots == {"contextEngine": "legacy", "memory": "memory-core"}, slots + assert plugin_statuses(feature_host).get("memory-core") == "loaded" + feature_host.openclaw("memory", "status", "--deep", timeout=600).require_ok() + + +def test_openclaw_qmd_profile_installs_and_uses_qmd( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + _activate(feature_host, gateway, "openclaw-qmd") + qmd = feature_host.home / ".bun" / "bin" / "qmd" + feature_host.run([str(qmd), "status"], timeout=120).require_ok() + memory = cast(dict[str, object], feature_host.rendered_config()["memory"]) + assert memory["backend"] == "qmd", memory + status = feature_host.openclaw("memory", "status", "--deep", timeout=600).require_ok() + assert "qmd" in status.output.lower(), status.describe() + + +def test_acp_profile_registers_acp_worker_agents( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + _activate(feature_host, gateway, "acp") + listed = _agent_ids(feature_host) + for agent_id in ACP_WORKER_AGENTS: + assert agent_id in listed, listed + agents = cast(dict[str, object], feature_host.rendered_config()["agents"]) + runtimes = { + str(cast(dict[str, object], agent)["id"]): cast(dict[str, object], agent).get("runtime") + for agent in cast(list[object], agents["list"]) + } + for agent_id in ACP_WORKER_AGENTS: + runtime = cast(dict[str, object], runtimes[agent_id]) + assert runtime["type"] == "acp", runtime + assert cast(dict[str, object], runtime["acp"])["backend"] == "acpx", runtime + + +def test_acp_worker_agent_runs_a_turn_through_acpx( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + _activate(feature_host, gateway, "acp") + agents = cast(dict[str, object], feature_host.rendered_config()["agents"]) + coder = next( + cast(dict[str, object], agent) + for agent in cast(list[object], agents["list"]) + if cast(dict[str, object], agent)["id"] == "coder-acp-codex" + ) + acp_cwd = Path( + str(cast(dict[str, object], cast(dict[str, object], coder["runtime"])["acp"])["cwd"]) + ) + acp_cwd.mkdir(parents=True, exist_ok=True) + stub = feature_host.repo_root / STUB_AGENT + (acp_cwd / ".acpxrc.json").write_text( + json.dumps( + {"agents": {"codex": {"command": f"{feature_host.python} {stub} --persona codex"}}} + ), + encoding="utf-8", + ) + token = f"FV-worker-{uuid.uuid4().hex[:10]}" + outcome = feature_host.openclaw( + "agent", + "--agent", + "coder-acp-codex", + "--session-id", + f"featver-{token}", + "--message", + f"Reply {token}", + "--json", + timeout=300, + ) + replies = reply_texts(outcome.require_ok().json()) + assert any(ACP_STUB_REPLY_PREFIX in reply and token in reply for reply in replies), ( + f"the ACP worker did not answer through acpx (bundled acpx plugin: " + f"{plugin_statuses(feature_host).get('acpx')!r}); agent replies: {replies}" + ) + + +def test_browser_lab_profile_points_at_the_cdp_endpoint( + feature_host: FeatureHost, gateway: GatewayController +) -> None: + _activate(feature_host, gateway, "browser-lab") + browser = cast(dict[str, object], feature_host.rendered_config()["browser"]) + profiles = cast(dict[str, object], browser["profiles"]) + lab = cast(dict[str, object], profiles[str(browser["defaultProfile"])]) + assert browser["enabled"] is True and lab["cdpUrl"] == "http://127.0.0.1:9222", browser + ssrf = cast(dict[str, object], browser["ssrfPolicy"]) + assert ssrf["dangerouslyAllowPrivateNetwork"] is False, ssrf diff --git a/tests/suites/e2e/feature_verification/test_sidecars.py b/tests/suites/e2e/feature_verification/test_sidecars.py new file mode 100644 index 00000000..856a1134 --- /dev/null +++ b/tests/suites/e2e/feature_verification/test_sidecars.py @@ -0,0 +1,329 @@ +"""Sidecar, observability and browser-lab features against the live compose stacks.""" + +from __future__ import annotations + +import time +import uuid +from typing import cast + +from clawops.strongclaw_runtime import set_env_assignment +from tests.utils.helpers.feature_verification import ( + OLLAMA_EMBEDDING_API_BASE, + OLLAMA_EMBEDDING_MODEL, + FeatureHost, + basic_auth, + http_request, + tcp_open, +) + +QDRANT = "http://127.0.0.1:6333" +LITELLM = "http://127.0.0.1:4000" +NEO4J_HTTP = "http://127.0.0.1:7474" +OTLP_HTTP = "http://127.0.0.1:4318" +OTEL_METRICS = "http://127.0.0.1:9464/metrics" +QDRANT_TEST_PREFIX = "memory-v2-int-" + + +def _container(feature_host: FeatureHost, fragment: str) -> str: + listing = feature_host.run(["docker", "ps", "--format", "{{.Names}}"], timeout=60).require_ok() + names = [name for name in listing.stdout.split() if fragment in name] + assert names, f"no running container matches {fragment!r}: {listing.stdout}" + return names[0] + + +def test_postgres_sidecar_accepts_queries(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars + container = _container(feature_host, "postgres") + outcome = feature_host.run( + [ + "docker", + "exec", + container, + "psql", + "-U", + "litellm", + "-d", + "litellm", + "-tAc", + "select count(*) from information_schema.tables", + ], + timeout=60, + ).require_ok() + assert int(outcome.stdout.strip()) > 0, outcome.describe() + + +def _embed_through_litellm(feature_host: FeatureHost, *, upstream_path: str) -> None: + """Embed through the hypermemory-embedding alias and prove LiteLLM called its upstream.""" + assert http_request("GET", f"{LITELLM}/health/liveliness", timeout=10).status == 200 + started = time.time() + response = http_request( + "POST", + f"{LITELLM}/v1/embeddings", + body={"model": "hypermemory-embedding", "input": [f"FV-litellm-{uuid.uuid4().hex[:10]}"]}, + headers={"Authorization": f"Bearer {feature_host.env_values()['LITELLM_MASTER_KEY']}"}, + timeout=120, + ) + assert response.status == 200, feature_host.redact(response.body[:2000]) + data = cast(list[dict[str, object]], cast(dict[str, object], response.json())["data"]) + assert data and len(cast(list[float], data[0]["embedding"])) > 0, response.body[:500] + upstream = [ + request + for request in feature_host.fake_model_requests() + if float(cast(float, request.get("at", 0.0))) >= started + and str(request.get("path", "")).startswith(upstream_path) + ] + assert upstream, f"litellm answered without calling its {upstream_path} upstream" + + +def _wait_for_litellm(timeout: float = 240.0) -> None: + deadline = time.monotonic() + timeout + while http_request("GET", f"{LITELLM}/health/liveliness", timeout=10).status != 200: + assert time.monotonic() < deadline, "litellm did not come back after recreation" + time.sleep(3) + + +def test_litellm_routes_embeddings_to_the_upstream( + feature_host: FeatureHost, sidecars: str +) -> None: + del sidecars + _embed_through_litellm(feature_host, upstream_path="/v1/embeddings") + + +def test_litellm_serves_the_default_ollama_embedding_lane( + feature_host: FeatureHost, sidecars: str +) -> None: + del sidecars + env_file = feature_host.env_file() + configured = feature_host.env_values() + keys = ("HYPERMEMORY_EMBEDDING_MODEL", "HYPERMEMORY_EMBEDDING_API_BASE") + # The lane `clawops varlock-env configure` selects when a local Ollama has the model. + for key, value in zip(keys, (OLLAMA_EMBEDDING_MODEL, OLLAMA_EMBEDDING_API_BASE), strict=True): + set_env_assignment(env_file, key, value) + try: + feature_host.clawops( + "ops", "--asset-root", ".", "sidecars", "up", timeout=1500 + ).require_ok() + _wait_for_litellm() + _embed_through_litellm(feature_host, upstream_path="/api/embed") + finally: + for key in keys: + set_env_assignment(env_file, key, configured.get(key, "")) + feature_host.clawops("ops", "--asset-root", ".", "sidecars", "up", timeout=1500) + _wait_for_litellm() + + +def test_qdrant_sidecar_round_trips_vectors(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars + del feature_host + collection = f"featver-{uuid.uuid4().hex[:8]}" + assert http_request("GET", f"{QDRANT}/healthz").status == 200 + created = http_request( + "PUT", + f"{QDRANT}/collections/{collection}", + body={"vectors": {"size": 4, "distance": "Cosine"}}, + ) + assert created.status == 200, created.body + try: + upsert = http_request( + "PUT", + f"{QDRANT}/collections/{collection}/points?wait=true", + body={"points": [{"id": 1, "vector": [0.1, 0.2, 0.3, 0.4], "payload": {"k": "fv"}}]}, + ) + assert upsert.status == 200, upsert.body + search = http_request( + "POST", + f"{QDRANT}/collections/{collection}/points/search", + body={"vector": [0.1, 0.2, 0.3, 0.4], "limit": 1, "with_payload": True}, + ) + hits = cast(list[dict[str, object]], cast(dict[str, object], search.json())["result"]) + assert hits and cast(dict[str, object], hits[0]["payload"])["k"] == "fv", search.body + finally: + http_request("DELETE", f"{QDRANT}/collections/{collection}") + + +def test_neo4j_sidecar_round_trips_cypher(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars + values = feature_host.env_values() + marker = f"fv-{uuid.uuid4().hex[:8]}" + response = http_request( + "POST", + f"{NEO4J_HTTP}/db/neo4j/tx/commit", + body={ + "statements": [ + { + "statement": "CREATE (n:FeatVer {id: $id}) RETURN n.id", + "parameters": {"id": marker}, + }, + { + "statement": "MATCH (n:FeatVer {id: $id}) DETACH DELETE n RETURN count(*)", + "parameters": {"id": marker}, + }, + ] + }, + headers=basic_auth(values.get("NEO4J_USERNAME", "neo4j"), values["NEO4J_PASSWORD"]), + timeout=60, + ) + payload = cast(dict[str, object], response.json()) + assert response.status == 200 and not payload.get("errors"), response.body[:2000] + results = cast(list[dict[str, object]], payload["results"]) + assert len(results) == 2, payload + + +def test_observability_pipeline_accepts_telemetry(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars + feature_host.clawops("verify-platform", "observability", "--asset-root", ".").require_ok() + trace = { + "resourceSpans": [ + { + "resource": { + "attributes": [{"key": "service.name", "value": {"stringValue": "featver"}}] + }, + "scopeSpans": [ + { + "spans": [ + { + "traceId": uuid.uuid4().hex, + "spanId": uuid.uuid4().hex[:16], + "name": "featver-span", + "kind": 1, + "startTimeUnixNano": str(time.time_ns() - 1_000_000), + "endTimeUnixNano": str(time.time_ns()), + } + ] + } + ], + } + ] + } + accepted = http_request("POST", f"{OTLP_HTTP}/v1/traces", body=trace) + assert accepted.status == 200, accepted.body + metric_name = f"featver_probe_{uuid.uuid4().hex[:8]}" + gauge = { + "resourceMetrics": [ + { + "resource": { + "attributes": [{"key": "service.name", "value": {"stringValue": "featver"}}] + }, + "scopeMetrics": [ + { + "metrics": [ + { + "name": metric_name, + "gauge": { + "dataPoints": [ + {"asDouble": 1.0, "timeUnixNano": str(time.time_ns())} + ] + }, + } + ] + } + ], + } + ] + } + assert http_request("POST", f"{OTLP_HTTP}/v1/metrics", body=gauge).status == 200 + deadline = time.monotonic() + 60 + metrics = http_request("GET", OTEL_METRICS) + while metric_name not in metrics.body and time.monotonic() < deadline: + time.sleep(2) + metrics = http_request("GET", OTEL_METRICS) + assert ( + metric_name in metrics.body + ), f"OTLP metric never reached the exporter: {metrics.body[:500]}" + + +def test_verify_platform_sidecars_runtime(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars + outcome = feature_host.clawops("verify-platform", "sidecars", "--asset-root", ".", timeout=300) + payload = cast(dict[str, object], outcome.json()) + assert outcome.ok and payload["ok"] is True, outcome.describe() + + +def test_ops_status_reports_required_sidecars_ready( + feature_host: FeatureHost, sidecars: str +) -> None: + del sidecars + payload = cast( + dict[str, object], + feature_host.clawops("ops", "--asset-root", ".", "status", timeout=300).require_ok().json(), + ) + readiness = cast(dict[str, object], payload["readiness"]) + assert payload["ok"] is True and readiness["requiredReady"] is True, payload + services = cast(dict[str, object], payload["services"]) + for name in ("postgres", "litellm", "qdrant", "neo4j"): + assert cast(dict[str, object], services[name])["state"] == "running", services + + +def test_prune_qdrant_test_collections(feature_host: FeatureHost, sidecars: str) -> None: + del sidecars + collection = f"{QDRANT_TEST_PREFIX}featver-{uuid.uuid4().hex[:8]}" + created = http_request( + "PUT", + f"{QDRANT}/collections/{collection}", + body={"vectors": {"size": 4, "distance": "Dot"}}, + ) + assert created.status == 200, created.body + dry_run = feature_host.clawops( + "ops", "--asset-root", ".", "prune-qdrant-test-collections", "--dry-run" + ).require_ok() + assert collection in dry_run.stdout, dry_run.describe() + assert http_request("GET", f"{QDRANT}/collections/{collection}").status == 200 + feature_host.clawops("ops", "--asset-root", ".", "prune-qdrant-test-collections").require_ok() + assert http_request("GET", f"{QDRANT}/collections/{collection}").status == 404 + + +def test_browser_lab_starts_and_exposes_cdp(feature_host: FeatureHost) -> None: + feature_host.clawops("ops", "--asset-root", ".", "browser-lab", "up", timeout=900).require_ok() + try: + verify = feature_host.clawops( + "verify-platform", "browser-lab", "--asset-root", ".", timeout=300 + ) + assert verify.ok and cast(dict[str, object], verify.json())["ok"] is True, verify.describe() + deadline = time.monotonic() + 120 + version = http_request("GET", "http://127.0.0.1:9222/json/version", timeout=10) + while version.status != 200 and time.monotonic() < deadline: + time.sleep(3) + version = http_request("GET", "http://127.0.0.1:9222/json/version", timeout=10) + assert version.status == 200, version.body + assert "webSocketDebuggerUrl" in version.body or "Browser" in version.body, version.body + assert tcp_open(3128), "browser-lab egress proxy is not listening on 3128" + finally: + feature_host.clawops("ops", "--asset-root", ".", "browser-lab", "down", timeout=600) + + +def test_reset_compose_state_clears_one_component(feature_host: FeatureHost) -> None: + state_root = feature_host.scratch("compose-state") + component_dir = state_root / "browser-lab" + component_dir.mkdir() + (component_dir / "profile.db").write_text("stale", encoding="utf-8") + outcome = feature_host.clawops( + "ops", + "--asset-root", + ".", + "reset-compose-state", + "--component", + "browser-lab", + "--state-dir", + str(state_root), + timeout=180, + ).require_ok() + payload = cast(dict[str, object], outcome.json()) + assert payload["ok"] is True and payload["stateDir"] == str(component_dir), payload + assert component_dir.is_dir() and not any(component_dir.iterdir()), list( + component_dir.iterdir() + ) + + +def test_optional_ollama_compose_serves_the_api(feature_host: FeatureHost) -> None: + compose = ["docker", "compose", "-f", "platform/compose/docker-compose.ollama.optional.yaml"] + env = {"STRONGCLAW_COMPOSE_STATE_DIR": str(feature_host.scratch("ollama-state"))} + feature_host.run([*compose, "up", "-d"], timeout=900, env=env).require_ok() + try: + deadline = time.monotonic() + 180 + version = http_request("GET", "http://127.0.0.1:11434/api/version", timeout=10) + while version.status != 200 and time.monotonic() < deadline: + time.sleep(3) + version = http_request("GET", "http://127.0.0.1:11434/api/version", timeout=10) + assert version.status == 200 and "version" in version.body, version.body + finally: + feature_host.run([*compose, "down", "-v"], timeout=300, env=env) diff --git a/tests/suites/unit/ci/test_feature_verification.py b/tests/suites/unit/ci/test_feature_verification.py new file mode 100644 index 00000000..3913e155 --- /dev/null +++ b/tests/suites/unit/ci/test_feature_verification.py @@ -0,0 +1,333 @@ +"""Unit coverage for the feature-verification harness and its fresh-host scenarios.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from typing import cast + +import pytest + +from tests.plugins.infrastructure.context import TestContext +from tests.utils.helpers import fresh_host +from tests.utils.helpers._fresh_host import linux as fresh_host_linux +from tests.utils.helpers.feature_verification import ( + ACP_STUB_REPLY_PREFIX, + REPLY_PREFIX, + FeatureResult, + FeatureVerificationError, + LaneResults, + build_report, + canned_reply, + deterministic_embedding, + extract_json, + load_lane_results, + load_manifest, + render_markdown, +) +from tests.utils.helpers.repo import REPO_ROOT + +_STUB_AGENT = ( + REPO_ROOT / "tests" / "utils" / "helpers" / "_feature_verification" / "acp_stub_agent.py" +) +_MANIFEST_HEADER = """\ +schema_version: 1 +lanes: + lane-a: {platform: linux, profile: hypermemory, scenario: linux-features, activate_services: true, description: a} + lane-b: {platform: macos, profile: openclaw-default, scenario: macos-features, activate_services: true, description: b} +features: +""" +_VALID_FEATURES = """\ + - id: alpha.one + category: cat-a + title: Alpha one + call_sites: [src/clawops/cli.py:1] + lanes: [lane-a, lane-b] + verification: live + test: tests/suites/e2e/feature_verification/test_x.py::test_alpha + - id: beta.two + category: cat-b + title: Beta two + call_sites: [src/clawops/cli.py:2] + status: unverifiable + reason: Needs a third-party account that CI does not have. +""" + + +def _write_manifest(tmp_path: Path, features: str = _VALID_FEATURES) -> Path: + path = tmp_path / "feature_matrix.yaml" + path.write_text(_MANIFEST_HEADER + features, encoding="utf-8") + return path + + +def test_canned_reply_echoes_tokens_from_the_latest_user_turn() -> None: + assert canned_reply("ctx", "please answer FV-abc123 now") == f"{REPLY_PREFIX} FV-abc123" + assert canned_reply("ctx", "no token here") == f"{REPLY_PREFIX} acknowledged" + + +def test_canned_reply_answers_reflection_and_extraction_prompts() -> None: + reflection = canned_reply("## Invariants\n## Derived\nMEMORY REFLECTION", "") + assert "FEATVER_REFLECTION_INVARIANT" in reflection and "## Derived" in reflection + extraction = canned_reply("Analyze ... extract memories worth long-term preservation.", "") + assert json.loads(extraction) == {"memories": []} + + +def test_deterministic_embedding_is_stable_and_unit_length() -> None: + first = deterministic_embedding("alpha", 64) + assert first == deterministic_embedding("alpha", 64) + assert first != deterministic_embedding("beta", 64) + assert len(first) == 64 + assert abs(sum(value * value for value in first) - 1.0) < 1e-3 + + +@pytest.mark.network_local +def test_fake_model_server_serves_openai_and_ollama_shapes() -> None: + import urllib.request + + from tests.utils.helpers.feature_verification import FakeModelServer + + server = FakeModelServer(host="127.0.0.1", ports=[0]) + server.start() + base = f"http://127.0.0.1:{server.ports[0]}" + try: + + def _post(path: str, body: object) -> bytes: + request = urllib.request.Request( + f"{base}{path}", data=json.dumps(body).encode(), method="POST" + ) + with urllib.request.urlopen(request, timeout=5) as response: # noqa: S310 + return cast(bytes, response.read()) + + chat = json.loads( + _post("/v1/chat/completions", {"messages": [{"role": "user", "content": "FV-1234"}]}) + ) + assert chat["choices"][0]["message"]["content"] == f"{REPLY_PREFIX} FV-1234" + stream = _post( + "/v1/chat/completions", + {"stream": True, "messages": [{"role": "user", "content": "FV-5678"}]}, + ).decode() + assert f"{REPLY_PREFIX} FV-5678" in stream and stream.rstrip().endswith("data: [DONE]") + embeddings = json.loads(_post("/v1/embeddings", {"input": ["a", "b"], "dimensions": 8})) + assert [len(item["embedding"]) for item in embeddings["data"]] == [8, 8] + ollama = json.loads(_post("/api/embed", {"input": ["a"], "dimensions": 4})) + assert len(ollama["embeddings"][0]) == 4 + with urllib.request.urlopen( + f"{base}/__featver/requests", timeout=5 + ) as response: # noqa: S310 + journal = json.loads(response.read())["requests"] + assert [entry["path"] for entry in journal] == [ + "/v1/chat/completions", + "/v1/chat/completions", + "/v1/embeddings", + "/api/embed", + ] + finally: + server.stop() + + +def test_acp_stub_agent_speaks_acp_and_reports_env_presence_only() -> None: + messages: list[dict[str, object]] = [ + {"jsonrpc": "2.0", "id": 0, "method": "initialize", "params": {"protocolVersion": 1}}, + { + "jsonrpc": "2.0", + "id": 1, + "method": "authenticate", + "params": {"methodId": "openai-api-key"}, + }, + { + "jsonrpc": "2.0", + "id": 2, + "method": "session/new", + "params": {"cwd": "/", "mcpServers": []}, + }, + { + "jsonrpc": "2.0", + "id": 3, + "method": "session/prompt", + "params": {"sessionId": "s", "prompt": [{"type": "text", "text": "FV-stub"}]}, + }, + ] + completed = subprocess.run( + [sys.executable, str(_STUB_AGENT), "--persona", "codex"], + input="\n".join(json.dumps(message) for message in messages) + "\n", + capture_output=True, + text=True, + timeout=30, + check=True, + env={"PATH": "/usr/bin:/bin", "OPENAI_API_KEY": "secret-value-never-echoed"}, + ) + replies = [json.loads(line) for line in completed.stdout.splitlines()] + methods = [method["id"] for method in replies[0]["result"]["authMethods"]] + assert methods == ["chatgpt", "codex-api-key", "openai-api-key"] + update = next(reply for reply in replies if reply.get("method") == "session/update") + text = update["params"]["update"]["content"]["text"] + assert text.startswith(ACP_STUB_REPLY_PREFIX) + assert "auth=openai-api-key" in text and "OPENAI_API_KEY:present" in text + assert "secret-value-never-echoed" not in completed.stdout + assert replies[-1] == {"jsonrpc": "2.0", "id": 3, "result": {"stopReason": "end_turn"}} + + +def test_extract_json_skips_leading_noise() -> None: + assert extract_json('warning: banner\n{"ok": true}\ntrailer') == {"ok": True} + with pytest.raises(AssertionError): + extract_json("no json at all") + + +def test_load_manifest_accepts_valid_matrix_and_derives_platforms(tmp_path: Path) -> None: + manifest = load_manifest(_write_manifest(tmp_path)) + alpha = manifest.feature("alpha.one") + assert manifest.platforms(alpha) == ("linux", "macos") + assert manifest.features_for_lane("lane-b") == (alpha,) + assert manifest.feature("beta.two").status == "unverifiable" + + +@pytest.mark.parametrize( + ("features", "message"), + [ + (_VALID_FEATURES.replace("lanes: [lane-a, lane-b]", "lanes: [lane-z]"), "unknown lanes"), + ( + _VALID_FEATURES.replace( + " reason: Needs", + " test: tests/suites/e2e/feature_verification/test_x.py::test_beta\n reason: Needs", + ), + "unverifiable", + ), + (_VALID_FEATURES.replace("verification: live", "verification: maybe"), "verification"), + (_VALID_FEATURES.replace("src/clawops/cli.py:1", "src/clawops/cli.py"), "file:line"), + (_VALID_FEATURES.replace("id: beta.two", "id: alpha.one"), "duplicate"), + ], +) +def test_load_manifest_rejects_invalid_entries(tmp_path: Path, features: str, message: str) -> None: + with pytest.raises(FeatureVerificationError, match=message): + load_manifest(_write_manifest(tmp_path, features)) + + +def _lane(lane: str, platform: str, outcome: str, message: str = "") -> LaneResults: + return LaneResults( + lane=lane, + platform=platform, + profile="p", + started_at="t0", + versions={"openclaw": "OpenClaw 2026.3.13", "acpx": "0.3.0"}, + results=[ + FeatureResult( + feature="alpha.one", + test="tests/suites/e2e/feature_verification/test_x.py::test_alpha", + outcome=outcome, # type: ignore[arg-type] + duration_seconds=1.0, + message=message, + ) + ], + ) + + +def test_report_requires_every_lane_to_pass(tmp_path: Path) -> None: + manifest = load_manifest(_write_manifest(tmp_path)) + passing = build_report( + manifest, + { + "lane-a": _lane("lane-a", "linux", "passed"), + "lane-b": _lane("lane-b", "macos", "passed"), + }, + ) + assert passing.counts == {"PASS": 1, "FAIL": 0, "UNVERIFIABLE": 1} + one_lane_missing = build_report(manifest, {"lane-a": _lane("lane-a", "linux", "passed")}) + assert one_lane_missing.counts == {"PASS": 0, "FAIL": 1, "UNVERIFIABLE": 1} + failing = build_report( + manifest, + { + "lane-a": _lane("lane-a", "linux", "failed", "boom | pipe"), + "lane-b": _lane("lane-b", "macos", "passed"), + }, + ) + markdown = render_markdown(failing) + assert "| `alpha.one` | cat-a | FAIL |" in markdown and "boom \\| pipe" in markdown + assert "OpenClaw 2026.3.13" in markdown + + +def test_report_maps_unverifiable_skips_and_plain_skips(tmp_path: Path) -> None: + manifest = load_manifest(_write_manifest(tmp_path)) + lanes = { + "lane-a": _lane("lane-a", "linux", "skipped", "unverifiable: provider outage"), + "lane-b": _lane("lane-b", "macos", "skipped", "unverifiable: provider outage"), + } + assert build_report(manifest, lanes).counts == {"PASS": 0, "FAIL": 0, "UNVERIFIABLE": 2} + lanes["lane-b"] = _lane("lane-b", "macos", "skipped", "some other skip") + assert build_report(manifest, lanes).counts["FAIL"] == 1 + + +def test_lane_results_round_trip(tmp_path: Path) -> None: + path = tmp_path / "feature-results-lane-a.json" + _lane("lane-a", "linux", "error", "teardown").write(path) + loaded = load_lane_results(path) + assert loaded.results[0].verdict == "FAIL" and loaded.versions["acpx"] == "0.3.0" + + +def test_feature_scenarios_install_into_the_runner_home( + tmp_path: Path, test_context: TestContext +) -> None: + test_context.apply_profiles("fresh_host_push") + workspace = tmp_path / "workspace" + workspace.mkdir() + context = fresh_host.prepare_context( + scenario_id="linux-features", + profile="hypermemory", + repo_root=workspace, + runner_temp=tmp_path / "runner-temp", + workspace=workspace, + github_env_file=None, + ) + assert Path(context.app_home) == Path.home() + assert context.xdg_runtime_dir is not None and context.xdg_runtime_dir.startswith("/run/user/") + reloaded = fresh_host.load_context(Path(context.context_path)) + assert reloaded.use_host_home is True + + +def test_legacy_context_files_default_the_new_fields( + tmp_path: Path, test_context: TestContext +) -> None: + test_context.apply_profiles("fresh_host_push") + workspace = tmp_path / "workspace" + workspace.mkdir() + context = fresh_host.prepare_context( + scenario_id="linux", + repo_root=workspace, + runner_temp=tmp_path / "runner-temp", + workspace=workspace, + github_env_file=None, + ) + payload = json.loads(Path(context.context_path).read_text(encoding="utf-8")) + del payload["use_host_home"] + Path(context.context_path).write_text(json.dumps(payload), encoding="utf-8") + reloaded = fresh_host.load_context(Path(context.context_path)) + assert reloaded.use_host_home is False + assert Path(reloaded.app_home) == Path(context.tmp_root) / "home" + + +@pytest.mark.parametrize( + ("scenario_id", "expects_flag"), [("linux", True), ("linux-features", False)] +) +def test_linux_setup_only_skips_activation_when_the_scenario_says_so( + tmp_path: Path, test_context: TestContext, scenario_id: str, expects_flag: bool +) -> None: + test_context.apply_profiles("fresh_host_push") + workspace = tmp_path / "workspace" + workspace.mkdir() + context = fresh_host.prepare_context( + scenario_id=scenario_id, + repo_root=workspace, + runner_temp=tmp_path / "runner-temp", + workspace=workspace, + github_env_file=None, + ) + calls: list[list[str]] = [] + + def _record(command: list[str], **_: object) -> None: + calls.append(command) + + test_context.patch.patch_object(fresh_host_linux, "run_command", new=_record) + command = fresh_host_linux.linux_setup(context) + assert calls == [command] + assert ("--no-activate-services" in command) is expects_flag diff --git a/tests/utils/helpers/_feature_verification/__init__.py b/tests/utils/helpers/_feature_verification/__init__.py new file mode 100644 index 00000000..a395d01d --- /dev/null +++ b/tests/utils/helpers/_feature_verification/__init__.py @@ -0,0 +1 @@ +"""Internal feature-verification helper package.""" diff --git a/tests/utils/helpers/_feature_verification/acp_stub_agent.py b/tests/utils/helpers/_feature_verification/acp_stub_agent.py new file mode 100644 index 00000000..db974e25 --- /dev/null +++ b/tests/utils/helpers/_feature_verification/acp_stub_agent.py @@ -0,0 +1,163 @@ +"""Minimal ACP (Agent Client Protocol) agent that stands in for vendor coding agents. + +Real ``acpx`` launches this over stdio in place of a vendor adapter (codex-acp, +claude-agent-acp, ``gemini --acp`` ...), so StrongClaw's auth-mode routing, env sanitization, +and acpx invocation are exercised end to end without vendor accounts. The reply reports which +auth-relevant env keys reached the agent process (presence only, never values) and which auth +method acpx selected, so tests can assert routing without the stub ever handling a secret. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import uuid +from collections.abc import Mapping +from typing import Final, TextIO, cast + +PROTOCOL_VERSION: Final[int] = 1 +REPLY_PREFIX: Final[str] = "FEATVER_ACP_STUB" +# Env keys whose presence (never value) the stub reports back. +OBSERVED_ENV_KEYS: Final[tuple[str, ...]] = ( + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "AWS_PROFILE", + "AWS_REGION", + "GOOGLE_APPLICATION_CREDENTIALS", + "VERTEXAI_PROJECT", + "ACPX_AUTH_OPENAI_API_KEY", +) +# Auth methods advertised by the real adapters each stub persona imitates. +AUTH_METHODS: Final[Mapping[str, tuple[str, ...]]] = { + "codex": ("chatgpt", "codex-api-key", "openai-api-key"), + "claude": ("claude-ai-login", "console-login"), +} + +type JsonObject = dict[str, object] + + +class StubAgent: + """Serve one ACP connection on stdio.""" + + def __init__(self, persona: str, *, out: TextIO) -> None: + self._persona = persona + self._out = out + self._authenticated: list[str] = [] + + def _send(self, payload: JsonObject) -> None: + self._out.write(json.dumps(payload) + "\n") + self._out.flush() + + def _result(self, request_id: object, result: JsonObject) -> None: + self._send({"jsonrpc": "2.0", "id": request_id, "result": result}) + + def _error(self, request_id: object, code: int, message: str) -> None: + self._send( + {"jsonrpc": "2.0", "id": request_id, "error": {"code": code, "message": message}} + ) + + def _env_report(self) -> str: + return ",".join( + f"{key}:{'present' if os.environ.get(key) else 'absent'}" for key in OBSERVED_ENV_KEYS + ) + + def _prompt_text(self, params: JsonObject) -> str: + prompt = params.get("prompt") + if not isinstance(prompt, list): + return "" + parts: list[str] = [] + for block in cast(list[object], prompt): + if isinstance(block, dict): + text = cast(Mapping[str, object], block).get("text") + if isinstance(text, str): + parts.append(text) + return " ".join(parts) + + def handle(self, message: JsonObject) -> None: + """Dispatch one JSON-RPC message.""" + method = message.get("method") + request_id = message.get("id") + params = message.get("params") + params_mapping = cast(JsonObject, params) if isinstance(params, dict) else {} + if method == "initialize": + self._result( + request_id, + { + "protocolVersion": PROTOCOL_VERSION, + "agentCapabilities": { + "loadSession": False, + "promptCapabilities": { + "image": False, + "audio": False, + "embeddedContext": False, + }, + }, + "authMethods": [ + {"id": method_id, "name": method_id, "description": None} + for method_id in AUTH_METHODS.get(self._persona, ()) + ], + "agentInfo": {"name": f"featver-{self._persona}", "version": "1.0.0"}, + }, + ) + elif method == "authenticate": + method_id = params_mapping.get("methodId") + self._authenticated.append(str(method_id)) + self._result(request_id, {}) + elif method == "session/new": + self._result(request_id, {"sessionId": f"featver-{uuid.uuid4().hex[:12]}"}) + elif method == "session/prompt": + session_id = params_mapping.get("sessionId") + reply = ( + f"{REPLY_PREFIX} persona={self._persona} " + f"auth={'+'.join(self._authenticated) or 'none'} " + f"env={self._env_report()} prompt={self._prompt_text(params_mapping)}" + ) + self._send( + { + "jsonrpc": "2.0", + "method": "session/update", + "params": { + "sessionId": session_id, + "update": { + "sessionUpdate": "agent_message_chunk", + "content": {"type": "text", "text": reply}, + }, + }, + } + ) + self._result(request_id, {"stopReason": "end_turn"}) + elif method in {"session/cancel"} or request_id is None: + return + elif str(method).startswith("session/"): + # Optional session controls (modes, config options): accept as no-ops. + self._result(request_id, {}) + else: + self._error(request_id, -32601, f"method not found: {method}") + + def serve(self, stream: TextIO) -> None: + """Read newline-delimited JSON-RPC messages until stdin closes.""" + for raw_line in stream: + line = raw_line.strip() + if not line: + continue + try: + message = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(message, dict): + self.handle(cast(JsonObject, message)) + + +def main(argv: list[str] | None = None) -> int: + """Run the stub agent on stdio.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--persona", required=True) + args = parser.parse_args(argv) + StubAgent(str(args.persona), out=sys.stdout).serve(sys.stdin) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/utils/helpers/_feature_verification/diagnostics.py b/tests/utils/helpers/_feature_verification/diagnostics.py new file mode 100644 index 00000000..c61ce88a --- /dev/null +++ b/tests/utils/helpers/_feature_verification/diagnostics.py @@ -0,0 +1,78 @@ +"""Best-effort lane diagnostics: service-manager state and gateway logs, secrets redacted.""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path +from typing import Final + +from clawops.strongclaw_runtime import varlock_local_env_file +from tests.utils.helpers._feature_verification.redaction import redact, secret_values +from tests.utils.helpers._fresh_host.shell import phase_env +from tests.utils.helpers._fresh_host.storage import load_context + +LAUNCHD_LABELS: Final[tuple[str, ...]] = ( + "ai.openclaw.gateway", + "ai.openclaw.sidecars", + "ai.openclaw.backup-create", + "ai.openclaw.backup-verify", + "ai.openclaw.maintenance", +) +_LOG_TAIL_CHARS: Final[int] = 400_000 +_COMMAND_TIMEOUT_SECONDS: Final[int] = 60 + + +def _diagnostic_commands(platform: str) -> dict[str, list[str]]: + if platform == "linux": + return { + "systemd-units.txt": ["systemctl", "--user", "list-units", "openclaw*", "--all"], + "systemd-timers.txt": ["systemctl", "--user", "list-timers", "--all"], + "systemd-status.txt": ["systemctl", "--user", "status", "openclaw*", "--no-pager"], + "systemd-journal.txt": ["journalctl", "--user", "--no-pager", "-n", "2000"], + "docker-ps.txt": ["docker", "ps", "-a"], + } + commands = { + f"launchctl-{label}.txt": ["launchctl", "print", f"gui/{os.getuid()}/{label}"] + for label in LAUNCHD_LABELS + } + commands["docker-ps.txt"] = ["docker", "ps", "-a"] + return commands + + +def collect_lane_diagnostics(*, context_file: Path, output_dir: Path) -> None: + """Capture service-manager state and gateway logs into *output_dir*.""" + context = load_context(context_file) + env = phase_env(context) + secrets = secret_values( + varlock_local_env_file( + Path(context.repo_root), home_dir=Path(context.app_home), environ=env + ) + ) + + output_dir.mkdir(parents=True, exist_ok=True) + for name, command in _diagnostic_commands(context.platform).items(): + try: + completed = subprocess.run( # noqa: S603 - fixed diagnostic argv + command, + env=env, + capture_output=True, + text=True, + timeout=_COMMAND_TIMEOUT_SECONDS, + check=False, + ) + text = ( + f"$ {' '.join(command)}\nexit={completed.returncode}\n" + f"{completed.stdout}\n{completed.stderr}" + ) + except (OSError, subprocess.TimeoutExpired) as exc: + text = f"$ {' '.join(command)}\nfailed: {exc}" + (output_dir / name).write_text(redact(text, secrets), encoding="utf-8") + for log_root in (Path("/tmp/openclaw"), Path(context.app_home) / ".openclaw" / "logs"): + if not log_root.is_dir(): + continue + for log_file in sorted(log_root.glob("*.log"))[-5:]: + tail = log_file.read_text(encoding="utf-8", errors="replace")[-_LOG_TAIL_CHARS:] + (output_dir / f"{log_root.name}-{log_file.name}").write_text( + redact(tail, secrets), encoding="utf-8" + ) diff --git a/tests/utils/helpers/_feature_verification/envvars.py b/tests/utils/helpers/_feature_verification/envvars.py new file mode 100644 index 00000000..92bb270d --- /dev/null +++ b/tests/utils/helpers/_feature_verification/envvars.py @@ -0,0 +1,10 @@ +"""Environment variables that carry lane state from ``run-lane`` into the pytest session.""" + +from typing import Final + +LANE_ENV: Final[str] = "FEATURE_VERIFICATION_LANE" +CONTEXT_ENV: Final[str] = "FEATURE_VERIFICATION_CONTEXT" +RESULTS_ENV: Final[str] = "FEATURE_VERIFICATION_RESULTS" +WORK_DIR_ENV: Final[str] = "FEATURE_VERIFICATION_WORK_DIR" +FAKE_MODEL_DIR_ENV: Final[str] = "FEATURE_VERIFICATION_FAKE_MODEL_DIR" +MANIFEST_ENV: Final[str] = "FEATURE_VERIFICATION_MANIFEST" diff --git a/tests/utils/helpers/_feature_verification/fake_model.py b/tests/utils/helpers/_feature_verification/fake_model.py new file mode 100644 index 00000000..37e1010b --- /dev/null +++ b/tests/utils/helpers/_feature_verification/fake_model.py @@ -0,0 +1,460 @@ +"""Deterministic OpenAI- and Ollama-compatible model server for feature verification. + +The server stands in for every model provider the product talks to on a runner: the OpenClaw +agent chain (an explicit OpenAI-compatible ``models.providers`` entry), the memory plugins' +embedding and extraction endpoints, and the LiteLLM sidecar's Ollama upstream. Replies are +canned, so assertions can rely on exact content, and every request is journaled so a test can +prove the product actually reached the model rather than short-circuiting. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +import threading +import time +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final, cast + +CHAT_MODEL_ID: Final[str] = "featver-chat" +EMBEDDING_MODEL_ID: Final[str] = "featver-embed" +DEFAULT_EMBEDDING_DIMENSIONS: Final[int] = 768 +DEFAULT_CONTEXT_WINDOW: Final[int] = 131_072 +REPLY_PREFIX: Final[str] = "FEATVER_REPLY" +ECHO_TOKEN_PATTERN: Final[re.Pattern[str]] = re.compile(r"\bFV-[A-Za-z0-9][A-Za-z0-9_-]{3,63}\b") +JOURNAL_PATH: Final[str] = "/__featver/requests" +HEALTH_PATH: Final[str] = "/__featver/health" +_MAX_JOURNAL_ENTRIES: Final[int] = 2000 +_PROMPT_EXCERPT_CHARS: Final[int] = 4000 +# Models advertised by /v1/models and /api/tags; requests naming any other model still succeed. +ADVERTISED_MODELS: Final[tuple[str, ...]] = ( + CHAT_MODEL_ID, + EMBEDDING_MODEL_ID, + "nomic-embed-text", + "qwen2.5:7b", +) + +type JsonObject = dict[str, object] +type ReplyRule = tuple[Callable[[str], bool], Callable[[str], str]] + + +def _as_object(value: object) -> JsonObject: + """Return *value* as a string-keyed mapping, or an empty mapping.""" + if not isinstance(value, dict): + return {} + return {str(key): item for key, item in cast(Mapping[object, object], value).items()} + + +def _content_text(content: object) -> str: + """Flatten OpenAI/Ollama message content (string or content parts) into text.""" + if isinstance(content, str): + return content + if isinstance(content, list): + parts: list[str] = [] + for part in cast(Sequence[object], content): + part_mapping = _as_object(part) + text = part_mapping.get("text") + if isinstance(text, str): + parts.append(text) + return "\n".join(parts) + return "" + + +def _messages_text(payload: JsonObject) -> tuple[str, str]: + """Return (all message text, latest user message text) for a chat payload.""" + messages = payload.get("messages") + if not isinstance(messages, list): + prompt = payload.get("prompt") + text = prompt if isinstance(prompt, str) else "" + return text, text + all_parts: list[str] = [] + latest_user = "" + for raw_message in cast(Sequence[object], messages): + message = _as_object(raw_message) + text = _content_text(message.get("content")) + all_parts.append(text) + if message.get("role") == "user": + latest_user = text + return "\n".join(all_parts), latest_user + + +def _is_reflection_prompt(text: str) -> bool: + """Return whether *text* is the memory-lancedb-pro reflection prompt.""" + return "## Invariants" in text and "## Derived" in text + + +def _reflection_reply(_: str) -> str: + """Return a reflection document the lancedb-pro parser accepts.""" + return ( + "## Context (session background)\n" + "- Feature verification session for the StrongClaw memory plugin.\n\n" + "## Decisions (durable)\n" + "- Keep feature verification replies deterministic.\n\n" + "## User model deltas (about the human)\n" + "- (none captured)\n\n" + "## Agent model deltas (about the assistant/system)\n" + "- (none captured)\n\n" + "## Lessons & pitfalls (symptom / cause / fix / prevention)\n" + "- (none captured)\n\n" + "## Learning governance candidates (.learnings / promotion / skill extraction)\n" + "- (none captured)\n\n" + "## Open loops / next actions\n" + "- (none captured)\n\n" + "## Retrieval tags / keywords\n" + "- featver, reflection\n\n" + "## Invariants\n" + "- Always verify features against the real runtime (FEATVER_REFLECTION_INVARIANT).\n\n" + "## Derived\n" + "- Prefer the embedded runner for reflection runs (FEATVER_REFLECTION_DERIVED).\n" + ) + + +def _is_extraction_prompt(text: str) -> bool: + """Return whether *text* is the memory-lancedb-pro smart-extraction prompt.""" + return "extract memories worth long-term preservation" in text + + +def _extraction_reply(_: str) -> str: + """Return an empty structured-extraction result.""" + return json.dumps({"memories": []}) + + +def _default_reply(latest_user: str) -> str: + """Return the canned agent reply, echoing any FV- token from the latest user turn.""" + tokens = ECHO_TOKEN_PATTERN.findall(latest_user) + suffix = " ".join(dict.fromkeys(tokens)) if tokens else "acknowledged" + return f"{REPLY_PREFIX} {suffix}" + + +REPLY_RULES: Final[tuple[ReplyRule, ...]] = ( + (_is_reflection_prompt, _reflection_reply), + (_is_extraction_prompt, _extraction_reply), +) + + +def canned_reply(all_text: str, latest_user: str) -> str: + """Return the deterministic reply for one chat request.""" + for matches, respond in REPLY_RULES: + if matches(all_text): + return respond(all_text) + return _default_reply(latest_user) + + +def deterministic_embedding( + text: str, dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS +) -> list[float]: + """Return a unit-length, content-derived embedding so equal texts embed identically.""" + values: list[float] = [] + counter = 0 + while len(values) < dimensions: + digest = hashlib.sha256(f"{counter}:{text}".encode()).digest() + values.extend((byte - 127.5) / 127.5 for byte in digest) + counter += 1 + vector = values[:dimensions] + norm = math.sqrt(sum(component * component for component in vector)) or 1.0 + return [round(component / norm, 6) for component in vector] + + +@dataclass(slots=True) +class RequestJournal: + """Thread-safe bounded record of the requests the server answered.""" + + entries: list[JsonObject] = field(default_factory=lambda: cast(list[JsonObject], [])) + lock: threading.Lock = field(default_factory=threading.Lock) + + def record(self, entry: JsonObject) -> None: + """Append one entry, dropping the oldest beyond the bound.""" + with self.lock: + self.entries.append(entry) + del self.entries[:-_MAX_JOURNAL_ENTRIES] + + def snapshot(self) -> list[JsonObject]: + """Return a copy of the recorded entries.""" + with self.lock: + return list(self.entries) + + +class FakeModelHandler(BaseHTTPRequestHandler): + """Route OpenAI-compatible and Ollama-native requests to canned responders.""" + + server_version = "featver-fake-model/1" + journal: RequestJournal = RequestJournal() + + def log_message(self, format: str, *args: object) -> None: # noqa: A002 + """Silence per-request stderr logging; the journal is the audit trail.""" + del format, args + + def _read_json(self) -> JsonObject: + length = int(self.headers.get("Content-Length") or 0) + if length <= 0: + return {} + raw = self.rfile.read(length) + try: + return _as_object(json.loads(raw.decode("utf-8"))) + except (UnicodeDecodeError, json.JSONDecodeError): + return {} + + def _send_json(self, payload: object, status: HTTPStatus = HTTPStatus.OK) -> None: + body = json.dumps(payload).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _record(self, payload: JsonObject, *, reply: str | None = None) -> None: + all_text, latest_user = _messages_text(payload) + self.journal.record( + { + "at": round(time.time(), 3), + "method": self.command, + "path": self.path.split("?", 1)[0], + "port": cast(tuple[str, int], self.server.server_address)[1], + "model": payload.get("model"), + "stream": bool(payload.get("stream")), + "latestUser": latest_user[:_PROMPT_EXCERPT_CHARS], + "prompt": all_text[:_PROMPT_EXCERPT_CHARS], + "reply": reply, + } + ) + + def do_GET(self) -> None: # noqa: N802 + """Serve discovery, health, and journal endpoints.""" + path = self.path.split("?", 1)[0] + if path == HEALTH_PATH: + self._send_json({"ok": True}) + elif path == JOURNAL_PATH: + self._send_json({"requests": self.journal.snapshot()}) + elif path in {"/v1/models", "/models"}: + self._send_json( + { + "object": "list", + "data": [ + {"id": model, "object": "model", "created": 0, "owned_by": "featver"} + for model in ADVERTISED_MODELS + ], + } + ) + elif path == "/api/tags": + self._send_json( + {"models": [{"name": model, "model": model} for model in ADVERTISED_MODELS]} + ) + elif path == "/api/version": + self._send_json({"version": "0.0.0-featver"}) + elif path in {"/", "/health", "/health/liveliness"}: + self._send_json({"status": "ok"}) + else: + self._send_json({"error": f"unknown path {path}"}, HTTPStatus.NOT_FOUND) + + def do_POST(self) -> None: # noqa: N802 + """Serve chat, completion, and embedding endpoints.""" + path = self.path.split("?", 1)[0] + payload = self._read_json() + if path in {"/v1/chat/completions", "/chat/completions"}: + self._openai_chat(payload) + elif path in {"/v1/completions", "/completions"}: + self._openai_completion(payload) + elif path in {"/v1/embeddings", "/embeddings"}: + self._openai_embeddings(payload) + elif path == "/api/chat": + self._ollama_chat(payload) + elif path in {"/api/embed", "/api/embeddings"}: + self._ollama_embeddings(payload, legacy=path == "/api/embeddings") + elif path == "/api/show": + self._send_json( + { + "details": {"family": "featver"}, + "model_info": {"featver.context_length": DEFAULT_CONTEXT_WINDOW}, + "capabilities": ["completion", "embedding"], + } + ) + else: + self._record(payload) + self._send_json({"error": f"unknown path {path}"}, HTTPStatus.NOT_FOUND) + + def _openai_chat(self, payload: JsonObject) -> None: + all_text, latest_user = _messages_text(payload) + reply = canned_reply(all_text, latest_user) + self._record(payload, reply=reply) + model = str(payload.get("model") or CHAT_MODEL_ID) + completion_id = f"chatcmpl-featver-{int(time.time() * 1000)}" + usage = {"prompt_tokens": 16, "completion_tokens": 8, "total_tokens": 24} + if not payload.get("stream"): + self._send_json( + { + "id": completion_id, + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": reply}, + "finish_reason": "stop", + } + ], + "usage": usage, + } + ) + return + chunks: list[JsonObject] = [ + {"index": 0, "delta": {"role": "assistant", "content": ""}, "finish_reason": None}, + {"index": 0, "delta": {"content": reply}, "finish_reason": None}, + {"index": 0, "delta": {}, "finish_reason": "stop"}, + ] + events: list[JsonObject] = [ + { + "id": completion_id, + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model, + "choices": [choice], + } + for choice in chunks + ] + events.append( + { + "id": completion_id, + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model, + "choices": [], + "usage": usage, + } + ) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Connection", "close") + self.end_headers() + for event in events: + self.wfile.write(f"data: {json.dumps(event)}\n\n".encode()) + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + self.close_connection = True + + def _openai_completion(self, payload: JsonObject) -> None: + prompt = payload.get("prompt") + text = prompt if isinstance(prompt, str) else "" + reply = canned_reply(text, text) + self._record(payload, reply=reply) + self._send_json( + { + "id": "cmpl-featver", + "object": "text_completion", + "created": int(time.time()), + "model": str(payload.get("model") or CHAT_MODEL_ID), + "choices": [{"index": 0, "text": reply, "finish_reason": "stop"}], + } + ) + + @staticmethod + def _embedding_inputs(raw: object) -> list[str]: + if isinstance(raw, str): + return [raw] + if isinstance(raw, list): + return [str(item) for item in cast(Sequence[object], raw)] + return [""] + + @staticmethod + def _dimensions(payload: JsonObject) -> int: + raw = payload.get("dimensions") + return raw if isinstance(raw, int) and raw > 0 else DEFAULT_EMBEDDING_DIMENSIONS + + def _openai_embeddings(self, payload: JsonObject) -> None: + self._record(payload) + inputs = self._embedding_inputs(payload.get("input")) + dimensions = self._dimensions(payload) + self._send_json( + { + "object": "list", + "model": str(payload.get("model") or EMBEDDING_MODEL_ID), + "data": [ + { + "object": "embedding", + "index": index, + "embedding": deterministic_embedding(text, dimensions), + } + for index, text in enumerate(inputs) + ], + "usage": {"prompt_tokens": len(inputs), "total_tokens": len(inputs)}, + } + ) + + def _ollama_embeddings(self, payload: JsonObject, *, legacy: bool) -> None: + self._record(payload) + dimensions = self._dimensions(payload) + if legacy: + prompt = payload.get("prompt") + text = prompt if isinstance(prompt, str) else "" + self._send_json({"embedding": deterministic_embedding(text, dimensions)}) + return + inputs = self._embedding_inputs(payload.get("input")) + self._send_json( + { + "model": str(payload.get("model") or EMBEDDING_MODEL_ID), + "embeddings": [deterministic_embedding(text, dimensions) for text in inputs], + } + ) + + def _ollama_chat(self, payload: JsonObject) -> None: + all_text, latest_user = _messages_text(payload) + reply = canned_reply(all_text, latest_user) + self._record(payload, reply=reply) + model = str(payload.get("model") or CHAT_MODEL_ID) + final: JsonObject = { + "model": model, + "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "message": {"role": "assistant", "content": reply}, + "done": True, + "done_reason": "stop", + "prompt_eval_count": 16, + "eval_count": 8, + } + if payload.get("stream") is False: + self._send_json(final) + return + body = json.dumps(final).encode("utf-8") + b"\n" + self.send_response(HTTPStatus.OK) + self.send_header("Content-Type", "application/x-ndjson") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +class FakeModelServer: + """Serve the canned model API on one or more ports from a shared journal.""" + + def __init__(self, *, host: str, ports: Sequence[int]) -> None: + if not ports: + raise ValueError("FakeModelServer needs at least one port") + self._servers = [ThreadingHTTPServer((host, port), FakeModelHandler) for port in ports] + for server in self._servers: + server.daemon_threads = True + self._threads: list[threading.Thread] = [] + + @property + def ports(self) -> list[int]: + """Return the bound ports (resolves ephemeral port 0 requests).""" + return [int(server.server_address[1]) for server in self._servers] + + def start(self) -> None: + """Serve every port on a daemon thread.""" + for server in self._servers: + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self._threads.append(thread) + + def stop(self) -> None: + """Shut down every listener.""" + for server in self._servers: + server.shutdown() + server.server_close() + for thread in self._threads: + thread.join(timeout=5) + self._threads.clear() diff --git a/tests/utils/helpers/_feature_verification/host.py b/tests/utils/helpers/_feature_verification/host.py new file mode 100644 index 00000000..0b063f57 --- /dev/null +++ b/tests/utils/helpers/_feature_verification/host.py @@ -0,0 +1,307 @@ +"""Black-box access to a provisioned feature-verification host. + +Tests drive the product only through its public entrypoints (``clawops``, ``openclaw`` via +Varlock, ``acpx``) as an operator would, with an environment scrubbed of harness variables so a +nested product run (for example ``clawops baseline verify`` running pytest) never re-enters +the feature suite. +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import time +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Final, cast + +from clawops.strongclaw_runtime import ( + load_env_assignments, + load_openclaw_config, + resolve_openclaw_config_path, + varlock_env_dir, + varlock_local_env_file, + wrap_command_with_varlock, +) +from tests.utils.helpers._feature_verification.envvars import ( + CONTEXT_ENV, + FAKE_MODEL_DIR_ENV, + LANE_ENV, + MANIFEST_ENV, + WORK_DIR_ENV, +) +from tests.utils.helpers._feature_verification.manifest import ( + FeatureManifest, + FeatureVerificationError, + Lane, + load_manifest, +) +from tests.utils.helpers._feature_verification.processes import fetch_journal, load_state +from tests.utils.helpers._feature_verification.redaction import redact, secret_values +from tests.utils.helpers._fresh_host.models import FreshHostContext +from tests.utils.helpers._fresh_host.storage import load_context + +_HARNESS_ENV_PREFIXES: Final[tuple[str, ...]] = ("FEATURE_VERIFICATION_", "PYTEST_") +_FRAMEWORK_ENV_KEYS: Final[frozenset[str]] = frozenset({"TEST_ID", "RESOURCE_PREFIX", "WORKER_ID"}) +_JSON_START: Final[re.Pattern[str]] = re.compile(r"(?m)^[ \t]*[\[{]") +_OUTPUT_TAIL_CHARS: Final[int] = 3000 + + +def default_manifest_path(repo_root: Path) -> Path: + """Return the checked-in feature matrix path.""" + return repo_root / "tests" / "feature_matrix.yaml" + + +def extract_json(text: str) -> object: + """Return the first JSON document embedded in mixed command output.""" + decoder = json.JSONDecoder() + for match in _JSON_START.finditer(text): + try: + payload, _ = decoder.raw_decode(text[match.start() :].lstrip()) + except json.JSONDecodeError: + continue + return cast(object, payload) + raise AssertionError(f"no JSON document in command output: {text[-_OUTPUT_TAIL_CHARS:]!r}") + + +def reply_texts(payload: object) -> list[str]: + """Collect every ``payloads[].text`` from an ``openclaw agent --json`` envelope.""" + texts: list[str] = [] + if isinstance(payload, dict): + mapping = cast(dict[str, object], payload) + for key, value in mapping.items(): + if key == "payloads" and isinstance(value, list): + texts.extend( + str(cast(dict[str, object], item).get("text")) + for item in cast(list[object], value) + if isinstance(item, dict) and cast(dict[str, object], item).get("text") + ) + else: + texts.extend(reply_texts(value)) + elif isinstance(payload, list): + for item in cast(list[object], payload): + texts.extend(reply_texts(item)) + return texts + + +@dataclass(frozen=True, slots=True) +class CommandOutcome: + """Captured result of one product command.""" + + argv: tuple[str, ...] + returncode: int + stdout: str + stderr: str + duration_seconds: float + + @property + def ok(self) -> bool: + """Return whether the command exited 0.""" + return self.returncode == 0 + + @property + def output(self) -> str: + """Return stdout and stderr combined.""" + return f"{self.stdout}\n{self.stderr}" + + def describe(self) -> str: + """Return a bounded, already-redacted description for assertion messages.""" + return ( + f"`{' '.join(self.argv)}` exited {self.returncode} after {self.duration_seconds:.1f}s\n" + f"stdout tail:\n{self.stdout[-_OUTPUT_TAIL_CHARS:]}\n" + f"stderr tail:\n{self.stderr[-_OUTPUT_TAIL_CHARS:]}" + ) + + def require_ok(self) -> CommandOutcome: + """Assert a zero exit status.""" + assert self.ok, self.describe() + return self + + def json(self) -> object: + """Parse the first JSON document from stdout.""" + return extract_json(self.stdout) + + +@dataclass(slots=True) +class FeatureHost: + """The provisioned host one feature-verification lane runs against.""" + + lane: Lane + manifest: FeatureManifest = field(repr=False) + context: FreshHostContext = field(repr=False) + work_dir: Path + fake_model_dir: Path | None + _secrets: tuple[str, ...] = field(default_factory=tuple, repr=False) + + @classmethod + def from_environment(cls, environ: Mapping[str, str] | None = None) -> FeatureHost: + """Build the host from the lane environment exported by ``run-lane``.""" + env = os.environ if environ is None else environ + lane_id = env.get(LANE_ENV, "").strip() + context_file = env.get(CONTEXT_ENV, "").strip() + if not lane_id or not context_file: + raise FeatureVerificationError(f"{LANE_ENV} and {CONTEXT_ENV} must both be set") + context = load_context(Path(context_file)) + manifest_path = Path( + env.get(MANIFEST_ENV, "").strip() + or default_manifest_path(Path(context.repo_root)).as_posix() + ) + manifest = load_manifest(manifest_path) + if lane_id not in manifest.lanes: + raise FeatureVerificationError(f"lane {lane_id!r} is not in {manifest_path}") + work_dir = Path(env.get(WORK_DIR_ENV, "").strip() or Path(context.tmp_root) / "featver") + work_dir.mkdir(parents=True, exist_ok=True) + fake_model_raw = env.get(FAKE_MODEL_DIR_ENV, "").strip() + return cls( + lane=manifest.lanes[lane_id], + manifest=manifest, + context=context, + work_dir=work_dir, + fake_model_dir=Path(fake_model_raw) if fake_model_raw else None, + ) + + @property + def repo_root(self) -> Path: + """Return the StrongClaw checkout under test.""" + return Path(self.context.repo_root) + + @property + def home(self) -> Path: + """Return the HOME the product was installed into.""" + return Path(self.context.app_home) + + @property + def python(self) -> Path: + """Return the managed-environment interpreter created by bootstrap.""" + return self.repo_root / ".venv" / "bin" / "python" + + def scratch(self, name: str) -> Path: + """Return a fresh per-test scratch directory.""" + path = self.work_dir / f"{name}-{time.time_ns()}" + path.mkdir(parents=True) + return path + + def env_file(self) -> Path: + """Return the Varlock local env file the operator edits.""" + return varlock_local_env_file(self.repo_root) + + def env_dir(self) -> Path: + """Return the Varlock env directory.""" + return varlock_env_dir(self.repo_root) + + def env_values(self) -> dict[str, str]: + """Return the operator env contract assignments.""" + return load_env_assignments(self.env_file()) + + def _refresh_secrets(self) -> None: + self._secrets = secret_values(self.env_file()) + + def redact(self, text: str) -> str: + """Replace known secret values in *text*.""" + return redact(text, self._secrets) + + def product_env(self, overrides: Mapping[str, str] | None = None) -> dict[str, str]: + """Return the operator environment with harness variables removed.""" + env = { + key: value + for key, value in os.environ.items() + if key not in _FRAMEWORK_ENV_KEYS and not key.startswith(_HARNESS_ENV_PREFIXES) + } + if overrides: + env.update(overrides) + return env + + def run( + self, + argv: Sequence[str], + *, + cwd: Path | None = None, + timeout: float = 300, + env: Mapping[str, str] | None = None, + input_text: str | None = None, + ) -> CommandOutcome: + """Run one command and capture its redacted output.""" + self._refresh_secrets() + started = time.monotonic() + command = tuple(str(part) for part in argv) + try: + completed = subprocess.run( # noqa: S603 - argv built by the harness + command, + cwd=cwd or self.repo_root, + env=self.product_env(env), + input=input_text, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout if isinstance(exc.stdout, str) else "" + stderr = exc.stderr if isinstance(exc.stderr, str) else "" + return CommandOutcome( + argv=command, + returncode=-9, + stdout=self.redact(stdout), + stderr=self.redact(f"{stderr}\n[harness] timed out after {timeout}s"), + duration_seconds=time.monotonic() - started, + ) + return CommandOutcome( + argv=command, + returncode=completed.returncode, + stdout=self.redact(completed.stdout), + stderr=self.redact(completed.stderr), + duration_seconds=time.monotonic() - started, + ) + + def clawops( + self, + *arguments: str, + cwd: Path | None = None, + timeout: float = 300, + env: Mapping[str, str] | None = None, + ) -> CommandOutcome: + """Run the managed ``clawops`` CLI.""" + return self.run( + [str(self.python), "-m", "clawops", *arguments], cwd=cwd, timeout=timeout, env=env + ) + + def openclaw( + self, + *arguments: str, + cwd: Path | None = None, + timeout: float = 300, + env: Mapping[str, str] | None = None, + ) -> CommandOutcome: + """Run ``openclaw`` exactly as StrongClaw does: inside ``varlock run``.""" + argv = wrap_command_with_varlock(self.repo_root, ["openclaw", *arguments]) + return self.run(argv, cwd=cwd, timeout=timeout, env=env) + + def clawops_managed( + self, + *arguments: str, + cwd: Path | None = None, + timeout: float = 300, + env: Mapping[str, str] | None = None, + ) -> CommandOutcome: + """Run ``clawops`` inside ``varlock run``, as StrongClaw's own runtime callers do.""" + argv = wrap_command_with_varlock( + self.repo_root, [str(self.python), "-m", "clawops", *arguments] + ) + return self.run(argv, cwd=cwd, timeout=timeout, env=env) + + def fake_model_requests(self) -> list[dict[str, object]]: + """Return every request the fake model server has answered so far.""" + if self.fake_model_dir is None: + raise FeatureVerificationError(f"{FAKE_MODEL_DIR_ENV} is not set for this lane") + state = load_state(self.fake_model_dir) + ports = state.get("ports") + if not isinstance(ports, list) or not ports: + raise FeatureVerificationError("fake model state has no ports") + return fetch_journal(str(state.get("host", "127.0.0.1")), int(cast(int, ports[0]))) + + def rendered_config(self) -> dict[str, object]: + """Return the rendered OpenClaw config.""" + return dict(load_openclaw_config(resolve_openclaw_config_path(self.repo_root))) diff --git a/tests/utils/helpers/_feature_verification/http.py b/tests/utils/helpers/_feature_verification/http.py new file mode 100644 index 00000000..82c542d9 --- /dev/null +++ b/tests/utils/helpers/_feature_verification/http.py @@ -0,0 +1,57 @@ +"""Loopback HTTP helpers for probing sidecars from feature tests.""" + +from __future__ import annotations + +import base64 +import json +import urllib.error +import urllib.request +from collections.abc import Mapping +from dataclasses import dataclass +from typing import cast + + +@dataclass(frozen=True, slots=True) +class HttpResponse: + """One HTTP response.""" + + status: int + body: str + + def json(self) -> object: + """Decode the body as JSON.""" + return cast(object, json.loads(self.body)) if self.body else {} + + +# Loopback probes must never route through system or environment proxies. +_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + +def basic_auth(user: str, password: str) -> dict[str, str]: + """Return a basic-auth header.""" + token = base64.b64encode(f"{user}:{password}".encode()).decode("ascii") + return {"Authorization": f"Basic {token}"} + + +def http_request( + method: str, + url: str, + *, + body: object = None, + headers: Mapping[str, str] | None = None, + timeout: float = 30.0, +) -> HttpResponse: + """Issue one loopback HTTP request and return status plus body, even for HTTP errors.""" + data = None if body is None else json.dumps(body).encode("utf-8") + request = urllib.request.Request(url, data=data, method=method) + request.add_header("Content-Type", "application/json") + for key, value in (headers or {}).items(): + request.add_header(key, value) + try: + with _OPENER.open(request, timeout=timeout) as response: + return HttpResponse(int(response.status), response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return HttpResponse(int(exc.code), exc.read().decode("utf-8", errors="replace")) + except (urllib.error.URLError, OSError) as exc: + # Status 0 marks "no HTTP response" so callers can poll and assert uniformly. + return HttpResponse(0, f"unreachable: {exc}") diff --git a/tests/utils/helpers/_feature_verification/lane.py b/tests/utils/helpers/_feature_verification/lane.py new file mode 100644 index 00000000..4180fe43 --- /dev/null +++ b/tests/utils/helpers/_feature_verification/lane.py @@ -0,0 +1,120 @@ +"""Lane provisioning and execution steps for the feature-verification workflow.""" + +from __future__ import annotations + +import os +import subprocess +import time +from pathlib import Path +from typing import Final + +from tests.utils.helpers._feature_verification.envvars import ( + CONTEXT_ENV, + FAKE_MODEL_DIR_ENV, + LANE_ENV, + MANIFEST_ENV, + RESULTS_ENV, + WORK_DIR_ENV, +) +from tests.utils.helpers._fresh_host.budget import ( + SYSTEM_CLOCK, + JobBudget, + scenario_phase_deadline, +) +from tests.utils.helpers._fresh_host.shell import phase_env +from tests.utils.helpers._fresh_host.storage import load_context + +FEATURE_SUITE_PATH: Final[str] = "tests/suites/e2e/feature_verification" +DEFAULT_LANE_TIMEOUT_SECONDS: Final[int] = 3600 + + +class LaneError(RuntimeError): + """Raised when a lane step cannot run.""" + + +LINGER_READY_TIMEOUT_SECONDS: Final[float] = 60.0 +_LINGER_POLL_SECONDS: Final[float] = 1.0 + + +def enable_systemd_user_manager() -> Path: + """Start the runner's systemd user manager (lingering) and wait for its bus.""" + user = ( + os.environ.get("USER") + or subprocess.run(["id", "-un"], check=True, capture_output=True, text=True).stdout.strip() + ) + subprocess.run(["sudo", "loginctl", "enable-linger", user], check=True) + runtime_dir = Path(f"/run/user/{os.getuid()}") + bus = runtime_dir / "bus" + deadline = time.monotonic() + LINGER_READY_TIMEOUT_SECONDS + while time.monotonic() < deadline: + if bus.exists(): + probe = subprocess.run( + ["systemctl", "--user", "is-system-running"], + env={**os.environ, "XDG_RUNTIME_DIR": str(runtime_dir)}, + capture_output=True, + text=True, + check=False, + ) + if probe.stdout.strip() in {"running", "degraded", "starting"}: + return runtime_dir + time.sleep(_LINGER_POLL_SECONDS) + raise LaneError(f"systemd user manager bus {bus} did not come up") + + +def _lane_timeout_seconds() -> int: + budget = JobBudget.from_env(os.environ) + if budget is None: + return DEFAULT_LANE_TIMEOUT_SECONDS + return max(60, int(scenario_phase_deadline(budget).remaining(SYSTEM_CLOCK))) + + +def run_lane( + *, + context_file: Path, + lane_id: str, + results_path: Path, + fake_model_dir: Path | None, + manifest_path: Path | None = None, +) -> int: + """Run the lane's feature tests against the provisioned host; return pytest's exit code.""" + context = load_context(context_file) + repo_root = Path(context.repo_root) + # The pytest session validates the lane against the matrix (this step runs without PyYAML). + manifest_file = manifest_path or repo_root / "tests" / "feature_matrix.yaml" + env = phase_env(context) + env.update( + { + # Fresh-host setup skips model auth; the lane's own commands run it for real. + "OPENCLAW_MODEL_SETUP_MODE": "auto", + LANE_ENV: lane_id, + CONTEXT_ENV: str(context_file), + RESULTS_ENV: str(results_path), + WORK_DIR_ENV: str(Path(context.tmp_root) / "featver"), + MANIFEST_ENV: str(manifest_file), + } + ) + # fresh-host injects an embedding-model default; the lane reads it from the env contract. + env.pop("HYPERMEMORY_EMBEDDING_MODEL", None) + if fake_model_dir is not None: + env[FAKE_MODEL_DIR_ENV] = str(fake_model_dir) + command = [ + str(repo_root / ".venv" / "bin" / "python"), + "-m", + "pytest", + FEATURE_SUITE_PATH, + "-m", + "feature_verification", + "-p", + "no:cacheprovider", + "-rA", + "--durations=15", + ] + timeout = _lane_timeout_seconds() + print(f"[feature-verification] lane={lane_id} timeout={timeout}s: {' '.join(command)}") + try: + completed = subprocess.run(command, cwd=repo_root, env=env, check=False, timeout=timeout) + except subprocess.TimeoutExpired as exc: + raise LaneError(f"lane {lane_id} exceeded its {timeout}s budget") from exc + if not results_path.is_file(): + raise LaneError(f"lane {lane_id} wrote no results to {results_path}") + return completed.returncode diff --git a/tests/utils/helpers/_feature_verification/manifest.py b/tests/utils/helpers/_feature_verification/manifest.py new file mode 100644 index 00000000..0aed6afd --- /dev/null +++ b/tests/utils/helpers/_feature_verification/manifest.py @@ -0,0 +1,235 @@ +"""Typed loader and validator for the checked-in feature matrix (``tests/feature_matrix.yaml``).""" + +from __future__ import annotations + +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal, cast + +import yaml + +type PlatformName = Literal["linux", "macos"] +type Verification = Literal["live", "fake-model", "stub"] +type FeatureStatus = Literal["verifiable", "unverifiable"] + +MANIFEST_SCHEMA_VERSION: Final[int] = 1 +PLATFORMS: Final[frozenset[str]] = frozenset({"linux", "macos"}) +VERIFICATIONS: Final[frozenset[str]] = frozenset({"live", "fake-model", "stub"}) +STATUSES: Final[frozenset[str]] = frozenset({"verifiable", "unverifiable"}) +FEATURE_ID_PATTERN: Final[re.Pattern[str]] = re.compile(r"^[a-z0-9]+(?:[.-][a-z0-9]+)*$") +CALL_SITE_PATTERN: Final[re.Pattern[str]] = re.compile(r"^(?P[^:\s]+):(?P[1-9]\d*)$") +TEST_NODE_PATTERN: Final[re.Pattern[str]] = re.compile( + r"^(?Ptests/suites/e2e/feature_verification/test_[a-z0-9_]+\.py)::(?Ptest_[a-z0-9_]+)$" +) + + +class FeatureVerificationError(RuntimeError): + """Raised when the feature matrix or a lane result is invalid.""" + + +@dataclass(frozen=True, slots=True) +class Lane: + """One provisioned CI host the suite runs on.""" + + lane_id: str + platform: PlatformName + profile: str + scenario: str + activate_services: bool + description: str + + +@dataclass(frozen=True, slots=True) +class Feature: + """One product feature and the test that verifies it.""" + + feature_id: str + category: str + title: str + call_sites: tuple[str, ...] + status: FeatureStatus + verification: Verification | None + lanes: tuple[str, ...] + test: str | None + reason: str | None + limitations: str | None + + @property + def test_path(self) -> str | None: + """Return the repo-relative test module path.""" + return None if self.test is None else self.test.split("::", 1)[0] + + @property + def test_name(self) -> str | None: + """Return the test function name.""" + return None if self.test is None else self.test.split("::", 1)[1] + + +@dataclass(frozen=True, slots=True) +class FeatureManifest: + """Validated feature matrix.""" + + lanes: Mapping[str, Lane] + features: tuple[Feature, ...] + + def feature(self, feature_id: str) -> Feature: + """Return one feature by id.""" + for feature in self.features: + if feature.feature_id == feature_id: + return feature + raise FeatureVerificationError(f"unknown feature {feature_id!r}") + + def features_for_lane(self, lane_id: str) -> tuple[Feature, ...]: + """Return the verifiable features scheduled on one lane.""" + return tuple(feature for feature in self.features if lane_id in feature.lanes) + + def feature_by_test(self) -> dict[str, Feature]: + """Map each test node id to the feature it verifies.""" + return {feature.test: feature for feature in self.features if feature.test is not None} + + def platforms(self, feature: Feature) -> tuple[PlatformName, ...]: + """Return the platforms one feature runs on, derived from its lanes.""" + return tuple(sorted({self.lanes[lane_id].platform for lane_id in feature.lanes})) + + +def _mapping(value: object, *, path: str) -> Mapping[str, object]: + if not isinstance(value, Mapping): + raise FeatureVerificationError(f"{path} must be a mapping") + return cast(Mapping[str, object], value) + + +def _string(value: object, *, path: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise FeatureVerificationError(f"{path} must be a non-empty string") + return value.strip() + + +def _optional_string(value: object, *, path: str) -> str | None: + return None if value is None else _string(value, path=path) + + +def _string_tuple(value: object, *, path: str) -> tuple[str, ...]: + if value is None: + return () + if not isinstance(value, Sequence) or isinstance(value, str): + raise FeatureVerificationError(f"{path} must be a list of strings") + return tuple( + _string(item, path=f"{path}[{index}]") + for index, item in enumerate(cast(Sequence[object], value)) + ) + + +def _choice(value: object, *, allowed: frozenset[str], path: str) -> str: + text = _string(value, path=path) + if text not in allowed: + raise FeatureVerificationError(f"{path} must be one of {sorted(allowed)}, got {text!r}") + return text + + +def _load_lane(lane_id: str, raw: object) -> Lane: + path = f"lanes.{lane_id}" + payload = _mapping(raw, path=path) + activate = payload.get("activate_services") + if not isinstance(activate, bool): + raise FeatureVerificationError(f"{path}.activate_services must be a boolean") + return Lane( + lane_id=lane_id, + platform=cast( + PlatformName, + _choice(payload.get("platform"), allowed=PLATFORMS, path=f"{path}.platform"), + ), + profile=_string(payload.get("profile"), path=f"{path}.profile"), + scenario=_string(payload.get("scenario"), path=f"{path}.scenario"), + activate_services=activate, + description=_string(payload.get("description"), path=f"{path}.description"), + ) + + +def _load_feature(index: int, raw: object, *, lanes: Mapping[str, Lane]) -> Feature: + path = f"features[{index}]" + payload = _mapping(raw, path=path) + feature_id = _string(payload.get("id"), path=f"{path}.id") + if FEATURE_ID_PATTERN.fullmatch(feature_id) is None: + raise FeatureVerificationError(f"{path}.id {feature_id!r} must be dotted kebab-case") + path = f"feature {feature_id}" + status = cast( + FeatureStatus, + _choice(payload.get("status", "verifiable"), allowed=STATUSES, path=f"{path}.status"), + ) + call_sites = _string_tuple(payload.get("call_sites"), path=f"{path}.call_sites") + for call_site in call_sites: + if CALL_SITE_PATTERN.fullmatch(call_site) is None: + raise FeatureVerificationError( + f"{path}.call_sites entry {call_site!r} is not file:line" + ) + feature_lanes = _string_tuple(payload.get("lanes"), path=f"{path}.lanes") + unknown_lanes = sorted(set(feature_lanes) - set(lanes)) + if unknown_lanes: + raise FeatureVerificationError(f"{path} references unknown lanes {unknown_lanes}") + test = _optional_string(payload.get("test"), path=f"{path}.test") + if test is not None and TEST_NODE_PATTERN.fullmatch(test) is None: + raise FeatureVerificationError(f"{path}.test {test!r} is not a feature-suite node id") + raw_verification = payload.get("verification") + verification = ( + None + if raw_verification is None + else cast( + Verification, + _choice(raw_verification, allowed=VERIFICATIONS, path=f"{path}.verification"), + ) + ) + reason = _optional_string(payload.get("reason"), path=f"{path}.reason") + if status == "verifiable": + if test is None or not feature_lanes or verification is None: + raise FeatureVerificationError( + f"{path} is verifiable, so it needs a test, at least one lane, and a verification" + ) + elif reason is None or test is not None or feature_lanes: + raise FeatureVerificationError( + f"{path} is unverifiable, so it needs a reason and must not schedule a test" + ) + if not call_sites: + raise FeatureVerificationError(f"{path} must cite at least one call site") + return Feature( + feature_id=feature_id, + category=_string(payload.get("category"), path=f"{path}.category"), + title=_string(payload.get("title"), path=f"{path}.title"), + call_sites=call_sites, + status=status, + verification=verification, + lanes=feature_lanes, + test=test, + reason=reason, + limitations=_optional_string(payload.get("limitations"), path=f"{path}.limitations"), + ) + + +def load_manifest(path: Path) -> FeatureManifest: + """Load and validate the feature matrix.""" + if not path.is_file(): + raise FeatureVerificationError(f"feature matrix not found at {path}") + document = _mapping(yaml.safe_load(path.read_text(encoding="utf-8")), path="manifest") + if document.get("schema_version") != MANIFEST_SCHEMA_VERSION: + raise FeatureVerificationError(f"manifest schema_version must be {MANIFEST_SCHEMA_VERSION}") + raw_lanes = _mapping(document.get("lanes"), path="lanes") + lanes = {str(lane_id): _load_lane(str(lane_id), raw) for lane_id, raw in raw_lanes.items()} + raw_features = document.get("features") + if not isinstance(raw_features, list) or not raw_features: + raise FeatureVerificationError("features must be a non-empty list") + features = tuple( + _load_feature(index, raw, lanes=lanes) + for index, raw in enumerate(cast(list[object], raw_features)) + ) + seen_ids: set[str] = set() + seen_tests: set[str] = set() + for feature in features: + if feature.feature_id in seen_ids: + raise FeatureVerificationError(f"duplicate feature id {feature.feature_id!r}") + seen_ids.add(feature.feature_id) + if feature.test is not None: + if feature.test in seen_tests: + raise FeatureVerificationError(f"test {feature.test!r} maps to several features") + seen_tests.add(feature.test) + return FeatureManifest(lanes=lanes, features=features) diff --git a/tests/utils/helpers/_feature_verification/processes.py b/tests/utils/helpers/_feature_verification/processes.py new file mode 100644 index 00000000..fb9d4d71 --- /dev/null +++ b/tests/utils/helpers/_feature_verification/processes.py @@ -0,0 +1,144 @@ +"""Detached lifecycle for the fake model server used by feature-verification lanes. + +Workflow steps are one-shot shell commands, so the server runs as a detached child that +outlives the step that started it; its pid and bound ports live in a small state file. +""" + +from __future__ import annotations + +import http.client +import json +import os +import signal +import subprocess +import sys +import time +from collections.abc import Sequence +from pathlib import Path +from typing import Final, cast + +from tests.utils.helpers._feature_verification.fake_model import HEALTH_PATH, JOURNAL_PATH + +STATE_FILE_NAME: Final[str] = "fake-model.json" +LOG_FILE_NAME: Final[str] = "fake-model.log" +READY_TIMEOUT_SECONDS: Final[float] = 45.0 +STOP_TIMEOUT_SECONDS: Final[float] = 10.0 +_POLL_SECONDS: Final[float] = 0.2 + + +class FakeModelProcessError(RuntimeError): + """Raised when the detached fake model server cannot be managed.""" + + +def _probe_host(host: str) -> str: + """Return the address a local client should use for *host*.""" + return "127.0.0.1" if host in {"0.0.0.0", "", "::"} else host + + +def _get(host: str, port: int, path: str, *, timeout: float) -> tuple[int, bytes]: + """GET a loopback path directly; http.client never consults system or env proxies.""" + connection = http.client.HTTPConnection(_probe_host(host), port, timeout=timeout) + try: + connection.request("GET", path) + response = connection.getresponse() + return response.status, response.read() + finally: + connection.close() + + +def _healthy(host: str, port: int) -> bool: + try: + status, _ = _get(host, port, HEALTH_PATH, timeout=2) + except OSError: + return False + return status == 200 + + +def start_detached( + *, + script_path: Path, + host: str, + ports: Sequence[int], + state_dir: Path, +) -> dict[str, object]: + """Start ``serve-fake-model`` detached and block until every port answers.""" + state_dir.mkdir(parents=True, exist_ok=True) + log_path = state_dir / LOG_FILE_NAME + command = [sys.executable, str(script_path), "serve-fake-model", "--host", host] + for port in ports: + command.extend(["--port", str(port)]) + with log_path.open("ab") as log_handle: + process = subprocess.Popen( # noqa: S603 - fixed argv built above + command, + stdin=subprocess.DEVNULL, + stdout=log_handle, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + deadline = time.monotonic() + READY_TIMEOUT_SECONDS + while time.monotonic() < deadline: + if process.poll() is not None: + raise FakeModelProcessError( + f"fake model server exited with {process.returncode}; see {log_path}" + ) + if all(_healthy(host, port) for port in ports): + state: dict[str, object] = { + "pid": process.pid, + "host": host, + "ports": list(ports), + "log": str(log_path), + } + (state_dir / STATE_FILE_NAME).write_text(json.dumps(state), encoding="utf-8") + return state + time.sleep(_POLL_SECONDS) + process.terminate() + log_tail = log_path.read_text(encoding="utf-8", errors="replace")[-2000:] + raise FakeModelProcessError( + f"fake model server not ready in {READY_TIMEOUT_SECONDS}s; log tail:\n{log_tail}" + ) + + +def load_state(state_dir: Path) -> dict[str, object]: + """Return the persisted server state.""" + state_path = state_dir / STATE_FILE_NAME + if not state_path.is_file(): + raise FakeModelProcessError(f"no fake model state at {state_path}") + payload: object = json.loads(state_path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise FakeModelProcessError(f"invalid fake model state at {state_path}") + return cast(dict[str, object], payload) + + +def fetch_journal(host: str, port: int) -> list[dict[str, object]]: + """Return the request journal recorded by a running server.""" + status, body = _get(host, port, JOURNAL_PATH, timeout=10) + if status != 200: + raise FakeModelProcessError(f"fake model journal returned HTTP {status}") + payload: object = json.loads(body.decode("utf-8")) + if not isinstance(payload, dict): + raise FakeModelProcessError("fake model journal is not a JSON object") + requests = cast(dict[str, object], payload).get("requests") + if not isinstance(requests, list): + raise FakeModelProcessError("fake model journal has no request list") + return [cast(dict[str, object], entry) for entry in cast(list[object], requests)] + + +def stop_detached(state_dir: Path) -> bool: + """Terminate the detached server; return whether a live process was stopped.""" + state = load_state(state_dir) + pid = state.get("pid") + if not isinstance(pid, int): + raise FakeModelProcessError("fake model state has no pid") + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + return False + deadline = time.monotonic() + STOP_TIMEOUT_SECONDS + while time.monotonic() < deadline: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + time.sleep(_POLL_SECONDS) + os.kill(pid, signal.SIGKILL) + return True diff --git a/tests/utils/helpers/_feature_verification/provider.py b/tests/utils/helpers/_feature_verification/provider.py new file mode 100644 index 00000000..235bb3e9 --- /dev/null +++ b/tests/utils/helpers/_feature_verification/provider.py @@ -0,0 +1,142 @@ +"""Wire StrongClaw's model chain to the fake model server, the way an operator would. + +The provider is declared as an explicit OpenAI-compatible entry under ``models.providers`` +(core OpenClaw config, independent of bundled provider plugins), and the operator env contract +selects it through StrongClaw's env-driven chain (``OPENCLAW_DEFAULT_MODEL``). ``clawops setup`` +re-renders ``openclaw.json``, so the provider is re-registered after every render. +""" + +from __future__ import annotations + +import json +import os +import subprocess +from collections.abc import Mapping +from pathlib import Path +from typing import Final, cast + +from clawops.strongclaw_runtime import ( + resolve_openclaw_config_path, + set_env_assignment, + varlock_local_env_file, +) +from tests.utils.helpers._fresh_host.shell import phase_env +from tests.utils.helpers._fresh_host.storage import load_context + +PROVIDER_ID: Final[str] = "featver" +CHAT_MODEL: Final[str] = "featver-chat" +FAKE_CHAT_MODEL_REF: Final[str] = f"{PROVIDER_ID}/{CHAT_MODEL}" +PROVIDER_BASE_URL: Final[str] = "http://127.0.0.1:18000/v1" +# A known provider env-var name is OpenClaw's env marker: resolved from the environment at +# runtime and never stored (or audited) as plaintext. The fake server accepts any value. +PROVIDER_API_KEY_ENV: Final[str] = "VLLM_API_KEY" +PROVIDER_API_KEY_VALUE: Final[str] = "featver-local" +# LiteLLM reaches the fake server from its container through host-gateway. +OPENAI_COMPATIBLE_EMBEDDING_MODEL: Final[str] = "openai/featver-embed" +OPENAI_COMPATIBLE_EMBEDDING_API_BASE: Final[str] = "http://host.docker.internal:11434/v1" +OLLAMA_EMBEDDING_MODEL: Final[str] = "ollama/nomic-embed-text" +OLLAMA_EMBEDDING_API_BASE: Final[str] = "http://host.docker.internal:11434" +ENV_CONTRACT: Final[dict[str, str]] = { + "OPENCLAW_DEFAULT_MODEL": FAKE_CHAT_MODEL_REF, + PROVIDER_API_KEY_ENV: PROVIDER_API_KEY_VALUE, + "HYPERMEMORY_EMBEDDING_MODEL": OPENAI_COMPATIBLE_EMBEDDING_MODEL, + "HYPERMEMORY_EMBEDDING_API_BASE": OPENAI_COMPATIBLE_EMBEDDING_API_BASE, +} +MODEL_AUTH_TIMEOUT_SECONDS: Final[int] = 900 +SYSTEMD_GATEWAY_UNIT: Final[str] = "openclaw-gateway.service" +LAUNCHD_GATEWAY_LABEL: Final[str] = "ai.openclaw.gateway" + + +class ProviderError(RuntimeError): + """Raised when the fake provider cannot be wired into the host.""" + + +def provider_entry() -> dict[str, object]: + """Return the ``models.providers`` entry for the fake model server.""" + return { + "baseUrl": PROVIDER_BASE_URL, + "apiKey": PROVIDER_API_KEY_ENV, + "api": "openai-completions", + "models": [ + { + "id": CHAT_MODEL, + "name": "Feature verification fake model", + "reasoning": False, + "input": ["text"], + "cost": {"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0}, + "contextWindow": 131072, + "maxTokens": 8192, + } + ], + } + + +def seed_env_contract(context_file: Path) -> Path: + """Select the fake provider in the operator env contract (before bootstrap).""" + context = load_context(context_file) + env_file = varlock_local_env_file( + Path(context.repo_root), home_dir=Path(context.app_home), environ=phase_env(context) + ) + env_file.parent.mkdir(parents=True, exist_ok=True) + for key, value in ENV_CONTRACT.items(): + set_env_assignment(env_file, key, value) + env_file.chmod(0o600) + return env_file + + +def register_provider(config_path: Path) -> None: + """Add the fake provider to a rendered ``openclaw.json``.""" + if not config_path.is_file(): + raise ProviderError(f"rendered OpenClaw config not found at {config_path}") + payload = cast(dict[str, object], json.loads(config_path.read_text(encoding="utf-8"))) + models = payload.setdefault("models", {}) + if not isinstance(models, dict): + raise ProviderError(f"{config_path}: `models` is not a mapping") + providers = cast(dict[str, object], models).setdefault("providers", {}) + if not isinstance(providers, dict): + raise ProviderError(f"{config_path}: `models.providers` is not a mapping") + cast(dict[str, object], providers)[PROVIDER_ID] = provider_entry() + config_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + +def _restart_supervised_gateway(platform: str, env: Mapping[str, str]) -> str: + if platform == "linux": + active = subprocess.run( + ["systemctl", "--user", "is-active", "--quiet", SYSTEMD_GATEWAY_UNIT], + env=dict(env), + check=False, + ) + if active.returncode != 0: + return "no supervised gateway" + command = ["systemctl", "--user", "restart", SYSTEMD_GATEWAY_UNIT] + else: + command = ["launchctl", "kickstart", "-k", f"gui/{os.getuid()}/{LAUNCHD_GATEWAY_LABEL}"] + restarted = subprocess.run(command, env=dict(env), capture_output=True, text=True, check=False) + return "restarted" if restarted.returncode == 0 else f"restart failed: {restarted.stderr}" + + +def apply_model_provider(context_file: Path) -> dict[str, object]: + """Register the provider, run ``clawops model-auth ensure``, and restart the gateway.""" + context = load_context(context_file) + env = {**phase_env(context), "OPENCLAW_MODEL_SETUP_MODE": "auto"} + repo_root = Path(context.repo_root) + config_path = resolve_openclaw_config_path( + repo_root, home_dir=Path(context.app_home), environ=env + ) + register_provider(config_path) + ensure = subprocess.run( + [str(repo_root / ".venv" / "bin" / "python"), "-m", "clawops", "model-auth", "ensure"], + cwd=repo_root, + env=env, + capture_output=True, + text=True, + timeout=MODEL_AUTH_TIMEOUT_SECONDS, + check=False, + ) + return { + "config": str(config_path), + "modelAuthExitCode": ensure.returncode, + "modelAuthOutput": ensure.stdout[-2000:], + "modelAuthErrors": ensure.stderr[-4000:], + "gateway": _restart_supervised_gateway(context.platform, env), + } diff --git a/tests/utils/helpers/_feature_verification/recorder.py b/tests/utils/helpers/_feature_verification/recorder.py new file mode 100644 index 00000000..75077bd3 --- /dev/null +++ b/tests/utils/helpers/_feature_verification/recorder.py @@ -0,0 +1,158 @@ +"""Collect pytest outcomes for one lane and map them onto manifest features.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import Final + +import pytest + +from clawops.strongclaw_runtime import varlock_local_env_file +from tests.utils.helpers._feature_verification.envvars import ( + CONTEXT_ENV, + LANE_ENV, + MANIFEST_ENV, + RESULTS_ENV, +) +from tests.utils.helpers._feature_verification.host import default_manifest_path +from tests.utils.helpers._feature_verification.manifest import ( + Feature, + FeatureManifest, + FeatureVerificationError, + load_manifest, +) +from tests.utils.helpers._feature_verification.redaction import redact, secret_values +from tests.utils.helpers._feature_verification.results import ( + MAX_MESSAGE_CHARS, + FeatureResult, + LaneResults, + TestOutcome, +) +from tests.utils.helpers.repo import REPO_ROOT + +FEATURE_SUITE_PARTS: Final[tuple[str, ...]] = ("suites", "e2e", "feature_verification") + + +def is_feature_suite_item(item: pytest.Item) -> bool: + """Return whether *item* belongs to the feature-verification suite.""" + parts = Path(str(item.fspath)).resolve().parts + return any( + parts[index : index + len(FEATURE_SUITE_PARTS)] == FEATURE_SUITE_PARTS + for index in range(len(parts)) + ) + + +def _message(report: pytest.TestReport) -> str: + if report.outcome == "skipped" and isinstance(report.longrepr, tuple): + return str(report.longrepr[2]).removeprefix("Skipped: ") + text = report.longreprtext or "" + return text[-MAX_MESSAGE_CHARS:] + + +@dataclass(slots=True) +class LaneRecorder: + """Accumulate one lane's feature outcomes and persist them at session end.""" + + manifest: FeatureManifest + lane_id: str + results_path: Path + scheduled: dict[str, Feature] = field(default_factory=lambda: dict[str, Feature]()) + results: dict[str, FeatureResult] = field(default_factory=lambda: dict[str, FeatureResult]()) + versions: dict[str, str] = field(default_factory=lambda: dict[str, str]()) + started_at: str = field(default_factory=lambda: datetime.now(tz=UTC).isoformat()) + + @classmethod + def from_environment(cls) -> LaneRecorder | None: + """Return a recorder when the lane environment is present, else ``None``.""" + lane_id = os.environ.get(LANE_ENV, "").strip() + if not lane_id: + return None + results_raw = os.environ.get(RESULTS_ENV, "").strip() + if not results_raw or not os.environ.get(CONTEXT_ENV, "").strip(): + raise FeatureVerificationError( + f"{LANE_ENV} is set, so {RESULTS_ENV} and {CONTEXT_ENV} are required" + ) + manifest_path = Path( + os.environ.get(MANIFEST_ENV, "").strip() or default_manifest_path(REPO_ROOT).as_posix() + ) + manifest = load_manifest(manifest_path) + if lane_id not in manifest.lanes: + raise FeatureVerificationError(f"unknown lane {lane_id!r}") + return cls(manifest=manifest, lane_id=lane_id, results_path=Path(results_raw)) + + def select(self, items: list[pytest.Item]) -> tuple[list[pytest.Item], list[pytest.Item]]: + """Split suite items into (scheduled on this lane, not scheduled).""" + by_test = self.manifest.feature_by_test() + keep: list[pytest.Item] = [] + drop: list[pytest.Item] = [] + for item in items: + feature = by_test.get(item.nodeid) + if feature is None: + raise pytest.UsageError(f"{item.nodeid} is not mapped in the feature matrix") + if self.lane_id in feature.lanes: + self.scheduled[item.nodeid] = feature + keep.append(item) + else: + drop.append(item) + return keep, drop + + def record(self, report: pytest.TestReport) -> None: + """Fold one phase report into the feature's outcome.""" + feature = self.scheduled.get(report.nodeid) + if feature is None: + return + existing = self.results.get(report.nodeid) + outcome: TestOutcome | None = None + if report.when == "call": + outcome = "passed" if report.passed else "skipped" if report.skipped else "failed" + elif report.failed: + outcome = "error" + elif report.when == "setup" and report.skipped: + outcome = "skipped" + if outcome is None: + if existing is not None: + existing.duration_seconds += report.duration + return + if existing is not None and existing.outcome in {"failed", "error"}: + existing.duration_seconds += report.duration + return + self.results[report.nodeid] = FeatureResult( + feature=feature.feature_id, + test=report.nodeid, + outcome=outcome, + duration_seconds=report.duration + (existing.duration_seconds if existing else 0.0), + message=_message(report), + ) + + def write(self) -> LaneResults: + """Persist lane results, marking scheduled-but-unreported features as not run.""" + lane = self.manifest.lanes[self.lane_id] + results = list(self.results.values()) + for feature in self.manifest.features_for_lane(self.lane_id): + if feature.test is not None and feature.test not in self.results: + results.append( + FeatureResult( + feature=feature.feature_id, + test=feature.test, + outcome="not-run", + duration_seconds=0.0, + message="test was scheduled on this lane but did not run", + ) + ) + secrets = secret_values(varlock_local_env_file(REPO_ROOT)) + for result in results: + result.message = redact(result.message, secrets) + lane_results = LaneResults( + lane=self.lane_id, + platform=lane.platform, + profile=lane.profile, + started_at=self.started_at, + finished_at=datetime.now(tz=UTC).isoformat(), + versions=dict(self.versions), + results=sorted(results, key=lambda result: result.feature), + ) + lane_results.write(self.results_path) + return lane_results diff --git a/tests/utils/helpers/_feature_verification/redaction.py b/tests/utils/helpers/_feature_verification/redaction.py new file mode 100644 index 00000000..ce9ebc80 --- /dev/null +++ b/tests/utils/helpers/_feature_verification/redaction.py @@ -0,0 +1,36 @@ +"""Redact generated env-contract secrets from anything the harness records or uploads.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Final + +from clawops.strongclaw_runtime import load_env_assignments + +REDACTED: Final[str] = "[REDACTED]" +_SECRET_KEY_MARKERS: Final[tuple[str, ...]] = ("TOKEN", "KEY", "PASSWORD", "SECRET") +_MIN_SECRET_LENGTH: Final[int] = 8 + + +def secret_values(env_file: Path) -> tuple[str, ...]: + """Return secret-looking values from an env contract, longest first.""" + values = load_env_assignments(env_file) if env_file.is_file() else {} + return tuple( + sorted( + { + value + for key, value in values.items() + if any(marker in key for marker in _SECRET_KEY_MARKERS) + and len(value) >= _MIN_SECRET_LENGTH + }, + key=len, + reverse=True, + ) + ) + + +def redact(text: str, secrets: tuple[str, ...]) -> str: + """Replace every known secret value in *text*.""" + for secret in secrets: + text = text.replace(secret, REDACTED) + return text diff --git a/tests/utils/helpers/_feature_verification/reporting.py b/tests/utils/helpers/_feature_verification/reporting.py new file mode 100644 index 00000000..9bbd1a69 --- /dev/null +++ b/tests/utils/helpers/_feature_verification/reporting.py @@ -0,0 +1,237 @@ +"""Join lane results with the feature matrix into a per-feature verdict report.""" + +from __future__ import annotations + +import json +from collections import Counter +from dataclasses import asdict, dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Final + +from tests.utils.helpers._feature_verification.manifest import ( + Feature, + FeatureManifest, + load_manifest, +) +from tests.utils.helpers._feature_verification.results import ( + FeatureVerdict, + LaneResults, + load_lane_results, +) + +LANE_RESULTS_GLOB: Final[str] = "feature-results-*.json" +VERDICTS: Final[tuple[FeatureVerdict, ...]] = ("PASS", "FAIL", "UNVERIFIABLE") +_DETAIL_CHARS: Final[int] = 180 + + +@dataclass(frozen=True, slots=True) +class LaneVerdict: + """One feature's outcome on one lane.""" + + verdict: FeatureVerdict + outcome: str + message: str + duration_seconds: float = 0.0 + + +@dataclass(frozen=True, slots=True) +class FeatureRow: + """One feature's overall verdict across its lanes.""" + + id: str + category: str + title: str + verdict: FeatureVerdict + verification: str | None + platforms: list[str] + lanes: dict[str, LaneVerdict] + reason: str | None + limitations: str | None + test: str | None + call_sites: list[str] + + +@dataclass(frozen=True, slots=True) +class LaneSummary: + """Where one lane ran and which tool versions it verified.""" + + platform: str + profile: str + reported: bool + versions: dict[str, str] + + +@dataclass(frozen=True, slots=True) +class VerificationReport: + """The full per-feature report.""" + + generated_at: str + lanes: dict[str, LaneSummary] + counts: dict[str, int] + categories: dict[str, dict[str, int]] + features: list[FeatureRow] + schema_version: int = 1 + + @property + def failures(self) -> int: + """Return the number of FAIL features.""" + return self.counts["FAIL"] + + def to_json(self) -> str: + """Serialize the report.""" + return json.dumps(asdict(self), indent=2, sort_keys=True) + "\n" + + +def _lane_results(results_dir: Path) -> dict[str, LaneResults]: + lanes: dict[str, LaneResults] = {} + for path in sorted(results_dir.rglob(LANE_RESULTS_GLOB)): + lane = load_lane_results(path) + lanes[lane.lane] = lane + return lanes + + +def _lane_verdict(feature: Feature, lane: LaneResults | None) -> LaneVerdict: + if lane is None: + return LaneVerdict("FAIL", "not-run", "lane reported no results") + match = next((result for result in lane.results if result.feature == feature.feature_id), None) + if match is None: + return LaneVerdict("FAIL", "not-run", "test did not run on lane") + return LaneVerdict( + verdict=match.verdict, + outcome=match.outcome, + message=match.message, + duration_seconds=round(match.duration_seconds, 3), + ) + + +def _feature_verdict(feature: Feature, lanes: dict[str, LaneVerdict]) -> FeatureVerdict: + if feature.status == "unverifiable": + return "UNVERIFIABLE" + verdicts = {entry.verdict for entry in lanes.values()} + if "FAIL" in verdicts: + return "FAIL" + return "PASS" if verdicts == {"PASS"} else "UNVERIFIABLE" + + +def build_report(manifest: FeatureManifest, lanes: dict[str, LaneResults]) -> VerificationReport: + """Return the report for every manifest feature.""" + rows: list[FeatureRow] = [] + counts: Counter[str] = Counter() + by_category: dict[str, Counter[str]] = {} + for feature in manifest.features: + lane_verdicts = { + lane_id: _lane_verdict(feature, lanes.get(lane_id)) for lane_id in feature.lanes + } + verdict = _feature_verdict(feature, lane_verdicts) + counts[verdict] += 1 + by_category.setdefault(feature.category, Counter())[verdict] += 1 + rows.append( + FeatureRow( + id=feature.feature_id, + category=feature.category, + title=feature.title, + verdict=verdict, + verification=feature.verification, + platforms=list(manifest.platforms(feature)), + lanes=lane_verdicts, + reason=feature.reason, + limitations=feature.limitations, + test=feature.test, + call_sites=list(feature.call_sites), + ) + ) + return VerificationReport( + generated_at=datetime.now(tz=UTC).isoformat(), + lanes={ + lane_id: LaneSummary( + platform=lane.platform, + profile=lane.profile, + reported=lane_id in lanes, + versions=dict(lanes[lane_id].versions) if lane_id in lanes else {}, + ) + for lane_id, lane in manifest.lanes.items() + }, + counts={verdict: counts.get(verdict, 0) for verdict in VERDICTS}, + categories={ + category: {verdict: tally.get(verdict, 0) for verdict in VERDICTS} + for category, tally in sorted(by_category.items()) + }, + features=rows, + ) + + +def _cell(text: str) -> str: + flattened = " ".join(text.split()) + if len(flattened) > _DETAIL_CHARS: + flattened = flattened[: _DETAIL_CHARS - 3] + "..." + return flattened.replace("|", "\\|") + + +def _detail(row: FeatureRow) -> str: + if row.verdict == "UNVERIFIABLE" and row.reason: + return _cell(row.reason) + for lane_id, entry in row.lanes.items(): + if entry.verdict != "PASS": + return _cell(f"{lane_id}: {entry.message}") + return _cell(row.limitations or "") + + +def render_markdown(report: VerificationReport) -> str: + """Render the report as a GitHub step-summary section.""" + lines = [ + "## Feature Verification", + "", + "| Lane | Platform | Profile | Versions |", + "| --- | --- | --- | --- |", + ] + for lane_id, lane in report.lanes.items(): + versions = ", ".join(f"{name} {value}" for name, value in lane.versions.items()) + lines.append( + f"| {lane_id} | {lane.platform} | {lane.profile} | {_cell(versions or 'not reported')} |" + ) + lines.extend( + [ + "", + " ยท ".join(f"**{verdict}** {report.counts[verdict]}" for verdict in VERDICTS), + "", + "| Category | PASS | FAIL | UNVERIFIABLE |", + "| --- | --- | --- | --- |", + ] + ) + for category, tally in report.categories.items(): + lines.append( + f"| {category} | {tally['PASS']} | {tally['FAIL']} | {tally['UNVERIFIABLE']} |" + ) + lines.extend( + [ + "", + "| Feature | Category | Verdict | Verification | Platforms | Detail |", + "| --- | --- | --- | --- | --- | --- |", + ] + ) + for row in report.features: + lines.append( + f"| `{row.id}` | {row.category} | {row.verdict} | {row.verification or '-'} " + f"| {', '.join(row.platforms) or '-'} | {_detail(row)} |" + ) + lines.append("") + return "\n".join(lines) + + +def write_lane_report( + *, + results_dir: Path, + manifest_path: Path, + output_path: Path, + summary_file: Path | None, +) -> int: + """Write the JSON report (and optional step summary); return the FAIL count.""" + report = build_report(load_manifest(manifest_path), _lane_results(results_dir)) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(report.to_json(), encoding="utf-8") + if summary_file is not None: + summary_file.parent.mkdir(parents=True, exist_ok=True) + with summary_file.open("a", encoding="utf-8") as handle: + handle.write(render_markdown(report)) + return report.failures diff --git a/tests/utils/helpers/_feature_verification/results.py b/tests/utils/helpers/_feature_verification/results.py new file mode 100644 index 00000000..03575667 --- /dev/null +++ b/tests/utils/helpers/_feature_verification/results.py @@ -0,0 +1,121 @@ +"""Per-lane feature results written by the pytest plugin and read by the reporter.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Final, Literal, cast + +from tests.utils.helpers._feature_verification.manifest import FeatureVerificationError + +type TestOutcome = Literal["passed", "failed", "error", "skipped", "not-run"] +type FeatureVerdict = Literal["PASS", "FAIL", "UNVERIFIABLE"] + +RESULT_SCHEMA_VERSION: Final[int] = 1 +TEST_OUTCOMES: Final[frozenset[str]] = frozenset( + {"passed", "failed", "error", "skipped", "not-run"} +) +# A test that discovers at runtime that its feature cannot be exercised skips with this prefix. +UNVERIFIABLE_SKIP_PREFIX: Final[str] = "unverifiable:" +MAX_MESSAGE_CHARS: Final[int] = 4000 + + +@dataclass(slots=True) +class FeatureResult: + """Outcome of one feature test on one lane.""" + + feature: str + test: str + outcome: TestOutcome + duration_seconds: float + message: str = "" + + @property + def verdict(self) -> FeatureVerdict: + """Map the pytest outcome onto the report vocabulary.""" + if self.outcome == "passed": + return "PASS" + if self.outcome == "skipped" and self.message.startswith(UNVERIFIABLE_SKIP_PREFIX): + return "UNVERIFIABLE" + return "FAIL" + + +def _empty_results() -> list[FeatureResult]: + return [] + + +def _empty_versions() -> dict[str, str]: + return {} + + +@dataclass(slots=True) +class LaneResults: + """All feature outcomes recorded on one lane.""" + + lane: str + platform: str + profile: str + started_at: str + finished_at: str = "" + versions: dict[str, str] = field(default_factory=_empty_versions) + results: list[FeatureResult] = field(default_factory=_empty_results) + schema_version: int = RESULT_SCHEMA_VERSION + + def write(self, path: Path) -> None: + """Persist the lane results as JSON.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(asdict(self), indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _str(value: object, *, path: str) -> str: + if not isinstance(value, str): + raise FeatureVerificationError(f"{path} must be a string") + return value + + +def load_lane_results(path: Path) -> LaneResults: + """Load one lane result file.""" + payload: object = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, Mapping): + raise FeatureVerificationError(f"{path} must hold a JSON object") + document = cast(Mapping[str, object], payload) + if document.get("schema_version") != RESULT_SCHEMA_VERSION: + raise FeatureVerificationError(f"{path} has an unsupported schema_version") + raw_results = document.get("results") + if not isinstance(raw_results, Sequence): + raise FeatureVerificationError(f"{path}.results must be a list") + results: list[FeatureResult] = [] + for index, raw in enumerate(cast(Sequence[object], raw_results)): + if not isinstance(raw, Mapping): + raise FeatureVerificationError(f"{path}.results[{index}] must be an object") + entry = cast(Mapping[str, object], raw) + outcome = _str(entry.get("outcome"), path=f"{path}.results[{index}].outcome") + if outcome not in TEST_OUTCOMES: + raise FeatureVerificationError(f"{path}.results[{index}] has outcome {outcome!r}") + duration = entry.get("duration_seconds", 0.0) + results.append( + FeatureResult( + feature=_str(entry.get("feature"), path=f"{path}.results[{index}].feature"), + test=_str(entry.get("test"), path=f"{path}.results[{index}].test"), + outcome=cast(TestOutcome, outcome), + duration_seconds=float(duration) if isinstance(duration, int | float) else 0.0, + message=str(entry.get("message", "")), + ) + ) + raw_versions = document.get("versions", {}) + versions = ( + {str(key): str(value) for key, value in cast(Mapping[object, object], raw_versions).items()} + if isinstance(raw_versions, Mapping) + else {} + ) + return LaneResults( + lane=_str(document.get("lane"), path=f"{path}.lane"), + platform=_str(document.get("platform"), path=f"{path}.platform"), + profile=_str(document.get("profile"), path=f"{path}.profile"), + started_at=_str(document.get("started_at"), path=f"{path}.started_at"), + finished_at=str(document.get("finished_at", "")), + versions=versions, + results=results, + ) diff --git a/tests/utils/helpers/_feature_verification/runtime.py b/tests/utils/helpers/_feature_verification/runtime.py new file mode 100644 index 00000000..1d67279c --- /dev/null +++ b/tests/utils/helpers/_feature_verification/runtime.py @@ -0,0 +1,242 @@ +"""Gateway and sidecar runtime control for feature-verification lanes. + +On lanes that activate host services the gateway belongs to systemd/launchd and the harness +only restarts it through the service manager. When activation is off (or failed, which the +setup feature reports on its own) the harness starts ``clawops ops gateway start`` itself, so +the remaining features still get a real gateway rather than cascading into failures. +""" + +from __future__ import annotations + +import os +import signal +import socket +import subprocess +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal, cast + +from tests.utils.helpers._feature_verification.host import FeatureHost + +type GatewayMode = Literal["systemd", "launchd", "harness"] + +SYSTEMD_GATEWAY_UNIT: Final[str] = "openclaw-gateway.service" +LAUNCHD_GATEWAY_LABEL: Final[str] = "ai.openclaw.gateway" +DEFAULT_GATEWAY_PORT: Final[int] = 18789 +GATEWAY_READY_TIMEOUT_SECONDS: Final[float] = 150.0 +_POLL_SECONDS: Final[float] = 2.0 +_STOP_TIMEOUT_SECONDS: Final[float] = 20.0 +_LOG_TAIL_CHARS: Final[int] = 2_000_000 +# OpenClaw writes its rolling file log here regardless of TMPDIR. +OPENCLAW_TMP_LOG_DIR: Final[str] = "/tmp/openclaw" + + +def tcp_open(port: int, host: str = "127.0.0.1", timeout: float = 2.0) -> bool: + """Return whether *host*:*port* accepts TCP connections.""" + try: + with socket.create_connection((host, port), timeout=timeout): + return True + except OSError: + return False + + +def gateway_port(host: FeatureHost) -> int: + """Return the gateway port from the rendered config.""" + gateway = host.rendered_config().get("gateway") + if isinstance(gateway, dict): + port = cast(dict[str, object], gateway).get("port") + if isinstance(port, int): + return port + return DEFAULT_GATEWAY_PORT + + +def gateway_status(host: FeatureHost) -> dict[str, object]: + """Return ``openclaw gateway status --json``.""" + payload = host.openclaw("gateway", "status", "--json", timeout=120).require_ok().json() + assert isinstance(payload, dict), payload + return cast(dict[str, object], payload) + + +def gateway_rpc_ok(host: FeatureHost) -> bool: + """Return whether the gateway answers RPC.""" + outcome = host.openclaw("gateway", "status", "--json", timeout=120) + if not outcome.ok: + return False + try: + payload = outcome.json() + except AssertionError: + return False + rpc = cast(dict[str, object], payload).get("rpc") if isinstance(payload, dict) else None + return isinstance(rpc, dict) and cast(dict[str, object], rpc).get("ok") is True + + +def plugin_statuses(host: FeatureHost) -> dict[str, str]: + """Return ``{plugin id: status}`` from ``openclaw plugins list --json``.""" + payload = host.openclaw("plugins", "list", "--json", timeout=300).require_ok().json() + assert isinstance(payload, dict), payload + plugins = cast(dict[str, object], payload).get("plugins") + assert isinstance(plugins, list), payload + statuses: dict[str, str] = {} + for raw in cast(list[object], plugins): + if isinstance(raw, dict): + entry = cast(dict[str, object], raw) + statuses[str(entry.get("id"))] = str(entry.get("status")) + return statuses + + +def service_active(host: FeatureHost) -> bool: + """Return whether the service manager reports the gateway as running.""" + if host.lane.platform == "linux": + return host.run( + ["systemctl", "--user", "is-active", "--quiet", SYSTEMD_GATEWAY_UNIT], timeout=30 + ).ok + outcome = host.run( + ["launchctl", "print", f"gui/{os.getuid()}/{LAUNCHD_GATEWAY_LABEL}"], timeout=30 + ) + return outcome.ok and "state = running" in outcome.stdout + + +@dataclass(slots=True) +class GatewayController: + """Own the gateway lifecycle for one lane session.""" + + host: FeatureHost + _process: subprocess.Popen[bytes] | None = None + _log_path: Path | None = None + + @property + def mode(self) -> GatewayMode: + """Return who supervises the gateway.""" + if self._process is not None or not self.host.lane.activate_services: + return "harness" + return "systemd" if self.host.lane.platform == "linux" else "launchd" + + @property + def log_path(self) -> Path | None: + """Return the harness-managed gateway log, when the harness started it.""" + return self._log_path + + def log_text(self) -> str: + """Return recent gateway output from every place the active supervisor writes it.""" + chunks: list[str] = [] + if self._log_path is not None and self._log_path.is_file(): + chunks.append(self._log_path.read_text(encoding="utf-8", errors="replace")) + if self.host.lane.platform == "linux": + journal = self.host.run( + ["journalctl", "--user", "-u", SYSTEMD_GATEWAY_UNIT, "--no-pager", "-n", "5000"], + timeout=60, + ) + chunks.append(journal.stdout) + state_logs = self.host.home / ".openclaw" / "logs" + file_logs = [ + *sorted(state_logs.glob("*gateway*.log")), + *sorted(Path(OPENCLAW_TMP_LOG_DIR).glob("openclaw-*.log")), + ] + for log_file in file_logs: + chunks.append(log_file.read_text(encoding="utf-8", errors="replace")[-_LOG_TAIL_CHARS:]) + return self.host.redact("\n".join(chunks)) + + def wait_ready(self, timeout: float = GATEWAY_READY_TIMEOUT_SECONDS) -> None: + """Block until the gateway port is open and RPC answers.""" + port = gateway_port(self.host) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if self._process is not None and self._process.poll() is not None: + raise AssertionError( + f"harness gateway exited with {self._process.returncode}; see {self._log_path}" + ) + if tcp_open(port) and gateway_rpc_ok(self.host): + return + time.sleep(_POLL_SECONDS) + raise AssertionError(f"gateway on port {port} not ready within {timeout:.0f}s") + + def _start_harness_process(self) -> None: + self._log_path = self.host.work_dir / f"gateway-{time.time_ns()}.log" + with self._log_path.open("ab") as log_handle: + self._process = subprocess.Popen( # noqa: S603 - fixed clawops argv + [ + str(self.host.python), + "-m", + "clawops", + "ops", + "--asset-root", + ".", + "gateway", + "start", + ], + cwd=self.host.repo_root, + env=self.host.product_env(), + stdin=subprocess.DEVNULL, + stdout=log_handle, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + + def ensure_running(self) -> GatewayMode: + """Make sure a gateway answers; start a harness gateway when none is supervised.""" + if gateway_rpc_ok(self.host): + return self.mode + if self.mode != "harness" and service_active(self.host): + self.wait_ready() + return self.mode + if self._process is None: + self._start_harness_process() + self.wait_ready() + return self.mode + + def restart(self) -> None: + """Restart the gateway so it reloads the rendered config.""" + if self.mode == "systemd": + self.host.run( + ["systemctl", "--user", "restart", SYSTEMD_GATEWAY_UNIT], timeout=60 + ).require_ok() + elif self.mode == "launchd": + self.host.run( + ["launchctl", "kickstart", "-k", f"gui/{os.getuid()}/{LAUNCHD_GATEWAY_LABEL}"], + timeout=60, + ).require_ok() + else: + self.stop() + self._start_harness_process() + time.sleep(_POLL_SECONDS) + self.wait_ready() + + def stop(self) -> None: + """Stop a harness-started gateway (its whole process group).""" + process = self._process + if process is None: + return + self._process = None + if process.poll() is not None: + return + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=_STOP_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=_STOP_TIMEOUT_SECONDS) + + +def sidecars_ready(host: FeatureHost) -> bool: + """Return whether ``clawops ops status`` reports every required sidecar ready.""" + outcome = host.clawops("ops", "--asset-root", ".", "status", timeout=300) + try: + payload = outcome.json() + except AssertionError: + return False + if not isinstance(payload, dict): + return False + readiness = cast(dict[str, object], payload).get("readiness") + return ( + isinstance(readiness, dict) + and cast(dict[str, object], readiness).get("requiredReady") is True + ) + + +def ensure_sidecars(host: FeatureHost) -> str: + """Bring the sidecar stack up with ``clawops ops sidecars up`` when no service did.""" + if sidecars_ready(host): + return "ready" + host.clawops("ops", "--asset-root", ".", "sidecars", "up", timeout=1800).require_ok() + return "started-by-harness" diff --git a/tests/utils/helpers/_fresh_host/context.py b/tests/utils/helpers/_fresh_host/context.py index 9dc6d3a0..5cd496a7 100644 --- a/tests/utils/helpers/_fresh_host/context.py +++ b/tests/utils/helpers/_fresh_host/context.py @@ -11,6 +11,7 @@ DEFAULT_DOCKER_PULL_PARALLELISM, FreshHostContext, FreshHostReport, + ScenarioSpec, ) from tests.utils.helpers._fresh_host.storage import ( load_context, @@ -70,6 +71,15 @@ def _ensure_cache_dirs() -> None: Path(raw_path).expanduser().resolve().mkdir(parents=True, exist_ok=True) +def _xdg_runtime_dir(spec: ScenarioSpec, *, tmp_root: Path) -> Path | None: + """Return the XDG runtime dir; host-home scenarios use the real user manager's.""" + if spec.platform != "linux": + return None + if spec.use_host_home: + return Path(f"/run/user/{os.getuid()}") + return tmp_root / "xdg-runtime" + + def prepare_context( *, scenario_id: str, @@ -84,8 +94,8 @@ def prepare_context( tmp_root = runner_temp / f"strongclaw-{spec.platform}-host" report_dir = runner_temp / "fresh-host-reports" / scenario_id context_dir = runner_temp / "fresh-host" / scenario_id - app_home = tmp_root / "home" - xdg_runtime_dir = tmp_root / "xdg-runtime" if spec.platform == "linux" else None + app_home = Path.home() if spec.use_host_home else tmp_root / "home" + xdg_runtime_dir = _xdg_runtime_dir(spec, tmp_root=tmp_root) diagnostics_dir = report_dir / "diagnostics" context_file = context_dir / "context.json" report_file = report_dir / "report.json" @@ -127,6 +137,7 @@ def prepare_context( compose_files=[ str((repo_root / relative_path).resolve()) for relative_path in spec.compose_files ], + use_host_home=spec.use_host_home, ) report = FreshHostReport( scenario_id=context.scenario_id, diff --git a/tests/utils/helpers/_fresh_host/linux.py b/tests/utils/helpers/_fresh_host/linux.py index 95ff4714..725bb763 100644 --- a/tests/utils/helpers/_fresh_host/linux.py +++ b/tests/utils/helpers/_fresh_host/linux.py @@ -73,8 +73,9 @@ def linux_setup(context: FreshHostContext) -> list[str]: context.profile, "--non-interactive", "--no-verify", - "--no-activate-services", ) + if not context.activate_services: + command.append("--no-activate-services") run_command(command, cwd=repo_root, env=phase_env(context)) return command diff --git a/tests/utils/helpers/_fresh_host/models.py b/tests/utils/helpers/_fresh_host/models.py index 88f967b8..e10c8965 100644 --- a/tests/utils/helpers/_fresh_host/models.py +++ b/tests/utils/helpers/_fresh_host/models.py @@ -5,7 +5,14 @@ from dataclasses import dataclass, field from typing import Final, Literal -ScenarioId = Literal["linux", "macos-sidecars", "macos-browser-lab"] +ScenarioId = Literal[ + "linux", + "macos-sidecars", + "macos-browser-lab", + "linux-features", + "linux-features-local", + "macos-features", +] PlatformName = Literal["linux", "macos"] PhaseStatus = Literal["success", "failure", "skipped"] @@ -40,6 +47,9 @@ class ScenarioSpec: verify_rendered_files: bool exercise_sidecars: bool exercise_browser_lab: bool + # Feature-verification lanes install into the runner's own HOME on Linux because the + # systemd user manager only reads user units from there. + use_host_home: bool = False @dataclass(slots=True) @@ -90,6 +100,7 @@ class FreshHostContext: exercise_browser_lab: bool phase_names: list[str] compose_files: list[str] + use_host_home: bool = False @dataclass(slots=True) @@ -176,4 +187,57 @@ class FreshHostReport: exercise_sidecars=False, exercise_browser_lab=True, ), + "linux-features": ScenarioSpec( + scenario_id="linux-features", + platform="linux", + job_name="Linux Feature Verification", + compose_files=( + "platform/compose/docker-compose.aux-stack.yaml", + "platform/compose/docker-compose.browser-lab.yaml", + ), + compose_variant=None, + phase_names=("bootstrap", "setup"), + activate_services=True, + ensure_images=True, + normalize_machine_name=False, + verify_launchd=False, + verify_rendered_files=False, + exercise_sidecars=False, + exercise_browser_lab=False, + use_host_home=True, + ), + "linux-features-local": ScenarioSpec( + scenario_id="linux-features-local", + platform="linux", + job_name="Linux Feature Verification (no host services)", + compose_files=( + "platform/compose/docker-compose.aux-stack.yaml", + "platform/compose/docker-compose.browser-lab.yaml", + ), + compose_variant=None, + phase_names=("bootstrap", "setup"), + activate_services=False, + ensure_images=False, + normalize_machine_name=False, + verify_launchd=False, + verify_rendered_files=False, + exercise_sidecars=False, + exercise_browser_lab=False, + use_host_home=True, + ), + "macos-features": ScenarioSpec( + scenario_id="macos-features", + platform="macos", + job_name="macOS Feature Verification", + compose_files=("platform/compose/docker-compose.aux-stack.ci-hosted-macos.yaml",), + compose_variant="ci-hosted-macos", + phase_names=("normalize-machine-name", "bootstrap", "setup", "verify-launchd"), + activate_services=True, + ensure_images=True, + normalize_machine_name=True, + verify_launchd=True, + verify_rendered_files=False, + exercise_sidecars=False, + exercise_browser_lab=False, + ), } diff --git a/tests/utils/helpers/_fresh_host/storage.py b/tests/utils/helpers/_fresh_host/storage.py index 9fe2e906..378b2b19 100644 --- a/tests/utils/helpers/_fresh_host/storage.py +++ b/tests/utils/helpers/_fresh_host/storage.py @@ -190,6 +190,7 @@ def load_context(path: Path) -> FreshHostContext: compose_files=list( as_string_list(payload.get("compose_files"), path="context.compose_files") ), + use_host_home=as_bool(payload.get("use_host_home", False), path="context.use_host_home"), ) diff --git a/tests/utils/helpers/feature_verification.py b/tests/utils/helpers/feature_verification.py new file mode 100644 index 00000000..b69a477c --- /dev/null +++ b/tests/utils/helpers/feature_verification.py @@ -0,0 +1,122 @@ +"""Public feature-verification helper facade.""" + +from tests.utils.helpers._feature_verification.acp_stub_agent import ( + REPLY_PREFIX as ACP_STUB_REPLY_PREFIX, +) +from tests.utils.helpers._feature_verification.diagnostics import collect_lane_diagnostics +from tests.utils.helpers._feature_verification.fake_model import ( + CHAT_MODEL_ID, + REPLY_PREFIX, + FakeModelServer, + canned_reply, + deterministic_embedding, +) +from tests.utils.helpers._feature_verification.host import ( + CommandOutcome, + FeatureHost, + extract_json, + reply_texts, +) +from tests.utils.helpers._feature_verification.http import ( + HttpResponse, + basic_auth, + http_request, +) +from tests.utils.helpers._feature_verification.lane import ( + LaneError, + enable_systemd_user_manager, + run_lane, +) +from tests.utils.helpers._feature_verification.manifest import ( + Feature, + FeatureManifest, + FeatureVerificationError, + Lane, + load_manifest, +) +from tests.utils.helpers._feature_verification.processes import ( + FakeModelProcessError, + start_detached, + stop_detached, +) +from tests.utils.helpers._feature_verification.provider import ( + FAKE_CHAT_MODEL_REF, + OLLAMA_EMBEDDING_API_BASE, + OLLAMA_EMBEDDING_MODEL, + ProviderError, + apply_model_provider, + register_provider, + seed_env_contract, +) +from tests.utils.helpers._feature_verification.recorder import LaneRecorder, is_feature_suite_item +from tests.utils.helpers._feature_verification.reporting import ( + build_report, + render_markdown, + write_lane_report, +) +from tests.utils.helpers._feature_verification.results import ( + UNVERIFIABLE_SKIP_PREFIX, + FeatureResult, + LaneResults, + load_lane_results, +) +from tests.utils.helpers._feature_verification.runtime import ( + GatewayController, + ensure_sidecars, + gateway_rpc_ok, + gateway_status, + plugin_statuses, + service_active, + tcp_open, +) + +__all__ = [ + "ACP_STUB_REPLY_PREFIX", + "CHAT_MODEL_ID", + "FAKE_CHAT_MODEL_REF", + "OLLAMA_EMBEDDING_API_BASE", + "OLLAMA_EMBEDDING_MODEL", + "REPLY_PREFIX", + "UNVERIFIABLE_SKIP_PREFIX", + "CommandOutcome", + "FakeModelProcessError", + "FakeModelServer", + "Feature", + "FeatureHost", + "FeatureManifest", + "FeatureResult", + "FeatureVerificationError", + "GatewayController", + "HttpResponse", + "Lane", + "LaneError", + "LaneRecorder", + "ProviderError", + "LaneResults", + "basic_auth", + "build_report", + "canned_reply", + "collect_lane_diagnostics", + "apply_model_provider", + "deterministic_embedding", + "enable_systemd_user_manager", + "extract_json", + "gateway_rpc_ok", + "ensure_sidecars", + "gateway_status", + "http_request", + "is_feature_suite_item", + "load_lane_results", + "load_manifest", + "plugin_statuses", + "register_provider", + "render_markdown", + "reply_texts", + "run_lane", + "seed_env_contract", + "service_active", + "start_detached", + "stop_detached", + "tcp_open", + "write_lane_report", +] diff --git a/tests/utils/helpers/fresh_host.py b/tests/utils/helpers/fresh_host.py index 2f261840..7873380b 100644 --- a/tests/utils/helpers/fresh_host.py +++ b/tests/utils/helpers/fresh_host.py @@ -8,6 +8,7 @@ from tests.utils.helpers._fresh_host.models import ( DEFAULT_DOCKER_PULL_MAX_ATTEMPTS, DEFAULT_DOCKER_PULL_PARALLELISM, + SCENARIO_SPECS, FreshHostContext, FreshHostError, FreshHostReport, @@ -33,6 +34,7 @@ "PhaseResult", "PlatformName", "PhaseStatus", + "SCENARIO_SPECS", "ScenarioId", "ScenarioSpec", "_deactivate_macos_host_services",