diff --git a/.github/workflows/device-tests.yml b/.github/workflows/device-tests.yml new file mode 100644 index 0000000..312f17d --- /dev/null +++ b/.github/workflows/device-tests.yml @@ -0,0 +1,106 @@ +name: Device Compute + +on: + pull_request: + branches: [main] + paths: + - '.github/workflows/device-tests.yml' + - 'scripts/ci/**' + - 'tests/compatibility/**' + - 'frontend/scripts/probe-browser-compute*' + - 'frontend/package.json' + - 'frontend/package-lock.json' + - '.nvmrc' + - 'docs/CI-MATRIX.md' + - 'docs/TESTING.md' + - 'docs/DEPENDENCY-STRATEGY.md' + - 'docs/SECURITY-*.md' + - 'docs/CODE-REVIEW.md' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + cpu: + name: CPU reference and browser capabilities / Linux x64 + runs-on: ubuntu-24.04 + timeout-minutes: 15 + env: + UV_NO_CONFIG: '1' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + with: + version: '0.12.24' + enable-cache: false + python-version: '3.12.15' + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: .nvmrc + package-manager-cache: false + - name: Prepare the standard-library Python runtime + run: | + mkdir -p "$RUNNER_TEMP/device-compute" + echo "RECORDS=$RUNNER_TEMP/device-compute" >> "$GITHUB_ENV" + uv venv --managed-python --python 3.12.15 + echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" + - name: Collect the real environment + id: collect + run: | + python -I scripts/ci/collect_environment.py --profile-id gha-linux-x64-device \ + --output "$RECORDS/environment.json" + - name: Prepare the existing locked browser tool + working-directory: frontend + run: | + npm install --global npm@11.21.0 + test "$(npm --version)" = 11.21.0 + npm ci --ignore-scripts + # Download the paired browser as the job user. No system package installation. + node node_modules/@playwright/test/cli.js install chromium + - name: Probe browser capabilities + id: browser + run: | + node frontend/scripts/probe-browser-compute.mjs --environment "$RECORDS/environment.json" \ + --output "$RECORDS/browser.json" + - name: Run CPU reference and attach compute evidence + if: ${{ !cancelled() && steps.collect.outcome == 'success' }} + run: | + args=() + if [ -f "$RECORDS/browser.json" ]; then args+=(--browser-report "$RECORDS/browser.json"); fi + python -I scripts/ci/probe_compute.py --environment "$RECORDS/environment.json" \ + --output "$RECORDS/result.json" --report "$RECORDS/compute-probe.json" \ + --log "$RECORDS/cpu-reference.log" --summary "$GITHUB_STEP_SUMMARY" "${args[@]}" + - name: Validate the common Result and profile + if: ${{ !cancelled() && steps.collect.outcome == 'success' }} + run: | + python -I scripts/ci/validate_compatibility.py --require-real \ + --dependency-strategy docs/DEPENDENCY-STRATEGY.md \ + --profile tests/compatibility/profiles/gha-linux-x64-device.json \ + --summary "$GITHUB_STEP_SUMMARY" "$RECORDS/result.json" + - name: Run device contract and negative regressions + if: ${{ !cancelled() }} + run: | + python -I scripts/ci/tests/test_device_contract.py --verbose 2>&1 | tee "$RECORDS/contract-tests.log" + python -I scripts/ci/tests/test_compute_probe.py --verbose 2>&1 | tee "$RECORDS/probe-tests.log" + python -I scripts/ci/tests/test_device_environments.py --verbose 2>&1 | tee "$RECORDS/environment-tests.log" + node --test frontend/scripts/probe-browser-compute.test.mjs 2>&1 | tee "$RECORDS/browser-tests.log" + # This bundle remains separate from the measured Result and is always synthetic. + python -I tests/compatibility/fixtures/device-environments/scenarios.py > "$RECORDS/synthetic-environments.json" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: ${{ !cancelled() }} + with: + name: device-compute-linux-x64-attempt-${{ github.run_attempt }} + path: ${{ runner.temp }}/device-compute/ + if-no-files-found: error + retention-days: 90 diff --git a/docs/CI-MATRIX.md b/docs/CI-MATRIX.md index 8562748..3ada7fe 100644 --- a/docs/CI-MATRIX.md +++ b/docs/CI-MATRIX.md @@ -2,7 +2,7 @@ Added 2026-10-11. This document defines how a compatibility lane is described, how a run on a lane is recorded, and what the validator checks. Commands and the CI job are listed in [TESTING.md](TESTING.md); which lanes are supported is recorded only in [DEPENDENCY-STRATEGY.md](DEPENDENCY-STRATEGY.md#platform-and-accelerator-lanes); CI permissions and trust boundaries are in [SECURITY-ARCHITECTURE.md](SECURITY-ARCHITECTURE.md) "CI workflow". -Implemented today: the schema, the validator, the environment collector (Linux, macOS, and Windows probes, and a Podman engine step on the host), the wheel and Python ABI probe, 24 real profiles, nine synthetic fixtures, their tests, and the CI jobs that record real runs: the `compatibility` job in `.github/workflows/ci.yml` on the checked lane, `.github/workflows/compatibility.yml` on seven GitHub-hosted runner lanes and four UBI container lanes, and `.github/workflows/python-abi.yml` with CPython 3.15.0 and 3.15t on six runners (added 2026-10-12; see "Current lanes and fixtures"). Nothing in this model is a support claim: a run is evidence about one runner image and one commit, and a synthetic fixture is not evidence that any platform works. +Implemented today: the schema, validator, environment collector (Linux, macOS, Windows, and a Podman engine step on the host), wheel/Python ABI probe, CPU/device probe, browser capability probe, real profiles, synthetic fixtures, and their tests. The CI jobs that record real runs are the `compatibility` job in `.github/workflows/ci.yml`, `.github/workflows/compatibility.yml` on seven GitHub-hosted runner lanes and four UBI container lanes, `.github/workflows/python-abi.yml` with CPython 3.15.0 and 3.15t on six runners, and `.github/workflows/device-tests.yml` with a CPU reference and browser capability checks on Ubuntu 24.04 (see "Current lanes and fixtures"). Nothing in this model is a support claim: a run is evidence about one runner image and one commit, and a synthetic fixture is not evidence that any platform works. ## Files @@ -14,9 +14,13 @@ Implemented today: the schema, the validator, the environment collector (Linux, | `scripts/ci/collect_environment.py` | Records the environment of the process that runs it as a result document. | | `scripts/ci/validate_compatibility.py` | Checks profiles and results: structure from the schema, then the rules under "Rules across fields". | | `scripts/ci/probe_python_abi.py` | Reports which wheel of each backend lock package the running interpreter accepts and installs, and decides a lane's `install` stage (see "Install stage in the lane jobs"). | +| `scripts/ci/probe_compute.py` | Runs the CPU reference, records native capability observations, imports the optional browser report, and attaches typed compute evidence to the collected Result. | +| `frontend/scripts/probe-browser-compute.mjs` | Uses existing Node Playwright/Chromium for WebAssembly execution and WebGL/WebGPU/WebNN capability detection. | +| `tests/compatibility/fixtures/device-environments/scenarios.py` | Builds 15 explicitly synthetic profile/result pairs for execution layers, managers, orchestration, and device access. | +| `scripts/ci/tests/test_device_contract.py`, `test_compute_probe.py`, `test_device_environments.py`; `frontend/scripts/probe-browser-compute.test.mjs` | Device evidence, CPU success/error, browser, and special-environment regressions. | | `scripts/ci/tests/test_compatibility_schema.py`, `scripts/ci/tests/test_environment_probe.py`, `scripts/ci/tests/test_python_abi_probe.py` | Tests of the validator, the collector, and the wheel probe, including documents and outcomes that must fail. | -All of these run without Django. All but the wheel probe and its tests use the Python standard library only; the wheel probe reads wheel tags with `packaging`, which the jobs install from the backend lock with its hash. +All run without Django. The Python device tools use only the standard library. The wheel probe reads wheel tags with `packaging`, installed from the backend lock with its hash; the browser probe reuses the locked Node Playwright dependency and adds no framework. ## Profiles and results @@ -44,7 +48,7 @@ Each axis is a separate field. Host, guest, container, and workload are never me | Run and revision | `run.ci` (system, event, repository, ref, run ID and attempt, run URL, workflow ref, job, runner environment, OS, architecture, image) and `run.revision` (see "Commits" below). | | Stages | `stages`: the seven stages below, each with its status. | -Documented values: architectures `x86_64`, `aarch64`, `x86`, `armv7`, `riscv64`, `ppc64le`, `s390x`, `loongarch64` (aliases such as `amd64` or `arm64` are errors; write the canonical name); OS families `linux`, `windows`, `macos`, `freebsd`; backend APIs `cpu`, `cuda`, `rocm`, `oneapi`, `level-zero`, `opencl`, `vulkan-compute`, `directcompute`, `directml`, `metal`, `mps`, `coreml`, `qnn`, `wasm`, `webgpu`, `webgl`, `webnn`; and the technologies, orchestrators, and service managers listed in the validator. Any other value is accepted with an `unrecognized-value` warning. Rules that need a documented value (which architectures run natively on which, pointer sizes, the targets of platform and ABI tags) do not apply to it, except that an ABI tag for an architecture outside the list disagrees with a listed arch fact; rules that only compare two values for equality, such as a container's OS family with its parent's, still do. An unrecognized value is never treated as supported. Node's module ABI and N-API version are recorded but not compared with its version. +Documented values: architectures `x86_64`, `aarch64`, `x86`, `armv7`, `riscv64`, `ppc64le`, `s390x`, `loongarch64` (aliases such as `amd64` or `arm64` are errors; write the canonical name); OS families `linux`, `windows`, `macos`, `freebsd`; backend APIs `cpu`, `cuda`, `rocm`, `oneapi`, `level-zero`, `opencl`, `vulkan-compute`, `directcompute`, `directml`, `metal`, `mps`, `coreml`, `mlx`, `qnn`, `wasm`, `webgpu`, `webgl`, `webnn`; and the technologies, orchestrators, and service managers listed in the validator. Any other value is accepted with an `unrecognized-value` warning. Rules that need a documented value (which architectures run natively on which, pointer sizes, the targets of platform and ABI tags) do not apply to it, except that an ABI tag for an architecture outside the list disagrees with a listed arch fact; rules that only compare two values for equality, such as a container's OS family with its parent's, still do. An unrecognized value is never treated as supported. Node's module ABI and N-API version are recorded but not compared with its version. `extensions` objects with `x-` keys may appear on documents, layers, backends, and stages for data the schema has no field for. The validator checks their strings for control characters and secret-shaped values and nothing else: data inside them that looks like a fact or an evidence list is not read as one. There are no per-technology validators. @@ -146,7 +150,7 @@ A profile that needs to break a rule records an exception with `id`, `rule`, `re Statuses: `passed`, `failed`, `blocked` (could not run because a prerequisite failed or something it needs is missing), `not-run` (nobody ran it), and `not-applicable`. Only `security` and `performance` can be `not-applicable`: every workload is detected, installed, initialized, run, and checked, so those stages are `not-run` or `blocked` when nobody ran them. A `passed` or `failed` stage cites the evidence that shows it and records its privilege escalation; in a real result that evidence includes a `log:` or `artifact:` reference, and `detect` may cite the collector (`tool:`) instead. Every status other than `passed` gives a reason; a `not-run` stage cites nothing; a stage can be `passed` or `failed` only when every stage it needs is `passed` or `not-applicable`; and `detect` cannot be `passed` while `probe_errors` lists a failed probe. -`overall` is derived: `failed` if any stage failed, `passed` if every stage is `passed` or `not-applicable` and at least one passed, and `incomplete` otherwise. So a `passed` run has passed every stage from `detect` to `correctness`. No CI job runs `security` or `performance`, which it records as `not-run`, so every record from CI is `incomplete` or `failed`: read its stages. The `compatibility` job in `ci.yml` runs `detect` only, the `python-abi.yml` lanes `detect` and `install`, and the `compatibility.yml` lanes `detect` through `correctness`. +`overall` is derived: `failed` if any stage failed, `passed` if every stage is `passed` or `not-applicable` and at least one passed, and `incomplete` otherwise. So a `passed` run has passed every stage from `detect` to `correctness`. No CI job runs `security` or `performance`, which it records as `not-run`, so every record from CI is `incomplete` or `failed`: read its stages. The `compatibility` job in `ci.yml` runs `detect` only, the `python-abi.yml` lanes `detect` and `install`, and the `compatibility.yml` and CPU `device-tests.yml` lanes `detect` through `correctness`. ### Install stage in the lane jobs @@ -161,7 +165,29 @@ In `compatibility.yml` and `python-abi.yml` a failed install is told apart from A `blocked` install leaves the job green, the later stages `blocked`, and the reason in the record and the step summary. A lane whose profile requires a stage that was `blocked` still fails at validation (`required-stage`). In `python-abi.yml` the probe's `packages` command installs each native package on its own, `--no-deps`, from the lock's hashes, and gives `install` the same split: a package with no wheel for the interpreter, or whose only compatible wheel is not in the lock, is `blocked`; an installer failure while a compatible wheel is in the lock, an install the reports contradict, or a lock dependency installed at a version a requirement excludes, is `failed`. The probe's import and call fail, and fail the job, when either raises, when a package enables the GIL in a build that started without it (the report's `gil_reenabled`), or when their process does not exit with status 0, even after it printed a passing result: the report keeps what the process printed and records how it ended under `exit` (`status`, `returncode`, and on POSIX the `signal` that ended it), and a process that prints no result is a failed import. -A stage that used a compute backend records `compute.requested` and `compute.effective` as backend IDs. A stage cannot pass when a `gpu` or `npu` backend was requested and another backend, or none, ran (`accelerator-fallback`): record such a run as `failed` with the reason, or request the CPU backend for a CPU run. A profile of an accelerator lane names its `gpu` or `npu` backend in `accelerator_backend`; then each of `execute`, `correctness`, and `performance` that the profile requires must record that backend as both requested and effective (`required-stage`), so a run that used only the CPU, or recorded no backend, does not satisfy the lane. +A stage that used a compute backend records `compute.requested` and `compute.effective` as backend IDs. A stage cannot pass when a `gpu` or `npu` backend was requested and another backend, or none, ran (`accelerator-fallback`): record such a run as `failed` with the reason, or request the CPU backend for a CPU run. A profile of an accelerator lane names its `gpu` or `npu` backend in `accelerator_backend`; then each of `execute`, `correctness`, and `performance` that the profile requires must record that backend as both requested and effective (`required-stage`), so a run that used only the CPU, or recorded no backend, does not satisfy the lane. A `device-v1` profile without `accelerator_backend` that declares CPU backends likewise requires those workload stages to request and execute the same CPU backend declared by the profile. Additional WASM or other backend evidence may accompany that CPU run; an optional workload stage may use another backend. + +### Device and compute evidence + +PR D keeps `schema_version: 1` and adds the optional `compute.probe_contract: "device-v1"` marker. A profile that requires it needs the same marker in its Result. The typed `compute.backends[].probe` is a Result-only observation, separate from `accelerator_inventory` and the existing backend ID, kind, API, availability, device, version, and features. An inventory entry, executable named `nvidia-smi`, `rocminfo`, or `clinfo`, or detected GPU name proves no operation ran. + +| Probe field | Fact established | +|---|---| +| `tool` | Probe that produced the observation. | +| `runtime_installed` | Runtime installation evidence, separately from API/device enumeration. Tool-file presence alone does not establish a working runtime or driver. | +| `api_enumerated`, `device_enumerated` | Whether an API/device was actually observed. An absent diagnostic tool does not prove absent hardware. | +| `device_accessible` | Whether required device access was established. It grants no host administrator privilege. | +| `execution.status`, `executed` | Outcome and whether an operation completed; an incorrect completed operation still has `executed: true`. | +| `execution.effective`, `correct`, `software_fallback` | Backend that computed, exact-result verification, and CPU/software fallback as separate facts. | +| `execution.runtime`, `process_id`, `evidence`, `run` | Measured runtime/process and log/artifact evidence bound to the Result's full run identity, including revisions and workflow/job. | + +Unknown facts retain `value: null`, provenance `unknown`, and a reason. `available: true` needs measured runtime installation, API enumeration, device enumeration, and access; a driver installation alone is insufficient. A passed probe needs a completed operation and measured correct result. A passed GPU/NPU operation also needs its requested backend as effective and measured `software_fallback: false`. A completed operation requires measured `runtime_installed: true` and identifies an available effective backend, runtime, and positive process ID. A real executed/failed probe cites logs or artifacts and the same run as the Result; workload stages agree with its requested/effective backend and cite overlapping execution evidence. Discovery alone cannot satisfy an execution/correctness stage, including real accelerator results without the new marker. Legacy CPU records remain valid. + +An unattempted operation is `not-run`; a missing required device or permission that prevents execution is `blocked`; an unexpected attempted-execution error, wrong result, or software fallback is `failed`. Neither `blocked` nor `not-run` carries an effective execution backend or a correctness result. A native accelerator backend currently remains `not-run`: PR D provides capability observations and a contract, but no accelerator kernel adapter. Browser WebGL context creation, WebGPU API/adapter discovery, and WebNN API discovery also leave computation `not-run`. WebAssembly alone executes a fixed `20 + 22 = 42` operation. A software WebGPU adapter never counts as hardware GPU execution, and WebNN detection never counts as NPU execution. + +The native CPU probe multiplies `((1, -2, 3), (4, 0, -1))` by `((2, 1), (-1, 3), (0, 5))` and compares against `((4, 10), (8, -1))`. Its requested and effective backend are `cpu`; successful execution and correctness are recorded separately, with the executing Python process and evidence. `--cpu-case mismatch` and `--cpu-case error` exercise false-result and execution-error paths and exit nonzero. This is an evidence-pipeline check, not model inference or performance qualification. + +`probe_compute.py` reads the existing collector's Result and adds the device contract and stages; it does not replace environment collection. The optional browser report includes the SHA-256 of that exact environment file, which the importer checks before attaching observations. A report or child reason ending in `-failed` requires aggregate browser status `failed`, even when the hardware operation remains honestly `not-run`. Browser execution runtime identifies Chromium and a `node-controller` version that must match the environment's measured Node version; a passing report, attempted operation, or observed boolean detection fact requires this runtime. An entirely unknown browser limitation may remain `not-run` without measured Node. Typed probe fields, unlike descriptive `extensions`, are validated. Neither run matching nor a digest authenticates a malicious probe: review the producing code and runner trust boundary. ### Stage status and support status @@ -208,6 +234,8 @@ Each finding has a stable code. Errors make the document invalid; warnings do no | `identity-contradiction` | error | `superuser` that disagrees with uid 0 on Linux, macOS, or FreeBSD, or an initial user namespace without the identity map. | | `backend-reference` | error | Duplicate backend IDs, a backend with the `cpu` API whose kind is not `cpu`, a stage naming a backend that is not listed, a backend used by a stage that did not run, a passing stage on a backend not recorded as available, a passing stage whose compute record names no backend that ran (on a GPU or NPU request that is an `accelerator-fallback`), or a profile's `accelerator_backend` that is not one of its `gpu` or `npu` backends. | | `accelerator-fallback` | error | A GPU or NPU stage that passed on another backend. | +| `compute-probe`, `compute-availability` | error | Missing or misplaced typed device probes; availability claimed without measured runtime installation, API/device enumeration, and access, or a completed operation without measured runtime installation. | +| `compute-execution`, `compute-evidence` | error | Discovery promoted to an operation, missing actual backend/runtime/process, incorrect or unexecuted operation passed, stage/probe disagreement, or execution evidence bound to another run/revision/workflow/job. | | `stage-fields`, `stage-order` | error | A stage without its evidence, reason, or privilege record, a real stage that cites no log or artifact, a not-run stage with evidence, a stage other than `security` or `performance` recorded as not-applicable, `detect` passed with probe errors, or a stage that ran before its prerequisites passed. | | `overall-mismatch` | error | `overall` that the stages do not give. | | `ci-evidence` | error | A GitHub Actions record without its run (its event, repository, run ID and attempt, run URL, workflow ref, and job), the commit its checkout holds, its `GITHUB_SHA`, or, for a pull request, its head and base, each observed or inferred (a declared value does not count), or with a run URL for another repository, run or attempt (a URL with no `/attempts/` names the run, whatever its attempt, and an attempt of more than 10 digits names none), or a run attempt of 0. | @@ -221,7 +249,7 @@ Each finding has a stable code. Errors make the document invalid; warnings do no | `evidence-source` | error | A real profile's evidence source that names a missing workflow file, one the validator cannot read (a file that is not UTF-8, or a path the system refuses, such as a name over its length limit), or a job its `jobs:` mapping does not define, or a real profile with no evidence source. | | `profile-link` | error | A result checked against a profile it does not name, a real result against a synthetic profile or the reverse, a real profile checked with a record not measured as a GitHub Actions run, or a GitHub Actions record whose workflow and job are not one of the profile's evidence sources: the workflow is the path `run.ci.workflow_ref` names inside `run.ci.repository`, so a record that does not name both, or whose workflow ref is outside that repository or names no workflow file or no ref, links to no profile. | | `profile-mismatch`, `profile-unverified` | error | A layer count that differs from the profile, a backend that is missing or has another kind or API, any other fact the profile declares (in a layer, a runtime, a backend matched by ID, the accelerator inventory, or the runner) that the result contradicts, a fact the profile declares `not-applicable` that the result measured, or a declared fact the run did not measure. | -| `required-stage` | error | A stage the profile requires that did not pass, or, under a profile with an `accelerator_backend`, a required `execute`, `correctness`, or `performance` stage that did not run on it. | +| `required-stage` | error | A stage the profile requires that did not pass, or a required `execute`, `correctness`, or `performance` stage that did not request and run on the profile's `accelerator_backend`, or on the same declared CPU backend under a `device-v1` CPU profile without `accelerator_backend`. | | `profile-unchecked` | warning | A result naming a profile that was not given; the default policy applies unless `--no-policy` is given. | The validator prints each finding on one line, with control characters replaced and `##[`, which the GitHub Actions runner reads as a workflow command anywhere in a line, written as `## [`, and `::`, which starts a workflow command at the start of a line, written as `: :`; a line it prints starts with a file name or with its own text, never with a value from a record, and a file name that starts with `::` cannot start a command either. A value shaped like a secret, and the rest of its line, is replaced by a note naming the shape, so a finding that echoes a rejected value, or a file name, never shows it. No shape needs a word boundary before it, so an escape the echo writes just before a value (`\n` from `repr`, `~1` in a JSON Pointer) does not hide one; the collector's check of its JSON text reads the shapes the same way. A finding names its place with a JSON Pointer. The `schema`, `control-character`, `secret-like-value`, `fact-provenance`, and `profile-provenance` rules, and `synthetic-marking` on an evidence reference, build the path as they walk the document, and that path is bounded: a key longer than 128 characters once escaped is written as its length before escaping and the first 8 hexadecimal digits of the SHA-256 of its UTF-8 bytes (a lone surrogate encoded as it is), such as `~(300 characters, sha256 1a2b3c4d)`, and a path of more than 16 keys keeps its first 8 and last 7 around `~...`. The other rules, such as `synthetic-marking` on a synthetic run, `profile-mismatch`, and `profile-unverified`, write the path of the place they check from the keys they read, without that bound. Outside `extensions`, the default schema allows only the property names it defines, which are short and hold no `/` or `~`, and no place deeper than 8 keys, and the other rules name no place inside `extensions`. Under the default schema, a long key or deep nesting therefore leaves the path of each finding at most 16 keys of at most 128 characters each, before the line's own escapes, though a document with many findings still prints a line for each. Under any other schema given with `--schema`, a path the other rules write can repeat a key as it is, unescaped and unshortened, and can hold more than 16 keys. An escaped key holds `~` only before `0` or `1`, so neither marker reads as a key, but a shortened path can name more than one place, also for two long keys of one length whose digests start alike: it names the place for a reader and is not resolved. Its `--summary` handles every table cell the same way and escapes Markdown. The collector writes its own lines, which name the output file, the same way. In both scripts, a command-line error, which repeats the rejected argument and the program name, is written the same way too. @@ -258,6 +286,7 @@ Each profile's `support` field copies its status from the lane table; this list | Document | Kind | Lane named in `support` | Run by | |---|---|---|---| | `profiles/linux-x64-gha.json` | real profile, required stage `detect` | Linux x86-64 glibc | `compatibility` job in `.github/workflows/ci.yml` | +| `profiles/gha-linux-x64-device.json` | real CPU/device profile, `device-v1`, required stages `detect` to `correctness` | none (`not-assessed`: a reference operation does not qualify the backend lock) | `cpu` job in `.github/workflows/device-tests.yml` | | `profiles/gha-linux-x64-cp312.json`, `gha-linux-arm64-cp312.json` | real, CPython 3.12.15 on `ubuntu-24.04` and the native Arm64 `ubuntu-24.04-arm`, required stages `detect` to `correctness` | Linux x86-64 glibc; Linux aarch64 glibc | `platform` job in `.github/workflows/compatibility.yml` | | `profiles/gha-macos-arm64-cp312.json` | real, `macos-15`, required stages `detect` to `correctness` | macOS Apple Silicon (`not-assessed`: the row's statuses depend on the macOS version, which each run records and the profile does not declare) | `platform` | | `profiles/gha-macos-x64-cp312.json` | real, `macos-15-intel`, required stage `detect` | macOS Intel | `platform` | @@ -271,6 +300,11 @@ Each profile's `support` field copies its status from the lane table; this list | `fixtures/windows-arm64-wsl2-podman.*` | synthetic, rootless Podman with a non-root user | Windows on Arm with WSL2 (Ubuntu arm64) | none | | `fixtures/vm-podman-quadlet-systemd.result.json` | synthetic, systemd in the container and on the VM | no profile | none | | `fixtures/kubernetes-arm64-cuda.*` | synthetic, CUDA as `accelerator_backend`, approved device exception | Linux aarch64 glibc (`not-assessed`, as a synthetic profile) | none | +| `fixtures/device-environments/scenarios.py` | 15 generated synthetic profile/result pairs; all stages `not-run` | none (`not-assessed`) | none; `test_device_environments.py` validates the generated fixtures | + +The device environment fixtures model bare metal and VM; Windows on Arm / WSL2 / Linux container; rootless Podman; VM / rootless Podman / non-root application; a process directly managed by host systemd; host systemd and Quadlet container management; container PID 1 systemd; Kubernetes, K3s, and OpenShift; GPU/NPU passthrough; and a CPU-only container. Systemd manager facts belong to their layer, and host unit scope belongs to container orchestration, not container PID 1. cgroup v2 delegation, engine rootlessness, non-root identity, and `no_new_privs` remain independent facts. Device-plugin metadata in `x-device-plugin` is descriptive only, not a validated access grant. + +All 15 pairs pass structure and cross-field checks; the three rootful cluster cases deliberately fail their required rootless policy, and the two device-passthrough cases deliberately fail `minimal-devices`. They contain no newly approved exception. Tests require those failures and reject host/container manager confusion, layer reordering, privileged/rootful mutations, and every synthetic record submitted with `--require-real`. None starts a service, container, WSL2 installation, or cluster. ## Adding a lane @@ -278,7 +312,7 @@ A later change that adds an OS, ABI, accelerator, or runtime lane: 1. Adds a profile under `tests/compatibility/profiles/` that declares only what its job can observe, copies its support status from the lane table (or uses `not-assessed` for a lane the table does not list, or for an interpreter other than the lock's), names its `gpu` or `npu` backend in `accelerator_backend` if it is an accelerator lane, declares its runner, and names the job in `evidence_sources`. 2. Adds the job, which runs the collector with `--profile-id` and passes the stages its other steps ran with the `--stage*` options, citing their logs or artifacts and their privilege escalation, then runs the validator with `--require-real`, `--profile`, and `--dependency-strategy`. The collector step runs with `if: ${{ !cancelled() }}` after the workload steps and takes each status from that step's `steps..outcome`: `success` gives `passed`, `failure` gives `failed`, and `skipped` gives `blocked` or `not-run` with the reason, each citing that step's log. The install stage takes its status from the wheel probe instead (see "Install stage in the lane jobs"). -3. Adds whatever probe the lane needs that the collector does not have, such as Windows token privileges beyond elevation and group membership, the engine-side facts of an engine other than Podman, or accelerator inventory, without per-technology validators. The `--stage*` options cannot record a stage's `compute`, and the collector lists only the `cpu` backend, so an accelerator lane also needs a way to record its backend and each stage's requested and effective backend before its profile can pass. Values outside the documented lists warn, and the rules that need a documented value skip them, until they are added to the lists; data without a field goes in `x-` extensions. A change that removes or narrows a field increments `schema_version`. The probe tests describe made-up machines, macOS and Windows among them, with the interpreter and native values fixed to match each one, so they do not depend on the host's interpreter; they are written for a Linux host and run on the Linux x86-64 and Arm64 lanes. +3. Adds whatever probe the lane needs that the collector does not have, such as Windows token privileges beyond elevation and group membership, the engine-side facts of an engine other than Podman, or accelerator inventory, without per-technology validators. The collector still records only CPU environment facts; `probe_compute.py` attaches the `device-v1` backend probes and requested/effective stage references to that Result. A future accelerator operation implements that same contract and supplies measured execution/correctness evidence before its profile can pass, rather than using `--stage*` status declarations alone. Values outside the documented lists warn, and the rules that need a documented value skip them, until they are added to the lists; data without a field goes in `x-` extensions. A change that removes or narrows a field increments `schema_version`. The probe tests describe made-up machines, macOS and Windows among them, with the interpreter and native values fixed to match each one, so they do not depend on the host's interpreter; they are written for a Linux host and run on the Linux x86-64 and Arm64 lanes. 4. Changes a lane's support status only by editing [DEPENDENCY-STRATEGY.md](DEPENDENCY-STRATEGY.md) with the evidence. ## What a passing job means @@ -290,3 +324,5 @@ It does not show that any other OS, architecture, Python or Node version, or acc A passing lane of `compatibility.yml` shows the same for its runner, plus what its record's stages say: for a stage that `passed`, that the hashed lock installed binary-only and `uv pip check` passed, that `manage.py check` and the migration check passed, that the Django tests ran and passed, and that pytest ran the same tests from the test lock; and, where the lane runs them, that the frontend lint, type check, build, and Vitest passed. A green lane can also hold a `blocked` install, which shows the lock cannot produce a working environment there and why; the profile's required stages decide whether that fails the lane. It does not show that the stages ran without an administrator token on Windows, that the lane's support status changed, or, for a UBI lane, anything about a RHEL host: the container shares the Ubuntu VM's kernel, has no SELinux in Enforcing mode, and runs no systemd, so a UBI pass is evidence about the image's user space with rootless Podman only. A passing lane of `python-abi.yml` shows that the interpreter was CPython 3.15.0 of the requested build, that the probe's tests passed on it, and, per package, which wheel was accepted and installed with the lock's hash and whether one import and one call passed in a fresh process, with the GIL state before and after. It does not show thread safety, which no step tests, nor that the backend works on 3.15: the lock is resolved for 3.12, Django is not run, and packages without a 3.15 wheel are not installed. + +A passing `cpu` job in `device-tests.yml` shows that the CPU reference completed correctly on `cpu`, its common Result passed the existing validator with the required device profile and default privilege policy, and the device, browser, and environment regressions passed. Browser observations are read individually from the report; an unavailable browser may be `not-run` with a reason. Accelerator operations remain `not-run` without an implementation and required hardware. `security` and `performance` are `not-run`, so overall remains `incomplete`. A green job qualifies no GPU/NPU, special execution environment, AI model, or inference engine. diff --git a/docs/CODE-REVIEW.md b/docs/CODE-REVIEW.md index dd1ba8b..f77a6e6 100644 --- a/docs/CODE-REVIEW.md +++ b/docs/CODE-REVIEW.md @@ -8,6 +8,7 @@ It is not a security policy (see [SECURITY-ARCHITECTURE.md](SECURITY-ARCHITECTUR - In effect for every code change, starting at HEAD 8ec5623 (committed 2026-09-21; the repository facts in this document were verified on 2026-10-08). The procedure depends on nothing that is Planned. (Updated 2026-10-10: the Frontend boundary checks were re-checked at 14b9109; unchanged through 3fe4d1a.) - Tooling: `.github/workflows/ci.yml` runs the baseline backend, frontend, and Playwright checks, plus the pytest and Vitest unit test jobs and the compatibility job of [CI-MATRIX.md](CI-MATRIX.md), on pull requests to `main` and pushes to `main`, on the checked lane only. Since 2026-10-12, `.github/workflows/compatibility.yml` and `.github/workflows/python-abi.yml` run the backend and frontend checks on other GitHub-hosted runner lanes and in UBI containers, and probe CPython 3.15.0 and 3.15t, on pull requests to `main` that change their inputs and on manual dispatch; they are not required checks, and a pull request that changes none of their paths has no run of them. Their Windows jobs run only for a pull request from a branch of this repository or a manual run on `main`, so a fork's pull request has no Windows result ([SECURITY-ARCHITECTURE.md](SECURITY-ARCHITECTURE.md#temporary-exception-windows-administrator-token)). [TESTING.md](TESTING.md) "Continuous integration" lists the jobs and what they do not cover. Its runs are evidence to read (see "Independence and authority" and step 8). GitHub-side integrations, such as review bots and required status checks, are configured outside the repository's files; whether one runs depends on the connected service and its settings, and its output is evidence to read. `.github/` also holds `copilot-instructions.md`, a bridge file that points to [CONTEXT.md](CONTEXT.md), and the advisory issue forms and pull request template described in [CONTEXT.md](CONTEXT.md) "GitHub issues and pull requests"; they are writing aids, not review tooling or evidence. Adding another workflow, or extending `ci.yml` beyond the checked lane, is a separate, explicitly requested change. (Updated 2026-10-12.) +- Device tooling: PR D adds `.github/workflows/device-tests.yml` on GitHub-hosted Ubuntu 24.04, the standard-library CPU/device probe, the existing Playwright-based browser probe, and special-environment fixture regressions. It leaves the three prior workflows unchanged; no GPU/NPU or special-environment support is implied. The compute/device review checks below cover this interface. - Verification baseline: the automated tests that exist are listed in [TESTING.md](TESTING.md) under "Tests that exist today". They were not executed in this documentation pass. "Exists" and "executed" stay distinct throughout this document. - The checks under "Boundary-specific checks" for Realtime, Task, and Runtime cover Planned areas. They apply to a change that introduces such code; they never imply the feature exists. @@ -119,6 +120,13 @@ Data (ORM, migrations, storage): - File access goes through the Django storage API. Object-name validation, no-overwrite saves, and presigned URL handling in `backend/core/storage/s3.py` are preserved; any change to them is a security review trigger. - `DATABASE_URL` parsing stays strict; a new silent fallback is a finding. +Compute/device validation (`scripts/ci/probe_compute.py`, `frontend/scripts/probe-browser-compute.mjs`, the compatibility schema/validator, `tests/compatibility/`, and `device-tests.yml`): + +- Read the requested/effective backend linkage, typed probe evidence, and actual reference operation. Runtime/tool presence, API enumeration, device access, completed operation, and correctness must remain separate. GPU-to-CPU/software fallback and detection-only success claims must fail; an unavailable SDK/tool alone must not prove absent hardware. Names in the API catalog do not qualify a runtime. +- Run the device contract, CPU probe, special-environment, and browser tests listed in [TESTING.md](TESTING.md#device-and-compute-workflow). Check intentional wrong-result/error inputs as well as the successful CPU operation. Verify SHA/workflow/job linkage and required-stage failures, and that synthetic results cannot satisfy `--require-real`. Inspect each optional browser outcome; WebGL/WebGPU/WebNN detection is not GPU/NPU execution. +- Check bare metal, VM/WSL2/container layer order; host systemd process management, host systemd/Quadlet container management, and container PID 1 independently; and rootless engine, non-root workload, cgroup delegation, device-plugin metadata, and policy denials independently. Generated fixtures test the model only. No cluster, WSL2 installation, service, or hardware execution can be claimed from them. +- Review the new workflow against the scope and security procedure, including action pins, token, runner, no escalation, public artifacts, failure propagation, and unchanged checks in the three earlier workflows. A CPU reference pass is pipeline evidence, not AI inference or hardware qualification. Unavailable hardware, unexecuted probes, and lower-priority follow-ups remain explicit in the review. Report independent code/security review separately from author tests and self-assessment; neither a green workflow nor a review bot is approval. + ### 7. Performance, concurrency, maintainability - No unprotected process-global mutable state. Shared in-process state uses explicit synchronization; `backend/core/storage/s3.py` uses a `threading.Lock` for its lazy client and is the existing reference. Nothing may rely on the GIL, and a native or third-party dependency the change adds is checked for free-threaded compatibility ([CONTEXT.md](CONTEXT.md) "Concurrency direction"). diff --git a/docs/DEPENDENCY-STRATEGY.md b/docs/DEPENDENCY-STRATEGY.md index 850a64d..935df11 100644 --- a/docs/DEPENDENCY-STRATEGY.md +++ b/docs/DEPENDENCY-STRATEGY.md @@ -221,6 +221,14 @@ Sources checked on 2026-10-10: - [Linux on Snapdragon X2 Series early developer preview](https://www.qualcomm.com/developer/blog/2026/09/announcing-linux-on-snapdragon-x2-series-early-developer-preview) and [Qualcomm OnQ: Snapdragon Summit 2026](https://www.qualcomm.com/news/onq/2026/09/snapdragon-summit-agentic-ai-pcs-linux) (Debian by the end of 2026; Ubuntu certification targeted for the first half of 2027). - [Playwright for Python system requirements](https://playwright.dev/python/docs/intro) and [microsoft/playwright#40202](https://github.com/microsoft/playwright/issues/40202). +### Compute evidence from PR D + +`device-tests.yml` adds a CPU reference and device capability lane on GitHub-hosted `ubuntu-24.04`; it changes no lock or support status in the table above. The standard-library Python probe runs a fixed integer matrix multiplication on `cpu` and checks the exact result. This validates the evidence pipeline, not LLM inference, accelerator performance, or an AI framework. The existing Node Playwright dependency also supplies a separate Chromium probe: WebAssembly performs a small operation, while WebGL, WebGPU, and WebNN report context/API/adapter observations only. + +The contract can name CUDA, ROCm, oneAPI, Level Zero, OpenCL, Vulkan Compute, DirectCompute, DirectML, Metal, MPS, Core ML, MLX, QNN, WebGPU, WebGL, WebNN, and WebAssembly. A name in that catalog is not an implementation or qualification. No CUDA/ROCm/oneAPI/QNN SDK, PyTorch, TensorFlow, or ONNX Runtime is introduced. The native accelerator probe observes only installed tool paths and fixed device-node classes/access; it runs no vendor utility or accelerator kernel. Missing tools leave runtime, driver, and hardware support `unknown`, and accelerator execution stays `not-run` with its reason. A future execution probe with an unavailable required device reports `blocked`; an unexpected execution error, wrong answer, or CPU/software fallback cannot pass the hardware lane. + +The result distinguishes requested and effective backends, detection from execution, correctness, device access, software fallback, and evidence tied to the environment record. A WebGPU API, adapter, or software renderer is not hardware GPU execution; WebNN presence is not NPU execution. Synthetic WSL2, VM, container, systemd, and Kubernetes-family fixtures exercise the contract only. Their validation cannot promote a hardware or platform lane to Checked. Commands and the exact CI scope are in [TESTING.md](TESTING.md#device-and-compute-workflow); the contract is in [CI-MATRIX.md](CI-MATRIX.md#device-and-compute-evidence). + ## Observed lane results, 2026-10-12 Added 2026-10-12. The first runs of `.github/workflows/compatibility.yml` (run 38210201779, 11 jobs) and `.github/workflows/python-abi.yml` (run 38210201628, 12 jobs), both green, on pull request [#19](https://github.com/P-R-N-D/Claus-AI/pull/19) at head `f7ccd3d`, tested as merge commit `aaaa80c1` onto `main` at `5f27b1c`. Each job's result record and probe reports are artifacts of its run, kept for 90 days. A run is evidence about one commit on one runner image; the lane table above remains the only record of status, and these runs change no status in it. Promoting a status, for example Linux aarch64 glibc to Checked, is a separate edit to that table. Every record is `incomplete` overall, because no lane runs the security and performance stages. diff --git a/docs/SECURITY-ARCHITECTURE.md b/docs/SECURITY-ARCHITECTURE.md index 7a304a0..1aaff69 100644 --- a/docs/SECURITY-ARCHITECTURE.md +++ b/docs/SECURITY-ARCHITECTURE.md @@ -171,6 +171,18 @@ Added 2026-10-12. Line numbers below are `compatibility.yml`'s unless `python-ab - Artifacts: each job uploads its result record, probe reports, and logs whether or not validation passed, kept for 90 days (lines 264-271, 545-551; `python-abi.yml` lines 167-174). They describe the runner and the lock's public packages, not the repository's users; the collector's and validator's secret-shaped value checks apply to the records, as in `ci.yml`, and nothing else is scanned. - The limits of the `ci.yml` checks apply here too: a pull request that changes these workflows, the probes, the validator, or their tests runs them as changed, so those changes need review in the diff. +### Device and compute workflow (`.github/workflows/device-tests.yml`) + +PR D adds one GitHub-hosted `ubuntu-24.04` job. It runs for path-filtered pull requests to `main` and `workflow_dispatch`, with `contents: read`, checkout credentials disabled, full commit SHA action pins, a bounded timeout, and no secrets, OIDC permission, self-hosted runner, privilege escalation, or `continue-on-error`. It applies the default compatibility privilege policy; there is no Windows job and no inherited policy exception. The existing three workflows and their checks remain in place. + +`scripts/ci/probe_compute.py` reads the collected environment, records only presence/access observations for allowlisted tool names and device-node classes, and executes a fixed CPU reference operation with the Python standard library. It does not run vendor diagnostic commands, install an SDK, open an accelerator for computation, or change device permissions. Its result records the tool/runtime/process, requested and effective backend, execution and correctness, and log/artifact references. The browser probe uses the existing frontend Playwright dependency and Chromium for a local fixed page; WebAssembly is the only browser operation checked for correctness. WebGL context creation and WebGPU/WebNN detection do not establish hardware computation. Published reports must exclude serial numbers, host/account names, addresses, raw environment dumps, and raw driver output. Reports and job summaries are public evidence, not confidential storage. + +Device access, host administrator rights, a rootful engine, a container's UID 0, capabilities, and application identity remain independent facts. Device access must never trigger fallback to `sudo`, rootful containers, or `--privileged`. A future lane must use a non-root application and rootless engine where possible, restrict access to named devices, and retain the existing exception contract for any approved deviation: scope, approval reference, verification, and end condition. Systemd as host process manager, host systemd/Quadlet managing a container, and systemd as container PID 1 are distinct configurations. cgroup v2 delegation is separate from engine mode, and container systemd does not imply a privileged container. Kubernetes workload user, no privilege escalation, engine mode, device-plugin access, and orchestration scope are likewise separate. + +Future hardware execution requires a suitable physical GPU/NPU, compatible runtime/driver, a reviewed operation with a known expected result, and narrowly granted device access. A trusted runner must be isolated from public pull request code, other tenants, credentials, and persistent host state; use an owner-controlled workflow and reviewed immutable revision with explicit authorization for that environment, ephemeral workspace/cleanup, and an artifact that binds the tested SHA, workflow/job/run, requested device, effective device, and correctness evidence. Merely gating a self-hosted `runs-on` expression in a pull request's editable workflow is not this boundary. PR D registers no runner, changes no GitHub setting, and never schedules public pull request code on hardware runners. The validator checks consistency of reported evidence; it cannot attest that a malicious runner or modified probe told the truth. + +The Windows administrator-token exception below remains confined to PR C's two jobs. The owner's disposition of PR #19's Codex finding accepted that risk; it did not verify a non-administrator Windows run. Non-administrator Windows execution and a token-privilege probe remain follow-up work, with the **2026-11-08** review date unchanged. + ### Temporary exception: Windows administrator token Approved 2026-10-12 by the repository owner for pull request #19 (PR C), which adds the two jobs below. It is a temporary security exception for the functional and ABI checks those jobs run on GitHub-hosted Windows runners. It covers every run of those jobs that meets their condition, before and after that pull request is merged, until an end condition below is met. It does not mean that a run without administrator rights was verified, and it does not relax the default privilege policy ([CI-MATRIX.md](CI-MATRIX.md) "Policy"), which every other job and record is still held to. diff --git a/docs/SECURITY-REVIEW.md b/docs/SECURITY-REVIEW.md index f46c3ec..8e1cb48 100644 --- a/docs/SECURITY-REVIEW.md +++ b/docs/SECURITY-REVIEW.md @@ -63,7 +63,7 @@ Map every file in the change to the boundaries in [SECURITY-ARCHITECTURE.md](SEC | `backend/agent/runtime/`, `backend/agent/llm/`, `backend/agent/rag/`, `backend/agent/orchestration/`, `backend/agent/tools/` | Runtimes, and model to content (docstring-only packages or a lazy factory today) | | `frontend/src/lib/api.ts`, `frontend/next.config.ts`, `frontend/src/proxy.ts`, `frontend/src/components/`, `frontend/src/app/` | Browser to control plane (the client side of it; `next.config.ts` and `src/proxy.ts` run in the Next.js server, which forwards `/core/*` and `/agent/*` to the backend and redirects UI paths, and authorizes nothing) | | `.env.example`, `.gitignore`, `backend/requirements.txt`, `backend/locks/*`, `frontend/package.json`, `frontend/package-lock.json`, `.nvmrc`, `.python-version` | No boundary: the "Secrets hygiene" implementation facts, plus the dependency surface | -| `.github/workflows/*`, `scripts/ci/*`, `tests/compatibility/*` | No product boundary: repository to CI runner, where pull request code runs on the runner and with the token the workflow gives it; the "CI workflow" and "Compatibility and Python ABI workflows" implementation facts. The compatibility collector reads runner state and allowlisted environment variables and its record is published as a run artifact; the wheel probe reads PyPI's JSON API and installs hash-locked wheels; the UBI lanes run a pulled image in rootless Podman | +| `.github/workflows/*`, `scripts/ci/*`, `tests/compatibility/*` | No product boundary: repository to CI runner, where pull request code runs on the runner and with the token the workflow gives it; the "CI workflow", "Compatibility and Python ABI workflows", and "Device and compute workflow" implementation facts. The compatibility collector reads runner state and allowlisted environment variables and its record is published as a run artifact; the wheel probe reads PyPI's JSON API and installs hash-locked wheels; the UBI lanes run a pulled image in rootless Podman | Record: @@ -77,6 +77,10 @@ For a change to a profile under `tests/compatibility/profiles/` that adds or cha A new file under `backend/agent/fastapi/routes/` or a new `path()` in `backend/core/urls.py` is always a new boundary. A dependency added to `requirements.txt` or `package.json` widens the trust surface and needs the explicit approval that [CONTEXT.md](CONTEXT.md) requires, cited in the change. A change to `backend/locks/*` or `frontend/package-lock.json` needs the same cited approval, because [CONTEXT.md](CONTEXT.md) requires it for lockfile changes. +For device/compute changes, also inspect [SECURITY-ARCHITECTURE.md](SECURITY-ARCHITECTURE.md) "Device and compute workflow" and trace the environment record, optional browser report, typed probe, stage linkage, summary, and public artifacts. Check that tool/runtime installation, API enumeration, device access, execution, correctness, and software fallback are separate facts; no detected name or driver file grants an execution pass; and SHA/workflow/job evidence agrees with the collected run. Check command and device allowlists, timeouts, bounded input, sanitization, and that no serial number, host/account name, raw environment, or driver dump reaches a public artifact. Digest binding detects a mismatched report, not a dishonest producer. + +Confirm that hardware access never falls back to rootful/privileged execution or host administration, and that non-root identity, engine rootlessness, capability sets, device access, no privilege escalation, and cgroup delegation are reviewed independently. Review all three systemd configurations by layer and Kubernetes device-plugin metadata separately from an actual grant. Synthetic fixtures test consistency and expected denials; they do not establish device availability or real enforcement. For future hardware runners, verify an owner-controlled trust boundary around reviewed code and the machine, rather than a PR-editable runner selector. PR D connects no such runner. PR C's Windows administrator exception applies only to its two named jobs; it supplies no permission for a new device job and no non-administrator result. Keep the 2026-11-08 review deadline. + ### 2. Trace input, output, and state-change flows For each entry point the change adds or alters, write the flow. When the change adds or alters none, write "none" and go on; steps 3 to 7 then examine only what the change cites or preserves. @@ -262,7 +266,7 @@ For each area marked `checked` in the coverage table, tests in these four classe - Boundary: inputs and states at the edge of what is allowed (scope and context, input size and shape, object names, configuration, embedded instructions). - Retry: repeated, replayed, resumed, or concurrent invocations. -The required cases per feature are in [TESTING.md](TESTING.md) ("Security-negative testing", "Approval and retry testing", "Personal and team boundary testing", "WebMCP testing", "Internationalization testing"); do not re-list them in the report. For the code that exists, `backend/core/tests.py`, `backend/agent/tests.py`, `backend/core/test_database.py`, and `backend/core/test_storage.py` are the baseline and must still pass unchanged. +The required cases per feature are in [TESTING.md](TESTING.md) ("Security-negative testing", "Approval and retry testing", "Personal and team boundary testing", "WebMCP testing", "Internationalization testing", "Device and compute workflow"); do not re-list them in the report. For the code that exists, `backend/core/tests.py`, `backend/agent/tests.py`, `backend/core/test_database.py`, and `backend/core/test_storage.py` are the baseline and must still pass unchanged. Rules: diff --git a/docs/TESTING.md b/docs/TESTING.md index a20757b..b98b9d4 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -20,6 +20,10 @@ The repository contains these automated tests. Their existence is a fact; whethe | `scripts/ci/tests/test_compatibility_schema.py` | `unittest`, 95 tests of `scripts/ci/validate_compatibility.py`. The committed profile and the nine synthetic fixtures validate, the fixtures are marked synthetic, and together they cover the scenarios [CI-MATRIX.md](CI-MATRIX.md) lists. The schema interpreter: every keyword it applies is exercised by a case that must fail, a schema with an unsupported keyword such as `oneOf`, `anyOf`, `allOf`, `not`, or `format` is rejected, and strict loading rejects duplicate keys, `NaN`, `Infinity`, overflowing numbers, a byte order mark, invalid UTF-8, deep nesting, and oversized files, read only up to the size limit. A schema is also rejected for another dialect, `$schema`, `$id`, or `$defs` below the root, a non-string annotation or type name, a pattern that Python and ECMA-262 read differently (escaped letters such as `\d`, `\A`, `\a`, `\N`, and `\U`, `\u` escapes of surrogates, `(?` groups, `{,n}`, possessive quantifiers, and a class that opens with `]`), or a `$ref` cycle. Each rule across fields, on edited copies of the fixtures: provenance and synthetic marking (including `--require-real`), control characters (named by a path that writes a key over 128 characters once escaped as its length and the first 8 hexadecimal digits of its SHA-256, so keys of one length are told apart unless those digits agree, keeps at most 16 keys around a `~...` that no key can be mistaken for, and is built only when the walk reaches its value), lone surrogates and secret-shaped values (a sample of every shape, and of both Slack prefixes, on its own and right after an ASCII or other letter, an AWS key ID followed by an underscore or a lowercase letter and a run of 17 key characters that is not one, the shortest JSON Web Token and one a character too short, and a long run of token characters searched in linear time), a collection time that is not a valid date and time, layer nesting (including a documented technology on a layer of another kind, a WSL2 layer with another technology, a host that names one, and a guest or container whose technology is `not-applicable`), OS, CPU, and ABI contradictions (with emulated and 32-bit cases that must pass, emulation inherited by every inner layer and runtime, the ABI tag's OS and architecture also when the version is unknown, an ABI tag architecture outside the list against a listed arch fact but not an unknown or unlisted one, a CPython version read whole (pre-release and `+` forms that must pass as CPython 3.12, though only a release of 3.12.13 or newer meets the checked lane's bound, text after the version and leading zeros, which also lose the lane, and another implementation's version left unchecked), a CPython platform tag and ABI tag compared with each other when the innermost layer's OS family is unknown, or the arch fact is unknown or not a listed name (by architecture only when both name one OS), and, when the layer is known, a platform tag for another OS reported against the layer once and not compared with the ABI tag, the ABI tag's C library against the libc fact (also when one of them names another C library, and not between two other names), Windows and macOS platform tags, and a glibc or musl libc fact on Windows or macOS, named by the innermost layer, the platform tag or the SOABI alone, or on FreeBSD, named by the layer or the platform tag alone), unrecognized values as warnings only, identity (also on macOS and FreeBSD), stage fields and order, which stages may be not-applicable, the log or artifact a real stage cites, `detect` with probe errors, failed versus not-run, `overall`, required stages, CPU fallback reported as a GPU pass, a passed stage whose compute record names no backend that ran, whatever was requested (reported once, as a fallback, on a GPU request, and not for a failed stage), an accelerator profile's stages, `performance` included, requested and run on its backend, backend references, each default policy rule (including capabilities on Linux or an unknown OS family, and a workload not shown to run outside a container, which a WSL2 distribution's own PID namespace does not trigger), rootless engine and non-root user as independent rules, policy decided by measured facts only (declared values do not show compliance, an unknown privilege state fails, off Linux an unmeasured layer kind fails, and a real or saved user ID of 0 fails), exceptions that are pending, out of scope, unused, missing fields, or approved on a day that does not exist, scoped by escalation method, or offered for a state that is not measured, the warnings for an applied exception and a skipped policy, a profile that weakens a rule, support claims against the lane table and bound to the lane's platform and the lock's interpreter (the C library from the libc fact before the ABI tag, `musl` included, `glibc` or `musl` read in any ASCII letter case at the start of the fact only and with no letter after it, with the error at that fact, and a libc fact or ABI tag that names another C library contradicting the lane whatever the status), version bounds (macOS Apple Silicon by macOS version, statically resolvable for 15 or newer and not supported for 14 or older; Linux x86-64 glibc checked for glibc 2.39 or newer, read from the libc fact in any ASCII letter case and not from the ABI tag, and CPython 3.12.13 or newer as a release; a version part with a leading zero or more than four digits, or more than one blank after `glibc`, is not read; a claim whose versions miss a bound or are not declared fails, and a real profile copies the status its versions meet or else records not-assessed, and on a lane without bounds copies the lane's status), the validator's lanes and version bounds kept equal to the table's, each bound naming a version the validator reads, a lane table that lists a lane twice (a usage error), evidence sources (with a comment line inside `jobs:`, job keys indented by four spaces, quoted, or followed by a tab or a comment, a quoted `jobs` key or one with a blank before its colon, and U+0085 inside a value, which name the job, and a deeper key, a longer name, no space after the colon, a comment right after a colon, an indented `jobs` key, a `jobs` block scalar, a `jobs` mapping in flow style, U+2028 inside a value, or a dot in the job name matched as another character, which name none, and a workflow file that is not UTF-8 or a path the system refuses, each reported as a finding that names why), the profile link (job, workflow, a workflow ref in another repository, including one whose name only starts with the record's, or with no workflow file or no ref, a ref holding `@`, which links, a record without its workflow ref or its repository, which links to no profile and is named by the field it lacks, and a real profile's GitHub Actions record) and the comparison of every declared fact (with the layer count, runtimes, backend API, runner, and `not-applicable` facts each checked on their own, and declared values in the result not counting), a blocked stage, fallback from an NPU and to another GPU, and CI evidence (each run and commit field, the workflow ref included, also when declared rather than observed or inferred, a run attempt of 0, and a run URL for another attempt or with an attempt longer than any `run_attempt`, while one with no attempt names the run). Command-line exit codes 0, 1, and 2, an unwritable or unusable schema, a schema given with `--schema` that accepts a document or profile the rules across fields cannot read (exit 2, not a traceback), and an exception in a rule under the default schema, however its path is written, which is raised, not reported as a usage error, an unwritable summary or one whose path loops, a summary that names the schema, the profile, the lane table, a document (the document also through a hard link or another spelling of its path), or a workflow file a real profile of the run names as its evidence source (also through a hard link, also when that file is missing, which it then does not create while it writes a summary elsewhere, whether that profile is given with `--profile` or as a document, and also when `synthetic` is a false value other than `false`), which leaves every input unchanged, a summary whose cells escape Markdown and that lists at most 200 findings, file names (including one that starts with `::`) and document text that cannot forge an output line, including the runner's `##[` command form anywhere in a line, a command-line error that repeats such an argument, a secret-shaped one, or such a program name, and a secret-shaped value or file name that the printed lines and the summary do not show, also after an escape the finding writes just before the value (`\n` from `repr`, `~1` in a JSON Pointer). | | `scripts/ci/tests/test_environment_probe.py` | `unittest`, 84 tests of `scripts/ci/collect_environment.py` on fake machines built in temporary directories, with the interpreter facts fixed to match each machine: a GitHub-hosted runner VM, a Hyper-V VM named only by its VMBus, a machine with no virtualization signal (not reported as bare metal), Docker in a VM, a container that leaves none of the markers (told apart from the VM by its PID namespace), an Apptainer container in the initial PID namespace, signals that do and do not show a VM (EC2 metal instances and an EC2 Arm VM, WSL 1 with the hypervisor flag), rootless Podman with a non-root user and with root, WSL2 with current and older kernels and a distribution in its own PID namespace, an emulated process (recorded on the runtime, not the container layer), an old kernel without `/proc/sys/kernel/arch`, the interpreter's build architecture (from the ELF header only when its byte order and class match the name, so a big-endian ppc64, aarch64 or armeb is not read as ppc64le, aarch64 or armv7, nor a 32-bit x32, aarch64 ILP32, riscv32, loongarch32 or 31-bit s390 build as its 64-bit name) and free-threading flag, a platform tag overridden by `_PYTHON_HOST_PLATFORM` or malformed (recorded as `unknown`), the C library version read only through `os.confstr`, Node on Power named from Node's own byte order, not Python's, kernel and AppArmor text with control characters, an AppArmor `enabled` value other than `Y` or `N` and a newline (also padded, or longer than the 16 bytes read), the service manager, a Kubernetes pod, systemd in a container, container markers that disagree, Docker in a user namespace of its own, the kernel release read from `/proc` with `uname` only as an inferred fallback, os-release values that fail their checks, a FIFO in place of a file, and the provenance each derived fact keeps; Windows x64 and Windows on Arm, with the machine named by `IsWow64Process2` and an x64 process emulated on Windows on Arm (Python and Node each compared with the machine), an elevated, administrator, or LocalSystem token recorded as the superuser and a token that cannot be read in full left `unknown` (both failing `policy-app-non-root`), the capabilities left `unknown` because the token's privileges are not read (so a token without administrator rights still fails `policy-minimal-capabilities`), and BIOS values that name Hyper-V, QEMU, Amazon, or VMware, while values that name no hypervisor record no layer kind; macOS on Apple Silicon and on Intel read through `sysctlbyname`, a process Rosetta translates, a root process, and values that are missing or fail their checks; the real and saved user IDs and a process that can regain root, cgroup version and delegation (not delegated without write access to the directory, `cgroup.procs`, `cgroup.threads` where it exists, or `cgroup.subtree_control`, with a `cgroup.threads` that cannot be looked up counted as present, and left unknown for real or saved root and for permitted capabilities, and for a cgroup path that leaves `/sys/fs/cgroup`), security modules, effective and permitted capabilities, missing files giving `unknown` values, and a failing probe kept apart from the other values; the user IDs on FreeBSD and macOS and none under Cygwin, the FreeBSD platform tag, and no Linux capability rule on macOS; pull request, `pull_request_target`, and push events, a run attempt up to the schema's largest integer, 4294967295 (a larger one is `unknown`), a manual dispatch from a branch with `#`, a comma, an apostrophe, `@`, a non-ASCII letter, or a trailing U+3000, checked out by its name as actions/checkout does, whose ref, workflow ref, and commit (from a loose ref or `packed-refs`) are recorded and which links to the profile, a workflow ref that is `unknown` for a branch with a space, `~`, `:`, `\`, a control or bidirectional formatting character, or a lone surrogate, or longer than 200 characters, a ref read in the same way, the characters a ref may hold checked against the validator's on every code point, malformed, unparsable, and missing event files and a missing event path, a checkout ref that leaves `.git`, that holds `..` or `@{`, a part that is empty, starts with `.`, or ends with `.lock` or `.`, a space, a control or bidirectional formatting character, or a byte that is not UTF-8 (not read as U+FFFD, the name of another branch), or that is longer than 200 characters, which is not followed even when `packed-refs` lists it, refs that hold `{`, `@`, or `.lock` elsewhere and are followed, `packed-refs` lines that end at `\n` only, a `packed-refs` line that is not UTF-8 (not read as U+FFFD, and not stopping the lookup), a ref that is not ASCII under a file system encoding other than UTF-8, which is not followed, and a `GITHUB_REF` or `GITHUB_WORKFLOW_REF` that is not ASCII under such an encoding, recorded as `unknown`, a loose ref that does not exist, looked up in `packed-refs`, and one that cannot be read or holds no commit ID (a symbolic ref), which is not, and the variable allowlist (which no longer holds the WSL variables); `node` run with an empty environment (on Windows, `SystemRoot` alone), looked for only on the `PATH` given (an empty `PATH` finds none, an unset one the default directories, and on Windows `node.exe` alone is looked for, not a `node.bat`, `node.cmd`, `node.com` or `node` earlier on `PATH`, without `shutil.which` and so in the same way on CPython 3.11 and 3.12, an empty entry names no directory, and the current directory is searched only when `PATH` names it, not when `PATH` is unset and the default directories list it first), failing, absent, skipped, reporting values that fail their checks, writing more than 4 KiB (and then ignoring the closed pipe), or not finishing in time (also after closing its output, by the deadline the read had), and a timer that fires after `node` was reaped, also after a failed read or right after the wait, which then sends no signal, or as the read ends, which holds back the reap until its signal is sent; stage option errors, including a value of every secret shape, which both the collector and the validator refuse; exit status 1 after a probe failure with the record still written; `--extension` files kept under their `x-` name, and a name without the prefix, a file that is missing, not a JSON object, not UTF-8, larger than 1 MiB, holds a secret-shaped value, or is given twice, each refused with exit status 2 and no record written; `--amend-container`, which adds what `podman info` and `podman container inspect` report to the container layer of a record collected inside it (rootless, privileged, and devices, also when they are rootful or privileged), changes nothing when the output does not describe that running container, and starts its two subprocesses without a shell; an unwritable output or summary; a summary that names the record's own file, also through a hard link or another spelling of its path, which leaves the record valid JSON; an output file name that would start a workflow command or is shaped like a token, which the printed line does not show, and a command-line error that repeats such an argument, a secret-shaped one, or such a program name, with the collector's line handling checked against the validator's; a record holding a secret-shaped value, which is not written, also when the JSON text writes the character before it as an escape such as `\n`, with the collector's secret shapes checked against the validator's; a summary with Markdown in a recorded value; and the tables both scripts keep (architecture names and which of them are 32-bit, platform tags, stages, the overall verdict, and the layer kind of every technology the collector writes), checked against each other and, for the platform tags, against expected values. One runs the probe's `node -p` expression with an empty environment on the `node` on `PATH`, when there is one, and checks that it gives every value the probe reads, the byte order included; a version-manager shim that needs `HOME` fails it. Two run the collector on the machine running the tests, with secret-shaped values in its environment, and check that no secret, host name, or home directory reaches the record and that the record is valid. Four read the collector's source and check its imports, that it never reads the whole environment or the host name, starts `node` without a shell, with an empty environment or, on Windows, `SystemRoot` alone, and discards its stderr, starts `podman` only in the amend mode, without a shell or input, and opens files for reading except its outputs. | | `scripts/ci/tests/test_python_abi_probe.py` | `unittest`, 35 tests of `scripts/ci/probe_python_abi.py`, which needs `packaging` (installed from the backend lock with its hash first). Most feed it made-up index metadata and tag lists, so they do not depend on the interpreter: reading the backend lock (its 50 pins, the `# via` graph followed transitively, requirement lines that keep every hash, and a pin given twice, without a hash, or in an unexpected form refused), the wheels of a release (a compressed tag set such as `abi3.abi3t` expanded to each tag, yanked files kept, and metadata for another release, with a malformed digest or file name, larger than its limit, or listing no files refused), the index asked for the locked release only, the order in which this interpreter accepts tags, wheels whose Requires-Python excludes this version dropped as installers drop them (an unreadable one excludes nothing, as in pip), the five ABI categories (`cp315-cp315`, `cp315-cp315t`, `cp315-abi3`, `cp315-abi3t`, and `cp315-abi3.abi3t`) and older `abi3` tags, the selected wheel found by its tags and the lock's hashes, the install status (a missing wheel is `no-wheel` or, when one was published after the lock, `not-in-lock`, both blocked, while an install that fails with a wheel in the lock is `error` and one that passes with none predicted is `contradiction`, both failed, and a package whose lock dependency is blocked or failed takes that outcome), the overall outcome (a package that enables the GIL again, or whose probe process did not exit with status 0, fails `execute` even when its import and call passed), the backend jobs' install stage (blocked only when a uv line that says a package has no wheel names one the wheel report shows uncovered, or when every problem `uv pip check` prints, as many as the count it prints, is a requirement that is not installed and that the lock does not pin, such as `tzdata` on Windows with the lines uv 0.12.24 printed there, and failed otherwise, including a network error, a failed download line, a missing report, an installed version a requirement excludes, a version conflict beside a missing `tzdata`, a count that does not match the lines, and a check failure nothing explains), unmet requirements read from fake installed distributions (missing, or installed at a version the requirement excludes, with an unrequested extra ignored), `probe_package` and the `packages` command with uv, the installed distributions, and the import probe replaced (a package installed with its lock dependency and probed, a dependency that fails or is blocked, a package with no wheel, which is installed once and not imported, and the report, GitHub outputs, and exit status, including exit status 1 when a package enables the GIL again or its probe process is ended by a signal, shown in the summary's Exit column), recorded text without this environment's paths or control characters, output lines that cannot be forged, and the import and call run in a fresh process with the GIL state before and after, with an import failure, a call failure, and a warning kept apart, and the process's exit checked after its result: status 0 passes; with `subprocess.run` replaced, a printed passing result followed by status 3, by -11 (`SIGSEGV`), by an unknown signal number, or by the Windows crash status 0xC0000005 fails, on every OS; in a real process, an `atexit` hook that flushes the result and then calls `os._exit(3)` fails, as does one that sends itself `SIGKILL` (POSIX only, skipped on Windows), and a process that exits with status 4 before printing a result fails as an import with no result. Four check this interpreter's own facts: SOABI, extension suffixes, and the GIL state agree, the tags match the build, and, only on CPython 3.15 with the GIL or on 3.15t, which tags and suffixes it accepts (3.15t takes `cp315t` and `abi3t` wheels and no `abi3` one) and that `sys._is_gil_enabled()` and `Py_GIL_DISABLED` match; on any other interpreter those two are skipped. Three read the probe's source: two `subprocess.run` calls without a shell or input and with a timeout, uv run with `--no-config`, `--require-hashes`, `--only-binary :all:`, and `--no-deps`, no network host but the PyPI JSON API (and `https://pypi.org/simple` as uv's index), and no environment variable read or changed. No test establishes thread safety. | +| `scripts/ci/tests/test_device_contract.py` | Device probe/stage consistency, GPU fallback, false availability/execution claims, and run/revision/workflow/job evidence regressions. | +| `scripts/ci/tests/test_compute_probe.py` | Standard-library CPU reference success, deliberate wrong result and execution error, capability observations, report import and output handling. | +| `scripts/ci/tests/test_device_environments.py` | Fifteen generated synthetic environment pairs, expected rootful/device-policy denials, layer/manager mutations, and synthetic-as-real rejection. | +| `frontend/scripts/probe-browser-compute.test.mjs` | Node built-in tests of browser result mapping, unavailable-browser handling, fixed WebAssembly operation, and no hardware-computation claims. | No test covers authentication, authorization, scope separation, approval, Tasks, retrieval, Browser sessions, realtime, i18n, or WebMCP, because none of those features exist. @@ -179,6 +183,40 @@ Added 2026-10-12. `.github/workflows/compatibility.yml` and `.github/workflows/p - The probe asks `https://pypi.org/pypi///json` for each pinned release, with a timeout and a size limit, and refuses metadata for another release or with a malformed file name or digest; what it installs still comes from the lock's hashes. - The first run of both workflows was on pull request #19, head `f7ccd3da0baf467bc6d336a946fd5a4d38107e5d`, base `5f27b1c23869e2afe7ba7cefb04154d296e1d6ad`, merge commit `aaaa80c1c0dcd2cabd0318f6ca29255bb540796b`: runs [38210201779](https://github.com/P-R-N-D/Claus-AI/actions/runs/38210201779) (`compatibility.yml`, 11 of 11 jobs passed) and [38210201628](https://github.com/P-R-N-D/Claus-AI/actions/runs/38210201628) (`python-abi.yml`, 12 of 12 passed). [DEPENDENCY-STRATEGY.md](DEPENDENCY-STRATEGY.md#observed-lane-results-2026-10-12) records what they observed. Cite the run that tested the commit in question, not this one. +### Device and compute workflow + +PR D adds `.github/workflows/device-tests.yml`, with one `cpu` job (`CPU reference and browser capabilities / Linux x64`) on GitHub-hosted `ubuntu-24.04`, a 15-minute timeout, path-filtered `pull_request` for `main`, and `workflow_dispatch`. It uses `contents: read`, checkout without persisted credentials, full commit SHA action pins, no secrets/OIDC, no self-hosted runner, no privilege escalation, and no `continue-on-error`. It adds no Windows administrator job or exception. `ci.yml`, `compatibility.yml`, and `python-abi.yml` remain unchanged, including all 12 E2E tests in each of dev and production mode. + +The job creates the same standard-library CPython 3.12.15 environment, collects its environment, uses the existing locked Node Playwright dependency (`npm ci --ignore-scripts`) and downloads paired Chromium without `--with-deps`, probes browser capabilities, executes the CPU matrix reference, attaches compute evidence to the common Result, and validates it with `--require-real`, `gha-linux-x64-device.json`, and the dependency strategy. Its required stages are `detect` through `correctness`; `security` and `performance` remain `not-run`. CPU execution, CPU correctness, malformed common evidence, and unexpected probe/test failures fail the job. Unexecuted accelerator lanes and unavailable optional browser probes do not become passes and do not by themselves fail the CPU lane. A browser download or workflow setup error still fails its step. + +To exercise the same probe interface from the repository root with Python and Node already available: + +```bash +device_records="$(mktemp -d)" +python -I scripts/ci/collect_environment.py --profile-id gha-linux-x64-device \ + --output "$device_records/environment.json" +node frontend/scripts/probe-browser-compute.mjs --environment "$device_records/environment.json" \ + --output "$device_records/browser-compute.json" +python -I scripts/ci/probe_compute.py --environment "$device_records/environment.json" \ + --browser-report "$device_records/browser-compute.json" --output "$device_records/result.json" \ + --report "$device_records/compute-probe.json" --log "$device_records/cpu-reference.log" +python -I scripts/ci/validate_compatibility.py --require-real \ + --dependency-strategy docs/DEPENDENCY-STRATEGY.md \ + --profile tests/compatibility/profiles/gha-linux-x64-device.json "$device_records/result.json" +python -I scripts/ci/tests/test_device_contract.py --verbose +python -I scripts/ci/tests/test_compute_probe.py --verbose +python -I scripts/ci/tests/test_device_environments.py --verbose +node --test frontend/scripts/probe-browser-compute.test.mjs +``` + +The real profile requires its GitHub workflow/job/runner identity: a local result normally fails that profile link. As with the original collector command, omitting `--profile` checks local facts without claiming a GitHub run; default privilege policy still applies. `--no-policy` is useful only for inspecting structure of an environment that does not satisfy that policy and is not used by the device job. Do not invent GitHub variables to make local evidence look like an Actions run. The CPU probe needs no package. Omit `--browser-report` when intentionally running CPU only; browser backends then record why they were not run. Missing Playwright/Chromium or unavailable sandbox leaves browser checks `not-run`. The browser probe keeps Chromium's sandbox enabled with no unsandboxed retry; an explicitly supplied `PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH` uses an already installed browser and is supplemental evidence rather than evidence for the paired binary. + +The CPU reference result is exactly `[[4, 10], [8, -1]]`, computed from fixed inputs. `--cpu-case mismatch` and `--cpu-case error` are deliberate negative runs: each writes failure evidence and exits nonzero; its Result must fail the required CPU profile. The tests exercise these paths as well as GPU-to-CPU fallback, detection-only passes, installed-driver-only availability, runtime installation missing from availability/completed-operation claims, required CPU stages replaced by extra WASM execution, absent/unrecorded devices, wrong backend IDs, non-execution promoted to success, synthetic-as-real records, SHA/workflow/job mismatches, and policy/layer/manager errors. They also retain valid extra WASM evidence, optional stages on another backend, profile-specific CPU IDs, and honest unknown/absent runtime facts for unexecuted optional backends. Special environments are 15 generated synthetic pairs under `tests/compatibility/fixtures/device-environments/`; three rootful cluster and two device-passthrough examples deliberately fail policy, without fabricated approvals. The fixture README explains those expected failures. + +The browser operation is WebAssembly `20 + 22 = 42`; WebGL context creation, WebGPU API/adapter availability, and WebNN API presence are detection only. No browser GPU or NPU computation runs. Browser importer regressions reject a passing aggregate with a producer failure reason, a `node-controller` version that differs from measured environment Node, and unknown or merely declared Node versions for observed browser facts. They retain genuine detection failures with hardware operations `not-run`, absent optional features with matching Node, and entirely unknown browser limitations without measured Node. CPU/software adapters do not verify hardware acceleration. Native GPU/NPU backends record optional tool/device-node observations, with runtime/API availability unknown where unmeasured and execution `not-run` because no kernel adapter is implemented. No SDK or inference framework is installed. + +The job writes its CPU/backend summary and validator result to the Actions summary and uploads `device-compute-linux-x64-attempt-` for 90 days, including environment and Result JSON, compute/browser reports, CPU log, and regression logs, after failures too unless cancelled. Cite the actual run URL, head/base/tested merge SHAs, job, and log summary when reporting a result; this workflow description alone proves no run occurred. + ## API collection testing Blocked, recorded 2026-10-10. No Postman collection, Postman CLI, or Newman is in the repository. The Postman CLI was checked as a candidate for the route checks above, in an isolated scratch install (`npm install --ignore-scripts`, a throwaway `HOME`, no login, API key, `postman init`, or cloud workspace) of `postman-cli` 1.73.0 and its `@postman/pm-bin-linux-x64` 1.73.0 binary package. Two conditions set for adopting it were not met: diff --git a/frontend/scripts/probe-browser-compute.mjs b/frontend/scripts/probe-browser-compute.mjs new file mode 100644 index 0000000..0eb20b1 --- /dev/null +++ b/frontend/scripts/probe-browser-compute.mjs @@ -0,0 +1,252 @@ +#!/usr/bin/env node +// Optional browser observations. Only WASM executes an operation; the other +// probes never qualify GPU/NPU hardware, including software/fallback adapters. +import { createHash } from "node:crypto"; +import { access, readFile, stat, writeFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; + +const ORIGIN = "https://compute-probe.invalid/"; +const APIS = ["wasm", "webgl", "webgpu", "webnn"]; + +export function emptyReport(reason, environmentSha256 = null) { + return { + schema_version: 1, + kind: "browser-compute-probe", + tool: "playwright-chromium", + environment_sha256: environmentSha256, + status: "not-run", + reason, + probes: APIS.map((api) => ({ + id: `browser-${api}`, + api, + kind: api === "wasm" ? "cpu" : api === "webnn" ? "npu" : "gpu", + runtime_installed: "unknown", + api_enumerated: "unknown", + device_enumerated: "unknown", + device_accessible: "unknown", + execution: { + status: "not-run", + executed: false, + effective: null, + correct: "unknown", + software_fallback: "unknown", + runtime: null, + process_id: null, + evidence: [], + reason, + }, + evidence: [], + })), + }; +} + +// Self-contained: Playwright serializes this function into the blank page. +export async function observeBrowser() { + const observations = {}; + const wasm = (observations.wasm = { api: typeof WebAssembly === "object" }); + if (wasm.api) { + try { + // (module (func (export "add") (param i32 i32) (result i32) + // local.get 0 local.get 1 i32.add)) + const bytes = new Uint8Array([ + 0, 97, 115, 109, 1, 0, 0, 0, 1, 7, 1, 96, 2, 127, 127, 1, 127, + 3, 2, 1, 0, 7, 7, 1, 3, 97, 100, 100, 0, 0, + 10, 9, 1, 7, 0, 32, 0, 32, 1, 106, 11, + ]); + const { instance } = await WebAssembly.instantiate(bytes); + wasm.actual = instance.exports.add(20, 22); + wasm.executed = true; + wasm.correct = wasm.actual === 42; + } catch { + wasm.error = "wasm-operation-failed"; + } + } + try { + const canvas = document.createElement("canvas"); + const context = canvas.getContext("webgl"); + observations.webgl = { + api: typeof WebGLRenderingContext === "function", + context: context !== null, + }; + context?.getExtension("WEBGL_lose_context")?.loseContext(); + } catch { + observations.webgl = { error: "webgl-context-probe-failed" }; + } + observations.webgpu = { api: typeof navigator.gpu === "object" }; + if (observations.webgpu.api) { + let timer; + try { + const adapter = await Promise.race([ + navigator.gpu.requestAdapter(), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error("timeout")), 5_000); + }), + ]); + observations.webgpu.adapter = adapter !== null; + // Read only the standard fallback boolean, never vendor, model, IDs, + // renderer strings, driver descriptions or device identifiers. + observations.webgpu.software = + typeof adapter?.info?.isFallbackAdapter === "boolean" + ? adapter.info.isFallbackAdapter + : typeof adapter?.isFallbackAdapter === "boolean" + ? adapter.isFallbackAdapter + : "unknown"; + } catch { + observations.webgpu.error = "webgpu-adapter-probe-failed"; + } finally { + clearTimeout(timer); + } + } + observations.webnn = { api: typeof navigator.ml === "object" }; + return observations; +} + +export function applyObservations(report, observations, browserVersion) { + // The process ID names the Node controller, not a Chromium renderer. + const runtime = `chromium ${browserVersion}; node-controller ${process.versions.node}`; + report.status = "passed"; + report.reason = "capability-probes-completed"; + for (const probe of report.probes) { + const observation = observations[probe.api]; + probe.runtime_installed = true; + probe.api_enumerated = observation.api ?? "unknown"; + probe.evidence = [`tool:playwright-chromium:${probe.api}`]; + probe.execution.runtime = runtime; + probe.execution.process_id = process.pid; + probe.execution.evidence = probe.evidence; + if (probe.api === "wasm") { + const ran = observation.executed === true; + probe.device_enumerated = ran ? true : "unknown"; + probe.device_accessible = ran ? true : "unknown"; + Object.assign(probe.execution, { + status: observation.error || (ran && !observation.correct) + ? "failed" : ran ? "passed" : "not-run", + executed: ran, + effective: ran ? probe.id : null, + correct: ran ? observation.correct : "unknown", + software_fallback: ran ? false : "unknown", + reason: observation.error ?? (ran ? "fixed-wasm-addition" : "wasm-api-unavailable"), + }); + probe.expected = 42; + probe.actual = ran ? observation.actual : null; + } else { + probe.execution.reason = observation.error ?? `${probe.api}-detection-only-no-kernel`; + // A context or an adapter can be software. Neither proves a usable + // physical device; WebNN API presence cannot identify an NPU either. + if (probe.api === "webgl") probe.context_available = observation.context ?? "unknown"; + if (probe.api === "webgpu") { + probe.adapter_available = observation.adapter ?? "unknown"; + probe.software_adapter = observation.software ?? "unknown"; + } + } + if (observation.error || probe.execution.status === "failed") { + report.status = "failed"; + report.reason = "browser-probe-failed"; + } + } + return report; +} + +async function bounded(promise, milliseconds) { + let timer; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error("probe-timeout")), milliseconds); + }), + ]); + } finally { + clearTimeout(timer); + } +} + +export async function probeBrowser(environmentSha256) { + const report = emptyReport("browser-not-started", environmentSha256); + let chromium; + try { + ({ chromium } = await import("@playwright/test")); + } catch { + return emptyReport("playwright-package-unavailable", environmentSha256); + } + const executablePath = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH || chromium.executablePath(); + try { + await access(executablePath); + } catch { + return emptyReport("chromium-binary-unavailable", environmentSha256); + } + let browser; + try { + browser = await chromium.launch({ + headless: true, + executablePath, + chromiumSandbox: true, + args: ["--disable-setuid-sandbox"], + env: { PATH: process.env.PATH || "", LANG: "C.UTF-8" }, + timeout: 15_000, + }); + } catch (error) { + // Do not print Chromium stderr: it can include user paths or identifiers. + if (/No usable sandbox|Failed to move to new namespace|Operation not permitted/.test(error.message)) { + return emptyReport("chromium-sandbox-unavailable", environmentSha256); + } + report.status = "failed"; + report.reason = "chromium-launch-failed"; + for (const probe of report.probes) probe.execution.reason = report.reason; + return report; + } + try { + const version = browser.version(); + if (!/^\d+(?:\.\d+){1,3}$/.test(version)) throw new Error("invalid-browser-version"); + const context = await browser.newContext({ serviceWorkers: "block" }); + await context.route("**/*", (route) => route.request().url() === ORIGIN + ? route.fulfill({ contentType: "text/html", body: "Compute probe" }) + : route.abort()); + const page = await context.newPage(); + await page.goto(ORIGIN, { waitUntil: "domcontentloaded", timeout: 10_000 }); + applyObservations(report, await bounded(page.evaluate(observeBrowser), 15_000), version); + } catch { + report.status = "failed"; + report.reason = "browser-observation-failed"; + for (const probe of report.probes) probe.execution.reason = report.reason; + } finally { + try { + await bounded(browser.close(), 5_000); + } catch { + report.status = "failed"; + report.reason = "browser-cleanup-failed"; + } + } + return report; +} + +async function main(args) { + const options = {}; + for (let index = 0; index < args.length; index += 2) { + const name = args[index]; + if (!["--environment", "--output"].includes(name) || !args[index + 1] || options[name]) { + throw new Error("invalid-options"); + } + options[name] = args[index + 1]; + } + if (!options["--environment"] || !options["--output"]) throw new Error("missing-options"); + const metadata = await stat(options["--environment"]); + if (!metadata.isFile() || metadata.size > 1024 * 1024) throw new Error("invalid-environment-file"); + const environment = await readFile(options["--environment"]); + const record = JSON.parse(environment.toString("utf8")); + if (record.schema_version !== 1 || record.kind !== "result" || record.synthetic !== false) { + throw new Error("expected-measured-environment"); + } + const report = await probeBrowser(createHash("sha256").update(environment).digest("hex")); + await writeFile(options["--output"], `${JSON.stringify(report, null, 2)}\n`, { flag: "wx" }); + console.log(`browser compute: ${report.status} (${report.reason})`); + process.exitCode = report.status === "failed" ? 1 : 0; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(process.argv.slice(2)).catch(() => { + // Never echo caller-supplied arguments, paths, JSON, or error messages. + console.error("browser compute: invalid input or output unavailable"); + process.exitCode = 2; + }); +} diff --git a/frontend/scripts/probe-browser-compute.test.mjs b/frontend/scripts/probe-browser-compute.test.mjs new file mode 100644 index 0000000..63c26d1 --- /dev/null +++ b/frontend/scripts/probe-browser-compute.test.mjs @@ -0,0 +1,79 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { applyObservations, emptyReport, observeBrowser } from "./probe-browser-compute.mjs"; + +function observations() { + return { + wasm: { api: true, actual: 42, executed: true, correct: true }, + webgl: { api: true, context: true }, + webgpu: { api: true, adapter: true, software: true }, + webnn: { api: true }, + }; +} + +test("only the fixed WASM operation passes; API, context, and software adapter do not", () => { + const report = applyObservations(emptyReport("initial", "a".repeat(64)), observations(), "151.0.0.0"); + assert.equal(report.status, "passed"); + assert.equal(report.probes[0].execution.status, "passed"); + assert.equal(report.probes[0].execution.effective, "browser-wasm"); + assert.match(report.probes[0].execution.runtime, /node-controller/); + for (const probe of report.probes.slice(1)) { + assert.equal(probe.execution.status, "not-run"); + assert.equal(probe.execution.executed, false); + assert.equal(probe.execution.effective, null); + assert.equal(probe.device_enumerated, "unknown"); + assert.equal(probe.device_accessible, "unknown"); + } + assert.equal(report.probes[2].software_adapter, true); +}); + +test("a real operation with the wrong answer fails", () => { + const values = observations(); + values.wasm.actual = 41; + values.wasm.correct = false; + const report = applyObservations(emptyReport("initial"), values, "151.0.0.0"); + assert.equal(report.status, "failed"); + assert.equal(report.probes[0].execution.status, "failed"); + assert.equal(report.probes[0].execution.executed, true); + assert.equal(report.probes[0].execution.correct, false); +}); + +test("missing browser means not run with unknown hardware", () => { + const report = emptyReport("chromium-binary-unavailable"); + for (const probe of report.probes) { + assert.equal(probe.runtime_installed, "unknown"); + assert.equal(probe.device_enumerated, "unknown"); + assert.equal(probe.execution.status, "not-run"); + } +}); + +test("an unexpected probe failure is not reported as a successful detection", () => { + const values = observations(); + values.webgpu = { api: true, error: "webgpu-adapter-probe-failed" }; + const report = applyObservations(emptyReport("initial"), values, "151.0.0.0"); + assert.equal(report.status, "failed"); + assert.equal(report.probes[2].execution.status, "not-run"); + assert.equal(report.probes[2].adapter_available, "unknown"); +}); + +test("the WASM bytes execute the expected operation in a CPU runtime", async () => { + // Exercise the actual bytecode independently from Chromium without replacing + // WebAssembly. Other globals are synthetic detection fixtures only. + const documentDescriptor = Object.getOwnPropertyDescriptor(globalThis, "document"); + const navigatorDescriptor = Object.getOwnPropertyDescriptor(globalThis, "navigator"); + Object.defineProperty(globalThis, "document", { configurable: true, value: { createElement: () => ({ getContext: () => null }) } }); + Object.defineProperty(globalThis, "navigator", { configurable: true, value: {} }); + try { + const result = await observeBrowser(); + assert.equal(result.wasm.actual, 42); + assert.equal(result.wasm.executed, true); + assert.equal(result.wasm.correct, true); + assert.equal(result.webgpu.api, false); + assert.equal(result.webnn.api, false); + } finally { + if (documentDescriptor) Object.defineProperty(globalThis, "document", documentDescriptor); + else delete globalThis.document; + if (navigatorDescriptor) Object.defineProperty(globalThis, "navigator", navigatorDescriptor); + else delete globalThis.navigator; + } +}); diff --git a/scripts/ci/probe_compute.py b/scripts/ci/probe_compute.py new file mode 100644 index 0000000..17ff2c5 --- /dev/null +++ b/scripts/ci/probe_compute.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 +"""Run a standard-library CPU reference and join device observations to a Result. + +No driver utility or SDK is executed. PATH tool presence and allowlisted Linux +device-node classes are discovery hints only: they never establish a driver, +runtime, usable device, or accelerator execution. No privileges are changed. +""" + +import copy +import datetime +import hashlib +import importlib.util +import json +import os +import pathlib +import platform +import re +import shutil +import stat +import sys + + +HERE = pathlib.Path(__file__).resolve().parent + + +def sibling(name): + spec = importlib.util.spec_from_file_location(name, HERE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +collector = sibling("collect_environment") +validator = sibling("validate_compatibility") +observed, unknown, not_applicable = collector.observed, collector.unknown, collector.not_applicable +TOOL = "tool:probe-compute" +# API catalog, not implemented hardware adapters. Presence tools are not run. +CATALOG = ( + ("cuda", "gpu", "nvidia-smi"), ("rocm", "gpu", "rocminfo"), + ("oneapi", "gpu", "sycl-ls"), ("level-zero", "gpu", "zeinfo"), + ("opencl", "gpu", "clinfo"), ("vulkan-compute", "gpu", "vulkaninfo"), + ("directcompute", "gpu", None), ("directml", "gpu", None), + ("metal", "gpu", None), ("mps", "gpu", None), ("coreml", "npu", None), + ("mlx", "gpu", None), ("qnn", "npu", None), +) +BROWSER_APIS = {"wasm": "cpu", "webgl": "gpu", "webgpu": "gpu", "webnn": "npu"} +EXPECTED = [[4, 10], [8, -1]] + + +def pending_execution(reason): + return { + "status": "not-run", "executed": False, "effective": None, + "correct": unknown(reason), "software_fallback": unknown("no operation ran"), + "runtime": unknown("no executing runtime"), "process_id": unknown("no executing process"), + "evidence": [], "reason": reason, + } + + +def untested_backend(backend_id, api, kind, reason): + return { + "id": backend_id, "kind": kind, "api": api, + "available": unknown("device usability has not been established"), + "device": unknown("no device identity is recorded"), + "version": unknown("runtime version is not observed"), + "features": unknown("features are not observed"), + "probe": { + "tool": "probe-compute", + "runtime_installed": unknown("no runtime installation probe ran"), + "api_enumerated": unknown("no API probe ran"), + "device_enumerated": unknown("no API device enumeration ran"), + "device_accessible": unknown("no API device access probe ran"), + "execution": pending_execution(reason), + }, + } + + +def device_hints(root=pathlib.Path("/"), system=None): + """Only coarse class/count information: no device names, serials or addresses.""" + if (system or platform.system()) != "Linux": + return {"status": "not-run", "reason": "allowlisted device-node hints are Linux-only"} + out = {} + for name, patterns in { + "nvidia": ("dev/nvidia[0-9]*",), "render": ("dev/dri/renderD[0-9]*",), + "kfd": ("dev/kfd",), "accelerator": ("dev/accel/accel[0-9]*",), + }.items(): + try: + nodes = [path for pattern in patterns for path in root.glob(pattern) + if stat.S_ISCHR(path.stat().st_mode)] + out[name] = {"visible_nodes": len(nodes), + "read_write_access": sum(os.access(path, os.R_OK | os.W_OK) for path in nodes)} + except OSError: + out[name] = {"status": "unknown", "reason": "device-node metadata unavailable"} + return out + + +def discover_backends(): + backends = [] + for api, kind, tool in CATALOG: + backend = untested_backend(api, api, kind, "no hardware kernel adapter is implemented or executed") + if tool: + backend["extensions"] = {"x-discovery": { + "tool": tool, "on_path": shutil.which(tool) is not None, + "meaning": "tool presence only; driver, runtime, device and execution remain unverified", + }} + backends.append(backend) + for api, kind in BROWSER_APIS.items(): + backends.append(untested_backend(f"browser-{api}", api, kind, "browser probe report was not supplied")) + return backends + + +def matrix_product(left, right): + if not left or not right or any(len(row) != len(right) for row in left): + raise ValueError("incompatible fixed matrix dimensions") + columns = len(right[0]) + if not columns or any(len(row) != columns for row in right): + raise ValueError("inconsistent fixed matrix dimensions") + return [[sum(row[k] * right[k][column] for k in range(len(right))) + for column in range(columns)] for row in left] + + +def cpu_reference(run, evidence, case="success"): + """Failure modes alter fixed inputs, and still perform the same real operation.""" + if case not in ("success", "mismatch", "error"): + raise ValueError("unknown CPU case") + left, right = [[1, -2, 3], [4, 0, -1]], [[2, 1], [-1, 3], [0, 5]] + if case == "mismatch": + left[0][0] = 2 + elif case == "error": + left[0].pop() + execution = { + "status": "failed", "executed": False, "effective": None, + "correct": unknown("the operation has not completed"), + "software_fallback": observed(False, evidence), + "runtime": observed(f"{sys.implementation.name} {platform.python_version()}", "python:sys.implementation", evidence), + "process_id": observed(os.getpid(), "python:os.getpid", evidence), + "evidence": [evidence], "run": copy.deepcopy(run), + } + log = [{"event": "cpu-reference-start", "requested": "cpu", "case": case, + "runtime": execution["runtime"]["value"], "process_id": os.getpid(), "run": run, + "left": left, "right": right, "expected": EXPECTED}] + try: + actual = matrix_product(left, right) + except Exception as exc: + # Fixed exception category only: exception messages can include host paths. + execution["reason"] = f"CPU reference operation raised {type(exc).__name__}" + log.append({"event": "cpu-reference-error", "error_type": type(exc).__name__}) + else: + correct = actual == EXPECTED + execution.update(status="passed" if correct else "failed", executed=True, effective="cpu", + correct=observed(correct, evidence)) + if not correct: + execution["reason"] = "CPU reference output differs from the fixed expected matrix" + log.append({"event": "cpu-reference-completed", "effective": "cpu", "actual": actual, + "expected": EXPECTED, "correct": correct}) + backend = { + "id": "cpu", "kind": "cpu", "api": "cpu", "available": observed(True, TOOL), + "device": observed("process CPU", TOOL), "version": not_applicable("native CPU has no runtime version"), + "features": observed(["fixed-integer-matrix-reference"], evidence), + "probe": {"tool": "python-standard-library", "runtime_installed": observed(True, TOOL), + "api_enumerated": observed(True, TOOL), "device_enumerated": observed(True, TOOL), + "device_accessible": observed(True, TOOL), "execution": execution}, + } + return backend, log + + +def stage(status, evidence, effective="cpu", reason=None): + entry = {"status": status, "compute": {"requested": "cpu", "effective": effective}} + if status in ("passed", "failed"): + entry.update(evidence=[evidence], privilege={"escalation": collector.declared("none", TOOL)}) + if reason: + entry["reason"] = reason + return entry + + +def build_result(environment, case="success", log_ref="log:cpu-reference.log"): + result = copy.deepcopy(environment) + cpu, log = cpu_reference(result["run"], log_ref, case) + result["compute"] = {"probe_contract": "device-v1", "backends": [cpu, *discover_backends()], + "accelerator_inventory": unknown("tool and device-node hints are not a complete hardware inventory")} + result.setdefault("extensions", {})["x-device-node-hints"] = device_hints() + result["generated_at"] = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + # Qualify the already installed standard library; this does not install a package. + log.insert(0, {"event": "cpu-runtime-qualified", "installed": True, "initialized": True, + "package_install_performed": False, "runtime": cpu["probe"]["execution"]["runtime"]["value"]}) + for name in ("install", "initialize"): + result["stages"][name] = stage("passed", log_ref) + execution = cpu["probe"]["execution"] + result["stages"]["execute"] = stage("passed" if execution["executed"] else "failed", log_ref, + execution["effective"], None if execution["executed"] else execution["reason"]) + result["stages"]["correctness"] = stage(execution["status"] if execution["executed"] else "blocked", + log_ref, execution["effective"], execution.get("reason")) + for name in ("security", "performance"): + result["stages"][name] = {"status": "not-run", "reason": f"CPU reference does not measure {name}"} + result["overall"] = collector.overall_of(result["stages"]) + return result, log + + +def check_environment(environment): + findings = validator.validate_document(environment, validator.load_schema(), evaluate_policy=False, require_real=True) + if findings.errors or environment["kind"] != "result" or environment["stages"]["detect"]["status"] != "passed": + raise ValueError("environment must be a valid measured Result with a passed detection stage") + current = collector.Probe(probe_node=False) + if environment["run"] != current.run(): + raise ValueError("environment revision or CI workflow/job/run differs from this process") + runtime = environment["workload"]["runtimes"]["python"] + if (validator.measured(runtime["implementation"]), validator.measured(runtime["version"])) != (sys.implementation.name, platform.python_version()): + raise ValueError("environment Python runtime differs from the CPU probe runtime") + if environment["workload"]["identity"] != current.identity(): + raise ValueError("environment identity differs from the CPU probe identity") + + +def browser_fact(value, evidence, reason): + if type(value) is bool: + return observed(value, evidence) + if value == "unknown": + return unknown(reason, evidence) + raise ValueError("browser observations must be boolean or unknown") + + +def import_browser(result, report, environment_sha256, artifact_ref): + """Import only the four known observations, bound to the collector's exact file. + + The browser report records its Node controller PID, explicitly named in the + runtime fact. It does not claim to know a Chromium renderer's process ID. + """ + if not isinstance(report, dict): + raise ValueError("browser report must be an object") + if (report.get("schema_version"), report.get("kind"), report.get("tool"), report.get("environment_sha256")) != ( + 1, "browser-compute-probe", "playwright-chromium", environment_sha256): + raise ValueError("browser report does not match this environment") + if report.get("status") not in ("passed", "failed", "not-run"): + raise ValueError("invalid browser probe status") + probes = report.get("probes") + if not isinstance(probes, list) or len(probes) != len(BROWSER_APIS): + raise ValueError("browser report must contain exactly the four API observations") + if not isinstance(report.get("reason"), str) or not re.fullmatch(r"[a-z0-9-]{1,100}", report["reason"]): + raise ValueError("invalid browser reason") + if report["status"] != "failed" and report["reason"].endswith("-failed"): + raise ValueError("a browser report with a failed probe must fail") + backends = {backend["id"]: backend for backend in result["compute"]["backends"]} + seen = set() + for raw in probes: + if not isinstance(raw, dict) or not isinstance(raw.get("execution"), dict): + raise ValueError("browser probe and execution must be objects") + api = raw["api"] + if api not in BROWSER_APIS or api in seen or (raw["id"], raw["kind"]) != (f"browser-{api}", BROWSER_APIS[api]): + raise ValueError("unexpected or duplicate browser backend") + seen.add(api) + backend = backends[raw["id"]] + evidence = f"{artifact_ref}#{api}" + probe = backend["probe"] + probe["tool"] = "playwright-chromium" + execution = raw["execution"] + reason = execution.get("reason") + if not isinstance(reason, str) or not re.fullmatch(r"[a-z0-9-]{1,100}", reason): + raise ValueError("invalid browser execution reason") + for field in ("runtime_installed", "api_enumerated", "device_enumerated", "device_accessible"): + probe[field] = browser_fact(raw[field], evidence, reason) + if execution["status"] not in ("passed", "failed", "not-run") or type(execution["executed"]) is not bool: + raise ValueError("invalid browser operation status") + if report["status"] == "not-run" and (execution["status"] != "not-run" or execution["executed"]): + raise ValueError("an unexecuted browser report cannot contain an attempted operation") + if report["status"] == "not-run": + unknown_fields = ("runtime_installed", "api_enumerated", "device_enumerated", "device_accessible", + "context_available", "adapter_available", "software_adapter") + if (any(raw.get(field, "unknown") != "unknown" for field in unknown_fields) + or execution.get("correct") != "unknown" or execution.get("software_fallback") != "unknown" + or any(execution.get(field) is not None for field in ("effective", "runtime", "process_id")) + or execution.get("evidence") or raw.get("evidence") or raw.get("actual") is not None): + raise ValueError("an unexecuted browser report cannot contain observed browser facts") + if report["status"] == "passed" and execution["status"] == "failed": + raise ValueError("a browser report with a failed operation cannot pass") + # Detection failures leave GPU/NPU operation status not-run. Their + # producer reason still records a failure of the browser probe. + if report["status"] != "failed" and reason.endswith("-failed"): + raise ValueError("a browser report with a failed probe must fail") + if api != "wasm" and (execution["status"] != "not-run" or execution["executed"]): + raise ValueError("this browser probe performs no GPU or NPU kernel") + runtime, pid = execution.get("runtime"), execution.get("process_id") + if runtime is not None: + match = re.fullmatch(r"chromium [0-9]+(?:\.[0-9]+){1,3}; node-controller ([0-9]+(?:\.[0-9]+){2})", runtime) if isinstance(runtime, str) else None + if not match: + raise ValueError("invalid browser runtime") + node_version = validator.measured(result["workload"]["runtimes"]["node"]["version"]) + if node_version is None or match.group(1) != node_version: + raise ValueError("browser controller runtime differs from the measured environment Node runtime") + elif (report["status"] == "passed" or execution["status"] != "not-run" + or any(type(raw.get(field)) is bool for field in ( + "runtime_installed", "api_enumerated", "device_enumerated", "device_accessible", + "context_available", "adapter_available", "software_adapter"))): + raise ValueError("browser observations require a controller runtime") + if pid is not None and (type(pid) is not int or not 1 <= pid <= 4294967295): + raise ValueError("invalid browser controller PID") + converted = { + "status": execution["status"], "executed": execution["executed"], "effective": execution["effective"], + "correct": browser_fact(execution["correct"], evidence, reason), + "software_fallback": browser_fact(execution["software_fallback"], evidence, reason), + "runtime": observed(runtime, evidence) if runtime else unknown(reason), + "process_id": observed(pid, evidence) if pid else unknown(reason), + "evidence": [evidence] if execution["status"] in ("passed", "failed") else [], "reason": reason, + } + if execution["status"] in ("passed", "failed"): + converted["run"] = copy.deepcopy(result["run"]) + probe["execution"] = converted + if api == "wasm" and execution["executed"]: + if raw.get("expected") != 42 or type(raw.get("actual")) is not int or execution["correct"] is not (raw["actual"] == 42): + raise ValueError("WASM correctness does not match its fixed arithmetic result") + if execution["effective"] != raw["id"]: + raise ValueError("WASM operation must record its actual WASM backend") + backend["available"] = observed(True, evidence) + backend["device"] = observed("browser CPU execution", evidence) + # Context and adapter discovery are useful, but cannot establish physical hardware. + observations = {} + for field in ("context_available", "adapter_available", "software_adapter"): + if field in raw: + observations[field] = browser_fact(raw[field], evidence, reason) + if observations: + backend["extensions"] = {"x-browser-observations": observations} + result.setdefault("extensions", {})["x-browser-probe"] = { + "status": report["status"], "reason": report["reason"], "environment_sha256": environment_sha256, + } + + +def main(argv=None): + parser = validator.ArgumentParser(description=__doc__.split("\n\n", 1)[0]) + for name in ("environment", "output", "report", "log"): + parser.add_argument(f"--{name}", type=pathlib.Path, required=True) + parser.add_argument("--browser-report", type=pathlib.Path) + parser.add_argument("--cpu-case", choices=("success", "mismatch", "error"), default="success") + parser.add_argument("--summary", type=pathlib.Path) + args = parser.parse_args(argv) + paths = [path for path in (args.environment, args.output, args.report, args.log, args.browser_report, args.summary) if path] + if any(validator.same_place(left, right) for index, left in enumerate(paths) for right in paths[index + 1:]): + parser.error("input, output, report, log and summary must be separate files") + if any(not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,100}", path.name) for path in (args.log, args.report, args.browser_report) if path): + parser.error("artifact filenames must use short ASCII letters, digits, dots, underscores or hyphens") + try: + environment = validator.load_json(args.environment) + check_environment(environment) + result, log = build_result(environment, args.cpu_case, f"log:{args.log.name}") + if args.browser_report: + import_browser(result, validator.load_json(args.browser_report), + hashlib.sha256(args.environment.read_bytes()).hexdigest(), f"artifact:{args.browser_report.name}") + findings = validator.validate_document(result, validator.load_schema(), evaluate_policy=False, require_real=True) + if findings.errors: + raise ValueError("generated compute Result violates the shared contract") + report = {"schema_version": 1, "kind": "compute-probe", "synthetic": False, "run": result["run"], + "compute": result["compute"], "device_node_hints": result["extensions"]["x-device-node-hints"], + "browser_probe": result["extensions"].get("x-browser-probe")} + args.log.write_text("".join(json.dumps(event, sort_keys=True) + "\n" for event in log), encoding="utf-8") + args.report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + args.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + cpu = result["compute"]["backends"][0]["probe"]["execution"] + lines = ["### Device and compute probe", "", f"CPU execution: {result['stages']['execute']['status']}; correctness: {result['stages']['correctness']['status']}.", + "", "| API | Execution |", "|---|---|"] + lines += [f"| {backend['api']} | {backend['probe']['execution']['status']} |" for backend in result["compute"]["backends"]] + lines += ["", "Native CPU fixed-matrix verification is pipeline evidence, not LLM inference or accelerator qualification.", ""] + if args.summary: + with args.summary.open("a", encoding="utf-8") as stream: + stream.write("\n".join(lines)) + print(f"CPU reference: {cpu['status']}; requested=cpu; effective={cpu['effective']}; correctness={cpu['correct']['value']}") + browser_failed = result["extensions"].get("x-browser-probe", {}).get("status") == "failed" + return int(browser_failed or any(backend["probe"]["execution"]["status"] == "failed" for backend in result["compute"]["backends"])) + except (OSError, ValueError, TypeError, KeyError) as exc: + print(f"compute probe could not produce a valid result ({type(exc).__name__})", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/tests/test_compute_probe.py b/scripts/ci/tests/test_compute_probe.py new file mode 100644 index 0000000..60e430f --- /dev/null +++ b/scripts/ci/tests/test_compute_probe.py @@ -0,0 +1,388 @@ +"""CPU reference, real CLI and discovery regression tests; standard library only. + + python -I scripts/ci/tests/test_compute_probe.py --verbose +""" + +import copy +import importlib.util +import json +import pathlib +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + + +ROOT = pathlib.Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts" / "ci" / "probe_compute.py" +spec = importlib.util.spec_from_file_location("probe_compute", SCRIPT) +probe = importlib.util.module_from_spec(spec) +spec.loader.exec_module(probe) + + +class CPUReferenceTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.environment = probe.collector.build_result(probe.collector.Probe(probe_node=False)) + cls.schema = probe.validator.load_schema() + + def validate(self, result): + # This environment's container engine may be invisible; these tests exercise + # the compute contract, while the real workflow applies the strict policy. + findings = probe.validator.validate_document(result, self.schema, evaluate_policy=False, require_real=True) + self.assertEqual(findings.errors, [], findings.errors) + + def test_real_cpu_operation_matches_fixed_expected_values(self): + result, log = probe.build_result(self.environment) + self.validate(result) + self.assertEqual(log[-1]["actual"], [[4, 10], [8, -1]]) + self.assertTrue(log[-1]["correct"]) + execution = result["compute"]["backends"][0]["probe"]["execution"] + self.assertEqual(execution["status"], "passed") + self.assertTrue(execution["executed"]) + self.assertEqual(execution["effective"], "cpu") + self.assertGreater(execution["process_id"]["value"], 0) + self.assertEqual(execution["run"], result["run"]) + self.assertEqual(result["stages"]["correctness"]["compute"], {"requested": "cpu", "effective": "cpu"}) + self.assertFalse(log[0]["package_install_performed"]) + + def test_changed_input_runs_but_fails_correctness(self): + result, log = probe.build_result(self.environment, "mismatch") + self.validate(result) + self.assertNotEqual(log[-1]["actual"], probe.EXPECTED) + self.assertFalse(log[-1]["correct"]) + self.assertEqual(result["stages"]["execute"]["status"], "passed") + self.assertEqual(result["stages"]["correctness"]["status"], "failed") + self.assertEqual(result["overall"], "failed") + + def test_bad_dimensions_fail_execution_and_block_correctness(self): + result, log = probe.build_result(self.environment, "error") + self.validate(result) + self.assertEqual(result["stages"]["execute"]["status"], "failed") + self.assertEqual(result["stages"]["correctness"]["status"], "blocked") + execution = result["compute"]["backends"][0]["probe"]["execution"] + self.assertFalse(execution["executed"]) + self.assertIsNone(execution["effective"]) + self.assertIsNone(execution["correct"]["value"]) + self.assertEqual(log[-1], {"event": "cpu-reference-error", "error_type": "ValueError"}) + + def test_unexpected_error_is_failed_without_exception_message(self): + with mock.patch.object(probe, "matrix_product", side_effect=RuntimeError("private-host-value")): + result, log = probe.build_result(self.environment) + self.validate(result) + self.assertEqual(result["stages"]["execute"]["status"], "failed") + self.assertNotIn("private-host-value", json.dumps([result, log])) + + def test_discovery_tool_presence_is_not_runtime_or_device_availability(self): + for present in (None, "/private/path/nvidia-smi"): + with self.subTest(present=bool(present)), mock.patch.object(probe.shutil, "which", return_value=present): + backends = probe.discover_backends() + cuda = next(backend for backend in backends if backend["id"] == "cuda") + self.assertEqual(cuda["extensions"]["x-discovery"]["on_path"], present is not None) + for backend in backends: + self.assertIsNone(backend["available"]["value"]) + self.assertIsNone(backend["probe"]["runtime_installed"]["value"]) + self.assertEqual(backend["probe"]["execution"]["status"], "not-run") + self.assertNotIn("/private", json.dumps(backends)) + + def test_catalog_covers_requested_apis_without_claiming_execution(self): + apis = {backend["api"] for backend in probe.discover_backends()} + self.assertEqual(apis, {"cuda", "rocm", "oneapi", "level-zero", "opencl", "vulkan-compute", + "directcompute", "directml", "metal", "mps", "coreml", "mlx", "qnn", + "wasm", "webgl", "webgpu", "webnn"}) + + def test_missing_nodes_leave_hardware_inventory_unknown(self): + with tempfile.TemporaryDirectory() as directory: + hints = probe.device_hints(pathlib.Path(directory), "Linux") + self.assertTrue(all(value["visible_nodes"] == 0 for value in hints.values())) + result, _ = probe.build_result(self.environment) + self.assertIsNone(result["compute"]["accelerator_inventory"]["value"]) + + def test_stale_revision_is_rejected(self): + environment = copy.deepcopy(self.environment) + environment["run"]["revision"]["tested_sha"] = probe.observed("1" * 40, "git:.git/HEAD") + with self.assertRaisesRegex(ValueError, "revision or CI"): + probe.check_environment(environment) + + def test_synthetic_environment_cannot_be_submitted_as_measurement(self): + environment = copy.deepcopy(self.environment) + environment["synthetic"] = True + with self.assertRaisesRegex(ValueError, "measured Result"): + probe.check_environment(environment) + + def test_runtime_changed_after_collection_is_rejected(self): + environment = copy.deepcopy(self.environment) + environment["workload"]["runtimes"]["python"]["version"]["value"] = "3.12.1" + with self.assertRaises(ValueError): + probe.check_environment(environment) + + +class CLITests(unittest.TestCase): + def cli(self, directory, case="success", extra=()): + return subprocess.run([sys.executable, "-I", str(SCRIPT), "--environment", str(directory / "environment.json"), + "--output", str(directory / "result.json"), "--report", str(directory / "compute-probe.json"), + "--log", str(directory / "cpu-reference.log"), "--cpu-case", case, *extra], + cwd=ROOT, capture_output=True, text=True, timeout=30) + + def write_environment(self, directory, probe_node=False): + environment = probe.collector.build_result(probe.collector.Probe(probe_node=probe_node)) + (directory / "environment.json").write_text(json.dumps(environment), encoding="utf-8") + return environment + + def browser_cli(self, directory, report, environment): + (directory / "environment.json").write_text(json.dumps(environment), encoding="utf-8") + report["environment_sha256"] = probe.hashlib.sha256((directory / "environment.json").read_bytes()).hexdigest() + browser = directory / "browser.json" + browser.write_text(json.dumps(report), encoding="utf-8") + return self.cli(directory, extra=("--browser-report", str(browser))) + + def browser_report(self, environment): + sample = BrowserImportTests() + sample.setUp() + report = sample.report + version = probe.validator.measured(environment["workload"]["runtimes"]["node"]["version"]) + self.assertIsNotNone(version, "these CLI cases require the installed Node runtime") + for raw in report["probes"]: + raw["execution"]["runtime"] = f"chromium 150.0.1.2; node-controller {version}" + return report + + def test_cli_rejects_pass_with_producer_failure_reasons(self): + failures = ((0, "wasm-operation-failed"), (1, "webgl-context-probe-failed"), + (2, "webgpu-adapter-probe-failed"), (None, "chromium-launch-failed"), + (None, "browser-observation-failed"), (None, "browser-cleanup-failed"), + (None, "browser-probe-failed")) + for index, reason in failures: + with self.subTest(reason=reason), tempfile.TemporaryDirectory() as temporary: + directory = pathlib.Path(temporary) + environment = self.write_environment(directory, probe_node=True) + report = self.browser_report(environment) + if index is None: + report["reason"] = reason + else: + report["probes"][index]["execution"]["reason"] = reason + run = self.browser_cli(directory, report, environment) + self.assertEqual(run.returncode, 2, run.stderr + run.stdout) + self.assertNotIn("Traceback", run.stderr) + self.assertFalse((directory / "result.json").exists()) + + def test_cli_failed_detection_retains_no_kernel_execution(self): + with tempfile.TemporaryDirectory() as temporary: + directory = pathlib.Path(temporary) + environment = self.write_environment(directory, probe_node=True) + report = self.browser_report(environment) + report.update(status="failed", reason="browser-probe-failed") + report["probes"][2]["execution"]["reason"] = "webgpu-adapter-probe-failed" + run = self.browser_cli(directory, report, environment) + self.assertEqual(run.returncode, 1, run.stderr + run.stdout) + result = json.loads((directory / "result.json").read_text()) + self.assertEqual(result["extensions"]["x-browser-probe"]["status"], "failed") + webgpu = next(backend for backend in result["compute"]["backends"] if backend["id"] == "browser-webgpu") + self.assertEqual(webgpu["probe"]["execution"]["status"], "not-run") + self.assertFalse(webgpu["probe"]["execution"]["executed"]) + + def test_cli_matching_node_with_absent_features_passes(self): + with tempfile.TemporaryDirectory() as temporary: + directory = pathlib.Path(temporary) + environment = self.write_environment(directory, probe_node=True) + report = self.browser_report(environment) + for raw in report["probes"][1:]: + raw["api_enumerated"] = False + report["probes"][1]["context_available"] = False + report["probes"][2].update(adapter_available=False, software_adapter="unknown") + run = self.browser_cli(directory, report, environment) + self.assertEqual(run.returncode, 0, run.stderr + run.stdout) + + def test_cli_rejects_node_controller_version_mismatch_for_every_api(self): + for index, api in enumerate(probe.BROWSER_APIS): + with self.subTest(api=api), tempfile.TemporaryDirectory() as temporary: + directory = pathlib.Path(temporary) + environment = self.write_environment(directory, probe_node=True) + report = self.browser_report(environment) + report["probes"][index]["execution"]["runtime"] = "chromium 150.0.1.2; node-controller 99.99.99" + run = self.browser_cli(directory, report, environment) + self.assertEqual(run.returncode, 2, run.stderr + run.stdout) + self.assertFalse((directory / "result.json").exists()) + + def test_cli_browser_observations_require_measured_node_version(self): + for fact in (probe.unknown("not probed"), probe.collector.declared("24.19.0", "tool:test")): + with self.subTest(provenance=fact["provenance"]), tempfile.TemporaryDirectory() as temporary: + directory = pathlib.Path(temporary) + environment = self.write_environment(directory, probe_node=True) + report = self.browser_report(environment) + environment["workload"]["runtimes"]["node"]["version"] = fact + run = self.browser_cli(directory, report, environment) + self.assertEqual(run.returncode, 2, run.stderr + run.stdout) + self.assertFalse((directory / "result.json").exists()) + + def test_cli_not_run_browser_limitation_accepts_unknown_node(self): + with tempfile.TemporaryDirectory() as temporary: + directory = pathlib.Path(temporary) + environment = self.write_environment(directory) + sample = BrowserImportTests() + sample.setUp() + sample.make_not_run_report() + run = self.browser_cli(directory, sample.report, environment) + self.assertEqual(run.returncode, 0, run.stderr + run.stdout) + + def test_cli_success_and_negative_inputs_write_real_artifacts(self): + for case, expected_status in (("success", 0), ("mismatch", 1), ("error", 1)): + with self.subTest(case=case), tempfile.TemporaryDirectory() as temporary: + directory = pathlib.Path(temporary) + self.write_environment(directory) + run = self.cli(directory, case) + self.assertEqual(run.returncode, expected_status, run.stderr + run.stdout) + result = json.loads((directory / "result.json").read_text()) + report = json.loads((directory / "compute-probe.json").read_text()) + log = [json.loads(line) for line in (directory / "cpu-reference.log").read_text().splitlines()] + self.assertEqual(report["compute"], result["compute"]) + self.assertEqual(log[1]["run"], result["run"]) + self.assertEqual(result["stages"]["correctness"]["status"], {"success": "passed", "mismatch": "failed", "error": "blocked"}[case]) + + def test_cli_does_not_overwrite_its_environment(self): + with tempfile.TemporaryDirectory() as temporary: + directory = pathlib.Path(temporary) + self.write_environment(directory) + before = (directory / "environment.json").read_bytes() + run = self.cli(directory, extra=("--output", str(directory / "environment.json"))) + self.assertEqual(run.returncode, 2) + self.assertEqual((directory / "environment.json").read_bytes(), before) + + def test_cli_does_not_rewrite_result_with_summary(self): + with tempfile.TemporaryDirectory() as temporary: + directory = pathlib.Path(temporary) + self.write_environment(directory) + run = self.cli(directory, extra=("--summary", str(directory / "result.json"))) + self.assertEqual(run.returncode, 2) + self.assertFalse((directory / "result.json").exists()) + + def test_cli_malformed_browser_objects_fail_without_tracebacks(self): + sample = BrowserImportTests() + sample.setUp() + malformed_execution = copy.deepcopy(sample.report) + malformed_execution["probes"][0]["execution"] = None + malformed_probe = copy.deepcopy(sample.report) + malformed_probe["probes"][0] = None + for report in ([], malformed_execution, malformed_probe): + with self.subTest(report_type=type(report).__name__), tempfile.TemporaryDirectory() as temporary: + directory = pathlib.Path(temporary) + self.write_environment(directory) + if isinstance(report, dict): + report["environment_sha256"] = probe.hashlib.sha256((directory / "environment.json").read_bytes()).hexdigest() + browser = directory / "browser.json" + browser.write_text(json.dumps(report), encoding="utf-8") + run = self.cli(directory, extra=("--browser-report", str(browser))) + self.assertEqual(run.returncode, 2) + self.assertNotIn("Traceback", run.stderr) + self.assertNotIn(str(directory), run.stderr) + self.assertFalse((directory / "result.json").exists()) + + +class BrowserImportTests(unittest.TestCase): + def setUp(self): + environment = probe.collector.build_result(probe.collector.Probe(probe_node=False)) + environment["workload"]["runtimes"]["node"]["version"] = probe.observed("24.19.0", "tool:test-node") + self.result, _ = probe.build_result(environment) + self.report = { + "schema_version": 1, "kind": "browser-compute-probe", "tool": "playwright-chromium", + "environment_sha256": "a" * 64, "status": "passed", "reason": "capability-probes-completed", + "probes": [{ + "id": f"browser-{api}", "api": api, "kind": kind, + "runtime_installed": True, "api_enumerated": True, + "device_enumerated": True if api == "wasm" else "unknown", + "device_accessible": True if api == "wasm" else "unknown", + "execution": { + "status": "passed" if api == "wasm" else "not-run", "executed": api == "wasm", + "effective": "browser-wasm" if api == "wasm" else None, + "correct": True if api == "wasm" else "unknown", + "software_fallback": False if api == "wasm" else "unknown", + "runtime": "chromium 150.0.1.2; node-controller 24.19.0", "process_id": 123, + "evidence": [f"tool:playwright-chromium:{api}"], "reason": "fixed-operation-or-discovery", + }, "expected": 42, "actual": 42, + } for api, kind in probe.BROWSER_APIS.items()], + } + + def import_report(self): + probe.import_browser(self.result, self.report, "a" * 64, "artifact:browser-compute.json") + + def test_wasm_import_preserves_binding_and_does_not_promote_gpu(self): + self.report["probes"][2].update(adapter_available=True, software_adapter=True) + self.import_report() + findings = probe.validator.validate_document(self.result, probe.validator.load_schema(), evaluate_policy=False, require_real=True) + self.assertEqual(findings.errors, []) + backends = {backend["id"]: backend for backend in self.result["compute"]["backends"]} + wasm = backends["browser-wasm"]["probe"]["execution"] + self.assertEqual(wasm["status"], "passed") + self.assertEqual(wasm["run"], self.result["run"]) + self.assertEqual(wasm["evidence"], ["artifact:browser-compute.json#wasm"]) + webgpu = backends["browser-webgpu"] + self.assertIsNone(webgpu["available"]["value"]) + self.assertEqual(webgpu["probe"]["execution"]["status"], "not-run") + self.assertEqual(webgpu["probe"]["execution"]["evidence"], []) + self.assertTrue(webgpu["extensions"]["x-browser-observations"]["software_adapter"]["value"]) + + def test_report_from_another_environment_is_rejected(self): + self.report["environment_sha256"] = "b" * 64 + with self.assertRaisesRegex(ValueError, "match this environment"): + self.import_report() + + def test_claimed_wasm_pass_requires_actual_correct_answer(self): + self.report["probes"][0]["actual"] = 41 + with self.assertRaisesRegex(ValueError, "fixed arithmetic"): + self.import_report() + + def test_gpu_detection_cannot_be_imported_as_execution(self): + self.report["probes"][2]["execution"].update(status="passed", executed=True, effective="browser-webgpu", correct=True) + with self.assertRaisesRegex(ValueError, "no GPU or NPU kernel"): + self.import_report() + + def test_not_run_browser_report_cannot_promote_wasm_execution(self): + self.report["status"] = "not-run" + with self.assertRaisesRegex(ValueError, "unexecuted browser report"): + self.import_report() + + def test_passed_browser_report_cannot_contain_failed_operation(self): + self.report["probes"][0]["execution"].update(status="failed", correct=False) + self.report["probes"][0]["actual"] = 41 + with self.assertRaisesRegex(ValueError, "failed operation cannot pass"): + self.import_report() + + def test_malicious_runtime_string_is_not_written(self): + self.report["probes"][0]["execution"]["runtime"] = "private host and secret" + with self.assertRaisesRegex(ValueError, "browser runtime"): + self.import_report() + + def test_duplicate_browser_backend_rejected(self): + self.report["probes"][3] = copy.deepcopy(self.report["probes"][2]) + with self.assertRaisesRegex(ValueError, "duplicate"): + self.import_report() + + def make_not_run_report(self): + self.report.update(status="not-run", reason="chromium-sandbox-unavailable") + for raw in self.report["probes"]: + for field in ("runtime_installed", "api_enumerated", "device_enumerated", "device_accessible"): + raw[field] = "unknown" + raw["execution"].update(status="not-run", executed=False, effective=None, correct="unknown", software_fallback="unknown", + runtime=None, process_id=None, evidence=[], reason="chromium-sandbox-unavailable") + raw["actual"] = None + + def test_browser_not_run_reason_remains_unexecuted(self): + self.make_not_run_report() + self.import_report() + self.assertEqual(self.result["extensions"]["x-browser-probe"]["status"], "not-run") + for backend in self.result["compute"]["backends"]: + if backend["id"].startswith("browser-"): + self.assertEqual(backend["probe"]["execution"]["status"], "not-run") + + def test_not_run_browser_report_cannot_contain_observed_api_facts(self): + self.report.update(status="not-run", reason="chromium-sandbox-unavailable") + for raw in self.report["probes"]: + raw["execution"].update(status="not-run", executed=False, effective=None, correct="unknown", software_fallback="unknown", + runtime=None, process_id=None, evidence=[]) + raw["actual"] = None + with self.assertRaisesRegex(ValueError, "observed browser facts"): + self.import_report() + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/tests/test_device_contract.py b/scripts/ci/tests/test_device_contract.py new file mode 100644 index 0000000..b2e1c3f --- /dev/null +++ b/scripts/ci/tests/test_device_contract.py @@ -0,0 +1,343 @@ +"""Device evidence regression inputs; these fabricated records are not run artifacts. + +The real CPU arithmetic and process tests live in test_compute_probe.py. These +tests mutate a valid input to prove the common v1 validator rejects false passes. +Run: python -I scripts/ci/tests/test_device_contract.py +""" + +import copy +import importlib.util +import pathlib +import unittest + +ROOT = pathlib.Path(__file__).resolve().parents[3] +spec = importlib.util.spec_from_file_location( + "compatibility_test_helpers", pathlib.Path(__file__).with_name("test_compatibility_schema.py")) +helpers = importlib.util.module_from_spec(spec) +spec.loader.exec_module(helpers) +validator = helpers.validator +fact = helpers.fact +unknown = helpers.unknown +LOG = "log:cpu-reference.log" + + +def device_pair(): + """Build a valid-looking unit-test input, never exported as measurement.""" + result, profile = helpers.real_result() + for document in (result, profile): + document["compute"]["probe_contract"] = "device-v1" + profile["required_stages"] = ["detect", "install", "initialize", "execute", "correctness"] + for name in profile["required_stages"][1:]: + result["stages"][name] = { + "status": "passed", "evidence": [LOG], + "privilege": {"escalation": fact("none", LOG)}, + } + for name in ("execute", "correctness"): + result["stages"][name]["compute"] = {"requested": "cpu", "effective": "cpu"} + result["compute"]["backends"][0]["probe"] = { + "tool": "python-standard-library", + "runtime_installed": fact(True, LOG), + "api_enumerated": fact(True, LOG), + "device_enumerated": fact(True, LOG), + "device_accessible": fact(True, LOG), + "execution": { + "status": "passed", "executed": True, "effective": "cpu", + "correct": fact(True, LOG), "software_fallback": fact(False, LOG), + "runtime": fact("cpython 3.12.15", LOG), "process_id": fact(123, LOG), + "evidence": [LOG], "run": copy.deepcopy(result["run"]), + }, + } + return result, profile + + +def add_gpu(result): + gpu = copy.deepcopy(result["compute"]["backends"][0]) + gpu.update(id="cuda", kind="gpu", api="cuda") + gpu["probe"]["execution"]["effective"] = "cuda" + result["compute"]["backends"].append(gpu) + return gpu + + +def add_browser_wasm(result): + browser = copy.deepcopy(result["compute"]["backends"][0]) + browser.update(id="browser-wasm", kind="cpu", api="wasm") + browser["probe"]["tool"] = "playwright-chromium" + browser["probe"]["execution"].update( + effective="browser-wasm", runtime=fact("chromium", LOG)) + result["compute"]["backends"].append(browser) + return browser + + +class DeviceContractTests(unittest.TestCase): + def setUp(self): + self.result, self.profile = device_pair() + self.cpu = self.result["compute"]["backends"][0] + self.execution = self.cpu["probe"]["execution"] + + def check(self): + return helpers.check(self.result, self.profile) + + def assertRejected(self, code): + findings = self.check() + self.assertIn(code, findings.codes(), findings.items) + self.assertNotIn("schema", findings.codes(), findings.items) + + def test_completed_cpu_reference_contract_is_valid(self): + self.assertEqual(self.check().errors, []) + + def test_available_backend_requires_measured_installed_runtime(self): + for runtime in (fact(False, LOG), unknown("runtime not inspected"), + fact(True, provenance="declared")): + with self.subTest(runtime=runtime): + self.cpu["probe"]["runtime_installed"] = runtime + self.assertRejected("compute-availability") + + def test_completed_failed_operation_requires_installed_runtime(self): + self.execution.update(status="failed", correct=fact(False, LOG), reason="reference mismatch") + self.result["stages"]["correctness"].update(status="failed", reason="reference mismatch") + self.result["overall"] = "failed" + self.cpu["probe"]["runtime_installed"] = unknown("runtime not inspected") + findings = self.check() + self.assertIn(("compute-availability", "/compute/backends/0/probe/runtime_installed"), + [(code, path) for _, code, path, _ in findings.errors], findings.items) + + def test_available_unexecuted_browser_requires_installed_runtime(self): + browser = add_browser_wasm(self.result) + browser["probe"]["runtime_installed"] = unknown("runtime not inspected") + browser["probe"]["execution"].update( + status="not-run", executed=False, effective=None, correct=unknown(), + runtime=unknown(), process_id=unknown(), evidence=[], reason="operation not attempted") + self.assertRejected("compute-availability") + + def test_completed_operation_requires_runtime_even_when_request_unavailable(self): + browser = add_browser_wasm(self.result) + browser["available"] = unknown("requested runtime unavailable") + browser["probe"]["runtime_installed"] = fact(False, LOG) + browser["probe"]["execution"].update(effective="cpu", software_fallback=fact(True, LOG)) + self.assertRejected("compute-availability") + + def test_inferred_runtime_installation_can_support_completed_operation(self): + self.cpu["probe"]["runtime_installed"] = fact(True, LOG, provenance="inferred") + self.assertEqual(self.check().errors, []) + + def test_required_cpu_stages_cannot_be_replaced_by_extra_browser_operation(self): + add_browser_wasm(self.result) + self.execution.update(status="not-run", executed=False, effective=None, + correct=unknown(), evidence=[], reason="CPU reference not attempted") + for name in ("execute", "correctness"): + self.result["stages"][name]["compute"] = { + "requested": "browser-wasm", "effective": "browser-wasm"} + findings = self.check() + for name in ("execute", "correctness"): + self.assertIn(("required-stage", f"/stages/{name}/compute"), + [(code, path) for _, code, path, _ in findings.errors], findings.items) + self.assertNotIn("schema", findings.codes(), findings.items) + + def test_required_performance_also_uses_profile_cpu(self): + add_browser_wasm(self.result) + self.profile["required_stages"].append("performance") + self.result["stages"]["performance"] = copy.deepcopy(self.result["stages"]["correctness"]) + self.result["stages"]["performance"]["compute"] = { + "requested": "browser-wasm", "effective": "browser-wasm"} + self.assertRejected("required-stage") + + def test_extra_completed_browser_operation_does_not_replace_cpu(self): + add_browser_wasm(self.result) + self.assertEqual(self.check().errors, []) + + def test_optional_stage_can_use_extra_browser_backend(self): + add_browser_wasm(self.result) + self.result["stages"]["performance"] = copy.deepcopy(self.result["stages"]["correctness"]) + self.result["stages"]["performance"]["compute"] = { + "requested": "browser-wasm", "effective": "browser-wasm"} + self.assertEqual(self.check().errors, []) + + def test_cpu_binding_uses_profile_id_instead_of_literal_cpu(self): + self.cpu["id"] = self.profile["compute"]["backends"][0]["id"] = "reference-cpu" + self.execution["effective"] = "reference-cpu" + for name in ("execute", "correctness"): + self.result["stages"][name]["compute"] = { + "requested": "reference-cpu", "effective": "reference-cpu"} + self.assertEqual(self.check().errors, []) + + def test_unexecuted_optional_browser_allows_unknown_or_absent_runtime(self): + browser = add_browser_wasm(self.result) + browser["available"] = unknown("browser capability not established") + for field in ("api_enumerated", "device_enumerated", "device_accessible"): + browser["probe"][field] = unknown("browser not inspected") + browser["probe"]["execution"] = { + "status": "not-run", "executed": False, "effective": None, + "correct": unknown(), "software_fallback": unknown(), + "runtime": unknown(), "process_id": unknown(), "evidence": [], + "reason": "browser not configured", + } + for status in ("not-run", "blocked"): + for runtime in (unknown("runtime not inspected"), fact(False, LOG)): + with self.subTest(status=status, runtime=runtime): + browser["probe"]["execution"]["status"] = status + browser["probe"]["runtime_installed"] = runtime + self.assertEqual(self.check().errors, []) + + def test_device_contract_cannot_be_stripped_from_profile_result(self): + self.result["compute"].pop("probe_contract") + self.cpu.pop("probe") + self.assertRejected("profile-mismatch") + + def test_device_contract_requires_every_backend_probe(self): + self.cpu.pop("probe") + self.assertRejected("compute-probe") + + def test_driver_presence_does_not_establish_device_availability(self): + gpu = add_gpu(self.result) + gpu["probe"]["api_enumerated"] = unknown("driver installed, API never called") + gpu["probe"]["device_accessible"] = unknown("no device opened") + self.assertRejected("compute-availability") + + def test_absent_device_cannot_be_reported_available_or_passed(self): + gpu = add_gpu(self.result) + gpu["available"] = fact(False, LOG) + gpu["probe"]["device_enumerated"] = fact(False, LOG) + self.assertRejected("compute-availability") + + def test_gpu_detection_is_not_operation_execution(self): + gpu = add_gpu(self.result) + gpu["probe"]["execution"].update(executed=False, effective=None, correct=unknown()) + self.assertRejected("compute-execution") + + def test_legacy_real_gpu_pass_still_requires_probe(self): + gpu = add_gpu(self.result) + gpu.pop("probe") + self.result["stages"]["execute"]["compute"] = {"requested": "cuda", "effective": "cuda"} + self.assertRejected("compute-execution") + + def test_gpu_request_cpu_execution_uses_existing_fallback_rule(self): + gpu = add_gpu(self.result) + gpu["probe"]["execution"].update(effective="cpu", software_fallback=fact(True, LOG)) + self.result["stages"]["execute"]["compute"] = {"requested": "cuda", "effective": "cpu"} + self.assertRejected("accelerator-fallback") + + def test_software_gpu_with_same_backend_id_is_not_hardware_pass(self): + gpu = add_gpu(self.result) + gpu["probe"]["execution"]["software_fallback"] = fact(True, LOG) + self.assertRejected("accelerator-fallback") + + def test_unknown_fallback_cannot_establish_hardware_execution(self): + gpu = add_gpu(self.result) + gpu["probe"]["execution"]["software_fallback"] = unknown("adapter type not observed") + self.assertRejected("accelerator-fallback") + + def test_failed_software_probe_cannot_support_hardware_execute_pass(self): + gpu = add_gpu(self.result) + gpu["probe"]["execution"].update( + status="failed", software_fallback=fact(True, LOG), reason="software adapter used") + self.result["stages"]["execute"]["compute"] = {"requested": "cuda", "effective": "cuda"} + self.assertRejected("accelerator-fallback") + + def test_backend_id_must_match_profile(self): + self.cpu["id"] = "other-cpu" + self.execution["effective"] = "other-cpu" + for name in ("execute", "correctness"): + self.result["stages"][name]["compute"] = {"requested": "other-cpu", "effective": "other-cpu"} + self.assertRejected("profile-mismatch") + + def test_actual_backend_cannot_be_omitted(self): + self.execution["effective"] = None + self.assertRejected("compute-execution") + + def test_stage_cannot_omit_compute_record(self): + self.result["stages"]["execute"].pop("compute") + self.assertRejected("compute-execution") + + def test_actual_stage_backend_must_match_probe(self): + add_gpu(self.result) + self.result["stages"]["execute"]["compute"]["effective"] = "cuda" + self.assertRejected("compute-execution") + + def test_not_run_probe_cannot_be_promoted_to_passed_stage(self): + self.execution.update(status="not-run", executed=False, effective=None, + correct=unknown(), evidence=[], reason="no operation attempted") + self.assertRejected("compute-execution") + + def test_completed_operation_requires_observed_runtime_and_process(self): + for field in ("runtime", "process_id"): + with self.subTest(field=field): + original = self.execution[field] + self.execution[field] = unknown("process not observed") + self.assertRejected("compute-evidence") + self.execution[field] = original + + def test_declared_correctness_is_not_a_measured_pass(self): + self.execution["correct"] = fact(True, provenance="declared") + self.assertRejected("compute-execution") + + def test_mismatched_cpu_result_cannot_pass_correctness(self): + self.execution.update(status="failed", correct=fact(False, LOG), reason="reference mismatch") + self.assertRejected("compute-execution") + self.result["stages"]["correctness"].update(status="failed", reason="reference mismatch") + self.result["overall"] = "failed" + findings = self.check() + self.assertEqual(findings.codes(), ["required-stage"], findings.items) + + def test_stage_evidence_cannot_cite_a_different_operation(self): + self.result["stages"]["execute"]["evidence"] = ["log:unrelated.log"] + self.assertRejected("compute-evidence") + + def test_discovery_tool_is_not_execution_evidence(self): + self.execution["evidence"] = ["tool:nvidia-smi"] + self.assertRejected("compute-evidence") + + def test_execution_must_link_same_github_sha_workflow_job_and_attempt(self): + changes = ( + ("revision", "tested_sha", "a" * 40), + ("revision", "event_sha", "b" * 40), + ("revision", "pr_head_sha", "c" * 40), + ("revision", "pr_base_sha", "d" * 40), + ("ci", "workflow_ref", "P-R-N-D/Claus-AI/.github/workflows/other.yml@refs/pull/1/merge"), + ("ci", "job", "another-job"), + ("ci", "run_id", "101"), + ("ci", "run_attempt", 2), + ) + for group, field, value in changes: + with self.subTest(field=field): + self.execution["run"] = copy.deepcopy(self.result["run"]) + self.execution["run"][group][field]["value"] = value + self.assertRejected("compute-evidence") + + def test_missing_execution_run_is_rejected(self): + self.execution.pop("run") + self.assertRejected("compute-evidence") + + def test_synthetic_fixture_is_not_measured_evidence(self): + result, profile = helpers.pair("kubernetes-arm64-cuda") + findings = helpers.check(result, profile, real=True) + self.assertIn("synthetic-marking", findings.codes()) + + def test_privileged_container_does_not_gain_a_device_exception(self): + result, profile = helpers.pair("kubernetes-arm64-cuda") + result["environment"]["layers"][-1]["container"]["privileged"]["value"] = True + findings = helpers.check(result, profile) + self.assertIn("policy-no-privileged-container", findings.codes(), findings.items) + + def test_rootful_engine_is_rejected_when_lane_requires_rootless(self): + result, profile = helpers.pair("windows-arm64-wsl2-podman") + profile["policy"]["rules"]["engine-rootless"] = "required" + result["environment"]["layers"][-1]["container"]["engine_mode"]["value"] = "rootful" + findings = helpers.check(result, profile) + self.assertIn("policy-engine-rootless", findings.codes(), findings.items) + + def test_not_run_accelerator_and_unknown_driver_do_not_fail_cpu_lane(self): + gpu = add_gpu(self.result) + gpu["available"] = unknown("device usability was not tested") + for field in ("runtime_installed", "api_enumerated", "device_enumerated", "device_accessible"): + gpu["probe"][field] = unknown("runtime and hardware were not inspected") + gpu["probe"]["execution"] = { + "status": "not-run", "executed": False, "effective": None, + "correct": unknown(), "software_fallback": unknown(), + "runtime": unknown(), "process_id": unknown(), "evidence": [], + "reason": "no trusted hardware executor configured", + } + self.assertEqual(self.check().errors, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/tests/test_device_environments.py b/scripts/ci/tests/test_device_environments.py new file mode 100644 index 0000000..eee09fa --- /dev/null +++ b/scripts/ci/tests/test_device_environments.py @@ -0,0 +1,140 @@ +"""Synthetic special environments and denial cases; Python standard library only.""" + +import copy +import importlib.util +import pathlib +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parents[3] + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +validator = load("device_environment_validator", ROOT / "scripts/ci/validate_compatibility.py") +fixtures = load("device_environment_scenarios", ROOT / "tests/compatibility/fixtures/device-environments/scenarios.py") +SCHEMA = validator.load_schema() + + +def validate(result, profile=None, **options): + return validator.validate_document(result, SCHEMA, profile=profile, **options) + + +def codes(findings): + return {finding[1] for finding in findings.errors} + + +class DeviceEnvironmentTests(unittest.TestCase): + def setUp(self): + self.examples = fixtures.scenarios() + + def test_all_static_profiles_and_environment_structures_are_valid(self): + self.assertEqual(len(self.examples), 15) + for name, (profile, result, expected_errors) in self.examples.items(): + with self.subTest(name=name): + self.assertEqual(validate(profile).errors, []) + self.assertEqual(validate(result, profile, evaluate_policy=False).errors, []) + self.assertEqual(codes(validate(result, profile)), expected_errors) + + def test_no_static_case_is_presented_as_an_executed_run(self): + for name, (profile, result, _) in self.examples.items(): + with self.subTest(name=name): + self.assertTrue(profile["synthetic"]) + self.assertTrue(result["synthetic"]) + self.assertIsNone(result["generated_at"]) + self.assertEqual(result["overall"], "incomplete") + self.assertEqual({stage["status"] for stage in result["stages"].values()}, {"not-run"}) + self.assertEqual(profile["policy"]["exceptions"], []) + self.assertTrue(all(validator.known(fact) is None for _, fact in validator.facts_of(result["run"]))) + self.assertIn("synthetic-marking", codes(validate(result, profile, require_real=True))) + + def test_unmarking_a_fixture_does_not_make_it_measured_evidence(self): + profile, result, _ = self.examples["bare-metal"] + result["synthetic"] = False + result["generated_at"] = "2026-10-12T00:00:00Z" + self.assertIn("synthetic-marking", codes(validate(result, profile))) + result["synthetic"] = True + result["run"]["revision"]["tested_sha"] = fixtures.fact("a" * 40, "invented commit") + self.assertIn("synthetic-marking", codes(validate(result))) + + def test_host_systemd_and_container_pid1_are_distinct(self): + direct = self.examples["host-systemd-process"][1]["environment"]["layers"] + self.assertEqual(len(direct), 1) + self.assertEqual(direct[0]["service_manager"]["scope"]["value"], "user") + for name in ("host-systemd-container", "host-quadlet-container"): + profile, result, _ = self.examples[name] + layers = result["environment"]["layers"] + self.assertEqual(layers[0]["service_manager"]["name"]["value"], "systemd") + self.assertEqual(layers[-1]["service_manager"]["name"]["value"], "none") + self.assertEqual(layers[-1]["container"]["orchestration"]["scope"]["value"], "user") + layers[-1]["service_manager"] = copy.deepcopy(layers[0]["service_manager"]) + self.assertIn("profile-mismatch", codes(validate(result, profile))) + + profile, result, _ = self.examples["container-pid1-systemd"] + layer = result["environment"]["layers"][-1] + self.assertEqual(layer["service_manager"]["name"]["value"], "systemd") + self.assertEqual(layer["service_manager"]["scope"]["value"], "system") + self.assertFalse(layer["container"]["privileged"]["value"]) + layer["service_manager"]["scope"] = fixtures.fact("user", "host scope incorrectly used for PID 1") + self.assertIn("profile-mismatch", codes(validate(result, profile))) + + def test_systemd_cgroup_delegation_and_rootless_engine_are_independent(self): + _, inside, _ = self.examples["container-pid1-systemd"] + _, plain, _ = self.examples["rootless-podman"] + for result in (inside, plain): + self.assertEqual(result["environment"]["layers"][-1]["container"]["engine_mode"]["value"], "rootless") + self.assertFalse(result["workload"]["identity"]["superuser"]["value"]) + self.assertEqual(result["workload"]["isolation"]["cgroup"]["version"]["value"], "v2") + self.assertTrue(inside["workload"]["isolation"]["cgroup"]["delegated"]["value"]) + self.assertFalse(plain["workload"]["isolation"]["cgroup"]["delegated"]["value"]) + + def test_wsl_vm_and_container_order_cannot_be_reversed(self): + for name in ("windows-arm-wsl2-container", "vm-rootless-podman-nonroot"): + _, original, _ = self.examples[name] + for order in ((2, 1, 0), (0, 2, 1)): + with self.subTest(name=name, order=order): + result = copy.deepcopy(original) + layers = result["environment"]["layers"] + result["environment"]["layers"] = [layers[index] for index in order] + # A VM inside a Linux container has no supported nested-lane profile; + # profile matching rejects that swap even if OS families agree. + errors = codes(validate(result, self.examples[name][0])) + self.assertTrue(errors & {"layer-structure", "os-contradiction", "profile-mismatch"}, errors) + + def test_cluster_nonroot_and_no_new_privileges_do_not_mean_rootless_engine(self): + for name in ("kubernetes", "k3s", "openshift"): + profile, result, _ = self.examples[name] + layer = result["environment"]["layers"][-1] + self.assertEqual(layer["container"]["orchestration"]["name"]["value"], name) + self.assertFalse(result["workload"]["identity"]["superuser"]["value"]) + self.assertTrue(result["workload"]["isolation"]["no_new_privs"]["value"]) + self.assertFalse(layer["extensions"]["x-device-plugin"]["requested"]) + self.assertEqual(layer["container"]["devices"]["value"], []) + self.assertIn("policy-engine-rootless", codes(validate(result, profile))) + + def test_unapproved_rootful_and_privileged_changes_are_rejected(self): + for field, value, expected in (("engine_mode", "rootful", "policy-engine-rootless"), + ("privileged", True, "policy-no-privileged-container")): + profile, result, _ = copy.deepcopy(self.examples["rootless-podman"]) + result["environment"]["layers"][-1]["container"][field] = fixtures.fact(value, "unauthorized change") + self.assertIn(expected, codes(validate(result, profile))) + + def test_device_passthrough_is_not_usability_or_execution(self): + for name in ("gpu-passthrough", "npu-passthrough"): + profile, result, _ = self.examples[name] + self.assertIsNone(result["compute"]["backends"][-1]["available"]["value"]) + self.assertEqual(result["stages"]["execute"]["status"], "not-run") + self.assertIn("policy-minimal-devices", codes(validate(result, profile))) + profile, cpu, _ = self.examples["cpu-only-container"] + self.assertEqual(cpu["environment"]["layers"][-1]["container"]["devices"]["value"], []) + self.assertEqual([backend["id"] for backend in cpu["compute"]["backends"]], ["cpu"]) + self.assertEqual(validate(cpu, profile).errors, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/validate_compatibility.py b/scripts/ci/validate_compatibility.py index 34e1664..513c40e 100644 --- a/scripts/ci/validate_compatibility.py +++ b/scripts/ci/validate_compatibility.py @@ -145,7 +145,7 @@ KNOWN_TECHNOLOGIES = frozenset(TECHNOLOGY_KINDS) KNOWN_BACKEND_APIS = frozenset({ "cpu", "cuda", "rocm", "oneapi", "level-zero", "opencl", "vulkan-compute", "directcompute", - "directml", "metal", "mps", "coreml", "qnn", "wasm", "webgpu", "webgl", "webnn", + "directml", "metal", "mps", "coreml", "mlx", "qnn", "wasm", "webgpu", "webgl", "webnn", }) KNOWN_ORCHESTRATION = frozenset({"none", "kubernetes", "k3s", "openshift", "nomad", "compose", "quadlet", "systemd"}) KNOWN_SERVICE_MANAGERS = frozenset({"none", "systemd", "launchd", "windows-scm", "openrc"}) @@ -836,6 +836,100 @@ def check_compute(document, findings): findings.error("backend-reference", f"/compute/backends/{index}/id", f"duplicate backend id {backend['id']!r}") if backend["api"] == "cpu" and backend["kind"] != "cpu": findings.error("backend-reference", f"/compute/backends/{index}", "the cpu API runs on the cpu backend kind") + check_device_probes(document, findings) + + +def check_device_probes(document, findings): + """The opt-in device contract augments v1; it does not reinterpret legacy CPU records.""" + required = document["compute"].get("probe_contract") == "device-v1" + backends = {backend["id"]: backend for backend in document["compute"]["backends"]} + for index, backend in enumerate(document["compute"]["backends"]): + base = f"/compute/backends/{index}/probe" + probe = backend.get("probe") + if document["kind"] == "profile": + if probe is not None: + findings.error("compute-probe", base, "profiles declare capabilities; execution probes belong to results") + continue + if probe is None: + if required: + findings.error("compute-probe", base, "device-v1 results record a probe for every backend, including unexecuted ones") + continue + if not required: + findings.error("compute-probe", base, "a result with device probes names compute.probe_contract device-v1") + execution = probe["execution"] + if (known(backend["available"]) is True or execution["executed"]) and measured(probe["runtime_installed"]) is not True: + findings.error("compute-availability", f"{base}/runtime_installed", "availability or a completed operation requires measured runtime installation") + # Runtime installation, API/device enumeration and access are separate prerequisites for availability. + if known(backend["available"]) is True: + for field in ("api_enumerated", "device_enumerated", "device_accessible"): + if measured(probe[field]) is not True: + findings.error("compute-availability", f"{base}/{field}", "available=true requires measured API enumeration, device enumeration and device access") + path = f"{base}/execution" + status = execution["status"] + effective = backends.get(execution["effective"]) + if execution["effective"] is not None and effective is None: + findings.error("backend-reference", f"{path}/effective", "the actual execution backend is absent from compute.backends") + if status != "passed" and not execution.get("reason"): + findings.error("compute-execution", f"{path}/reason", "an unpassed probe records the reason or error category") + if status in ("not-run", "blocked"): + if execution["executed"] or execution["effective"] is not None or known(execution["correct"]) is not None: + findings.error("compute-execution", path, "an unexecuted probe has no completed operation, actual backend or correctness result") + if status == "not-run" and execution["evidence"]: + findings.error("compute-execution", f"{path}/evidence", "an operation that was not run has no execution evidence") + if status in RAN: + if execution.get("run") != document["run"]: + findings.error("compute-evidence", f"{path}/run", "execution evidence must name the same run, tested/event/head/base SHAs, workflow and job as the Result") + if not execution["evidence"] or (not document["synthetic"] and not any(ref.startswith(STAGE_EVIDENCE) for ref in execution["evidence"])): + findings.error("compute-evidence", f"{path}/evidence", "an attempted operation cites its execution log or artifact") + if execution["executed"]: + if status not in RAN or effective is None: + findings.error("compute-execution", path, "a completed operation names its actual backend and passed/failed status") + pid = measured(execution["process_id"]) + if measured(execution["runtime"]) is None or pid is None or pid < 1: + findings.error("compute-evidence", path, "a completed operation records its measured runtime and positive process ID") + if effective is not None and measured(effective["available"]) is not True: + findings.error("compute-availability", f"{path}/effective", "a completed operation's actual backend is measured available") + elif execution["effective"] is not None or known(execution["correct"]) is not None: + findings.error("compute-execution", path, "an operation that did not complete names no actual backend or correctness result") + if status == "passed": + if not execution["executed"] or measured(execution["correct"]) is not True: + findings.error("compute-execution", path, "passed requires a completed operation and measured correct result") + if backend["kind"] in ("gpu", "npu") and (execution["effective"] != backend["id"] or measured(execution["software_fallback"]) is not False): + findings.error("accelerator-fallback", path, "a hardware pass requires that requested backend to execute with software fallback measured false") + + +def check_device_stage(document, name, stage, backends, findings): + """Bind workload stage claims to the requested backend's recorded operation, not discovery.""" + if name not in WORKLOAD_STAGES or stage["status"] not in RAN: + return + compute = stage.get("compute") + required = document["compute"].get("probe_contract") == "device-v1" + if compute is None: + if required: + findings.error("compute-execution", f"/stages/{name}/compute", "a device workload stage records requested and actual backends") + return + backend = backends.get(compute["requested"]) + if backend is None: + return # The existing backend-reference rule reports this. + probe = backend.get("probe") + # Legacy synthetic fixtures remain shape examples. Real accelerator claims always need execution evidence. + if probe is None: + if required or (not document["synthetic"] and backend["kind"] in ("gpu", "npu") and stage["status"] == "passed"): + findings.error("compute-execution", f"/stages/{name}/compute", "a compute stage cannot pass from inventory or driver discovery alone") + return + execution = probe["execution"] + path = f"/stages/{name}" + if execution["status"] not in RAN or compute["effective"] != execution["effective"]: + findings.error("compute-execution", f"{path}/compute", "stage execution and actual backend must agree with the requested backend's probe") + if not set(stage.get("evidence", ())).intersection(execution["evidence"]): + findings.error("compute-evidence", f"{path}/evidence", "the stage must cite its backend probe's execution evidence") + if stage["status"] == "passed" and not execution["executed"]: + findings.error("compute-execution", f"{path}/status", "a stage cannot pass when its probe completed no operation") + if stage["status"] == "passed" and backend["kind"] in ("gpu", "npu") and measured(execution["software_fallback"]) is not False: + findings.error("accelerator-fallback", f"{path}/compute", "a hardware stage pass requires software fallback measured false, even if its probe failed correctness") + if name in ("correctness", "performance") and stage["status"] == "passed": + if execution["status"] != "passed" or measured(execution["correct"]) is not True: + findings.error("compute-execution", f"{path}/status", "correctness and performance passes require a passed, correct operation") # --------------------------------------------------------------------------- @@ -876,6 +970,7 @@ def check_stages(document, findings): before = stages[prerequisite]["status"] if status in RAN and before not in ("passed", "not-applicable"): findings.error("stage-order", f"{base}/status", f"{name} cannot have {status} while {prerequisite} is {before}; record it as blocked or not-run") + check_device_stage(document, name, stage, backends, findings) compute = stage.get("compute") if compute is None: continue @@ -1313,6 +1408,8 @@ def check_against_profile(result, profile, findings): findings.error("profile-link", "/profile_id", f"the result names profile {result['profile_id']!r}, not {profile['id']!r}") if result["synthetic"] != profile["synthetic"]: findings.error("profile-link", "/synthetic", "a real result is checked against a real profile, and a synthetic one against a synthetic one") + if profile["compute"].get("probe_contract") != result["compute"].get("probe_contract"): + findings.error("profile-mismatch", "/compute/probe_contract", "the result must use its profile's compute probe contract") ci = result["run"]["ci"] job = known(ci["job"]) if not profile["synthetic"] and measured(ci["system"]) != "github-actions": @@ -1370,15 +1467,19 @@ def compare(path, declared_fact, observed_fact): for name, fact in profile.get("runner", {}).items(): compare(f"/run/ci/{name}", fact, ci[name]) accelerator = profile.get("accelerator_backend") + required_backends = {accelerator} if accelerator is not None else set() + if accelerator is None and profile["compute"].get("probe_contract") == "device-v1": + required_backends = {backend["id"] for backend in profile["compute"]["backends"] if backend["kind"] == "cpu"} for name in profile["required_stages"]: stage = result["stages"][name] if stage["status"] != "passed": findings.error("required-stage", f"/stages/{name}/status", f"profile {profile['id']} requires {name} to pass; it is {stage['status']}") - elif accelerator is not None and name in WORKLOAD_STAGES: + elif required_backends and name in WORKLOAD_STAGES: compute = stage.get("compute") or {} - if compute.get("requested") != accelerator or compute.get("effective") != accelerator: + if compute.get("requested") not in required_backends or compute.get("effective") != compute.get("requested"): ran = compute.get("effective") or "no recorded backend" - findings.error("required-stage", f"/stages/{name}/compute", f"profile {profile['id']} requires {name} to run on {accelerator}; it ran on {ran}") + allowed = ", ".join(sorted(required_backends)) + findings.error("required-stage", f"/stages/{name}/compute", f"profile {profile['id']} requires {name} to request and run on the same declared backend ({allowed}); it ran on {ran}") STATUS_WORDS = ( diff --git a/tests/compatibility/fixtures/device-environments/README.md b/tests/compatibility/fixtures/device-environments/README.md new file mode 100644 index 0000000..a8ca8fc --- /dev/null +++ b/tests/compatibility/fixtures/device-environments/README.md @@ -0,0 +1,38 @@ +# Synthetic device environment fixtures + +`scenarios.py` builds 15 profile/result pairs from the existing synthetic fixtures. +These are static schema and policy examples. No host, WSL2 installation, VM, +container, cluster, systemd service, GPU or NPU is started. Every stage is +`not-run`; every result is `synthetic: true`, has no timestamp, run or commit, +and remains `incomplete`. Export the bundle for inspection with: + +```sh +python -I tests/compatibility/fixtures/device-environments/scenarios.py > /tmp/device-environments.json +python -I scripts/ci/tests/test_device_environments.py --verbose +``` + +The cases cover bare metal, a VM, Windows on Arm / WSL2 / Linux container, +rootless Podman, VM / rootless Podman / non-root application, a directly +managed host process, host systemd and Quadlet container management, container +PID 1 systemd, Kubernetes, K3s, OpenShift, GPU/NPU passthrough and a CPU-only +container. Service manager facts belong to their own layer. The container +orchestration scope describes its host's unit; it does not describe container +PID 1. cgroup v2 delegation, engine rootlessness, and workload identity remain +independent facts. The systemd container example is unprivileged. + +The three cluster examples deliberately use rootful node engines with non-root +workloads and `no_new_privs: true`. Their profiles require rootless engines, +so policy validation must reject them with `policy-engine-rootless`. A normal +rootless engine fixture must also be rejected if mutated to rootful. GPU/NPU +passthrough examples must be rejected with `policy-minimal-devices`: they model +device access without granting approval for it. They leave backend usability +unknown. None has a policy exception or claims hardware execution. All 15 pairs +pass structural and cross-field validation when policy evaluation is disabled; +that check makes no privilege-policy claim. + +The `x-device-plugin` extension describes the synthetic request/access route +separately from `container.devices`, engine mode, identity and `no_new_privs`. +Like other `extensions`, it is descriptive metadata, not a validated grant or a +security-policy input. A real device plugin request and grant would need +independent engine/cluster evidence and a scoped approved device exception. +Synthetic records cannot be submitted with `--require-real`. diff --git a/tests/compatibility/fixtures/device-environments/scenarios.py b/tests/compatibility/fixtures/device-environments/scenarios.py new file mode 100644 index 0000000..fa74195 --- /dev/null +++ b/tests/compatibility/fixtures/device-environments/scenarios.py @@ -0,0 +1,167 @@ +"""Build synthetic environment examples; never inspect or execute a platform. + +Existing fixtures supply the schema's shared fields. These small transformations +keep the distinctions under test visible without duplicating thousands of lines +of runtime and identity facts. Every returned result is synthetic and incomplete. +""" + +import copy +import json +import pathlib + + +FIXTURES = pathlib.Path(__file__).resolve().parent.parent + + +def fact(value, source): + return {"value": value, "provenance": "observed", "evidence": [f"synthetic:{source}"]} + + +def unknown(reason): + return {"value": None, "provenance": "unknown", "reason": reason} + + +def na(reason): + return {"value": None, "provenance": "not-applicable", "reason": reason} + + +def template(name): + return json.loads((FIXTURES / f"{name}.result.json").read_text(encoding="utf-8")) + + +def manager(name="none", scope=None, source="service-manager scenario"): + return {"name": fact(name, source), "scope": fact(scope, source) if scope else na("no service manager")} + + +def orchestration(name="none", scope=None): + return { + "name": fact(name, "engine-side orchestration scenario"), + "version": unknown("version not specified by this scenario") if name != "none" else na("no orchestrator"), + "scope": fact(scope, "engine-side unit scope") if scope else na("no user/system unit scope"), + } + + +def bare_metal(): + result = template("linux-x64-vm") + host = result["environment"]["layers"][-1] + host.update(label="Synthetic bare metal", kind=fact("host", "bare-metal scenario"), + technology=na("the outermost host runs on hardware"), service_manager=manager()) + host["cpu"]["execution"] = fact("native", "bare-metal scenario") + result["environment"]["layers"] = [host] + return result + + +def podman(vm=False): + result = template("vm-podman-quadlet-systemd") + layers = result["environment"]["layers"] + layers[-1]["service_manager"] = manager() + layers[-1]["container"]["orchestration"] = orchestration() + if not vm: + result["environment"]["layers"] = [bare_metal()["environment"]["layers"][0], layers[-1]] + result["workload"]["isolation"]["cgroup"]["delegated"] = fact(False, "delegation is independent of rootless engine") + return result + + +def declared(node): + """Turn synthetic result facts into profile declarations; do not invent observations.""" + if isinstance(node, dict): + if "provenance" in node: + if node["provenance"] in ("observed", "inferred"): + return {"value": copy.deepcopy(node["value"]), "provenance": "declared"} + return copy.deepcopy(node) + return {key: declared(value) for key, value in node.items()} + if isinstance(node, list): + return [declared(value) for value in node] + return node + + +def pair(name, result): + """No stage in these examples ran, including detection; approval is never fabricated.""" + result["profile_id"] = f"synthetic-device-{name}" + result["generated_at"] = None + result["synthetic"] = True + result["overall"] = "incomplete" + result["collector"] = {"name": "synthetic-device-environments", "version": 1} + result["stages"] = { + stage: {"status": "not-run", "reason": "static environment fixture; no process ran on this platform"} + for stage in result["stages"] + } + result["extensions"] = {"x-note": "Synthetic environment contract fixture, never a measured platform result."} + profile = { + "schema_version": 1, "kind": "profile", "synthetic": True, + "id": result["profile_id"], "title": f"Synthetic device environment: {name}", + "support": {"status": "not-assessed", "lane": None, + "reference": "docs/DEPENDENCY-STRATEGY.md#platform-and-accelerator-lanes"}, + "evidence_sources": [], "required_stages": [], + "environment": declared(result["environment"]), + "workload": {"runtimes": declared(result["workload"]["runtimes"])}, + "compute": declared(result["compute"]), + "policy": {"rules": {"engine-rootless": "required"}, "exceptions": []}, + } + return profile, result + + +def scenarios(): + """Map scenario to (profile, result, expected policy error codes).""" + examples = {} + + def add(name, result, errors=()): + examples[name] = (*pair(name, result), frozenset(errors)) + + add("bare-metal", bare_metal()) + add("vm", template("linux-x64-vm")) + add("windows-arm-wsl2-container", template("windows-arm64-wsl2-podman")) + add("rootless-podman", podman()) + add("vm-rootless-podman-nonroot", podman(vm=True)) + + direct = bare_metal() + direct["environment"]["layers"][0]["service_manager"] = manager("systemd", "user", "host systemd user unit") + add("host-systemd-process", direct) + + for controller in ("systemd", "quadlet"): + controlled = podman() + controlled["environment"]["layers"][0]["service_manager"] = manager("systemd", "user", "host systemd user manager") + controlled["environment"]["layers"][-1]["container"]["orchestration"] = orchestration(controller, "user") + add(f"host-{controller}-container", controlled) + + inside = podman() + inside["environment"]["layers"][-1]["service_manager"] = manager("systemd", "system", "container /proc/1/comm") + inside["workload"]["isolation"]["cgroup"]["delegated"] = fact(True, "container cgroup v2 delegation scenario") + add("container-pid1-systemd", inside) + + for cluster in ("kubernetes", "k3s", "openshift"): + result = template("kubernetes-arm64-cuda") + layer = result["environment"]["layers"][-1] + layer["service_manager"] = manager() + layer["container"]["devices"] = fact([], "no device plugin allocation") + layer["container"]["orchestration"] = orchestration(cluster) + layer["extensions"] = {"x-device-plugin": {"requested": False, "access": "not-requested"}} + result["compute"]["backends"] = result["compute"]["backends"][:1] + result["compute"]["accelerator_inventory"] = unknown("device plugin absence does not establish node hardware absence") + # Non-root workload and allowPrivilegeEscalation=false do not make the node engine rootless. + add(cluster, result, ("policy-engine-rootless",)) + + for kind, api, device in (("gpu", "cuda", "nvidia-gpu"), ("npu", "qnn", "qualcomm-npu")): + result = podman() + layer = result["environment"]["layers"][-1] + layer["container"]["devices"] = fact([device], "engine-side passed device scenario") + layer["extensions"] = {"x-device-plugin": {"requested": False, "access": "engine-device-allowlist"}} + result["compute"]["backends"].append({ + "id": api, "kind": kind, "api": api, + "available": unknown("device passed through; no usable runtime or kernel execution established"), + "device": fact(device, "synthetic device class only; no serial number"), + "version": unknown("driver not probed"), "features": unknown("features not probed"), + }) + result["compute"]["accelerator_inventory"] = fact(True, "engine-side passed device scenario") + add(f"{kind}-passthrough", result, ("policy-minimal-devices",)) + + cpu_only = podman() + cpu_only["compute"]["accelerator_inventory"] = fact(False, "synthetic CPU-only container inventory") + add("cpu-only-container", cpu_only) + return examples + + +if __name__ == "__main__": + # A single JSON bundle on stdout is easy to inspect/archive; it is not a Result document. + print(json.dumps({name: {"profile": profile, "result": result, "expected_policy_errors": sorted(errors)} + for name, (profile, result, errors) in scenarios().items()}, indent=2)) diff --git a/tests/compatibility/profiles/gha-linux-x64-device.json b/tests/compatibility/profiles/gha-linux-x64-device.json new file mode 100644 index 0000000..8e507f0 --- /dev/null +++ b/tests/compatibility/profiles/gha-linux-x64-device.json @@ -0,0 +1,276 @@ +{ + "schema_version": 1, + "kind": "profile", + "synthetic": false, + "id": "gha-linux-x64-device", + "title": "CPU reference and device capability evidence on GitHub-hosted Ubuntu 24.04", + "description": "Standard-library CPU reference correctness and optional browser capability detection; no accelerator hardware qualification or inference benchmark.", + "support": { + "status": "not-assessed", + "lane": null, + "reference": "docs/DEPENDENCY-STRATEGY.md#platform-and-accelerator-lanes" + }, + "evidence_sources": [ + { + "workflow": ".github/workflows/device-tests.yml", + "job": "cpu" + } + ], + "runner": { + "runner_environment": { + "value": "github-hosted", + "provenance": "declared" + }, + "runner_os": { + "value": "Linux", + "provenance": "declared" + }, + "runner_arch": { + "value": "X64", + "provenance": "declared" + }, + "image_os": { + "value": "ubuntu24", + "provenance": "declared" + } + }, + "required_stages": [ + "detect", + "install", + "initialize", + "execute", + "correctness" + ], + "environment": { + "layers": [ + { + "label": "Physical host, not visible to the runner", + "kind": { + "value": "host", + "provenance": "declared" + }, + "technology": { + "value": null, + "provenance": "not-applicable", + "reason": "the outermost host runs on hardware" + }, + "os": { + "family": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not document the physical hosts, and a VM guest cannot see them" + }, + "name": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not document the physical hosts, and a VM guest cannot see them" + }, + "version": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not document the physical hosts, and a VM guest cannot see them" + }, + "kernel": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not document the physical hosts, and a VM guest cannot see them" + } + }, + "cpu": { + "arch": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not document the physical hosts, and a VM guest cannot see them" + }, + "execution": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not document the physical hosts, and a VM guest cannot see them" + } + }, + "service_manager": { + "name": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not document the physical hosts, and a VM guest cannot see them" + }, + "scope": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not document the physical hosts, and a VM guest cannot see them" + } + } + }, + { + "label": "GitHub-hosted runner virtual machine (runs-on: ubuntu-24.04)", + "kind": { + "value": "vm", + "provenance": "declared" + }, + "technology": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not document the hypervisor; each run records what DMI shows" + }, + "os": { + "family": { + "value": "linux", + "provenance": "declared" + }, + "name": { + "value": "ubuntu", + "provenance": "declared" + }, + "version": { + "value": "24.04", + "provenance": "declared" + }, + "kernel": { + "value": null, + "provenance": "unknown", + "reason": "changes with runner image updates; each run records it" + } + }, + "cpu": { + "arch": { + "value": "x86_64", + "provenance": "declared" + }, + "execution": { + "value": null, + "provenance": "unknown", + "reason": "whether the hypervisor emulates the CPU is not visible from inside the VM" + } + }, + "service_manager": { + "name": { + "value": "systemd", + "provenance": "declared" + }, + "scope": { + "value": "system", + "provenance": "declared" + } + } + } + ] + }, + "workload": { + "runtimes": { + "python": { + "implementation": { + "value": "cpython", + "provenance": "declared" + }, + "version": { + "value": "3.12.15", + "provenance": "declared" + }, + "cache_tag": { + "value": "cpython-312", + "provenance": "declared" + }, + "soabi": { + "value": "cpython-312-x86_64-linux-gnu", + "provenance": "declared" + }, + "platform": { + "value": "linux-x86_64", + "provenance": "declared" + }, + "arch": { + "value": "x86_64", + "provenance": "declared" + }, + "pointer_bits": { + "value": 64, + "provenance": "declared" + }, + "free_threaded_build": { + "value": false, + "provenance": "declared" + }, + "gil_enabled": { + "value": true, + "provenance": "declared" + }, + "libc": { + "value": "glibc 2.39", + "provenance": "declared" + }, + "execution": { + "value": "native", + "provenance": "declared" + } + }, + "node": { + "version": { + "value": "24.21.0", + "provenance": "declared" + }, + "modules_abi": { + "value": null, + "provenance": "unknown", + "reason": "follows from the Node version; each run records it" + }, + "napi_version": { + "value": null, + "provenance": "unknown", + "reason": "follows from the Node version; each run records it" + }, + "platform": { + "value": "linux", + "provenance": "declared" + }, + "arch": { + "value": "x86_64", + "provenance": "declared" + }, + "execution": { + "value": "native", + "provenance": "declared" + } + } + } + }, + "compute": { + "backends": [ + { + "id": "cpu", + "kind": "cpu", + "api": "cpu", + "available": { + "value": true, + "provenance": "declared" + }, + "device": { + "value": null, + "provenance": "unknown", + "reason": "GitHub does not fix the CPU model of hosted runners" + }, + "version": { + "value": null, + "provenance": "not-applicable", + "reason": "a CPU has no backend version" + }, + "features": { + "value": null, + "provenance": "unknown", + "reason": "not declared" + } + } + ], + "accelerator_inventory": { + "value": null, + "provenance": "unknown", + "reason": "absence of a visible accelerator does not establish physical hardware absence" + }, + "probe_contract": "device-v1" + }, + "policy": { + "rules": { + "engine-rootless": "required" + }, + "exceptions": [] + } +} diff --git a/tests/compatibility/schema.json b/tests/compatibility/schema.json index 6cd3183..aa32c75 100644 --- a/tests/compatibility/schema.json +++ b/tests/compatibility/schema.json @@ -945,11 +945,46 @@ "features": { "$ref": "#/$defs/fact_slug_list" }, + "probe": { + "$ref": "#/$defs/backend_probe" + }, "extensions": { "$ref": "#/$defs/extensions" } } }, + "backend_probe": { + "description": "Optional device-v1 observations. Tool/runtime presence, API/device enumeration, access and actual computation are independent facts. Execution is never inferred from discovery.", + "type": "object", + "required": ["tool", "runtime_installed", "api_enumerated", "device_enumerated", "device_accessible", "execution"], + "additionalProperties": false, + "properties": { + "tool": {"$ref": "#/$defs/text"}, + "runtime_installed": {"$ref": "#/$defs/fact_boolean"}, + "api_enumerated": {"$ref": "#/$defs/fact_boolean"}, + "device_enumerated": {"$ref": "#/$defs/fact_boolean"}, + "device_accessible": {"$ref": "#/$defs/fact_boolean"}, + "execution": {"$ref": "#/$defs/compute_execution"} + } + }, + "compute_execution": { + "description": "One operation attempt. executed means the operation completed, even when its result was incorrect. A failed attempt may have executed=false. run is copied from the enclosing Result to prevent evidence from another run being attached as a pass.", + "type": "object", + "required": ["status", "executed", "effective", "correct", "software_fallback", "runtime", "process_id", "evidence"], + "additionalProperties": false, + "properties": { + "status": {"enum": ["passed", "failed", "blocked", "not-run"]}, + "executed": {"type": "boolean"}, + "effective": {"type": ["string", "null"], "pattern": "^[a-z0-9][a-z0-9._+-]{0,63}$"}, + "correct": {"$ref": "#/$defs/fact_boolean"}, + "software_fallback": {"$ref": "#/$defs/fact_boolean"}, + "runtime": {"$ref": "#/$defs/fact_string"}, + "process_id": {"$ref": "#/$defs/fact_integer"}, + "evidence": {"$ref": "#/$defs/evidence_list"}, + "reason": {"$ref": "#/$defs/text"}, + "run": {"$ref": "#/$defs/run"} + } + }, "compute": { "type": "object", "required": [ @@ -958,6 +993,10 @@ ], "additionalProperties": false, "properties": { + "probe_contract": { + "description": "Profiles opt into the device evidence contract; their results must keep this marker and record every backend probe.", + "const": "device-v1" + }, "backends": { "type": "array", "items": {