diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78e9e3dc..debb54e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,14 +20,9 @@ jobs: uses: actions/setup-go@v6 with: go-version: 'stable' - - - name: Cache Go modules - uses: actions/cache@v5 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- + cache-dependency-path: | + go.sum + tools/go.sum - name: Build binary run: make build @@ -46,6 +41,12 @@ jobs: - name: Run linters run: make lint + - name: Test CLI harness reporting and coverage gates + run: python3 -m unittest discover -s test/cli -p 'test_*.py' -v + + - name: Test Terraform harness evidence gates + run: python3 -m unittest discover -s test/terraform -p 'test_*.py' -v + - name: Upload binary artifact uses: actions/upload-artifact@v7 with: @@ -84,14 +85,9 @@ jobs: uses: actions/setup-go@v6 with: go-version: 'stable' - - - name: Cache Go modules - uses: actions/cache@v5 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- + cache-dependency-path: | + go.sum + tools/go.sum - name: Install qemu-img for image normalization tests run: sudo apt-get update && sudo apt-get install -y --no-install-recommends qemu-utils @@ -129,14 +125,9 @@ jobs: uses: actions/setup-go@v6 with: go-version: 'stable' - - - name: Cache Go modules - uses: actions/cache@v5 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- + cache-dependency-path: | + go.sum + tools/go.sum - name: Start lightstack services run: | @@ -250,14 +241,9 @@ jobs: uses: actions/setup-go@v6 with: go-version: 'stable' - - - name: Cache Go modules - uses: actions/cache@v5 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- + cache-dependency-path: | + go.sum + tools/go.sum - name: Build run: make build @@ -423,6 +409,113 @@ jobs: # TODO: Add e2e-fast and e2e-full jobs once E2E test scenarios are implemented. # Planned: VM lifecycle, volume attach/detach, network isolation, Terraform validation, Horizon UI tests. + cli-conformance: + name: CLI Conformance + runs-on: ubuntu-latest + needs: build + timeout-minutes: 40 + + # SPEC-028: drive the build artifact with the pinned python OpenStack CLI. + # The downloaded binary runs natively against a PostgreSQL service (no + # docker), so the suite always exercises exactly what the build job made. + services: + postgres: + image: postgres:18.4-alpine + env: + POSTGRES_USER: lightstack + POSTGRES_PASSWORD: secret + POSTGRES_DB: lightstack_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U lightstack -d lightstack_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Download lightstack binary + uses: actions/download-artifact@v5 + with: + name: lightstack-binary + path: bin/ + + - name: Make binary executable + run: chmod +x bin/lightstack + + - name: Install local storage tools + run: | + sudo apt-get update + sudo apt-get install -y qemu-utils + + - name: Start lightstack server (native) + run: | + # The native server runs as the unprivileged runner. Keep its local + # image and volume backing in the job workspace, not /var/lib. + sed '/^state_path:/d' config/lightstack.yaml > "$RUNNER_TEMP/lightstack.yaml" + printf '\nstate_path: "%s"\n' "$RUNNER_TEMP/lightstack-state" >> "$RUNNER_TEMP/lightstack.yaml" + LIGHTSTACK_DB_URL="postgres://lightstack:secret@localhost:5432/lightstack_test?sslmode=disable" \ + LIGHTSTACK_JWT_SECRET="ci-secret" LIGHTSTACK_ENDPOINT_HOST="localhost" \ + nohup ./bin/lightstack --config "$RUNNER_TEMP/lightstack.yaml" --migrations migrations > /tmp/lightstack.log 2>&1 & + + - name: Wait for lightstack to be ready + run: | + echo "Waiting for all Lightstack services to be ready..." + + # Check with proper API version paths + timeout 120 bash -c ' + until curl -sf http://localhost:35357/v3 > /dev/null 2>&1 && \ + curl -sf http://localhost:8774/v2.1 > /dev/null 2>&1 && \ + curl -sf http://localhost:9696/v2.0 > /dev/null 2>&1 && \ + curl -sf http://localhost:8776/v3 > /dev/null 2>&1 && \ + curl -sf http://localhost:9292/v2 > /dev/null 2>&1; do + echo "⏳ Services not ready, retrying... ($(date +%H:%M:%S))" + sleep 3 + done + ' + + echo "✅ All Lightstack services are ready" + + - name: Install pinned OpenStack CLI + # A venv, not the system python: ubuntu-latest is PEP 668 + # externally-managed, and the harness resolves the CLI from PATH + # (command -v openstack), so the venv's bin dir joins GITHUB_PATH. + run: | + python3 -m venv /tmp/osc-venv + /tmp/osc-venv/bin/pip install -c test/cli/constraints.txt python-openstackclient + echo "/tmp/osc-venv/bin" >> "$GITHUB_PATH" + + - name: Run CLI conformance suite (fast subset) + run: ./test/cli/bin/run.sh --fast + env: + OS_AUTH_URL: "http://localhost:35357/v3" + OS_USERNAME: "admin" + OS_PASSWORD: "secret" + OS_PROJECT_NAME: "default" + OS_USER_DOMAIN_NAME: "Default" + OS_PROJECT_DOMAIN_NAME: "Default" + + - name: Check CLI conformance coverage + run: ./test/cli/bin/coverage.sh + + - name: Dump server log on failure + if: failure() + run: tail -n 300 /tmp/lightstack.log + + - name: Upload CLI conformance results + if: always() + uses: actions/upload-artifact@v7 + with: + name: cli-conformance-results + path: test/cli/results/ + retention-days: 7 + + # The e2e surface is the CLI conformance suite: cli-conformance here (fast + # subset) and the nightly full + terraform runs (.github/workflows/nightly.yml). + helm-check: name: Helm Charts runs-on: ubuntu-latest @@ -493,7 +586,7 @@ jobs: status-check: name: CI Status runs-on: ubuntu-latest - needs: [build, unit-tests, contract-tests, contract-tests-sqlite, integration-tests, dco-check, rgw-integration-tests, helm-check] + needs: [build, unit-tests, contract-tests, contract-tests-sqlite, integration-tests, cli-conformance, dco-check, rgw-integration-tests, helm-check] if: always() steps: @@ -504,6 +597,7 @@ jobs: echo "Contract Tests: ${{ needs.contract-tests.result }}" echo "Contract Tests (SQLite): ${{ needs.contract-tests-sqlite.result }}" echo "Integration Tests: ${{ needs.integration-tests.result }}" + echo "CLI Conformance: ${{ needs.cli-conformance.result }}" echo "SAP Sign-off: ${{ needs.dco-check.result }}" echo "RGW Integration Tests: ${{ needs.rgw-integration-tests.result }}" echo "Helm Charts: ${{ needs.helm-check.result }}" @@ -535,7 +629,12 @@ jobs: exit 1 fi - if [ "${{ needs.dco-check.result }}" != "success" ]; then + if [ "${{ needs.cli-conformance.result }}" != "success" ]; then + echo "❌ CLI conformance suite failed" + exit 1 + fi + + if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ needs.dco-check.result }}" != "success" ]; then echo "❌ SAP sign-off check failed — all commits need Signed-off-by: name@sap.com" exit 1 fi diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000..5326da72 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,223 @@ +name: Nightly + +on: + schedule: + - cron: "0 3 * * *" # 03:00 UTC daily + workflow_dispatch: + +# The repository guard sits at job level (not workflow level) so manual +# workflow_dispatch runs stay possible on forks while the schedule only ever +# runs on the upstream repo. +jobs: + cli-conformance-full: + name: CLI Conformance (Full) + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' || github.repository == 'cobaltcore-dev/lightstack' + timeout-minutes: 60 + + # SPEC-028 nightly: the FULL suite (workflows included, so tier C) plus the + # strict coverage gate. No PR context here, so the binary is built in the + # job instead of downloaded from the build job's artifact. + services: + postgres: + image: postgres:18.4-alpine + env: + POSTGRES_USER: lightstack + POSTGRES_PASSWORD: secret + POSTGRES_DB: lightstack_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U lightstack -d lightstack_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version: '1.27.3' + cache-dependency-path: | + go.sum + tools/go.sum + + - name: Build binary + run: make build + + - name: Install local storage tools + run: | + sudo apt-get update + sudo apt-get install -y qemu-utils + + - name: Start lightstack server (native) + run: | + # Match the PR job: local backing must be writable by the runner. + sed '/^state_path:/d' config/lightstack.yaml > "$RUNNER_TEMP/lightstack.yaml" + printf '\nstate_path: "%s"\n' "$RUNNER_TEMP/lightstack-state" >> "$RUNNER_TEMP/lightstack.yaml" + LIGHTSTACK_DB_URL="postgres://lightstack:secret@localhost:5432/lightstack_test?sslmode=disable" \ + LIGHTSTACK_JWT_SECRET="ci-secret" LIGHTSTACK_ENDPOINT_HOST="localhost" \ + nohup ./bin/lightstack --config "$RUNNER_TEMP/lightstack.yaml" --migrations migrations > /tmp/lightstack.log 2>&1 & + + - name: Wait for lightstack to be ready + run: | + echo "Waiting for all Lightstack services to be ready..." + + # Check with proper API version paths + timeout 120 bash -c ' + until curl -sf http://localhost:35357/v3 > /dev/null 2>&1 && \ + curl -sf http://localhost:8774/v2.1 > /dev/null 2>&1 && \ + curl -sf http://localhost:9696/v2.0 > /dev/null 2>&1 && \ + curl -sf http://localhost:8776/v3 > /dev/null 2>&1 && \ + curl -sf http://localhost:9292/v2 > /dev/null 2>&1; do + echo "⏳ Services not ready, retrying... ($(date +%H:%M:%S))" + sleep 3 + done + ' + + echo "✅ All Lightstack services are ready" + + - name: Install pinned OpenStack CLI + # venv for PEP 668 + PATH resolution — see the ci.yml cli-conformance job. + run: | + python3 -m venv /tmp/osc-venv + /tmp/osc-venv/bin/pip install -c test/cli/constraints.txt python-openstackclient + echo "/tmp/osc-venv/bin" >> "$GITHUB_PATH" + + - name: Run CLI conformance suite (full, incl. workflows) + run: ./test/cli/bin/run.sh --strict + env: + OS_AUTH_URL: "http://localhost:35357/v3" + OS_USERNAME: "admin" + OS_PASSWORD: "secret" + OS_PROJECT_NAME: "default" + OS_USER_DOMAIN_NAME: "Default" + OS_PROJECT_DOMAIN_NAME: "Default" + + - name: Check CLI conformance coverage (strict, incl. tier C) + run: ./test/cli/bin/coverage.sh --strict + + - name: Dump server log on failure + if: failure() + run: tail -n 300 /tmp/lightstack.log + + - name: Upload CLI conformance results + if: always() + uses: actions/upload-artifact@v7 + with: + name: cli-conformance-full-results + path: test/cli/results/ + retention-days: 7 + + terraform-conformance: + name: Terraform Conformance + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' || github.repository == 'cobaltcore-dev/lightstack' + timeout-minutes: 45 + + # SPEC-028 nightly: apply + destroy the UNMODIFIED terraform-provider- + # openstack compat module (test/terraform/compat) against a native server + # and assert a clean final state. `make test-terraform` wraps + # test/terraform/run.sh; the module reads auth from the OS_* variables. + services: + postgres: + image: postgres:18.4-alpine + env: + POSTGRES_USER: lightstack + POSTGRES_PASSWORD: secret + POSTGRES_DB: lightstack_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U lightstack -d lightstack_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version: '1.27.3' + cache-dependency-path: | + go.sum + tools/go.sum + + - name: Build binary + run: make build + + - name: Install local storage tools + run: | + sudo apt-get update + sudo apt-get install -y qemu-utils + + - name: Start lightstack server (native) + run: | + # Match the PR job: local backing must be writable by the runner. + sed '/^state_path:/d' config/lightstack.yaml > "$RUNNER_TEMP/lightstack.yaml" + printf '\nstate_path: "%s"\n' "$RUNNER_TEMP/lightstack-state" >> "$RUNNER_TEMP/lightstack.yaml" + LIGHTSTACK_DB_URL="postgres://lightstack:secret@localhost:5432/lightstack_test?sslmode=disable" \ + LIGHTSTACK_JWT_SECRET="ci-secret" LIGHTSTACK_ENDPOINT_HOST="localhost" \ + nohup ./bin/lightstack --config "$RUNNER_TEMP/lightstack.yaml" --migrations migrations > /tmp/lightstack.log 2>&1 & + + - name: Wait for lightstack to be ready + run: | + echo "Waiting for all Lightstack services to be ready..." + + # Check with proper API version paths + timeout 120 bash -c ' + until curl -sf http://localhost:35357/v3 > /dev/null 2>&1 && \ + curl -sf http://localhost:8774/v2.1 > /dev/null 2>&1 && \ + curl -sf http://localhost:9696/v2.0 > /dev/null 2>&1 && \ + curl -sf http://localhost:8776/v3 > /dev/null 2>&1 && \ + curl -sf http://localhost:9292/v2 > /dev/null 2>&1; do + echo "⏳ Services not ready, retrying... ($(date +%H:%M:%S))" + sleep 3 + done + ' + + echo "✅ All Lightstack services are ready" + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.13.4" + + - name: Install pinned OpenStack CLI + # venv for PEP 668 + PATH resolution — see the ci.yml cli-conformance job. + run: | + python3 -m venv /tmp/osc-venv + /tmp/osc-venv/bin/pip install -c test/cli/constraints.txt python-openstackclient + echo "/tmp/osc-venv/bin" >> "$GITHUB_PATH" + + - name: Run Terraform conformance module + run: make test-terraform + env: + OS_AUTH_URL: "http://localhost:35357/v3" + OS_USERNAME: "admin" + OS_PASSWORD: "secret" + OS_PROJECT_NAME: "default" + OS_USER_DOMAIN_NAME: "Default" + OS_PROJECT_DOMAIN_NAME: "Default" + + - name: Dump server log on failure + if: failure() + run: tail -n 300 /tmp/lightstack.log + + - name: Upload Terraform run artifacts on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: terraform-conformance-results + # Upload diagnostic logs. Recovery state remains in the job-local + # workspace on failed cleanup; do not publish credential-bearing state. + path: | + test/terraform/results/ + retention-days: 7 + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index 3ac69b33..fe91aded 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ # Binaries bin/ +# The CLI conformance harness lives in test/cli/bin/ — scripts, not +# binaries; re-include it so the gate scripts ship with the repo. +!test/cli/bin/ *.exe *.dll *.so @@ -85,6 +88,9 @@ cirros*.img # HOST_TESTING_REAL_MODE.md generates this (ideally under /tmp); guard the repo-root path too. /lightstack-real-values.yaml +# Python harness regression tests +__pycache__/ + # Keep important root docs !/README.md !/CLAUDE.md diff --git a/Makefile b/Makefile index 22c94242..f5f2bee9 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build run test lint chart-test test-endpoint-migration test-compose-identity test-compose-kvm clean install-deps install-tools migrate db-up db-down build-ebpf bench bench-quick horizon-chart-sync horizon-chart-check test-horizon-ui +.PHONY: build run test lint chart-test test-endpoint-migration test-compose-identity test-compose-kvm clean install-deps install-tools migrate db-up db-down build-ebpf bench bench-quick horizon-chart-sync horizon-chart-check test-horizon-ui test-cli cli-coverage test-terraform # Build variables BINARY_NAME=lightstack @@ -125,6 +125,29 @@ test-compose-kvm: @docker compose version >/dev/null 2>&1 || (echo "ERROR: 'docker compose' not found." && exit 1) @bash test/compose_kvm_state_test.sh +# Run the OpenStack CLI conformance suite (SPEC-028, issue #105) against a +# lightstack server. Builds bin/lightstack first if absent — the suite tests +# the built artifact. Uses a server at OS_AUTH_URL if one is running, +# otherwise starts the binary-test compose stack (requires docker). +test-cli: + @echo "Running OpenStack CLI conformance suite..." + @test -x bin/lightstack || $(MAKE) build + @bash test/cli/bin/run.sh + +# CLI conformance coverage gate (SPEC-028): cross-reference manifest/*.json +# with results/results.json; fails on untested/failed/skipped tier A+B +# commands. Run after test-cli (or a CI job that stores its results there). +cli-coverage: + @echo "Checking CLI conformance coverage..." + @bash test/cli/bin/coverage.sh + +# Run the Terraform provider conformance module (SPEC-028) against a running +# lightstack server. Requires terraform and network access to the provider +# registry on first run. +test-terraform: + @echo "Running Terraform conformance tests..." + @bash test/terraform/run.sh + # Run Helm chart tests (helm lint + template + unittest; no cluster or running Lightstack needed) chart-test: @echo "Running Helm chart tests (lint + template + unittest)..." diff --git a/cmd/lightstack/main.go b/cmd/lightstack/main.go index 7224f5d4..1c1b56c9 100644 --- a/cmd/lightstack/main.go +++ b/cmd/lightstack/main.go @@ -741,6 +741,13 @@ func newEngine() *gin.Engine { func createKeystoneServer(cfg *common.Config, svc *keystone.Service, authService *keystone.AuthService, checker *health.Checker) *http.Server { r := newEngine() + // FIRST, and on the engine rather than only the /v3 group (which keeps + // its own copy for the test routers): keystone answers in its own error + // envelope, and NoRoute/NoMethod are engine-level — without this, an + // unknown path under :35357 rendered the compute envelope and + // keystoneauth1 degraded it to "Unrecognized schema in response body." + // (SPEC-028; same placement Neutron's ErrorEnvelope uses.) + r.Use(common.MarkIdentityErrors) r.Use(middleware.ErrorHandlingMiddleware()) r.Use(middleware.LoggingMiddleware()) r.Use(middleware.RecoveryMiddleware()) diff --git a/deployments/docker-compose.binary-test.yml b/deployments/docker-compose.binary-test.yml new file mode 100644 index 00000000..2cfe051c --- /dev/null +++ b/deployments/docker-compose.binary-test.yml @@ -0,0 +1,84 @@ +# Binary-test stack (SPEC-028): run the artifact from `make build`, not a +# container-internal rebuild. +# +# The lightstack service uses the same runtime image as docker-compose.yml, +# but bind-mounts this checkout's bin/lightstack over the in-image binary. +# The binary self-applies migrations on startup (cmd/lightstack/main.go), so +# lightstack-migrate is not invoked. Build the image once from the normal +# compose file (layers are cached), then: +# +# make build +# docker compose -f deployments/docker-compose.binary-test.yml up -d +# ./test/cli/bin/run.sh --keep-server +# +# The distinct project name (lightstack-binary-test) and volume prefix keep +# this stack from colliding with a developer's docker-compose.yml deployment. +version: "3.8" + +services: + postgres: + image: postgres:18.4-alpine + container_name: lightstack-binary-test-postgres + environment: + POSTGRES_DB: ${POSTGRES_DB:-lightstack} + POSTGRES_USER: ${POSTGRES_USER:-lightstack} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-secret} + volumes: + - binary_test_postgres_data:/var/lib/postgresql + ports: + - "5433:5432" # offset from the dev stack to allow both at once + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-lightstack}"] + interval: 5s + timeout: 5s + retries: 5 + networks: + - lightstack-binary-test-network + + lightstack: + image: lightstack-binary-test:latest + build: + context: .. + dockerfile: build/package/Dockerfile + container_name: lightstack-binary-test + depends_on: + postgres: + condition: service_healthy + ports: + - "35357:35357" # Keystone (Identity) + - "8774:8774" # Nova (Compute) + - "9696:9696" # Neutron (Network) + - "8776:8776" # Cinder (Block Storage) + - "9292:9292" # Glance (Image) + - "8775:8775" # Metadata Service + volumes: + # THE POINT OF THIS FILE: test the binary this checkout builds. The + # image's own copy is shadowed; CI downloads the build artifact into + # bin/ before starting this stack. + - ../bin/lightstack:/app/lightstack:ro + environment: + LIGHTSTACK_DB_URL: "postgres://${POSTGRES_USER:-lightstack}:${POSTGRES_PASSWORD:-secret}@postgres:5432/${POSTGRES_DB:-lightstack}?sslmode=disable" + LIGHTSTACK_JWT_SECRET: "${LIGHTSTACK_JWT_SECRET:-change-me-in-production}" + LIGHTSTACK_ENDPOINT_HOST: "localhost" # service catalog URLs reachable from the host + # Stub-mode stack, no real guests: safe to trust the instance header + # from the Docker bridge. Never set this where real guests run. + LIGHTSTACK_METADATA_TRUST_INSTANCE_HEADER: "true" + networks: + - lightstack-binary-test-network + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:35357/v3 || exit 1"] + interval: 10s + timeout: 5s + retries: 3 + # The image CMD runs lightstack-migrate then lightstack; the mounted + # binary self-migrates, so run it directly. + command: ["/bin/sh", "-c", "/app/lightstack --config /app/config/lightstack.yaml"] + +volumes: + binary_test_postgres_data: + driver: local + +networks: + lightstack-binary-test-network: + driver: bridge diff --git a/docs/specs/028-cli-conformance-test-environment/plan.md b/docs/specs/028-cli-conformance-test-environment/plan.md new file mode 100644 index 00000000..ef9913c9 --- /dev/null +++ b/docs/specs/028-cli-conformance-test-environment/plan.md @@ -0,0 +1,304 @@ +# Implementation Plan: OpenStack CLI Conformance Test Environment + +**Branch**: `feature/105` (feature dir `028-cli-conformance-test-environment`) | **Date**: 2026-10-15 +**Spec input**: GitHub issue **#105 — "[Enhancement] Testing"** ("we need proper testing for the existing +Openstack API calls and terraform use cases") +**Goal**: Be sure the **working binary** (the artifact `make build` produces) works against the +**python OpenStack CLI** (`openstack` command) **in all cases** — every CLI-exposed operation for all +five services, happy path and negative path, with a machine-auditable coverage matrix — and that this +is enforced in CI, not run by hand. + +--- + +## 1. Analysis — current state (why #105 is real) + +### 1.1 What exists + +| Layer | What | Where | Runs in CI? | +|---|---|---|---| +| Unit tests | `go test -race`, DB-backed via `LIGHTSTACK_TEST_DB_URL` | `internal/**` | ✅ `unit-tests` job | +| Contract tests | gophercloud + testify, live server, no mocks | `test/contract/**` (336 `Test*`, 14 files with `t.Skip`) | ⚠️ soft gate: `continue-on-error: true`, ≥85% pass-rate floor (`ci.yml:156,201`) | +| Integration scripts | 75 bash scripts, expect an already-running server | `test/*.sh` | ⚠️ only `quick_test.sh` (curl-only) + `horizon_create_ui_test.sh` (curl-only) | +| CLI-driven scripts | ~24 scripts that do run `openstack …` | `test/validation_suite.sh`, `horizon_full_test.sh`, nova/cinder/glance waves, … | ❌ **none** (except `rgw_integration_test.sh` in the RGW job, Glance-only) | +| Terraform modules | unmodified provider apply/destroy (compat) + real-mode deep scenario (sc007) | `test/terraform/` | ❌ manual only; ~13 provider resources never exercised (`docs/testing/BLUEPRINT_TEST_COVERAGE.md`) | +| Helm | lint/render/helm-unittest | `make chart-test` | ✅ | + +### 1.2 The gaps that block the goal + +1. **No systematic CLI conformance suite.** The `openstack` CLI (python-openstackclient on + python-openstacksdk) is a *different client* from gophercloud: different request shapes, + microversion headers, name→ID resolution, and `-f json` output parsing. Nothing in the repo + systematically drives it. The best CLI suites (`validation_suite.sh` ~40 ops, + `horizon_full_test.sh` ~30 ops) are ad-hoc, unrunner'd, and happy-path-heavy — far short of "all + cases" for 342 endpoints. +2. **CI installs the CLI and then never uses it.** The `integration-tests` job + (`ci.yml:235-237`) runs `pip install python-openstackclient` and then executes `quick_test.sh`, + which is curl-only. The install is dead weight; `openstack server list` is never validated in CI. +3. **CI never tests the built binary.** The `build` job uploads `bin/lightstack` as artifact + `lightstack-binary` — nothing consumes it. The contract/integration jobs bring up + `deployments/docker-compose.yml`, which **rebuilds the binary inside Docker**. So the thing we + ship is not the thing we test. +4. **`test/run_contract_tests.sh` is broken off the original author's machine**: hardcoded + `/Users/I761222/git/lightstack` path and `nova` missing from its service loop. +5. **~65 of 75 `test/*.sh` scripts have no runner at all.** `make test-all` covers only 5 live-server + scripts. Results are bash counters, exit codes only — no structured (JUnit/JSON) reporting. +6. **No test image is seeded.** Nothing in migrations/compose seeds a `cirros` image; CLI and + Terraform tests that boot servers need one and currently rely on it being preloaded by hand. +7. **Coverage bookkeeping is inconsistent.** Four status docs disagree on totals (157/191, 223/233, + 258/286, 375-endpoint basis); `BLUEPRINT_TEST_COVERAGE.md` counts **25 endpoints with zero + coverage**; 19 Cinder contract tests are known-failing with no machine-readable list. +8. **Terraform conformance is manual** (issue #105 explicitly names Terraform use cases). + +### 1.3 Assets we can build on (the change is shallow where it counts) + +- The stack bootstrap is already solved: `deployments/docker-compose.yml` (postgres + lightstack, + healthcheck, `LIGHTSTACK_ENDPOINT_HOST=localhost`), migrations + admin bootstrap run on startup. +- A CLI-shaped test vocabulary already exists in ~24 scripts — commands, cleanup patterns, wait + loops. The harness extracts these into a shared library instead of re-inventing them. +- `test/terraform/compat` already applies the **unmodified** upstream provider — moving it into CI + is wiring, not new design. +- CI already knows how to install the CLI (`pip install python-openstackclient`) and upload/download + artifacts. + +--- + +## 2. Goals / Non-Goals + +### Goals + +1. **G1 — Test the shipped binary.** CI gates run against the `lightstack-binary` artifact produced + by the `build` job (plus a Postgres service), not a second Docker-rebuilt binary. +2. **G2 — CLI conformance suite ("all cases").** A maintained, machine-readable matrix of + `openstack` CLI commands × expected outcomes covering every CLI-exposed operation of the five + services; happy path, output-shape (`-f json` field presence/type), and negative paths (404/400/ + 409 semantics). Suite runs against the artifact binary in stub mode. +3. **G3 — CI enforcement.** New `cli-conformance` job, **hard gate** (no `continue-on-error`), fast + enough for PRs (<10 min); full matrix runs nightly. +4. **G4 — Terraform in CI.** The `compat` module runs nightly against the artifact binary; provider + resource coverage grows from 4 categories toward the full set the product claims. +5. **G5 — Auditable results.** JUnit/JSON output, a generated coverage report (command → endpoint → + test status), one source of truth replacing the four disagreeing status docs. +6. **G6 — Fix the broken runner** (`run_contract_tests.sh`) and seed a test image so every suite can + boot servers. + +### Non-Goals + +- Real-mode/libvirt/KVM testing (existing wave scripts and k3s verification remain the mechanism). +- Horizon browser automation (covered by the Horizon compatibility program, SPEC-018/019/020). +- Making the 19 known-failing Cinder contract tests pass (tracked separately; the new suite must not + inherit the soft-gate habit, but fixing Cinder gaps is product work, not this feature). +- SDK conformance beyond the CLI and Terraform (gophercloud contract tests already exist). +- Rewriting the 75 existing bash scripts (they stay; the harness supersedes the ad-hoc CLI ones). + +--- + +## 3. Technical approach + +### 3.1 Architecture of the CLI conformance harness + +``` +test/cli/ +├── bin/run.sh # entrypoint: preflight, env, fixture setup, suite exec, report +├── lib/ +│ ├── common.sh # OS_* env handling, `osc` wrapper (timeout + -f json), PASS/FAIL counters +│ ├── fixtures.sh # idempotent setup: test project+user, image, flavor, network; teardown +│ ├── assert.sh # jq-based field/type asserts on CLI JSON output +│ └── report.sh # JSON + JUnit emission, per-command result records +├── manifest/ +│ └── .json # THE matrix (one file per service, jq-native): +│ # service → commands → endpoints → tier → status +├── suites/ +│ ├── keystone.sh # Tier A: identity commands +│ ├── compute.sh # Tier A: server/flavor/keypair/image-action commands +│ ├── network.sh # Tier A: network/subnet/port/router/secgroup/fip/qos +│ ├── volume.sh # Tier A: volume/snapshot/backup/quotas +│ ├── image.sh # Tier A: image commands +│ ├── negative.sh # Tier B: 404/400/409 semantics via CLI +│ └── workflows.sh # Tier C: cross-service scenarios (boot with volume+net, snapshot→boot, …) +└── README.md # how to run, how to add a case, tier definitions +``` + +Design rules: + +- **The manifest is the definition of "all cases".** `commands.yaml` enumerates every + `openstack` command lightstack claims to support (intersect `openstack --help` output of the + pinned CLI version with the 342-endpoint inventory). Each entry carries: command template, + endpoint(s), tier, and test status. `make cli-coverage` regenerates a report from the manifest and + fails if an entry is `untested`. This makes "all cases" auditable instead of aspirational, and + gives the one source of truth G5 asks for. +- **Tiers** (mirroring `test/contract/README.md`'s tier model): + - **A — happy path + shape**: run command, exit 0, required JSON fields present and typed. + - **B — negative**: wrong ID → 404-equivalent CLI error; malformed input → 400; duplicate → 409. + - **C — workflows**: multi-command, cross-service scenarios with state assertions. +- **`osc` wrapper**: every CLI call goes through one function that enforces `--os-*` env, a + per-command timeout (Constitution fail-fast: 10s), `-f json`, captures stderr, and records the + result — no naked `openstack` calls in suites. +- **Fixtures, not luck**: `fixtures.sh` creates (and tears down) a dedicated project/user, a tiny + qcow2 test image (generated or committed, ~100KB dummy — no download in CI), and relies on seeded + flavors (m1.*) from migration `002`. +- **Idempotent teardown**: every suite cleans what it creates; runner supports `--no-cleanup` for + debugging. +- **Pinning**: `python-openstackclient` is installed at a pinned version from a constraints file + (`test/cli/constraints.txt`), so a CLI release can't flip CI red overnight; upgrades are a + deliberate PR. + +### 3.2 Testing the shipped binary (G1) + +New compose-lite bootstrap for CI jobs, `deployments/docker-compose.binary-test.yml`: + +- `postgres:18-alpine` service (unchanged credentials scheme). +- lightstack container built `FROM alpine` (or reuse the existing runtime stage) that **bind-mounts + `bin/lightstack` and `bin/lightstack-migrate` from the build artifact** instead of building them. + One Dockerfile target, no second compile. +- Same env contract as the existing compose (`LIGHTSTACK_ENDPOINT_HOST=localhost`, stub mode, + `LIGHTSTACK_DB_URL`, JWT secret), same ports. + +CI flow: `build` job → artifact → `cli-conformance` job downloads artifact, `docker compose up` the +binary-test stack, readiness probe (reuse the existing 5-port curl loop), run `test/cli/bin/run.sh`. + +Local parity: `make test-cli` does `make build` + the same compose file + the same runner — a +developer reproduces CI exactly. + +### 3.3 CI wiring + +- **`cli-conformance` job (PR + push, hard gate)**: artifact stack, pinned CLI, Tier A+B fast subset + + Tier C core workflows. Target < 10 min (Tier A suites are parallelizable per service; the CLI's + ~1-2s python startup is the cost driver, so the PR subset stays curated). +- **`nightly.yml` (new workflow)**: full manifest (every tier, every command), Terraform compat + module apply/destroy, plus the contract suite via the *fixed* `run_contract_tests.sh` for + cross-checking. Publishes JUnit + the coverage report as artifacts; opens an issue on regression + (optional, later). +- **Fix the existing job while we're there**: in `integration-tests`, either drop the unused + `pip install python-openstackclient` or (better) swap `quick_test.sh` for the Tier A suites once + green, and delete the stale TODO at `ci.yml:318-319`. + +### 3.4 Terraform conformance (G4) + +- Nightly job: artifact stack + pinned terraform + pinned `terraform-provider-openstack`; + `test/terraform/compat` gets a `Makefile`/script wrapper (`make test-terraform`) doing + `init → validate → apply → destroy`, failing on any non-clean exit or non-empty final state. +- Extend `compat/` with configs for the currently unexercised resources (13 per + `BLUEPRINT_TEST_COVERAGE.md`): floating IP, router + interface, port, volume snapshot/backup, + security-group rule variants, server groups, image members, etc. — one TF config file per resource + family, added incrementally. +- sc007 (real-mode SSH-verified scenario) stays manual/host-based — out of scope per Non-Goals. + +--- + +## 4. Implementation phases + +### Phase 0 — Fix the broken foundation (small, unblocks everything) + +| # | Task | Files | +|---|---|---| +| 0.1 | Fix `run_contract_tests.sh`: portable path, add `nova` to the service loop, `set -euo pipefail` | `test/run_contract_tests.sh` | +| 0.2 | Seed a test image: commit a tiny qcow2 (or generate at test time via a fixture) + add to `fixtures.sh`; document that CI no longer depends on a hand-preloaded `cirros` | `test/cli/lib/fixtures.sh` (new), `deployments/` | +| 0.3 | Fix stale paths/references in `test/contract/README.md` (dead `contract-tests.yml`, `/Users/…`) | `test/contract/README.md` | +| 0.4 | Consolidate test-status docs: mark the three older reports superseded by the manifest-generated report | `docs/testing/*` | + +### Phase 1 — CLI harness core (the feature) + +| # | Task | +|---|---| +| 1.1 | `test/cli/lib/` — `common.sh` (`osc` wrapper), `assert.sh` (jq), `report.sh` (JSON+JUnit) | +| 1.2 | `manifest/*.json` — one file per service (keystone, compute, network, volume, image, workflows), jq-native, derived from pinned CLI `--help` ∩ endpoint inventory; every entry starts `untested` | +| 1.3 | `fixtures.sh` — project/user/image/network setup + teardown, idempotent | +| 1.4 | Tier A suites: `keystone.sh`, `compute.sh`, `network.sh`, `volume.sh`, `image.sh` — write cases until no manifest entry in these services is `untested` | +| 1.5 | Tier B `negative.sh` — 404/400/409 semantics via CLI per service | +| 1.6 | Tier C `workflows.sh` — cross-service scenarios (boot with volume+network; snapshot→new server; secgroup lifecycle; floating-IP associate/disassociate) | +| 1.7 | `make test-cli`, `make cli-coverage` Makefile targets; `test/cli/README.md` | + +**TDD note (Constitution Art. III)**: for the harness itself the "red" step is running each new case +against a stub-mode server and confirming it exercises the endpoint (manifest entry flips to +`tested`). Product defects the cases uncover are filed and either fixed or recorded as +`known-failing` in the suite. Every known-failing case must also appear in the checked-in baseline +`test/cli/baseline/known_failures.txt` with the reason it cannot pass against the pinned environment +(stub-mode console; the OSC 7.0.0 backup-by-name client bug). `coverage.sh` and `run.sh --strict` +fail on any known-failing case missing from that ledger, so a regression cannot hide as tracked +debt; the "known-failing count" is reported and trended instead. A ledger entry that starts passing +is a stale-baseline warning (never a failure). **Maintenance:** when a genuine divergence is fixed, +remove its baseline entry — and if the suite probe self-heals, convert it to a plain assertion. + +### Phase 2 — CI integration + +| # | Task | +|---|---| +| 2.1 | `deployments/docker-compose.binary-test.yml` (artifact-mounting stack) + runtime-image Dockerfile target | +| 2.2 | `cli-conformance` job in `ci.yml`: artifact download, stack up, pinned CLI install, fast subset, **hard gate**, JUnit artifact upload | +| 2.3 | `nightly.yml`: full manifest + Terraform compat + fixed contract runner; artifacts + generated `CLI_CONFORMANCE_REPORT.md` | +| 2.4 | Clean up `integration-tests` job (drop/replace the dead pip install; resolve the `ci.yml:318` TODO) | +| 2.5 | `status-check` job: add `cli-conformance` to the required-needs list | + +### Phase 3 — Terraform conformance + +| # | Task | +|---|---| +| 3.1 | `make test-terraform` wrapper: pinned terraform + provider, apply/destroy with clean-state assertion | +| 3.2 | Nightly job runs it against the artifact stack | +| 3.3 | Incrementally add configs for the 13 unexercised provider resources (one PR-sized batch each) | + +### Phase 4 — Coverage closure & hardening + +| # | Task | +|---|---| +| 4.1 | Close the 25 zero-coverage endpoints from `BLUEPRINT_TEST_COVERAGE.md` with CLI or contract cases (they are itemized there) | +| 4.2 | Migrate the strongest ad-hoc scripts' unique scenarios into Tier C (then retire the duplicates) | +| 4.3 | CLI-version upgrade policy documented; scheduled monthly bump PR of `constraints.txt` | + +--- + +## 5. Success criteria (definition of done) + +1. `make build && make test-cli` runs the shipped `bin/lightstack` against the pinned OpenStack CLI + and passes with **zero `untested` Tier A/B manifest entries**. +2. A PR that breaks any CLI-visible behavior fails `cli-conformance` in CI — hard gate, no + threshold math. +3. `make cli-coverage` prints a per-service matrix (commands × status) and matches the artifact + published by nightly CI. +4. Nightly runs the full matrix **and** `test/terraform/compat` apply/destroy green. +5. The binary artifact from `build` is what all live-server jobs consume. +6. Issue #105's two asks are answered: OpenStack API calls (CLI conformance, G2/G3) and Terraform + use cases (G4). + +## 6. Risks & mitigations + +| Risk | Mitigation | +|---|---| +| CLI invocations are slow (~1-2s each; full matrix could be 600+ calls) | Tier split: curated PR subset, full matrix nightly; per-service parallel suites | +| New CLI release changes output/behavior | Pinned `constraints.txt`; deliberate upgrade PRs | +| Cases uncover real product bugs (likely — Cinder metadata/transfers already known weak) | Every `known-failing` case is listed in the checked-in baseline with a reason; both gates fail on a new one, a stale entry only warns, and the count is visible — not a silent 85% floor | +| Alpine runtime image differs from developers' environments | Same image already used in prod compose; local `make test-cli` uses the identical stack | +| bash harness complexity grows | `report.sh` + strict `common.sh` from day one; revisit a Go/python runner only if suites exceed bash's comfort (~Tier C growth) | + +## 7. Estimate + +| Phase | Size | +|---|---| +| 0 — foundation fixes | ~1 day | +| 1 — CLI harness + full manifest | ~4-5 days (largest item; mostly writing cases) | +| 2 — CI wiring | ~1-2 days | +| 3 — Terraform in CI | ~1 day + incremental resource batches | +| 4 — coverage closure | ongoing, batched | + +## 8. Open decisions (defaults chosen, flag now to override) + +1. **Bash vs Go for the harness** — default bash (Constitution Art. VIII anti-abstraction; builds on + the existing script vocabulary; strategy doc Option A). A Go runner would give parallelism and + typed asserts but adds a framework; revisit at Phase 4. +2. **What "the working binary" means in CI** — default: the `build`-job artifact mounted into the + runtime image (G1). Alternative: keep testing the compose-built image and drop G1 — rejected + because the issue's wording ("working binary") points at the artifact. +3. **PR-time scope** — default: Tier A+B + 2-3 Tier C workflows (<10 min). Alternative: everything + nightly-only — rejected; PRs should catch CLI breakage before merge. +4. **Known-failing policy** — default: `known-failing` cases are explicit in the suite and must be + listed in `test/cli/baseline/known_failures.txt` with a reason; new ones fail the gate, stale ones + warn, no pass-rate floor. Alternative: an 85%-style threshold — rejected, it is how the Cinder + debt went invisible. + +## 9. Merge dependencies + +- This PR claims **SPEC-028**. Main already uses migrations 074–075, so this PR's schema changes + use **migrations 076–080**. The overlapping SPEC-028/029 registry claims in #108/#109 still need + the merge order recorded; never silently share a spec number. +- This specification is **plan.md-only** today (no `spec.md`/`tasks.md`). It needs those files or a + documented plan-first carve-out before the registry can treat it as complete. diff --git a/docs/testing/CONTRACT_TESTS_FINAL.md b/docs/testing/CONTRACT_TESTS_FINAL.md index f21684f5..7536b830 100644 --- a/docs/testing/CONTRACT_TESTS_FINAL.md +++ b/docs/testing/CONTRACT_TESTS_FINAL.md @@ -1,3 +1,10 @@ +> **⚠️ SUPERSEDED — historical snapshot (2026-03-30).** The counts below were frozen on the date +> shown and no longer reflect the current test suite. The current coverage matrix is +> [BLUEPRINT_TEST_COVERAGE.md](BLUEPRINT_TEST_COVERAGE.md). Going forward, per-command status will +> come from the machine-generated `test/cli` manifest report (`test/cli/results/`, produced by +> `make cli-coverage`), currently being built in parallel under +> [SPEC-028](../../docs/specs/028-cli-conformance-test-environment/plan.md). + # Contract Test Results - Final Summary (2026-03-30) ## Overall Results diff --git a/docs/testing/CONTRACT_TESTS_FINAL_REPORT.md b/docs/testing/CONTRACT_TESTS_FINAL_REPORT.md index e9a8fddc..9c04c11b 100644 --- a/docs/testing/CONTRACT_TESTS_FINAL_REPORT.md +++ b/docs/testing/CONTRACT_TESTS_FINAL_REPORT.md @@ -1,3 +1,10 @@ +> **⚠️ SUPERSEDED — historical snapshot (2026-03-25).** The counts below were frozen on the date +> shown and no longer reflect the current test suite. The current coverage matrix is +> [BLUEPRINT_TEST_COVERAGE.md](BLUEPRINT_TEST_COVERAGE.md). Going forward, per-command status will +> come from the machine-generated `test/cli` manifest report (`test/cli/results/`, produced by +> `make cli-coverage`), currently being built in parallel under +> [SPEC-028](../../docs/specs/028-cli-conformance-test-environment/plan.md). + # Contract Test Execution Report - All Services **Date**: 2026-03-25 diff --git a/docs/testing/CONTRACT_TEST_RESULTS.md b/docs/testing/CONTRACT_TEST_RESULTS.md index c1926593..28496f6c 100644 --- a/docs/testing/CONTRACT_TEST_RESULTS.md +++ b/docs/testing/CONTRACT_TEST_RESULTS.md @@ -1,3 +1,10 @@ +> **⚠️ SUPERSEDED — historical snapshot (2026-03-31).** The counts below were frozen on the date +> shown and no longer reflect the current test suite. The current coverage matrix is +> [BLUEPRINT_TEST_COVERAGE.md](BLUEPRINT_TEST_COVERAGE.md). Going forward, per-command status will +> come from the machine-generated `test/cli` manifest report (`test/cli/results/`, produced by +> `make cli-coverage`), currently being built in parallel under +> [SPEC-028](../../docs/specs/028-cli-conformance-test-environment/plan.md). + # Contract Test Results **Overall Status: 258/286 passing (90.2%)** diff --git a/internal/cinder/backups.go b/internal/cinder/backups.go index f5112356..0a0a3c86 100644 --- a/internal/cinder/backups.go +++ b/internal/cinder/backups.go @@ -167,6 +167,15 @@ func (svc *Service) GetBackup(c *gin.Context) { backupID := c.Param("id") projectID := c.GetString("project_id") + // A non-UUID reference (a backup name — cinderclient resolves show and + // delete by name with no list fallback, so the name itself reaches this + // lookup) can only name nothing; the uuid-typed query used to surface + // the cast error as 500 instead of 404 (SPEC-028). + if _, err := uuid.Parse(backupID); err != nil { + common.SendError(c, common.NewNotFoundError("backup")) + return + } + var ( volumeID string name *string @@ -212,6 +221,12 @@ func (svc *Service) DeleteBackup(c *gin.Context) { backupID := c.Param("id") projectID := c.GetString("project_id") + // See GetBackup: a non-UUID reference must 404, not 500 (SPEC-028). + if _, err := uuid.Parse(backupID); err != nil { + common.SendError(c, common.NewNotFoundError("backup")) + return + } + result, err := svc.pool.Exec(c.Request.Context(), "DELETE FROM volume_backups WHERE id = $1 AND project_id = $2", backupID, projectID, diff --git a/internal/cinder/spec028_conformance_test.go b/internal/cinder/spec028_conformance_test.go new file mode 100644 index 00000000..a3e32547 --- /dev/null +++ b/internal/cinder/spec028_conformance_test.go @@ -0,0 +1,359 @@ +package cinder + +// SPEC-028 conformance fixes for the volume surface, found by the CLI +// conformance suites (issue #105). + +import ( + "context" + "net/http" + "strings" + "testing" + + "github.com/google/uuid" +) + +// seedSpec028Volume inserts an available volume owned by the seeded project +// and returns its id. The caller owns cleanup. +func seedSpec028Volume(t *testing.T, name string, bootable bool) string { + t.Helper() + id := uuid.NewString() + if _, err := testPool.Exec(t.Context(), + `INSERT INTO volumes (id, name, project_id, user_id, size_gb, status, bootable, created_at, updated_at) + VALUES ($1, $2, $3, $4, 1, 'available', $5, now(), now())`, + id, name, seededProjectID, seededUserID, bootable); err != nil { + t.Fatalf("seed volume %s: %v", name, err) + } + t.Cleanup(func() { + testPool.Exec(context.Background(), "DELETE FROM volume_metadata WHERE volume_id=$1", id) + testPool.Exec(context.Background(), "DELETE FROM volumes WHERE id=$1", id) + }) + return id +} + +// TestUpdateVolumeV2Aliases — python-cinderclient still PUTs the legacy v2 +// field names (display_name / display_description) for `volume set +// --name/--description`; only the v3 names were bound, so the request +// succeeded but persisted nothing (SPEC-028). +func TestUpdateVolumeV2Aliases(t *testing.T) { + requireDB(t) + r := newCinderTestRouter() + hdr := cinderAuthHdr(seededProjectID, seededUserID) + id := seedSpec028Volume(t, "spec028-v2alias", false) + + code, body := doReq(r, http.MethodPut, "/v3/volumes/"+id, + `{"volume":{"display_name":"spec028-v2alias-renamed","display_description":"set via v2 alias"}}`, hdr) + if code != http.StatusOK { + t.Fatalf("update with v2 aliases: code=%d body=%s", code, body) + } + + var name, description string + if err := testPool.QueryRow(t.Context(), + "SELECT name, COALESCE(description,'') FROM volumes WHERE id=$1", id).Scan(&name, &description); err != nil { + t.Fatalf("read back: %v", err) + } + if name != "spec028-v2alias-renamed" { + t.Errorf("display_name was not persisted: name=%q", name) + } + if description != "set via v2 alias" { + t.Errorf("display_description was not persisted: description=%q", description) + } + + // The v3 names keep precedence when both shapes are sent. + code, _ = doReq(r, http.MethodPut, "/v3/volumes/"+id, + `{"volume":{"name":"v3-wins","display_name":"v2-loses"}}`, hdr) + if code != http.StatusOK { + t.Fatalf("update with both shapes: code=%d", code) + } + if err := testPool.QueryRow(t.Context(), + "SELECT name FROM volumes WHERE id=$1", id).Scan(&name); err != nil { + t.Fatalf("read back: %v", err) + } + if name != "v3-wins" { + t.Errorf("v3 name must win over the v2 alias: name=%q", name) + } +} + +// TestVolumeMetadataOnObject — volume show and list-detail serialized +// metadata: {} however much was set, so 'volume show' always reported empty +// properties (SPEC-028). +func TestVolumeMetadataOnObject(t *testing.T) { + requireDB(t) + r := newCinderTestRouter() + hdr := cinderAuthHdr(seededProjectID, seededUserID) + id := seedSpec028Volume(t, "spec028-meta", false) + + code, body := doReq(r, http.MethodPost, "/v3/volumes/"+id+"/metadata", + `{"metadata":{"spec028key":"spec028value"}}`, hdr) + if code != http.StatusOK { + t.Fatalf("set metadata: code=%d body=%s", code, body) + } + + code, body = doReq(r, http.MethodGet, "/v3/volumes/"+id, "", hdr) + if code != http.StatusOK { + t.Fatalf("show: code=%d body=%s", code, body) + } + if !strings.Contains(body, `"metadata":{"spec028key":"spec028value"}`) { + t.Errorf("show response lacks the volume's metadata: %s", body) + } + + code, body = doReq(r, http.MethodGet, "/v3/volumes/detail", "", hdr) + if code != http.StatusOK { + t.Fatalf("list detail: code=%d body=%s", code, body) + } + if !strings.Contains(body, `"metadata":{"spec028key":"spec028value"}`) { + t.Errorf("list-detail response lacks the volume's metadata: %s", body) + } +} + +// TestVolumeSetBootableAction — `volume set --bootable/--non-bootable` PUTs +// os-set_bootable, which answered 400 unknown action (SPEC-028). +func TestVolumeSetBootableAction(t *testing.T) { + requireDB(t) + r := newCinderTestRouter() + hdr := cinderAuthHdr(seededProjectID, seededUserID) + id := seedSpec028Volume(t, "spec028-bootable", false) + + code, body := doReq(r, http.MethodPost, "/v3/volumes/"+id+"/action", + `{"os-set_bootable":{"bootable":true}}`, hdr) + if code != http.StatusAccepted { + t.Fatalf("os-set_bootable: code=%d body=%s; want 202", code, body) + } + _, body = doReq(r, http.MethodGet, "/v3/volumes/"+id, "", hdr) + if !strings.Contains(body, `"bootable":"true"`) { + t.Errorf("bootable was not persisted: %s", body) + } + + code, body = doReq(r, http.MethodPost, "/v3/volumes/"+id+"/action", + `{"os-set_bootable":{"bootable":false}}`, hdr) + if code != http.StatusAccepted { + t.Fatalf("os-set_bootable false: code=%d body=%s; want 202", code, body) + } + _, body = doReq(r, http.MethodGet, "/v3/volumes/"+id, "", hdr) + if !strings.Contains(body, `"bootable":"false"`) { + t.Errorf("bootable was not reset: %s", body) + } + + // Malformed payloads are client errors, and an unknown volume 404s. + code, _ = doReq(r, http.MethodPost, "/v3/volumes/"+id+"/action", + `{"os-set_bootable":{"bootable":"yes"}}`, hdr) + if code != http.StatusBadRequest { + t.Errorf("non-boolean bootable: code=%d; want 400", code) + } + code, _ = doReq(r, http.MethodPost, "/v3/volumes/"+uuid.NewString()+"/action", + `{"os-set_bootable":{"bootable":true}}`, hdr) + if code != http.StatusNotFound { + t.Errorf("unknown volume: code=%d; want 404", code) + } +} + +// TestSnapshotMetadataKey — snapshot create/show/list omitted the metadata +// key entirely, and the pinned OSC 7 formatter crashes with KeyError +// 'metadata' on a snapshot without one (SPEC-028). +func TestSnapshotMetadataKey(t *testing.T) { + requireDB(t) + r := newCinderTestRouter() + hdr := cinderAuthHdr(seededProjectID, seededUserID) + volID := seedSpec028Volume(t, "spec028-snapmeta-vol", false) + + code, body := doReq(r, http.MethodPost, "/v3/snapshots", + `{"snapshot":{"name":"spec028-snapmeta","volume_id":"`+volID+`"}}`, hdr) + if code != http.StatusAccepted { + t.Fatalf("create snapshot: code=%d body=%s", code, body) + } + if !strings.Contains(body, `"metadata":{}`) { + t.Errorf("create response lacks the metadata key: %s", body) + } + var snapID string + if err := testPool.QueryRow(t.Context(), + "SELECT id::text FROM snapshots WHERE name='spec028-snapmeta' AND volume_id=$1", volID).Scan(&snapID); err != nil { + t.Fatalf("resolve snapshot: %v", err) + } + t.Cleanup(func() { + testPool.Exec(context.Background(), "DELETE FROM snapshot_metadata WHERE snapshot_id=$1", snapID) + testPool.Exec(context.Background(), "DELETE FROM snapshots WHERE id=$1", snapID) + }) + + // Set metadata through the endpoint, then read it back on the object. + code, body = doReq(r, http.MethodPost, "/v3/snapshots/"+snapID+"/metadata", + `{"metadata":{"snapkey":"snapval"}}`, hdr) + if code != http.StatusOK { + t.Fatalf("set snapshot metadata: code=%d body=%s", code, body) + } + _, body = doReq(r, http.MethodGet, "/v3/snapshots/"+snapID, "", hdr) + if !strings.Contains(body, `"metadata":{"snapkey":"snapval"}`) { + t.Errorf("show response lacks the snapshot's metadata: %s", body) + } + _, body = doReq(r, http.MethodGet, "/v3/snapshots/detail", "", hdr) + if !strings.Contains(body, `"metadata":{"snapkey":"snapval"}`) { + t.Errorf("list-detail response lacks the snapshot's metadata: %s", body) + } + code, body = doReq(r, http.MethodPut, "/v3/snapshots/"+snapID, + `{"snapshot":{"description":"updated"}}`, hdr) + if code != http.StatusOK || !strings.Contains(body, `"metadata":{"snapkey":"snapval"}`) { + t.Errorf("update response lost snapshot metadata: code=%d body=%s", code, body) + } +} + +// TestVolumeTypeConformance — three defects in the type surface (SPEC-028): +// - a type without a description (NULL) made GET /v3/types fail with 500 +// for every caller until the type was deleted; +// - the type object omitted extra_specs, so 'volume type show' always +// reported properties: {} however much was set; +// - a non-UUID reference (a name) 500'd the uuid-typed lookups instead of +// answering 404. +func TestVolumeTypeConformance(t *testing.T) { + requireDB(t) + r := newCinderTestRouter() + hdr := cinderAuthHdr(seededProjectID, seededUserID) + + // No description — that is the row shape that broke the listing. + typeID := uuid.NewString() + if _, err := testPool.Exec(t.Context(), + `INSERT INTO volume_types (id, name, is_public, extra_specs, created_at) + VALUES ($1, 'spec028-type-nodesc', true, '{"spec028k":"spec028v"}'::jsonb, now())`, + typeID); err != nil { + t.Fatalf("seed type: %v", err) + } + t.Cleanup(func() { testPool.Exec(context.Background(), "DELETE FROM volume_types WHERE id=$1", typeID) }) + + code, body := doReq(r, http.MethodGet, "/v3/types", "", hdr) + if code != http.StatusOK { + t.Fatalf("list types with a NULL-description member: code=%d body=%s; want 200", code, body) + } + if !strings.Contains(body, `"name":"spec028-type-nodesc"`) { + t.Errorf("listing dropped the NULL-description type: %s", body) + } + + code, body = doReq(r, http.MethodGet, "/v3/types/"+typeID, "", hdr) + if code != http.StatusOK { + t.Fatalf("show type: code=%d body=%s", code, body) + } + if !strings.Contains(body, `"extra_specs":{"spec028k":"spec028v"}`) { + t.Errorf("show response lacks the type's extra_specs: %s", body) + } + + // A non-UUID reference and an unknown UUID both 404 — in show and delete. + for _, ref := range []string{"spec028-type-nodesc", uuid.NewString()} { + code, _ = doReq(r, http.MethodGet, "/v3/types/"+ref, "", hdr) + if code != http.StatusNotFound { + t.Errorf("show type by %q: code=%d; want 404", ref, code) + } + code, _ = doReq(r, http.MethodDelete, "/v3/types/"+ref, "", hdr) + if code != http.StatusNotFound { + t.Errorf("delete type by %q: code=%d; want 404", ref, code) + } + } +} + +// TestBackupNonUUIDIs404 — cinderclient resolves backup show/delete by name +// with no list fallback, so the name itself reached the uuid-typed lookup +// and surfaced the cast error as 500; clients need the 404 (SPEC-028). +func TestBackupNonUUIDIs404(t *testing.T) { + requireDB(t) + r := newCinderTestRouter() + hdr := cinderAuthHdr(seededProjectID, seededUserID) + + for _, ref := range []string{"my-backup-name", "not-a-uuid", uuid.NewString()} { + code, _ := doReq(r, http.MethodGet, "/v3/backups/"+ref, "", hdr) + if code != http.StatusNotFound { + t.Errorf("show backup by %q: code=%d; want 404", ref, code) + } + code, _ = doReq(r, http.MethodDelete, "/v3/backups/"+ref, "", hdr) + if code != http.StatusNotFound { + t.Errorf("delete backup by %q: code=%d; want 404", ref, code) + } + } +} + +// TestVolumeListAllTenantsAndProjectFilter_DB — cinder's cross-project +// listing semantics were absent: an admin's `volume list --project X` +// (project_id filter) and `--all-projects` (all_tenants) were silently +// ignored and the ADMIN's volumes were presented as the target project's. +// Both are admin-only upstream, and ignored for non-admins (SPEC-028). +func TestVolumeListAllTenantsAndProjectFilter_DB(t *testing.T) { + requireDB(t) + + otherProject := uuid.NewString() + if _, err := testPool.Exec(t.Context(), + `INSERT INTO projects (id, name, domain_id, enabled) VALUES ($1,$2,NULL,true)`, + otherProject, "spec028-vol-tenant-"+otherProject[:8]); err != nil { + t.Fatalf("insert other project: %v", err) + } + otherVol := uuid.NewString() + if _, err := testPool.Exec(t.Context(), + `INSERT INTO volumes (id, name, project_id, user_id, size_gb, status, bootable, created_at, updated_at) + VALUES ($1,$2,$3,$4,1,'available',false,now(),now())`, + otherVol, "spec028-vol-tenant-other", otherProject, seededUserID); err != nil { + t.Fatalf("insert other volume: %v", err) + } + ownVol := uuid.NewString() + if _, err := testPool.Exec(t.Context(), + `INSERT INTO volumes (id, name, project_id, user_id, size_gb, status, bootable, created_at, updated_at) + VALUES ($1,$2,$3,$4,1,'available',false,now(),now())`, + ownVol, "spec028-vol-tenant-own", seededProjectID, seededUserID); err != nil { + t.Fatalf("insert own volume: %v", err) + } + t.Cleanup(func() { + testPool.Exec(context.Background(), "DELETE FROM volumes WHERE id = ANY($1)", []string{otherVol, ownVol}) + testPool.Exec(context.Background(), "DELETE FROM projects WHERE id=$1", otherProject) + }) + + r := newCinderTestRouter() + adminHdr := cinderAuthHdr(seededProjectID, seededUserID) + memberHdr := map[string]string{ + "X-Test-Project": seededProjectID, "X-Test-User": seededUserID, "X-Test-Roles": "member", + } + + // Admin + project_id: the OTHER project's volume appears, the own one does not. + code, body := doReq(r, http.MethodGet, "/v3/volumes/detail?project_id="+otherProject, "", adminHdr) + if code != http.StatusOK || !strings.Contains(body, otherVol) || strings.Contains(body, ownVol) { + t.Errorf("admin volumes?project_id=: code=%d body=%s; want only the other project's volume", code, body) + } + // OSC 7.0.0's --project sends BOTH parameters. The explicit filter must + // still select only that project's volumes on both list endpoints. + for _, path := range []string{"/v3/volumes", "/v3/volumes/detail"} { + query := path + "?all_tenants=1&project_id=" + otherProject + code, body = doReq(r, http.MethodGet, query, "", adminHdr) + if code != http.StatusOK || !strings.Contains(body, otherVol) || strings.Contains(body, ownVol) { + t.Errorf("admin %s: code=%d body=%s; want only the other project's volume", query, code, body) + } + code, body = doReq(r, http.MethodGet, query, "", memberHdr) + if code != http.StatusOK || strings.Contains(body, otherVol) || !strings.Contains(body, ownVol) { + t.Errorf("member %s: code=%d body=%s; want only the member's volume", query, code, body) + } + } + // Admin + all_tenants: both appear. + code, body = doReq(r, http.MethodGet, "/v3/volumes/detail?all_tenants", "", adminHdr) + if code != http.StatusOK || !strings.Contains(body, otherVol) || !strings.Contains(body, ownVol) { + t.Errorf("admin volumes?all_tenants: code=%d body=%s; want both volumes", code, body) + } + // A non-admin's all_tenants is IGNORED (upstream FR-005b): only their own. + code, body = doReq(r, http.MethodGet, "/v3/volumes/detail?all_tenants", "", memberHdr) + if code != http.StatusOK || strings.Contains(body, otherVol) || !strings.Contains(body, ownVol) { + t.Errorf("member volumes?all_tenants: code=%d body=%s; must contain only the member's volume", code, body) + } + + // Round 8, MAJOR-1: a foreign row must carry ITS OWN tenant_id — the + // detail listing used to stamp every row with the caller's project, so + // an admin's cross-project walk labeled project X's volumes as the + // admin's (os-vol-tenant-attr:tenant_id is consumed by tooling). + code, body = doReq(r, http.MethodGet, "/v3/volumes/detail?project_id="+otherProject, "", adminHdr) + if code != http.StatusOK || !strings.Contains(body, `"tenant_id":"`+otherProject+`"`) { + t.Errorf("admin volumes?project_id=: tenant_id must be the row's owner %s (code=%d body=%s)", otherProject, code, body) + } + if strings.Contains(body, `"tenant_id":"`+seededProjectID+`"`) { + t.Errorf("admin volumes?project_id=: a row carries the ADMIN's tenant_id: %s", body) + } + + // Round 8, MINOR-3: the show must not dead-end the listing's rows — an + // admin's follow-up `volume show` on the foreign volume answers 200 with + // the row's own tenant_id; a member's answers 404. + code, body = doReq(r, http.MethodGet, "/v3/volumes/"+otherVol, "", adminHdr) + if code != http.StatusOK || !strings.Contains(body, `"tenant_id":"`+otherProject+`"`) { + t.Errorf("admin shows foreign volume: code=%d; want 200 with the row's own tenant_id (body=%s)", code, body) + } + code, body = doReq(r, http.MethodGet, "/v3/volumes/"+otherVol, "", memberHdr) + if code != http.StatusNotFound { + t.Errorf("member shows foreign volume: code=%d; want 404 (body=%s)", code, body) + } +} diff --git a/internal/cinder/volume_types.go b/internal/cinder/volume_types.go index c2d5b431..461babf4 100644 --- a/internal/cinder/volume_types.go +++ b/internal/cinder/volume_types.go @@ -150,6 +150,12 @@ func (svc *Service) DeleteVolumeType(c *gin.Context) { typeID := c.Param("id") + // See GetVolumeType: a non-UUID reference must 404, not 500 (SPEC-028). + if _, err := uuid.Parse(typeID); err != nil { + common.SendError(c, common.NewNotFoundError("volume type")) + return + } + result, err := svc.pool.Exec(c.Request.Context(), "DELETE FROM volume_types WHERE id = $1", typeID, diff --git a/internal/cinder/volumes.go b/internal/cinder/volumes.go index 2bfef1ba..55bb56a6 100644 --- a/internal/cinder/volumes.go +++ b/internal/cinder/volumes.go @@ -384,9 +384,13 @@ const ( // rather than a positional argument list because the fields are mostly strings and // a transposed pair (tenant/user, name/description) would be silently wrong. type volumeRow struct { - ID string - Name string - Description string + ID string + Name string + // Description is any so a NULL column renders as JSON null — the value + // upstream emits for an unset description — instead of the "" that + // NullString.String would produce (SPEC-028). Request-driven sites pass + // the string the caller supplied. + Description any TenantID string UserID string VolumeType string @@ -394,8 +398,12 @@ type volumeRow struct { Size int Bootable bool Attachments []any - CreatedAt time.Time - UpdatedAt time.Time + // Metadata is the volume's user metadata; nil means none, which + // serializes as the empty object — the key itself is mandatory + // (SPEC-028: OSC reads volume.properties from it). + Metadata map[string]string + CreatedAt time.Time + UpdatedAt time.Time } // serializeVolume renders a volume as the full upstream Cinder v3 object, with a @@ -416,6 +424,10 @@ func serializeVolume(v volumeRow) gin.H { if attachments == nil { attachments = []any{} } + metadata := v.Metadata + if metadata == nil { + metadata = map[string]string{} + } return gin.H{ "id": v.ID, "name": v.Name, @@ -428,7 +440,7 @@ func serializeVolume(v volumeRow) gin.H { "created_at": v.CreatedAt.Format(volumeTSLayout), "updated_at": v.UpdatedAt.Format(volumeTSLayout), "attachments": attachments, - "metadata": gin.H{}, + "metadata": metadata, "volume_type": volumeType, "volume_type_id": volumeType, "availability_zone": "nova", @@ -743,6 +755,25 @@ func (svc *Service) CreateVolume(c *gin.Context) { } // ListVolumes lists all volumes (brief) +// volumeListScope decides the project predicate for the volume listings (the +// summary and the detail variant share it): cinder's cross-project semantics +// are all_tenants (every project, admin-only — `volume list --all-projects`) +// and project_id (that one project, admin-only — `volume list --project`). +// Both are IGNORED for non-administrators, whose listing stays their own +// project — upstream ignores administrator-only query parameters the same +// way nova's all_tenants does (FR-005b, SPEC-028). Before this, an admin's +// --project filter was silently dropped and the ADMIN's volumes were +// presented as the target project's. +func volumeListScope(c *gin.Context) (tenant string, allTenants bool) { + if !common.HasRole(c, common.RoleAdmin) { + return "", false + } + // OSC sends all_tenants together with project_id for --project. Keep + // the explicit project filter: all_tenants enables cross-project access, + // it does not discard the requested scope. + return c.Query("project_id"), common.QueryFlag(c, "all_tenants") +} + func (svc *Service) ListVolumes(c *gin.Context) { projectID := c.GetString("project_id") @@ -761,19 +792,32 @@ func (svc *Service) ListVolumes(c *gin.Context) { } f := &common.SQLFilter{} - f.Eq("project_id", projectID) + volTenant, volAllTenants := volumeListScope(c) + volCrossProject := volAllTenants || volTenant != "" + switch { + case volTenant != "": + f.Eq("project_id::text", volTenant) + case !volAllTenants: + f.Eq("project_id", projectID) + } // Marker-based pagination over the SAME total order the ORDER BY uses. Resolving the marker to // `created_at < $n` excluded every volume sharing its timestamp rather than only those before it, // so a page walk silently dropped the marker's siblings — volumes created by one script or // Terraform apply share created_at. Same defect and same fix as the compute listings in #549. + // The marker resolves under the listing's own scope, or a cross-project walk would stall on + // the first page forever. if marker := c.Query("marker"); marker != "" { var markerCreatedAt time.Time var markerID string - err := svc.pool.QueryRow(c.Request.Context(), - "SELECT created_at, id FROM volumes WHERE id = $1 AND project_id = $2", - marker, projectID, - ).Scan(&markerCreatedAt, &markerID) + markerQuery := "SELECT created_at, id FROM volumes WHERE id = $1" + markerArgs := []any{marker} + if !volCrossProject { + markerQuery += " AND project_id = $2" + markerArgs = append(markerArgs, projectID) + } + err := svc.pool.QueryRow(c.Request.Context(), markerQuery, markerArgs...). + Scan(&markerCreatedAt, &markerID) if err == nil { f.Cond(fmt.Sprintf("(created_at, id) < (%s, %s)", f.Arg(markerCreatedAt), f.Arg(markerID))) } @@ -841,19 +885,32 @@ func (svc *Service) ListVolumesDetail(c *gin.Context) { } f := &common.SQLFilter{} - f.Eq("v.project_id", projectID) + volTenant, volAllTenants := volumeListScope(c) + volCrossProject := volAllTenants || volTenant != "" + switch { + case volTenant != "": + f.Eq("v.project_id::text", volTenant) + case !volAllTenants: + f.Eq("v.project_id", projectID) + } // Marker paging over a TOTAL order. This ordered by v.created_at alone, which is not unique, so // ties had no defined relative order and two identical requests could return them differently — // and `v.created_at < $n` then excluded the marker's tied siblings entirely. Adding v.id to both // the ORDER BY and the comparison makes the boundary exact (as #549 did for compute). + // The marker resolves under the listing's own scope, or a cross-project walk would stall on + // the first page forever. if marker := c.Query("marker"); marker != "" { var markerCreatedAt time.Time var markerID string - err := svc.pool.QueryRow(c.Request.Context(), - "SELECT created_at, id FROM volumes WHERE id = $1 AND project_id = $2", - marker, projectID, - ).Scan(&markerCreatedAt, &markerID) + markerQuery := "SELECT created_at, id FROM volumes WHERE id = $1" + markerArgs := []any{marker} + if !volCrossProject { + markerQuery += " AND project_id = $2" + markerArgs = append(markerArgs, projectID) + } + err := svc.pool.QueryRow(c.Request.Context(), markerQuery, markerArgs...). + Scan(&markerCreatedAt, &markerID) if err == nil { f.Cond(fmt.Sprintf("(v.created_at, v.id) < (%s, %s)", f.Arg(markerCreatedAt), f.Arg(markerID))) } @@ -862,7 +919,7 @@ func (svc *Service) ListVolumesDetail(c *gin.Context) { where := f.Where() limitOffset := fmt.Sprintf(" LIMIT %s OFFSET %s", f.Arg(limit), f.Arg(offset)) rows, err := svc.pool.Query(c.Request.Context(), ` - SELECT v.id, v.name, v.description, v.size_gb, v.status, v.bootable, v.user_id, v.volume_type, v.created_at, v.updated_at + SELECT v.id, v.name, v.description, v.size_gb, v.status, v.bootable, v.user_id, v.volume_type, v.project_id, v.created_at, v.updated_at FROM volumes v`+where+` ORDER BY v.created_at DESC, v.id DESC`+limitOffset, f.Args()...) @@ -881,15 +938,21 @@ func (svc *Service) ListVolumesDetail(c *gin.Context) { var bootable bool var userID, volumeType, description sql.NullString var createdAt, updatedAt time.Time - - if err := rows.Scan(&id, &name, &description, &size, &status, &bootable, &userID, &volumeType, &createdAt, &updatedAt); err != nil { + // The row's OWN project, not the caller's: the admin cross-project + // listing (all_tenants / project_id) serves other projects' rows, + // and stamping them with the admin's project would tell tooling a + // resource is theirs when it is not (os-vol-tenant-attr:tenant_id + // is consumed by `volume list --long --all-projects`). + var ownerID string + + if err := rows.Scan(&id, &name, &description, &size, &status, &bootable, &userID, &volumeType, &ownerID, &createdAt, &updatedAt); err != nil { log.Warn().Err(err).Msg("failed to scan volume detail row") continue } volumes = append(volumes, serializeVolume(volumeRow{ - ID: id, Name: name, Description: description.String, - TenantID: projectID, UserID: userID.String, VolumeType: volumeType.String, + ID: id, Name: name, Description: common.NullString(description), + TenantID: ownerID, UserID: userID.String, VolumeType: volumeType.String, Status: status, Size: size, Bootable: bootable, CreatedAt: createdAt, UpdatedAt: updatedAt, })) @@ -914,6 +977,19 @@ func (svc *Service) ListVolumesDetail(c *gin.Context) { } } + // User metadata for every listed volume in one query (SPEC-028). + metadata, err := volumeMetadataRows(c.Request.Context(), svc.pool, ids) + if err != nil { + log.Error().Err(err).Str("operation", "list_volumes_detail_metadata").Msg("database error") + common.SendError(c, common.NewInternalServerError("failed to list volumes")) + return + } + for i, id := range ids { + if m, ok := metadata[id]; ok { + volumes[i]["metadata"] = m + } + } + if volumes == nil { volumes = []gin.H{} } @@ -931,13 +1007,25 @@ func (svc *Service) GetVolume(c *gin.Context) { var bootable bool var userID, volumeType, description sql.NullString var createdAt, updatedAt time.Time - - // Support lookup by ID or name - err := svc.pool.QueryRow(c.Request.Context(), ` - SELECT id, name, description, size_gb, status, bootable, user_id, volume_type, created_at, updated_at + // The row's OWN project, not the caller's — see the detail listing. + var ownerID string + + // Support lookup by ID or name. An administrator may show any volume — + // the same carve-out the listings have for their cross-project walks + // (`volume list --project X` then `volume show ` must not dead-end + // with 404 on the very rows the listing returned; SPEC-028). A + // non-admin keeps the project scope. + getQuery := ` + SELECT id, name, description, size_gb, status, bootable, user_id, volume_type, project_id, created_at, updated_at FROM volumes - WHERE project_id = $2 AND ((id::text = $1) OR (name = $1)) - `, volumeID, projectID).Scan(&id, &name, &description, &size, &status, &bootable, &userID, &volumeType, &createdAt, &updatedAt) + WHERE ((id::text = $1) OR (name = $1))` + getArgs := []any{volumeID} + if !common.HasRole(c, common.RoleAdmin) { + getQuery += ` AND project_id = $2` + getArgs = append(getArgs, projectID) + } + err := svc.pool.QueryRow(c.Request.Context(), getQuery, getArgs...). + Scan(&id, &name, &description, &size, &status, &bootable, &userID, &volumeType, &ownerID, &createdAt, &updatedAt) if errors.Is(err, database.ErrNoRows) { common.SendError(c, common.NewNotFoundError("volume")) @@ -956,12 +1044,21 @@ func (svc *Service) GetVolume(c *gin.Context) { return } + // User metadata on the object, not only via the metadata endpoint + // (SPEC-028: 'volume show' read its properties column from {}). + metadata, err := volumeMetadataRows(c.Request.Context(), svc.pool, []string{id}) + if err != nil { + log.Error().Err(err).Str("operation", "get_volume_metadata").Str("volume_id", volumeID).Msg("database error") + common.SendError(c, common.NewInternalServerError("failed to get volume")) + return + } + c.JSON(http.StatusOK, gin.H{ "volume": serializeVolume(volumeRow{ - ID: id, Name: name, Description: description.String, - TenantID: projectID, UserID: userID.String, VolumeType: volumeType.String, + ID: id, Name: name, Description: common.NullString(description), + TenantID: ownerID, UserID: userID.String, VolumeType: volumeType.String, Status: status, Size: size, Bootable: bootable, Attachments: attachments[id], - CreatedAt: createdAt, UpdatedAt: updatedAt, + Metadata: metadata[id], CreatedAt: createdAt, UpdatedAt: updatedAt, }), }) } @@ -1137,6 +1234,95 @@ func volumeAttachments(ctx context.Context, pool database.DB, volumeIDs []string return result, nil } +// volumeMetadataRows loads the user metadata of many volumes in one query. +// Volume show and list-detail serialize it into the volume object — the +// metadata endpoint returns it, but the object itself answered {} however +// much was set, so 'volume show' always reported empty properties +// (SPEC-028). +func volumeMetadataRows(ctx context.Context, pool database.DB, volumeIDs []string) (map[string]map[string]string, error) { + result := make(map[string]map[string]string, len(volumeIDs)) + if len(volumeIDs) == 0 { + return result, nil + } + rows, err := pool.Query(ctx, ` + SELECT volume_id::text, meta_key, meta_value + FROM volume_metadata + WHERE volume_id = ANY($1::uuid[]) + `, volumeIDs) + if err != nil { + return nil, fmt.Errorf("query volume metadata: %w", err) + } + defer rows.Close() + for rows.Next() { + var volumeID, key, value string + if err := rows.Scan(&volumeID, &key, &value); err != nil { + return nil, fmt.Errorf("scan volume metadata: %w", err) + } + if result[volumeID] == nil { + result[volumeID] = map[string]string{} + } + result[volumeID][key] = value + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate volume metadata: %w", err) + } + return result, nil +} + +// snapshotMetadataRows loads the user metadata of many snapshots in one +// query. Snapshot show and list-detail serialize it into the snapshot object +// (SPEC-028): the metadata endpoint returned it, but the object itself +// omitted the key entirely, and the pinned OSC 7 formatter crashes with +// KeyError 'metadata' on a snapshot without one. +func snapshotMetadataRows(ctx context.Context, pool database.DB, snapshotIDs []string) (map[string]map[string]string, error) { + result := make(map[string]map[string]string, len(snapshotIDs)) + if len(snapshotIDs) == 0 { + return result, nil + } + rows, err := pool.Query(ctx, ` + SELECT snapshot_id::text, meta_key, meta_value + FROM snapshot_metadata + WHERE snapshot_id = ANY($1::uuid[]) + `, snapshotIDs) + if err != nil { + return nil, fmt.Errorf("query snapshot metadata: %w", err) + } + defer rows.Close() + for rows.Next() { + var snapshotID, key, value string + if err := rows.Scan(&snapshotID, &key, &value); err != nil { + return nil, fmt.Errorf("scan snapshot metadata: %w", err) + } + if result[snapshotID] == nil { + result[snapshotID] = map[string]string{} + } + result[snapshotID][key] = value + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate snapshot metadata: %w", err) + } + return result, nil +} + +// serializeSnapshot renders a snapshot as the upstream Cinder v3 object. The +// metadata key is mandatory — OSC's formatter dereferences it unconditionally +// (SPEC-028). +func serializeSnapshot(id, name, description, volumeID, status string, size int, createdAt time.Time, metadata map[string]string) gin.H { + if metadata == nil { + metadata = map[string]string{} + } + return gin.H{ + "id": id, + "name": name, + "description": description, + "volume_id": volumeID, + "size": size, + "status": status, + "created_at": createdAt.Format("2006-01-02T15:04:05.000000"), + "metadata": metadata, + } +} + // DeleteVolume deletes a volume func (svc *Service) DeleteVolume(c *gin.Context) { volumeID := c.Param("id") @@ -1620,6 +1806,39 @@ func (svc *Service) VolumeAction(c *gin.Context) { return } + // Handle set/unset bootable. python-cinderclient's 'volume set + // --bootable/--non-bootable' PUTs {"os-set_bootable": {"bootable": true}} + // (SPEC-028). Bootable is attach metadata rather than a state + // transition, so it is accepted in any status — matching upstream. + if setBootableData, ok := req["os-set_bootable"]; ok { + bootableMap, ok := setBootableData.(map[string]any) + if !ok { + common.SendError(c, common.NewBadRequestError("os-set_bootable must be an object")) + return + } + bootable, ok := bootableMap["bootable"].(bool) + if !ok { + common.SendError(c, common.NewBadRequestError("os-set_bootable.bootable must be a boolean")) + return + } + + result, err := svc.pool.Exec(c.Request.Context(), ` + UPDATE volumes SET bootable = $1, updated_at = $2 WHERE id::text = $3 AND project_id = $4 + `, bootable, time.Now(), volumeID, projectID) + if err != nil { + log.Error().Err(err).Str("operation", "set_volume_bootable").Str("volume_id", volumeID).Msg("failed to set volume bootable") + common.SendError(c, common.NewInternalServerError("failed to set volume bootable")) + return + } + if result.RowsAffected() == 0 { + common.SendError(c, common.NewNotFoundError("volume")) + return + } + + c.Status(http.StatusAccepted) + return + } + common.SendError(c, common.NewBadRequestError("unknown action")) } @@ -1703,15 +1922,8 @@ func (svc *Service) CreateSnapshot(c *gin.Context) { } c.JSON(http.StatusAccepted, gin.H{ - "snapshot": gin.H{ - "id": snapshotID, - "name": req.Snapshot.Name, - "description": req.Snapshot.Description, - "volume_id": volumeID, - "size": size, - "status": "creating", - "created_at": now.Format("2006-01-02T15:04:05.000000"), - }, + "snapshot": serializeSnapshot(snapshotID, req.Snapshot.Name, req.Snapshot.Description, + volumeID, "creating", size, now, nil), }) } @@ -1787,6 +1999,7 @@ func (svc *Service) ListSnapshotsDetail(c *gin.Context) { defer rows.Close() var snapshots []gin.H + var ids []string for rows.Next() { var id, name, description, volumeID, status string var size int @@ -1797,15 +2010,8 @@ func (svc *Service) ListSnapshotsDetail(c *gin.Context) { continue } - snapshots = append(snapshots, gin.H{ - "id": id, - "name": name, - "description": description, - "volume_id": volumeID, - "size": size, - "status": status, - "created_at": createdAt.Format("2006-01-02T15:04:05.000000"), - }) + snapshots = append(snapshots, serializeSnapshot(id, name, description, volumeID, status, size, createdAt, nil)) + ids = append(ids, id) } if err := rows.Err(); err != nil { log.Error().Err(err).Msg("rows iteration error") @@ -1813,6 +2019,19 @@ func (svc *Service) ListSnapshotsDetail(c *gin.Context) { return } + // User metadata for every listed snapshot in one query (SPEC-028). + metadata, err := snapshotMetadataRows(c.Request.Context(), svc.pool, ids) + if err != nil { + log.Error().Err(err).Str("operation", "list_snapshots_metadata").Msg("database error") + common.SendError(c, common.NewInternalServerError("failed to list snapshots")) + return + } + for i, id := range ids { + if m, ok := metadata[id]; ok { + snapshots[i]["metadata"] = m + } + } + if snapshots == nil { snapshots = []gin.H{} } @@ -1853,6 +2072,7 @@ func (svc *Service) ListSnapshots(c *gin.Context) { defer rows.Close() var snapshots []gin.H + var ids []string for rows.Next() { var id, name, description, volumeID, status string var size int @@ -1863,15 +2083,8 @@ func (svc *Service) ListSnapshots(c *gin.Context) { continue } - snapshots = append(snapshots, gin.H{ - "id": id, - "name": name, - "description": description, - "volume_id": volumeID, - "size": size, - "status": status, - "created_at": createdAt.Format("2006-01-02T15:04:05.000000"), - }) + snapshots = append(snapshots, serializeSnapshot(id, name, description, volumeID, status, size, createdAt, nil)) + ids = append(ids, id) } if err := rows.Err(); err != nil { log.Error().Err(err).Msg("rows iteration error") @@ -1879,6 +2092,19 @@ func (svc *Service) ListSnapshots(c *gin.Context) { return } + // User metadata for every listed snapshot in one query (SPEC-028). + metadata, err := snapshotMetadataRows(c.Request.Context(), svc.pool, ids) + if err != nil { + log.Error().Err(err).Str("operation", "list_snapshots_metadata").Msg("database error") + common.SendError(c, common.NewInternalServerError("failed to list snapshots")) + return + } + for i, id := range ids { + if m, ok := metadata[id]; ok { + snapshots[i]["metadata"] = m + } + } + if snapshots == nil { snapshots = []gin.H{} } @@ -1891,6 +2117,14 @@ func (svc *Service) GetSnapshot(c *gin.Context) { snapshotID := c.Param("id") projectID := c.GetString("project_id") + // A non-UUID reference can only name nothing here — the lookup is + // uuid-typed, and the cast error used to surface as 500 instead of the + // 404 clients need (SPEC-028). + if _, err := uuid.Parse(snapshotID); err != nil { + common.SendError(c, common.NewNotFoundError("snapshot")) + return + } + var id, name, description, volumeID, status string var size int var createdAt time.Time @@ -1911,16 +2145,17 @@ func (svc *Service) GetSnapshot(c *gin.Context) { return } + // User metadata on the object, not only via the metadata endpoint + // (SPEC-028). + metadata, err := snapshotMetadataRows(c.Request.Context(), svc.pool, []string{id}) + if err != nil { + log.Error().Err(err).Str("operation", "get_snapshot_metadata").Str("snapshot_id", snapshotID).Msg("database error") + common.SendError(c, common.NewInternalServerError("failed to get snapshot")) + return + } + c.JSON(http.StatusOK, gin.H{ - "snapshot": gin.H{ - "id": id, - "name": name, - "description": description, - "volume_id": volumeID, - "size": size, - "status": status, - "created_at": createdAt.Format("2006-01-02T15:04:05.000000"), - }, + "snapshot": serializeSnapshot(id, name, description, volumeID, status, size, createdAt, metadata[id]), }) } @@ -1929,6 +2164,12 @@ func (svc *Service) DeleteSnapshot(c *gin.Context) { snapshotID := c.Param("id") projectID := c.GetString("project_id") + // See GetSnapshot: a non-UUID reference must 404, not 500 (SPEC-028). + if _, err := uuid.Parse(snapshotID); err != nil { + common.SendError(c, common.NewNotFoundError("snapshot")) + return + } + // Get volume ID var volumeID, status string err := svc.pool.QueryRow(c.Request.Context(), @@ -2026,7 +2267,7 @@ func (svc *Service) SnapshotAction(c *gin.Context) { // ListVolumeTypes lists all volume types func (svc *Service) ListVolumeTypes(c *gin.Context) { rows, err := svc.pool.Query(c.Request.Context(), ` - SELECT id, name, COALESCE(description, ''), is_public, created_at + SELECT id, name, description, is_public, COALESCE(extra_specs, '{}'::jsonb), created_at FROM volume_types ORDER BY name `) @@ -2040,20 +2281,35 @@ func (svc *Service) ListVolumeTypes(c *gin.Context) { var types []gin.H for rows.Next() { - var id, name, description string + var id, name string + // description is NULL for a type created without one; scanning it + // into a plain string failed the row, and rows.Err() then failed the + // WHOLE listing with 500 "failed to read results" until the type was + // deleted (SPEC-028). + var description sql.NullString var isPublic bool + // jsonb: scan as any-valued so non-string values (none are + // written today) pass through instead of failing the row. + var extraSpecs map[string]any var createdAt time.Time - if err := rows.Scan(&id, &name, &description, &isPublic, &createdAt); err != nil { + if err := rows.Scan(&id, &name, &description, &isPublic, &extraSpecs, &createdAt); err != nil { log.Warn().Err(err).Msg("failed to scan volume type row") continue } + if extraSpecs == nil { + extraSpecs = map[string]any{} + } types = append(types, gin.H{ "id": id, "name": name, - "description": description, + "description": common.NullString(description), "is_public": isPublic, + // The type object carries its extra_specs — the endpoint returns + // them, but the object omitted the key, so 'volume type show' + // always reported properties: {} (SPEC-028). + "extra_specs": extraSpecs, }) } if err := rows.Err(); err != nil { @@ -2073,14 +2329,24 @@ func (svc *Service) ListVolumeTypes(c *gin.Context) { func (svc *Service) GetVolumeType(c *gin.Context) { typeID := c.Param("id") - var id, name, description string + // A non-UUID reference (a type name, or garbage) can only name nothing; + // the uuid-typed lookup used to surface the cast error as 500 instead of + // the 404 clients need (SPEC-028). + if _, err := uuid.Parse(typeID); err != nil { + common.SendError(c, common.NewNotFoundError("volume type")) + return + } + + var id, name string + var description sql.NullString var isPublic bool + var extraSpecs map[string]any err := svc.pool.QueryRow(c.Request.Context(), ` - SELECT id, name, COALESCE(description, ''), is_public + SELECT id, name, description, is_public, COALESCE(extra_specs, '{}'::jsonb) FROM volume_types WHERE id = $1 - `, typeID).Scan(&id, &name, &description, &isPublic) + `, typeID).Scan(&id, &name, &description, &isPublic, &extraSpecs) if errors.Is(err, database.ErrNoRows) { common.SendError(c, common.NewNotFoundError("volume type")) @@ -2091,13 +2357,17 @@ func (svc *Service) GetVolumeType(c *gin.Context) { common.SendError(c, common.NewInternalServerError("failed to get volume type")) return } + if extraSpecs == nil { + extraSpecs = map[string]any{} + } c.JSON(http.StatusOK, gin.H{ "volume_type": gin.H{ "id": id, "name": name, - "description": description, + "description": common.NullString(description), "is_public": isPublic, + "extra_specs": extraSpecs, }, }) } @@ -2111,6 +2381,13 @@ func (svc *Service) UpdateVolume(c *gin.Context) { Volume struct { Name *string `json:"name"` Description *string `json:"description"` + // python-cinderclient still PUTs the legacy v2 field names for + // set --name/--description (captured with --debug against OSC 7: + // {"volume": {"display_description": ...}}); real Cinder accepts + // both aliases, and only the v3 names were bound here, so the + // request succeeded but persisted nothing (SPEC-028). + DisplayName *string `json:"display_name"` + DisplayDescription *string `json:"display_description"` } `json:"volume"` } if err := c.ShouldBindJSON(&req); err != nil { @@ -2145,6 +2422,12 @@ func (svc *Service) UpdateVolume(c *gin.Context) { if req.Volume.Description != nil { currentDesc = *req.Volume.Description } + if req.Volume.DisplayName != nil && req.Volume.Name == nil { + currentName = *req.Volume.DisplayName + } + if req.Volume.DisplayDescription != nil && req.Volume.Description == nil { + currentDesc = *req.Volume.DisplayDescription + } now := time.Now() _, err = svc.pool.Exec(c.Request.Context(), @@ -2225,16 +2508,13 @@ func (svc *Service) UpdateSnapshot(c *gin.Context) { return } + metadata, err := snapshotMetadataRows(c.Request.Context(), svc.pool, []string{snapshotID}) + if err != nil { + common.SendError(c, common.NewInternalServerError("failed to read snapshot metadata")) + return + } c.JSON(http.StatusOK, gin.H{ - "snapshot": gin.H{ - "id": snapshotID, - "name": currentName, - "description": currentDesc, - "volume_id": volumeID, - "size": sizeGB, - "status": status, - "created_at": createdAt.Format("2006-01-02T15:04:05.000000"), - }, + "snapshot": serializeSnapshot(snapshotID, currentName, currentDesc, volumeID, status, sizeGB, createdAt, metadata[snapshotID]), }) } diff --git a/internal/common/errors.go b/internal/common/errors.go index 9e7c318d..5b2fbc5b 100644 --- a/internal/common/errors.go +++ b/internal/common/errors.go @@ -59,6 +59,10 @@ func SendError(c *gin.Context, err *OpenStackError) { c.JSON(err.StatusCode, err.NeutronJSON()) return } + if usesIdentityEnvelope(c) { + c.JSON(err.StatusCode, err.IdentityJSON()) + return + } c.JSON(err.StatusCode, err.ToJSON()) } diff --git a/internal/common/identity_errors.go b/internal/common/identity_errors.go new file mode 100644 index 00000000..29d7df6f --- /dev/null +++ b/internal/common/identity_errors.go @@ -0,0 +1,67 @@ +package common + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" +) + +// Keystone's error envelope differs from every other service's, exactly as +// Neutron's does (see neutron_errors.go): upstream renders +// +// {"error": {"code": 403, "message": "...", "title": "Forbidden"}} +// +// while this codebase's default renders the compute shape +// ({"forbidden": {"message": ...}}). The compute shape is fine for the SDK — +// openstacksdk's raise_from_response walks the body's values for a "message" +// key — but the identity client is keystoneauth1, whose from_response parses +// ONLY the {"error": {...}} (or API-SIG {"errors": [...]}) shape. Every other +// body degrades to "Unrecognized schema in response body.", which throws away +// the reason: 'domain delete ' told the operator nothing about +// disabling it first (SPEC-028). +// +// Marked once per engine by the identity service, so all handler sites and +// every middleware path render Keystone's shape without touching any of them. + +// identityEnvelopeKey marks a request whose errors render in Keystone's envelope. +const identityEnvelopeKey = "lightstack.error_envelope.identity" + +// MarkIdentityErrors flags the request so SendError renders Keystone's envelope +// instead of the default. +func MarkIdentityErrors(c *gin.Context) { + c.Set(identityEnvelopeKey, true) +} + +// usesIdentityEnvelope reports whether this request was marked. A nil context +// is treated as unmarked so a handler invoked outside a request (tests, +// background work) keeps the default rendering. +func usesIdentityEnvelope(c *gin.Context) bool { + if c == nil { + return false + } + marked, ok := c.Get(identityEnvelopeKey) + if !ok { + return false + } + on, _ := marked.(bool) + return on +} + +// IdentityJSON renders the error in Keystone's envelope. `title` is the HTTP +// reason phrase, as upstream's is ("Forbidden", "Not Found", …); `details` +// is folded into the message — upstream has no details field, and the message +// is the only thing keystoneauth shows the user. +func (e *OpenStackError) IdentityJSON() gin.H { + message := e.Message + if e.Details != "" { + message = fmt.Sprintf("%s %s", message, e.Details) + } + return gin.H{ + "error": gin.H{ + "code": e.StatusCode, + "message": message, + "title": http.StatusText(e.StatusCode), + }, + } +} diff --git a/internal/common/nullstring.go b/internal/common/nullstring.go new file mode 100644 index 00000000..17e3f481 --- /dev/null +++ b/internal/common/nullstring.go @@ -0,0 +1,15 @@ +package common + +import "database/sql" + +// NullString renders a sql.NullString the way the OpenStack APIs do: null +// when the column is NULL, the string otherwise. Emitting ns.String +// unconditionally turns a NULL into "" — a value the APIs never produce for +// optional string fields, and one the CLI renders differently from a real +// empty string (SPEC-028). +func NullString(ns sql.NullString) any { + if !ns.Valid { + return nil + } + return ns.String +} diff --git a/internal/database/migration_080_sqlite_test.go b/internal/database/migration_080_sqlite_test.go new file mode 100644 index 00000000..fa7d0ed8 --- /dev/null +++ b/internal/database/migration_080_sqlite_test.go @@ -0,0 +1,76 @@ +package database + +import ( + "context" + "database/sql" + "path/filepath" + "testing" +) + +func migratedSQLite(t *testing.T, target uint) (*sql.DB, string) { + t.Helper() + ctx := context.Background() + path := filepath.Join(t.TempDir(), "migration.db") + url := "sqlite://" + path + if err := MigrateToVersion(url, "../../migrations", target); err != nil { + t.Fatalf("migrate SQLite to %d: %v", target, err) + } + db, err := openSQLiteHandle(ctx, path, defaultSQLiteTxLock, true, PoolConfig{MaxConns: 1}) + if err != nil { + t.Fatalf("open migrated SQLite database: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + return db, url +} + +func TestSQLiteServerGroupMigrationPreservesDuplicates(t *testing.T) { + ctx := context.Background() + db, url := migratedSQLite(t, 77) + var projectID string + if err := db.QueryRowContext(ctx, "SELECT id FROM projects ORDER BY id LIMIT 1").Scan(&projectID); err != nil { + t.Fatalf("find seeded project: %v", err) + } + for _, row := range []struct{ id, members string }{ + {"duplicate-a", "[\"member-a\",\"shared\"]"}, + {"duplicate-b", "[\"member-b\",\"shared\"]"}, + } { + if _, err := db.ExecContext(ctx, "INSERT INTO server_groups (id, name, policies, members, project_id) VALUES (?, 'migration-duplicate', '[]', ?, ?)", + row.id, row.members, projectID); err != nil { + t.Fatalf("seed duplicate server group: %v", err) + } + } + if err := MigrateToVersion(url, "../../migrations", 78); err != nil { + t.Fatalf("migrate server-group uniqueness: %v", err) + } + for _, row := range []struct{ id, members string }{ + {"duplicate-a", "[\"member-a\",\"shared\"]"}, + {"duplicate-b", "[\"member-b\",\"shared\"]"}, + } { + var members string + if err := db.QueryRowContext(ctx, "SELECT members FROM server_groups WHERE id=?", row.id).Scan(&members); err != nil || members != row.members { + t.Fatalf("group %s changed during upgrade: members=%s err=%v", row.id, members, err) + } + } + if _, err := db.ExecContext(ctx, "INSERT INTO server_groups (id, name, policies, members, project_id) VALUES ('duplicate-c', 'migration-duplicate', '[]', '[]', ?)", projectID); err != nil { + t.Fatalf("duplicate group name rejected: %v", err) + } + +} + +func TestSQLiteOwnerMigrationsRejectNullProjectIDs(t *testing.T) { + ctx := context.Background() + db, _ := migratedSQLite(t, 80) + for _, tc := range []struct{ table, query string }{ + {"volumes", "INSERT INTO volumes (project_id, size_gb) VALUES (NULL, 1)"}, + {"ports", "INSERT INTO ports (project_id) VALUES (NULL)"}, + {"networks", "INSERT INTO networks (name, project_id) VALUES ('null-owner-network', NULL)"}, + {"routers", "INSERT INTO routers (name, project_id) VALUES ('null-owner-router', NULL)"}, + {"floating_ips", "INSERT INTO floating_ips (floating_ip_address, project_id) VALUES ('192.0.2.1', NULL)"}, + } { + t.Run(tc.table, func(t *testing.T) { + if _, err := db.ExecContext(ctx, tc.query); err == nil { + t.Fatalf("%s accepted a NULL project_id", tc.table) + } + }) + } +} diff --git a/internal/database/schema_model_test.go b/internal/database/schema_model_test.go index ae4919ab..1fcc8997 100644 --- a/internal/database/schema_model_test.go +++ b/internal/database/schema_model_test.go @@ -209,6 +209,38 @@ func (t *modelTable) sorted() *modelTable { return t } +// sqliteTriggerNotNullColumns reports ownership columns whose NOT NULL invariant is +// enforced by SQLite triggers. SQLite cannot ALTER these referenced tables to add a +// column constraint without rebuilding them, which would cascade-delete child rows. +func sqliteTriggerNotNullColumns(ctx context.Context, db *sql.DB) (map[string]map[string]bool, error) { + rows, err := db.QueryContext(ctx, "SELECT name FROM sqlite_schema WHERE type = 'trigger'") + if err != nil { + return nil, err + } + defer rows.Close() + const suffix = "_project_id_not_null_insert" + const updateSuffix = "_project_id_not_null_update" + result := map[string]map[string]bool{} + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + table := strings.TrimSuffix(name, suffix) + if table == name { + table = strings.TrimSuffix(name, updateSuffix) + } + if table == name { + continue + } + if result[table] == nil { + result[table] = map[string]bool{} + } + result[table]["project_id"] = true + } + return result, rows.Err() +} + // sqliteModel builds the model of a SQLite database's schema from its catalog. func sqliteModel(ctx context.Context, db *sql.DB, source string) (schemaModel, error) { type table struct{ name, sql string } @@ -228,6 +260,10 @@ func sqliteModel(ctx context.Context, db *sql.DB, source string) (schemaModel, e } rows.Close() + enforcedNotNull, err := sqliteTriggerNotNullColumns(ctx, db) + if err != nil { + return nil, err + } m := schemaModel{} for _, t := range tables { mt := &modelTable{} @@ -250,7 +286,7 @@ func sqliteModel(ctx context.Context, db *sql.DB, source string) (schemaModel, e return nil, err } mt.columns = append(mt.columns, modelColumn{name: name, family: strings.ToLower(typ), - notNull: notNull == 1 || pk > 0, def: normalizeDefault(def.String), maxLen: maxLen[name]}) + notNull: notNull == 1 || pk > 0 || enforcedNotNull[t.name][name], def: normalizeDefault(def.String), maxLen: maxLen[name]}) if pk > 0 { pkOrder[pk] = name } diff --git a/internal/glance/image_list_filters_test.go b/internal/glance/image_list_filters_test.go new file mode 100644 index 00000000..922b3f58 --- /dev/null +++ b/internal/glance/image_list_filters_test.go @@ -0,0 +1,268 @@ +package glance + +import ( + "context" + "encoding/json" + "net/http" + "strings" + "testing" + + "github.com/google/uuid" +) + +// insertListFilterImage seeds one image row the filter tests can see. The +// caller owns cleanup. +func insertListFilterImage(t *testing.T, name, diskFormat string, sizeBytes int64, tags ...string) string { + t.Helper() + ctx := context.Background() + id := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO images (id, name, project_id, status, visibility, disk_format, container_format, size_bytes) + VALUES ($1, $2, $3, 'active', 'public', $4, 'bare', $5)`, + id, name, seededProjectID, diskFormat, sizeBytes); err != nil { + t.Fatalf("insert image %s: %v", name, err) + } + t.Cleanup(func() { testPool.Exec(ctx, "DELETE FROM images WHERE id=$1", id) }) + for _, tag := range tags { + if _, err := testPool.Exec(ctx, + `INSERT INTO image_tags (image_id, tag) VALUES ($1, $2) ON CONFLICT DO NOTHING`, + id, tag); err != nil { + t.Fatalf("insert tag %s: %v", tag, err) + } + } + return id +} + +// listedNames parses a /v2/images response and returns the names in it. +func listedNames(t *testing.T, body string) []string { + t.Helper() + var resp struct { + Images []struct { + Name string `json:"name"` + } `json:"images"` + } + if err := json.Unmarshal([]byte(body), &resp); err != nil { + t.Fatalf("parse list response %q: %v", body, err) + } + names := make([]string, 0, len(resp.Images)) + for _, img := range resp.Images { + names = append(names, img.Name) + } + return names +} + +func contains(haystack []string, needle string) bool { + for _, s := range haystack { + if s == needle { + return true + } + } + return false +} + +// TestListImagesNameFilter — the exact-match `name` query parameter. This is +// the filter the unmodified terraform-provider-openstack data source relies +// on (images.ListOpts{Name} with no client-side fallback), so without it any +// server holding more than one image breaks `data "openstack_images_image_v2"` +// with "Your query returned more than one result" (SPEC-028 terraform +// conformance finding). +func TestListImagesNameFilter(t *testing.T) { + requireDB(t) + insertListFilterImage(t, "spec028-name-a", "raw", 1024) + insertListFilterImage(t, "spec028-name-b", "raw", 2048) + r := newGlanceTestRouter(t) + + code, body := doReq(r, http.MethodGet, "/images?name=spec028-name-a", hdr(seededProjectID, "admin")) + if code != http.StatusOK { + t.Fatalf("list by name: code=%d body=%s; want 200", code, body) + } + names := listedNames(t, body) + if len(names) != 1 || names[0] != "spec028-name-a" { + t.Errorf("name filter: got %v; want exactly [spec028-name-a]", names) + } + + // A name that matches nothing must return an empty list, not everything. + code, body = doReq(r, http.MethodGet, "/images?name=spec028-name-nope", hdr(seededProjectID, "admin")) + if code != http.StatusOK { + t.Fatalf("list by missing name: code=%d body=%s; want 200", code, body) + } + if names := listedNames(t, body); len(names) != 0 { + t.Errorf("missing name filter: got %v; want empty", names) + } +} + +// TestListImagesPropertyFilters — the other standard Glance v2 list +// parameters (glance ALLOWED_PARAMS): disk_format, size_min/max, owner, tag. +// Silently ignoring them makes `openstack image list --tag X` / +// `--disk-format qcow2` return unfiltered results (SPEC-028 CLI conformance +// finding). +// +// The assertions are written for a SHARED server (other suites create images +// concurrently): each case asserts the target image is present, the +// non-matching sibling is absent, and — where the response carries the +// filtered field — that no returned image violates the predicate. +func TestListImagesPropertyFilters(t *testing.T) { + requireDB(t) + insertListFilterImage(t, "spec028-prop-raw", "raw", 54321, "spec028-tagged") + insertListFilterImage(t, "spec028-prop-qcow2", "qcow2", 12345) + r := newGlanceTestRouter(t) + + listImages := func(query string) []map[string]any { + t.Helper() + code, body := doReq(r, http.MethodGet, "/images?"+query, hdr(seededProjectID, "admin")) + if code != http.StatusOK { + t.Fatalf("list %s: code=%d body=%s; want 200", query, code, body) + } + var resp struct { + Images []map[string]any `json:"images"` + } + if err := json.Unmarshal([]byte(body), &resp); err != nil { + t.Fatalf("parse list response: %v", err) + } + return resp.Images + } + names := func(images []map[string]any) []string { + out := make([]string, 0, len(images)) + for _, img := range images { + if n, _ := img["name"].(string); n != "" { + out = append(out, n) + } + } + return out + } + + // disk_format: target present, sibling absent, nothing foreign leaks in + // with the wrong format. + if got := names(listImages("disk_format=raw")); !contains(got, "spec028-prop-raw") || contains(got, "spec028-prop-qcow2") { + t.Errorf("disk_format=raw: got %v; want spec028-prop-raw present, spec028-prop-qcow2 absent", got) + } + for _, img := range listImages("disk_format=qcow2") { + if df, _ := img["disk_format"].(string); df != "qcow2" { + t.Errorf("disk_format=qcow2 returned image %v with disk_format %q", img["name"], df) + } + } + + // size bounds chosen to exclude both the sibling and the 64KiB fixtures + // other suites create. + if got := names(listImages("size_min=50000&size_max=60000")); len(got) != 1 || got[0] != "spec028-prop-raw" { + t.Errorf("size 50000..60000: got %v; want exactly [spec028-prop-raw]", got) + } + + // tag: tagged target present, untagged sibling absent; an unused tag + // matches neither. + if got := names(listImages("tag=spec028-tagged")); !contains(got, "spec028-prop-raw") || contains(got, "spec028-prop-qcow2") { + t.Errorf("tag=spec028-tagged: got %v; want spec028-prop-raw present, spec028-prop-qcow2 absent", got) + } + if got := names(listImages("tag=spec028-unused")); contains(got, "spec028-prop-raw") || contains(got, "spec028-prop-qcow2") { + t.Errorf("tag=spec028-unused: got %v; want neither spec028 image", got) + } + + // owner: our images are owned by the seeded project; an owner that owns + // nothing of ours must not list them. + if got := names(listImages("owner=" + seededProjectID)); !contains(got, "spec028-prop-raw") { + t.Errorf("owner=seeded: got %v; want spec028-prop-raw present", got) + } + if got := names(listImages("owner=ffffffff-ffff-ffff-ffff-ffffffffffff")); contains(got, "spec028-prop-raw") || contains(got, "spec028-prop-qcow2") { + t.Errorf("owner=unknown: got %v; want neither spec028 image", got) + } +} + +// TestCreateImagePersistsTags — `POST /v2/images` accepts a `tags` array and +// echoes it in the 201 response, but the rows were never written: an +// immediate `GET /v2/images/{id}` returned `tags: []`. The create/echo/show +// round-trip must agree (SPEC-028 CLI conformance finding: +// `openstack image create --tag` silently drops the tag). +func TestCreateImagePersistsTags(t *testing.T) { + requireDB(t) + r := newGlanceTestRouter(t) + ctx := context.Background() + + code, body := doReqBody(r, http.MethodPost, "/images", + `{"name":"spec028-tags-img","disk_format":"raw","container_format":"bare","tags":["alpha","beta"]}`, + hdr(seededProjectID, "member")) + if code != http.StatusCreated { + t.Fatalf("create image: code=%d body=%s; want 201", code, body) + } + var created struct { + ID string `json:"id"` + Tags []string `json:"tags"` + } + if err := json.Unmarshal([]byte(body), &created); err != nil { + t.Fatalf("parse create response: %v", err) + } + t.Cleanup(func() { testPool.Exec(ctx, "DELETE FROM images WHERE id=$1", created.ID) }) + if len(created.Tags) != 2 { + t.Errorf("create response tags: got %v; want [alpha beta]", created.Tags) + } + + code, body = doReq(r, http.MethodGet, "/images/"+created.ID, hdr(seededProjectID, "member")) + if code != http.StatusOK { + t.Fatalf("show image: code=%d body=%s; want 200", code, body) + } + var shown struct { + Tags []string `json:"tags"` + } + if err := json.Unmarshal([]byte(body), &shown); err != nil { + t.Fatalf("parse show response: %v", err) + } + if !contains(shown.Tags, "alpha") || !contains(shown.Tags, "beta") { + t.Errorf("show after create: tags %v; want [alpha beta] persisted", shown.Tags) + } + + // Duplicate tags in one request must not violate image_tags' UNIQUE + // constraint — the dedicated PUT uses ON CONFLICT DO NOTHING, and a + // create request with repeats is legal input. + code, _ = doReqBody(r, http.MethodPost, "/images", + `{"name":"spec028-tags-dup","tags":["same","same"]}`, + hdr(seededProjectID, "member")) + if code != http.StatusCreated { + t.Errorf("create image with duplicate tags: code=%d; want 201", code) + } +} + +// TestImageTagLengthValidated — upstream's image schema caps a tag at 255 +// characters and answers 400; the image_tags column is VARCHAR(255), so +// without the check an over-long tag surfaced as a 500 after (create path) +// the image row was already committed (SPEC-028). +func TestImageTagLengthValidated(t *testing.T) { + requireDB(t) + r := newGlanceTestRouter(t) + hdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "member"} + longTag := strings.Repeat("t", 256) + + // Create path: refused before anything is written. + code, body := doReqBody(r, http.MethodPost, "/images", + `{"name":"spec028-longtag","tags":["`+longTag+`"]}`, hdr) + if code != http.StatusBadRequest { + t.Fatalf("create with over-long tag: code=%d; want 400 (body=%s)", code, body) + } + + // Dedicated PUT path: same refusal. + id := insertListFilterImage(t, "spec028-longtag-put", "qcow2", 0) + code, body = doReqBody(r, http.MethodPut, "/images/"+id+"/tags/"+longTag, "", hdr) + if code != http.StatusBadRequest { + t.Fatalf("put over-long tag: code=%d; want 400 (body=%s)", code, body) + } + + // A maximal tag stays legal on both paths. + code, body = doReqBody(r, http.MethodPost, "/images", + `{"name":"spec028-maxtag","tags":["`+strings.Repeat("t", 255)+`"]}`, hdr) + if code != http.StatusCreated { + t.Errorf("create with 255-char tag: code=%d; want 201 (body=%s)", code, body) + } else { + t.Cleanup(func() { + var created struct { + ID string `json:"id"` + } + if err := json.Unmarshal([]byte(body), &created); err == nil && created.ID != "" { + testPool.Exec(context.Background(), + "DELETE FROM image_tags WHERE image_id=$1", created.ID) + testPool.Exec(context.Background(), "DELETE FROM images WHERE id=$1", created.ID) + } + }) + } + code, _ = doReqBody(r, http.MethodPut, "/images/"+id+"/tags/"+strings.Repeat("t", 255), "", hdr) + if code != http.StatusNoContent { + t.Errorf("put 255-char tag: code=%d; want 204", code) + } +} diff --git a/internal/glance/images.go b/internal/glance/images.go index 29d6d028..e45e6a67 100644 --- a/internal/glance/images.go +++ b/internal/glance/images.go @@ -14,6 +14,7 @@ import ( "strconv" "strings" "time" + "unicode/utf8" "github.com/cobaltcore-dev/lightstack/internal/database" @@ -239,12 +240,42 @@ func (svc *Service) CreateImage(c *gin.Context) { containerFormat = req.ContainerFormat } + // Name length is validated with the tags, for the same reason: the + // column is VARCHAR(255) and upstream's schema answers 400, not a 500 + // from the overflow after (part of) the write failed. + if utf8.RuneCountInString(req.Name) > 255 { + common.SendError(c, common.NewBadRequestError("image name too long (max 255 characters)")) + return + } + + // Tag length is validated BEFORE the image row is written — upstream's + // schema caps a tag at 255 characters and answers 400, while the column's + // VARCHAR(255) would turn an over-long tag into a 500 after the image was + // already committed. + for _, tag := range req.Tags { + if utf8.RuneCountInString(tag) > 255 { + common.SendError(c, common.NewBadRequestError("image tag too long (max 255 characters)")) + return + } + } + // Insert into database. Record the owning project for images of EVERY visibility, // not just private ones — visibility governs who can see an image, ownership governs // who can delete it (F6, FR-014). Storing public images ownerless (project_id NULL) // was the root cause of the delete no-op. now := time.Now() - _, err := svc.pool.Exec(c.Request.Context(), ` + // Image and tags land in ONE transaction: a tag-insert failure used to + // answer 500 with the image row already committed, so the client's + // retry created a duplicate image (upstream creates them atomically). + tx, err := svc.pool.Begin(c.Request.Context()) + if err != nil { + log.Error().Err(err).Str("operation", "create_image").Msg("failed to begin transaction") + common.SendError(c, common.NewInternalServerError("failed to create image")) + return + } + defer func() { _ = tx.Rollback(c.Request.Context()) }() // no-op once committed + + _, err = tx.Exec(c.Request.Context(), ` INSERT INTO images (id, name, project_id, status, visibility, disk_format, container_format, min_disk_gb, min_ram_mb, rbd_pool, rbd_image, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) `, imageID, req.Name, sql.NullString{String: projectID, Valid: projectID != ""}, "queued", visibility, diskFormat, containerFormat, req.MinDisk, req.MinRAM, svc.cephPool, "image-"+imageID, now, now) @@ -255,6 +286,29 @@ func (svc *Service) CreateImage(c *gin.Context) { return } + // Persist the requested tags. The 201 response below echoes req.Tags, but + // the rows were never written before, so a follow-up show returned `tags: []` + // — `openstack image create --tag` silently dropped them (SPEC-028). The + // dedicated PUT uses the same ON CONFLICT DO NOTHING, so a request repeating + // a tag stays legal. Length was validated above, before the image row. + for _, tag := range req.Tags { + if _, err := tx.Exec(c.Request.Context(), ` + INSERT INTO image_tags (image_id, tag) + VALUES ($1, $2) + ON CONFLICT (image_id, tag) DO NOTHING + `, imageID, tag); err != nil { + log.Error().Err(err).Str("operation", "create_image").Str("image_id", imageID).Msg("failed to insert image tag") + common.SendError(c, common.NewInternalServerError("failed to create image")) + return + } + } + + if err := tx.Commit(c.Request.Context()); err != nil { + log.Error().Err(err).Str("operation", "create_image").Msg("failed to commit image transaction") + common.SendError(c, common.NewInternalServerError("failed to create image")) + return + } + c.JSON(http.StatusCreated, gin.H{ "id": imageID, "name": req.Name, @@ -341,6 +395,47 @@ func (svc *Service) ListImages(c *gin.Context) { f.Eq("i.status", status) } + // The remaining standard Glance v2 list parameters (glance's ALLOWED_PARAMS): + // name, disk_format, container_format, size_min, size_max, owner, tag. All are + // exact matches upstream. They were silently ignored before, which broke the + // terraform-provider-openstack image data source (it sends `name` server-side + // and does no client-side matching) and `openstack image list --tag` (SPEC-028). + if name := c.Query("name"); name != "" { + f.Eq("i.name", name) + } + if diskFormat := c.Query("disk_format"); diskFormat != "" { + f.Eq("i.disk_format", diskFormat) + } + if containerFormat := c.Query("container_format"); containerFormat != "" { + f.Eq("i.container_format", containerFormat) + } + // size_min/size_max are byte bounds; a NULL size (a queued image) matches + // neither, which is upstream's behaviour too. + if sizeMin := c.Query("size_min"); sizeMin != "" { + if n, err := strconv.ParseInt(sizeMin, 10, 64); err == nil && n >= 0 { + f.Cond("i.size_bytes >= " + f.Arg(n)) + } else { + common.SendError(c, common.NewBadRequestError("Invalid size_min value: "+sizeMin)) + return + } + } + if sizeMax := c.Query("size_max"); sizeMax != "" { + if n, err := strconv.ParseInt(sizeMax, 10, 64); err == nil && n >= 0 { + f.Cond("i.size_bytes <= " + f.Arg(n)) + } else { + common.SendError(c, common.NewBadRequestError("Invalid size_max value: "+sizeMax)) + return + } + } + if owner := c.Query("owner"); owner != "" { + // Dash-insensitive, like the membership comparison above — callers hold + // the id in either spelling. + f.Cond("replace(i.project_id::text, '-', '') = replace(" + f.Arg(owner) + ", '-', '')") + } + if tag := c.Query("tag"); tag != "" { + f.Cond("EXISTS (SELECT 1 FROM image_tags t WHERE t.image_id = i.id AND t.tag = " + f.Arg(tag) + ")") + } + // Marker-based pagination over a TOTAL order. // // This used to order by created_at alone and resolve the marker to `created_at < $n`. created_at @@ -663,9 +758,10 @@ func (svc *Service) DeleteImage(c *gin.Context) { // Invalidate cache. Entries are keyed per caller (imageCacheKey), so evicting this image // means evicting every project's view of it, not just the deleter's. + // (Only `image:::` keys are ever written — imageCacheKey — + // so this one pattern is the complete eviction.) if svc.cache != nil { svc.cache.TryDeletePattern(ctx, "image:*:"+imageID) - svc.cache.TryDeletePattern(ctx, "images:*") } c.Status(http.StatusNoContent) @@ -790,7 +886,6 @@ func (svc *Service) UpdateImage(c *gin.Context) { // every project's view of it, not just the patcher's — same as DeleteImage. if svc.cache != nil { svc.cache.TryDeletePattern(c.Request.Context(), "image:*:"+imageID) - svc.cache.TryDeletePattern(c.Request.Context(), "images:*") } // The response reports the applied state (FR-016). @@ -1533,6 +1628,13 @@ func (svc *Service) AddImageTag(c *gin.Context) { return } + // Upstream's schema caps a tag at 255 characters and answers 400; the + // column's VARCHAR(255) would instead surface as a 500. + if utf8.RuneCountInString(tag) > 255 { + common.SendError(c, common.NewBadRequestError("image tag too long (max 255 characters)")) + return + } + // Add tag (ignore if already exists due to UNIQUE constraint) _, err = svc.pool.Exec(c.Request.Context(), ` INSERT INTO image_tags (image_id, tag) @@ -1546,6 +1648,13 @@ func (svc *Service) AddImageTag(c *gin.Context) { return } + // Evict like every other image mutation (UpdateImage, members): GetImage + // caches the object INCLUDING its tags, so without this a tag added here + // stays invisible to `image show` for the cache's whole TTL (SPEC-028). + // evictImage drops every caller's view via the one key family that + // exists (image:::). + svc.evictImage(c.Request.Context(), imageID) + c.Status(http.StatusNoContent) } @@ -1589,6 +1698,10 @@ func (svc *Service) DeleteImageTag(c *gin.Context) { return } + // Evict like every other image mutation — the cached image body carries + // the tags array, and a removal must not linger in `image show` either. + svc.evictImage(c.Request.Context(), imageID) + c.Status(http.StatusNoContent) } diff --git a/internal/glance/spec028_round8_test.go b/internal/glance/spec028_round8_test.go new file mode 100644 index 00000000..bd0676d2 --- /dev/null +++ b/internal/glance/spec028_round8_test.go @@ -0,0 +1,183 @@ +package glance + +// Round-8 delta review pins for PR #107 (issue #105): the tag-mutation cache +// eviction and the name-length validation, both from the round-7 fix set. + +import ( + "context" + "encoding/json" + "net/http" + "strings" + "testing" + "time" + + "github.com/cobaltcore-dev/lightstack/pkg/cache" + "github.com/cobaltcore-dev/lightstack/pkg/storage" + "github.com/gin-gonic/gin" + "github.com/google/uuid" +) + +// newGlanceCacheRouter builds a router whose service carries a REAL cache — +// newGlanceTestRouter passes nil, and the eviction this file pins only exists +// with one behind the service. Skips when no local Redis is available (the +// CI unit-tests job provides one). +func newGlanceCacheRouter(t *testing.T) (*gin.Engine, *cache.Cache) { + t.Helper() + cacheInst, err := cache.NewCache(cache.Config{ + RedisURL: "redis://127.0.0.1:6379", Enabled: true, + KeyPrefix: "spec028-r8glance:", DefaultTTL: time.Minute, + }) + if err != nil { + t.Skipf("no local Redis for the cache regression: %v", err) + } + t.Cleanup(func() { + // DeletePattern prepends the instance's KeyPrefix itself, so the + // pattern is just "*" — a prefixed pattern would double the prefix + // and match nothing (round 9). + cacheInst.DeletePattern(context.Background(), "*") + }) + + gin.SetMode(gin.TestMode) + r := gin.New() + r.Use(func(c *gin.Context) { + if p := c.GetHeader("X-Test-Project"); p != "" { + c.Set("project_id", p) + } + if roles := c.GetHeader("X-Test-Roles"); roles != "" { + c.Set("roles", strings.Split(roles, ",")) + } + }) + svc, err := NewService(testPool, "stub", "", "", storage.S3Options{}, cacheInst) + if err != nil { + t.Fatalf("NewService: %v", err) + } + svc.RegisterRoutes(&r.RouterGroup) + return r, cacheInst +} + +// TestAddImageTagEvictsCache — GetImage caches the object INCLUDING its tags; +// the dedicated tag PUT used to skip the eviction every other image mutation +// performs, so a tag added after a show stayed invisible to `image show` for +// the cache's whole TTL (SPEC-028, round-7 F3). +func TestAddImageTagEvictsCache(t *testing.T) { + requireDB(t) + r, _ := newGlanceCacheRouter(t) + + hdr := map[string]string{"X-Test-Project": "00000000-0000-0000-0000-000000000002"} + // The tag is deliberately NOT a substring of the image name: the first + // version of this test asserted on a tag that matched the name in every + // response, so the check passed with the eviction deleted (round 9). + name := "spec028-r8-addtag-" + uuid.NewString()[:8] + const tag = "round8-evict-pin" + code, body := doReqBody(r, http.MethodPost, "/images", + `{"name":"`+name+`","disk_format":"raw","container_format":"bare"}`, hdr) + if code != http.StatusCreated { + t.Fatalf("create image: code=%d body=%s; want 201", code, body) + } + var created struct { + ID string `json:"id"` + } + if err := json.Unmarshal([]byte(body), &created); err != nil || created.ID == "" { + t.Fatalf("create response carries no image id: %s", body) + } + imageID := created.ID + t.Cleanup(func() { + _, _ = testPool.Exec(context.Background(), "DELETE FROM image_tags WHERE image_id=$1", imageID) + _, _ = testPool.Exec(context.Background(), "DELETE FROM images WHERE id=$1", imageID) + }) + + // Prime the cache, then mutate via the dedicated tag endpoint. + if code, body := doReq(r, http.MethodGet, "/images/"+imageID, hdr); code != http.StatusOK { + t.Fatalf("prime show: code=%d body=%s", code, body) + } + code, body = doReq(r, http.MethodPut, "/images/"+imageID+"/tags/"+tag, hdr) + if code != http.StatusNoContent { + t.Fatalf("add tag: code=%d body=%s; want 204", code, body) + } + + // The follow-up show must see the tag — a stale cache would answer the + // primed body with tags: []. The assertion is the exact single-element + // array: a loose substring could match unrelated response fields. + code, body = doReq(r, http.MethodGet, "/images/"+imageID, hdr) + if code != http.StatusOK { + t.Fatalf("post-tag show: code=%d body=%s", code, body) + } + if !strings.Contains(body, `"tags":["`+tag+`"]`) { + t.Errorf("post-tag show does not see the tag — cache not evicted: %s", body) + } +} + +// TestDeleteImageTagEvictsCache — the same pin for the DELETE half. +func TestDeleteImageTagEvictsCache(t *testing.T) { + requireDB(t) + r, _ := newGlanceCacheRouter(t) + + hdr := map[string]string{"X-Test-Project": "00000000-0000-0000-0000-000000000002"} + name := "spec028-r8-untag-" + uuid.NewString()[:8] + code, body := doReqBody(r, http.MethodPost, "/images", + `{"name":"`+name+`","disk_format":"raw","container_format":"bare","tags":["spec028-r8-gone"]}`, hdr) + if code != http.StatusCreated { + t.Fatalf("create image: code=%d body=%s; want 201", code, body) + } + var created struct { + ID string `json:"id"` + } + if err := json.Unmarshal([]byte(body), &created); err != nil || created.ID == "" { + t.Fatalf("create response carries no image id: %s", body) + } + imageID := created.ID + t.Cleanup(func() { + _, _ = testPool.Exec(context.Background(), "DELETE FROM image_tags WHERE image_id=$1", imageID) + _, _ = testPool.Exec(context.Background(), "DELETE FROM images WHERE id=$1", imageID) + }) + + if code, body := doReq(r, http.MethodGet, "/images/"+imageID, hdr); code != http.StatusOK { + t.Fatalf("prime show: code=%d body=%s", code, body) + } + code, body = doReq(r, http.MethodDelete, "/images/"+imageID+"/tags/spec028-r8-gone", hdr) + if code != http.StatusNoContent { + t.Fatalf("delete tag: code=%d body=%s; want 204", code, body) + } + + code, body = doReq(r, http.MethodGet, "/images/"+imageID, hdr) + if code != http.StatusOK { + t.Fatalf("post-untag show: code=%d body=%s", code, body) + } + // Exact-array form, symmetric with the add-tag pin above. + if strings.Contains(body, `"tags":["spec028-r8-gone"]`) { + t.Errorf("post-untag show still sees the tag — cache not evicted: %s", body) + } +} + +// TestCreateImageNameTooLong — the name column is VARCHAR(255) and upstream's +// schema answers 400; the overflow used to surface as a 500 from the INSERT +// (SPEC-028, round-7 F7). +func TestCreateImageNameTooLong(t *testing.T) { + requireDB(t) + r := newGlanceTestRouter(t) + hdr := map[string]string{"X-Test-Project": "00000000-0000-0000-0000-000000000002"} + + long := strings.Repeat("n", 256) + code, body := doReqBody(r, http.MethodPost, "/images", + `{"name":"`+long+`","disk_format":"raw","container_format":"bare"}`, hdr) + if code != http.StatusBadRequest { + t.Errorf("create with 256-char name: code=%d body=%s; want 400", code, body) + } +} + +func TestImageNameAndTagLengthCountCharacters(t *testing.T) { + requireDB(t) + r := newGlanceTestRouter(t) + hdr := map[string]string{"X-Test-Project": "00000000-0000-0000-0000-000000000002"} + value := strings.Repeat("é", 255) + payload, _ := json.Marshal(map[string]any{"name": value, "disk_format": "raw", "container_format": "bare", "tags": []string{value}}) + code, body := doReqBody(r, http.MethodPost, "/images", string(payload), hdr) + if code != http.StatusCreated { + t.Fatalf("255 Unicode characters: %d %s", code, body) + } + var image map[string]any + if err := json.Unmarshal([]byte(body), &image); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { testPool.Exec(context.Background(), "DELETE FROM images WHERE id=$1", image["id"]) }) +} diff --git a/internal/keystone/application_credentials.go b/internal/keystone/application_credentials.go index 4b536b37..9be79c0e 100644 --- a/internal/keystone/application_credentials.go +++ b/internal/keystone/application_credentials.go @@ -68,15 +68,19 @@ func (svc *Service) ListApplicationCredentials(c *gin.Context) { credential["expires_at"] = expiresAt.Format(time.RFC3339) } - // Get roles + // Get roles. Upstream always includes the key — an empty array when + // the credential has none — so a lookup failure must not drop it + // (SPEC-028). + roles := []map[string]any{} roleRows, err := svc.pool.Query(c.Request.Context(), ` SELECT r.id, r.name FROM application_credential_roles acr JOIN roles r ON acr.role_id = r.id WHERE acr.application_credential_id = $1 `, id) - if err == nil { - roles := []map[string]any{} + if err != nil { + log.Warn().Err(err).Msg("failed to query application credential roles") + } else { for roleRows.Next() { var roleID, roleName string if err := roleRows.Scan(&roleID, &roleName); err == nil { @@ -90,8 +94,8 @@ func (svc *Service) ListApplicationCredentials(c *gin.Context) { log.Warn().Err(err).Msg("roleRows iteration error; list may be truncated") } roleRows.Close() - credential["roles"] = roles } + credential["roles"] = roles credentials = append(credentials, credential) } @@ -158,31 +162,75 @@ func (svc *Service) CreateApplicationCredential(c *gin.Context) { } } - _, err = svc.pool.Exec(c.Request.Context(), ` + // Resolve the requested roles BEFORE writing anything. A role entry may + // name an id or a name (upstream accepts both); the old code type-asserted + // role["id"] unconditionally, so a role given by name only panicked — + // recovered by gin as a 500 after the credential row was already committed + // (SPEC-028). Unresolvable roles are the caller's error now, and nothing + // is written. + type resolvedRole struct{ id, name string } + var resolved []resolvedRole + seenRoles := map[string]bool{} + for _, role := range req.ApplicationCredential.Roles { + idStr, _ := role["id"].(string) + nameStr, _ := role["name"].(string) + var rid, rname string + var err error + switch { + case idStr != "": + err = svc.pool.QueryRow(c.Request.Context(), + "SELECT id::text, name FROM roles WHERE id = $1", idStr).Scan(&rid, &rname) + case nameStr != "": + err = svc.pool.QueryRow(c.Request.Context(), + "SELECT id::text, name FROM roles WHERE name = $1", nameStr).Scan(&rid, &rname) + default: + common.SendError(c, common.NewBadRequestError("each role must have an id or name")) + return + } + if err != nil { + common.SendError(c, common.NewBadRequestError("application credential references a role that does not exist")) + return + } + if !seenRoles[rid] { + resolved = append(resolved, resolvedRole{rid, rname}) + seenRoles[rid] = true + } + } + + err = database.WithTx(c.Request.Context(), svc.pool, func(tx database.Tx) error { + _, err := tx.Exec(c.Request.Context(), ` INSERT INTO application_credentials (id, user_id, project_id, name, secret_hash, description, expires_at, unrestricted, created_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) `, credID, userID, projectID, req.ApplicationCredential.Name, string(secretHash), req.ApplicationCredential.Description, expiresAt, req.ApplicationCredential.Unrestricted, now) + if err != nil { + return err + } + for _, role := range resolved { + if _, err := tx.Exec(c.Request.Context(), ` + INSERT INTO application_credential_roles (application_credential_id, role_id) + VALUES ($1, $2) + `, credID, role.id); err != nil { + return err + } + } + + return nil + }) + if err != nil { + // application_credentials is UNIQUE(user_id, name); a duplicate name + // is the caller's conflict, not a server fault — upstream answers + // 409 (SPEC-028). + if database.IsUniqueViolation(err) { + common.SendError(c, common.NewConflictError("application credential already exists")) + return + } log.Error().Err(err).Str("operation", "create_application_credential").Str("user_id", userID).Msg("Failed to create application credential") common.SendError(c, common.NewInternalServerError("failed to create application credential")) return } - // Associate roles. A failed role insert is logged but doesn't - // abort: the credential row already exists; missing roles will - // limit what it can do but the API caller still gets a working - // credential they can re-grant manually. - for _, role := range req.ApplicationCredential.Roles { - roleID := role["id"].(string) - if _, err := svc.pool.Exec(c.Request.Context(), ` - INSERT INTO application_credential_roles (application_credential_id, role_id) - VALUES ($1, $2) - `, credID, roleID); err != nil { - log.Warn().Err(err).Str("credential_id", credID.String()).Str("role_id", roleID).Msg("failed to associate role with application credential") - } - } - credential := map[string]any{ "id": credID.String(), "user_id": userID, @@ -201,10 +249,16 @@ func (svc *Service) CreateApplicationCredential(c *gin.Context) { credential["expires_at"] = expiresAt.(time.Time).Format(time.RFC3339) } - // Add roles to response - if len(req.ApplicationCredential.Roles) > 0 { - credential["roles"] = req.ApplicationCredential.Roles + // Upstream always includes roles (an empty array when none were + // requested) — the CLI's formatter requires the key (SPEC-028). + responseRoles := []map[string]any{} + for _, role := range resolved { + responseRoles = append(responseRoles, map[string]any{ + "id": role.id, + "name": role.name, + }) } + credential["roles"] = responseRoles c.JSON(201, gin.H{"application_credential": credential}) } @@ -261,15 +315,19 @@ func (svc *Service) GetApplicationCredential(c *gin.Context) { credential["expires_at"] = expiresAt.Format(time.RFC3339) } - // Get roles + // Get roles. Upstream always includes the key — an empty array when + // the credential has none — so a lookup failure must not drop it + // (SPEC-028). + roles := []map[string]any{} roleRows, err := svc.pool.Query(c.Request.Context(), ` SELECT r.id, r.name FROM application_credential_roles acr JOIN roles r ON acr.role_id = r.id WHERE acr.application_credential_id = $1 `, id) - if err == nil { - roles := []map[string]any{} + if err != nil { + log.Warn().Err(err).Msg("failed to query application credential roles") + } else { for roleRows.Next() { var roleID, roleName string if err := roleRows.Scan(&roleID, &roleName); err == nil { @@ -283,8 +341,8 @@ func (svc *Service) GetApplicationCredential(c *gin.Context) { log.Warn().Err(err).Msg("roleRows iteration error; list may be truncated") } roleRows.Close() - credential["roles"] = roles } + credential["roles"] = roles c.JSON(200, gin.H{"application_credential": credential}) } @@ -371,15 +429,19 @@ func (svc *Service) GetApplicationCredentialByID(c *gin.Context) { credential["expires_at"] = expiresAt.Format(time.RFC3339) } - // Get roles + // Get roles. Upstream always includes the key — an empty array when + // the credential has none — so a lookup failure must not drop it + // (SPEC-028). + roles := []map[string]any{} roleRows, err := svc.pool.Query(c.Request.Context(), ` SELECT r.id, r.name FROM application_credential_roles acr JOIN roles r ON acr.role_id = r.id WHERE acr.application_credential_id = $1 `, id) - if err == nil { - roles := []map[string]any{} + if err != nil { + log.Warn().Err(err).Msg("failed to query application credential roles") + } else { for roleRows.Next() { var roleID, roleName string if err := roleRows.Scan(&roleID, &roleName); err == nil { @@ -393,8 +455,8 @@ func (svc *Service) GetApplicationCredentialByID(c *gin.Context) { log.Warn().Err(err).Msg("roleRows iteration error; list may be truncated") } roleRows.Close() - credential["roles"] = roles } + credential["roles"] = roles c.JSON(200, gin.H{"application_credential": credential}) } diff --git a/internal/keystone/domains.go b/internal/keystone/domains.go index 7d4c9e80..6036039e 100644 --- a/internal/keystone/domains.go +++ b/internal/keystone/domains.go @@ -62,6 +62,11 @@ func (svc *Service) ListDomains(c *gin.Context) { if description != nil { domain["description"] = *description } + // See CreateDomain: links.self is mandatory for the CLI's formatter. + // This list is also how the CLI resolves a domain BY NAME (show and + // delete fall back to ?name= after the by-id GET 404s), so every item + // needs it, not just the direct GET (SPEC-028). + domain["links"] = identityLink(c, "domains", id.String()) domains = append(domains, domain) } @@ -126,6 +131,9 @@ func (svc *Service) CreateDomain(c *gin.Context) { if description != nil { domain["description"] = *description } + // Upstream always embeds links.self; the pinned OSC 7 formatter aborts + // with KeyError 'links' without it (SPEC-028). + domain["links"] = identityLink(c, "domains", domainID.String()) c.JSON(201, gin.H{"domain": domain}) } @@ -181,6 +189,8 @@ func (svc *Service) GetDomain(c *gin.Context) { if description != nil { domain["description"] = *description } + // See CreateDomain: links.self is mandatory for the CLI's formatter. + domain["links"] = identityLink(c, "domains", id.String()) c.JSON(200, gin.H{"domain": domain}) } diff --git a/internal/keystone/handlers.go b/internal/keystone/handlers.go index b7153adf..04146862 100644 --- a/internal/keystone/handlers.go +++ b/internal/keystone/handlers.go @@ -4,6 +4,7 @@ import ( "database/sql" "fmt" "net/http" + "strings" "time" "github.com/cobaltcore-dev/lightstack/internal/database" @@ -34,6 +35,14 @@ func NewService(pool database.DB, authService *AuthService, cacheInstance *cache // RegisterRoutes registers Keystone routes. // Public routes (auth) require no token. Management routes require a valid token. func (svc *Service) RegisterRoutes(r *gin.RouterGroup, authMiddleware gin.HandlerFunc) { + // Keystone's error envelope is {"error": {...}} (see + // common/identity_errors.go): keystoneauth1 — the client behind every + // identity command — parses only that shape, and every other body + // degrades to "Unrecognized schema in response body." Marked once here so + // every route below, plus the auth middleware and panic recovery inside, + // renders it. + r.Use(common.MarkIdentityErrors) + v3 := r.Group("/v3") // Version discovery — no auth required @@ -321,8 +330,20 @@ func (svc *Service) ValidateToken(c *gin.Context) { // RevokeToken revokes a token (DELETE /v3/auth/tokens) func (svc *Service) RevokeToken(c *gin.Context) { - // In JWT implementation, we don't maintain token blacklist - // Tokens expire naturally based on TTL + // Upstream validates the subject token and answers 404 when it names + // nothing revocable; the previous unconditional 204 hid typos and + // expired ids, so clients could not detect them (SPEC-028). Tokens are + // JWTs that expire naturally, so revoking a valid token is acknowledged + // with 204 — there is no server-side session to destroy. + tokenString := c.GetHeader("X-Subject-Token") + if tokenString == "" { + common.SendError(c, common.NewBadRequestError("X-Subject-Token header required")) + return + } + if _, err := svc.authService.ValidateToken(tokenString); err != nil { + common.SendError(c, common.NewIdentityNotFoundError("token", "requested")) + return + } c.Status(http.StatusNoContent) } @@ -376,7 +397,7 @@ func (svc *Service) ListAuthProjects(c *gin.Context) { projects = append(projects, gin.H{ "id": id, "name": name, - "description": description.String, + "description": common.NullString(description), "enabled": enabled, "domain_id": domainID.String, "links": identityLink(c, "projects", id), @@ -411,11 +432,11 @@ func (svc *Service) ListUsers(c *gin.Context) { f.eqDomain(c, "domain_id", "domain_id") f.eqBool(c, "enabled", "enabled") rows, err := svc.pool.Query(c.Request.Context(), - "SELECT id, name, enabled, domain_id FROM users"+f.Where()+" ORDER BY name", f.Args()..., + "SELECT id, name, enabled, domain_id, email, description FROM users"+f.Where()+" ORDER BY name", f.Args()..., ) if err != nil { log.Error().Err(err).Str("operation", "list_users").Msg("Failed to query users") - common.SendError(c, common.NewInternalServerError("failed to query users")) + common.SendError(c, common.NewInternalServerError("failed to list users")) return } defer rows.Close() @@ -424,17 +445,25 @@ func (svc *Service) ListUsers(c *gin.Context) { for rows.Next() { var id, name, domainID string var enabled bool - if err := rows.Scan(&id, &name, &enabled, &domainID); err != nil { + // email and description are nullable; scanning them into plain + // strings would turn a user without either into a 500 for the whole + // collection (SPEC-028: both were also missing from the response + // entirely, so 'user show' printed email: null for a user created + // with one). + var email, description sql.NullString + if err := rows.Scan(&id, &name, &enabled, &domainID, &email, &description); err != nil { log.Error().Err(err).Str("operation", "list_users_scan").Msg("Failed to scan user row") common.SendError(c, common.NewInternalServerError("failed to read users")) return } users = append(users, gin.H{ - "id": id, - "name": name, - "enabled": enabled, - "domain_id": domainID, - "links": identityLink(c, "users", id), + "id": id, + "name": name, + "enabled": enabled, + "domain_id": domainID, + "email": common.NullString(email), + "description": common.NullString(description), + "links": identityLink(c, "users", id), }) } if err := rows.Err(); err != nil { @@ -459,10 +488,13 @@ func (svc *Service) GetUser(c *gin.Context) { var id, name, domainID string var enabled bool + // Nullable for the same reason as in ListUsers; SPEC-028: they were + // dropped entirely from the show response. + var email, description sql.NullString err := svc.pool.QueryRow(c.Request.Context(), - "SELECT id, name, enabled, domain_id FROM users WHERE id = $1", + "SELECT id, name, enabled, domain_id, email, description FROM users WHERE id = $1", userID, - ).Scan(&id, &name, &enabled, &domainID) + ).Scan(&id, &name, &enabled, &domainID, &email, &description) if err != nil { notFoundOr500(c, err, "user", userID) @@ -470,11 +502,13 @@ func (svc *Service) GetUser(c *gin.Context) { } c.JSON(http.StatusOK, gin.H{"user": gin.H{ - "id": id, - "name": name, - "enabled": enabled, - "domain_id": domainID, - "links": identityLink(c, "users", id), + "id": id, + "name": name, + "enabled": enabled, + "domain_id": domainID, + "email": common.NullString(email), + "description": common.NullString(description), + "links": identityLink(c, "users", id), }}) } @@ -526,7 +560,7 @@ func (svc *Service) ListProjects(c *gin.Context) { projects = append(projects, gin.H{ "id": id, "name": name, - "description": description.String, + "description": common.NullString(description), "enabled": enabled, "domain_id": domainID.String, "links": identityLink(c, "projects", id), @@ -588,7 +622,7 @@ func (svc *Service) GetProject(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"project": gin.H{ "id": id, "name": name, - "description": description.String, + "description": common.NullString(description), "enabled": enabled, "domain_id": domainID.String, "links": identityLink(c, "projects", id), @@ -745,7 +779,7 @@ func (svc *Service) UpdateProject(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"project": gin.H{ "id": projectID, "name": name, - "description": description.String, + "description": common.NullString(description), "domain_id": domainID.String, "enabled": enabled, "links": identityLink(c, "projects", projectID), @@ -895,6 +929,12 @@ func (svc *Service) CreateRole(c *gin.Context) { roleID, req.Role.Name, now, ) if err != nil { + // roles.name is UNIQUE; a duplicate is the caller's conflict, not a + // server fault — upstream answers 409 (SPEC-028). + if database.IsUniqueViolation(err) { + common.SendError(c, common.NewConflictError("role already exists")) + return + } log.Error().Err(err).Str("operation", "create_role").Msg("Failed to create role") common.SendError(c, common.NewInternalServerError("failed to create role")) return @@ -1198,39 +1238,78 @@ func (svc *Service) ListRoleAssignments(c *gin.Context) { f.eqID(c, "scope.project.id", "ra.project_id") f.eqID(c, "role.id", "ra.role_id") - query := `SELECT ra.user_id, ra.project_id, ra.role_id, r.name as role_name + // include_names=true embeds entity names alongside the ids, the shape + // upstream returns and the one `role assignment list --names` (and + // openstacksdk's name-normalizing consumer) requires: user.name, + // user.domain.{id,name}, scope.project.name, scope.project.domain.{id,name}, + // role.name. Without it only ids are emitted, as before (SPEC-028). + includeNames := strings.EqualFold(c.Query("include_names"), "true") + + query := `SELECT ra.user_id, ra.project_id, ra.role_id, r.name FROM role_assignments ra - INNER JOIN roles r ON ra.role_id = r.id` + f.Where() + INNER JOIN roles r ON ra.role_id = r.id` + if includeNames { + query = `SELECT ra.user_id, u.name, ud.id, ud.name, + ra.project_id, p.name, pd.id, pd.name, + ra.role_id, r.name + FROM role_assignments ra + INNER JOIN roles r ON ra.role_id = r.id + LEFT JOIN users u ON ra.user_id = u.id + LEFT JOIN domains ud ON u.domain_id = ud.id + LEFT JOIN projects p ON ra.project_id = p.id + LEFT JOIN domains pd ON p.domain_id = pd.id` + } + query += f.Where() rows, err := svc.pool.Query(c.Request.Context(), query, f.Args()...) if err != nil { log.Error().Err(err).Str("operation", "list_role_assignments").Msg("Failed to query role assignments") - common.SendError(c, common.NewInternalServerError("failed to query role assignments")) + common.SendError(c, common.NewInternalServerError("failed to list role assignments")) return } defer rows.Close() + baseURL := common.BaseURL(c, 35357) var assignments []gin.H for rows.Next() { - var uid, pid, rid, rname string - if err := rows.Scan(&uid, &pid, &rid, &rname); err != nil { + var uid, pid, rid, rname sql.NullString + var uname, udomID, udomName, pname, pdomID, pdomName sql.NullString + if includeNames { + if err := rows.Scan(&uid, &uname, &udomID, &udomName, + &pid, &pname, &pdomID, &pdomName, &rid, &rname); err != nil { + log.Warn().Err(err).Msg("failed to scan role assignment row") + continue + } + } else if err := rows.Scan(&uid, &pid, &rid, &rname); err != nil { log.Warn().Err(err).Msg("failed to scan role assignment row") continue } - assignments = append(assignments, gin.H{ - "user": gin.H{ - "id": uid, - }, - "scope": gin.H{ - "project": gin.H{ - "id": pid, - }, - }, + + user := gin.H{"id": uid.String} + project := gin.H{"id": pid.String} + if includeNames { + user["name"] = uname.String + user["domain"] = gin.H{"id": udomID.String, "name": udomName.String} + project["name"] = pname.String + project["domain"] = gin.H{"id": pdomID.String, "name": pdomName.String} + } + assignment := gin.H{ + "user": user, + "scope": gin.H{"project": project}, "role": gin.H{ - "id": rid, - "name": rname, + "id": rid.String, + "name": rname.String, }, - }) + } + // The self link names the user and project; a row without either + // (neither is creatable through the API today) would render empty + // path segments, so it gets no link at all. + if uid.Valid && pid.Valid && rid.Valid { + assignment["links"] = gin.H{ + "assignment": baseURL + "/v3/projects/" + pid.String + "/users/" + uid.String + "/roles/" + rid.String, + } + } + assignments = append(assignments, assignment) } if err := rows.Err(); err != nil { log.Error().Err(err).Msg("rows iteration error") @@ -1630,7 +1709,7 @@ func (svc *Service) GetUserProjects(c *gin.Context) { projects = append(projects, gin.H{ "id": id, "name": name, - "description": description.String, + "description": common.NullString(description), "enabled": enabled, "domain_id": domainID.String, "links": identityLink(c, "projects", id), diff --git a/internal/keystone/pr107_review_test.go b/internal/keystone/pr107_review_test.go new file mode 100644 index 00000000..bc940d23 --- /dev/null +++ b/internal/keystone/pr107_review_test.go @@ -0,0 +1,75 @@ +package keystone + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "strings" + "testing" + + "github.com/cobaltcore-dev/lightstack/internal/database" + "github.com/gin-gonic/gin" + "github.com/google/uuid" +) + +type failCredentialDB struct{ database.DB } +type failCredentialTx struct{ database.Tx } + +func (d failCredentialDB) Begin(ctx context.Context) (database.Tx, error) { + tx, err := d.DB.Begin(ctx) + return failCredentialTx{tx}, err +} +func (tx failCredentialTx) Exec(ctx context.Context, query string, args ...any) (database.Result, error) { + if strings.Contains(query, "INSERT INTO application_credential_roles") { + return nil, errors.New("injected role failure") + } + return tx.Tx.Exec(ctx, query, args...) +} + +func TestCredentialRolesAreAtomic(t *testing.T) { + requireDB(t) + ctx := context.Background() + userID, roleID, name := uuid.NewString(), uuid.NewString(), uuid.NewString() + if _, err := testPool.Exec(ctx, "INSERT INTO users (id,name,password_hash,domain_id) VALUES ($1,$2,'x',$3)", userID, userID, defaultDomainID); err != nil { + t.Fatal(err) + } + if _, err := testPool.Exec(ctx, "INSERT INTO roles (id,name) VALUES ($1,$2)", roleID, roleID); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + testPool.Exec(ctx, "DELETE FROM application_credentials WHERE user_id=$1", userID) + testPool.Exec(ctx, "DELETE FROM roles WHERE id=$1", roleID) + testPool.Exec(ctx, "DELETE FROM users WHERE id=$1", userID) + }) + svc := NewService(failCredentialDB{testPool}, nil, nil) + r := gin.New() + r.POST("/users/:id/application_credentials", func(c *gin.Context) { c.Set("user_id", userID) }, svc.CreateApplicationCredential) + payload := func(roles []map[string]string) string { + b, err := json.Marshal(map[string]any{"application_credential": map[string]any{"name": name, "roles": roles}}) + if err != nil { + t.Fatal(err) + } + return string(b) + } + path := "/users/" + userID + "/application_credentials" + body := payload([]map[string]string{{"id": roleID}}) + if code, resp := doReqBody(r, http.MethodPost, path, body, nil); code != 500 { + t.Fatalf("injected role failure: %d %s", code, resp) + } + var count int + if err := testPool.QueryRow(ctx, "SELECT COUNT(*) FROM application_credentials WHERE user_id=$1", userID).Scan(&count); err != nil || count != 0 { + t.Fatalf("credential survived failed role write: %d %v", count, err) + } + svc.pool = testPool + if code, resp := doReqBody(r, http.MethodPost, path, payload([]map[string]string{{}}), nil); code != 400 { + t.Fatalf("malformed role: %d %s", code, resp) + } + body = payload([]map[string]string{{"id": roleID}, {"name": roleID}}) + if code, resp := doReqBody(r, http.MethodPost, path, body, nil); code != 201 { + t.Fatalf("role aliases: %d %s", code, resp) + } + if err := testPool.QueryRow(ctx, "SELECT COUNT(*) FROM application_credential_roles WHERE role_id=$1", roleID).Scan(&count); err != nil || count != 1 { + t.Fatalf("stored roles: %d %v", count, err) + } +} diff --git a/internal/keystone/spec028_conformance_test.go b/internal/keystone/spec028_conformance_test.go new file mode 100644 index 00000000..12229574 --- /dev/null +++ b/internal/keystone/spec028_conformance_test.go @@ -0,0 +1,426 @@ +package keystone + +// SPEC-028 conformance fixes for the identity surface, found by the CLI +// conformance suites (issue #105). + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/golang-jwt/jwt/v5" + "github.com/google/uuid" +) + +// doReqBody is doReq with a request body, for the POST paths these tests drive. +func doReqBody(r *gin.Engine, method, path, body string, headers map[string]string) (int, string) { + var req *http.Request + if body == "" { + req = httptest.NewRequest(method, path, nil) + } else { + req = httptest.NewRequest(method, path, strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + } + for k, v := range headers { + req.Header.Set(k, v) + } + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + return w.Code, w.Body.String() +} + +// spec028Router wires the real routes over the test pool with a real +// AuthService, for the paths (token revoke) that need one. +func spec028Router() (*gin.Engine, *AuthService) { + gin.SetMode(gin.TestMode) + r := gin.New() + authSvc := NewAuthService(testPool, "spec028-test-secret", time.Hour, nil) + NewService(testPool, authSvc, nil).RegisterRoutes(&r.RouterGroup, func(c *gin.Context) { + if roles := c.GetHeader("X-Test-Roles"); roles != "" { + c.Set("roles", strings.Split(roles, ",")) + } + if uid := c.GetHeader("X-Test-User"); uid != "" { + c.Set("user_id", uid) + } + c.Next() + }) + return r, authSvc +} + +// TestUserShowCarriesEmailAndDescription — the users table has both columns, +// create/set accept both, but the show and list queries selected neither, so +// `openstack user show` printed email: null for a user created with one +// (SPEC-028). +func TestUserShowCarriesEmailAndDescription(t *testing.T) { + requireDB(t) + r, _ := spec028Router() + + userID := uuid.NewString() + name := "spec028-email-" + userID[:8] + if _, err := testPool.Exec(context.Background(), + `INSERT INTO users (id, name, password_hash, domain_id, email, description) + VALUES ($1,$2,'x',$3,'spec028@example.com','spec028 description')`, + userID, name, defaultDomainID); err != nil { + t.Fatalf("seed user: %v", err) + } + t.Cleanup(func() { testPool.Exec(context.Background(), "DELETE FROM users WHERE id=$1", userID) }) + + code, body := doReq(r, http.MethodGet, "/v3/users/"+userID, adminHdr) + if code != http.StatusOK { + t.Fatalf("show user: code=%d body=%s", code, body) + } + if !strings.Contains(body, `"email":"spec028@example.com"`) { + t.Errorf("show response lacks the persisted email: %s", body) + } + if !strings.Contains(body, `"description":"spec028 description"`) { + t.Errorf("show response lacks the persisted description: %s", body) + } + + _, body = doReq(r, http.MethodGet, "/v3/users?name="+name, adminHdr) + if !strings.Contains(body, `"email":"spec028@example.com"`) { + t.Errorf("list response lacks the persisted email: %s", body) + } +} + +// TestRoleCreateDuplicateIsConflict — roles.name is UNIQUE; a duplicate name +// was surfaced as 500 instead of upstream's 409 (SPEC-028). +func TestRoleCreateDuplicateIsConflict(t *testing.T) { + requireDB(t) + r, _ := spec028Router() + + name := "spec028-dup-role-" + uuid.NewString()[:8] + body := `{"role":{"name":"` + name + `"}}` + t.Cleanup(func() { testPool.Exec(context.Background(), "DELETE FROM roles WHERE name=$1", name) }) + + code, resp := doReqBody(r, http.MethodPost, "/v3/roles", body, adminHdr) + if code != http.StatusCreated { + t.Fatalf("first create: code=%d body=%s", code, resp) + } + code, resp = doReqBody(r, http.MethodPost, "/v3/roles", body, adminHdr) + if code != http.StatusConflict { + t.Errorf("duplicate create: code=%d body=%s; want 409", code, resp) + } +} + +// TestRoleAssignmentIncludeNames — `GET /v3/role_assignments` ignored +// include_names=true and returned bare ids; openstacksdk's consumer dies with +// KeyError 'name', so `role assignment list --names` exited 1 (SPEC-028). +func TestRoleAssignmentIncludeNames(t *testing.T) { + requireDB(t) + r, _ := spec028Router() + ctx := context.Background() + domID := defaultDomainID + + userID, projectID := uuid.NewString(), uuid.NewString() + roleID := uuid.NewString() + roleName := "spec028-ra-role-" + roleID[:8] + for _, ins := range []struct { + sql string + args []any + }{ + {`INSERT INTO users (id, name, password_hash, domain_id) VALUES ($1,$2,'x',$3)`, + []any{userID, "spec028-ra-user-" + userID[:8], domID}}, + {`INSERT INTO projects (id, name, domain_id) VALUES ($1,$2,$3)`, + []any{projectID, "spec028-ra-proj-" + projectID[:8], domID}}, + {`INSERT INTO roles (id, name) VALUES ($1,$2)`, []any{roleID, roleName}}, + {`INSERT INTO role_assignments (user_id, project_id, role_id) VALUES ($1,$2,$3)`, + []any{userID, projectID, roleID}}, + } { + if _, err := testPool.Exec(ctx, ins.sql, ins.args...); err != nil { + t.Fatalf("seed: %v", err) + } + } + t.Cleanup(func() { + testPool.Exec(ctx, "DELETE FROM role_assignments WHERE user_id=$1", userID) + testPool.Exec(ctx, "DELETE FROM roles WHERE id=$1", roleID) + testPool.Exec(ctx, "DELETE FROM projects WHERE id=$1", projectID) + testPool.Exec(ctx, "DELETE FROM users WHERE id=$1", userID) + }) + + // Filtered to the seeded assignment so the shared database's other rows + // cannot make the assertions below flaky. + code, body := doReq(r, http.MethodGet, "/v3/role_assignments?include_names=true&user.id="+userID, adminHdr) + if code != http.StatusOK { + t.Fatalf("list: code=%d body=%s", code, body) + } + var resp struct { + RoleAssignments []struct { + User struct { + ID string `json:"id"` + Name string `json:"name"` + Domain struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"domain"` + } `json:"user"` + Scope struct { + Project struct { + ID string `json:"id"` + Name string `json:"name"` + Domain struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"domain"` + } `json:"project"` + } `json:"scope"` + Role struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"role"` + } `json:"role_assignments"` + } + if err := json.Unmarshal([]byte(body), &resp); err != nil { + t.Fatalf("unmarshal: %v (body=%s)", err, body) + } + if len(resp.RoleAssignments) != 1 { + t.Fatalf("want exactly the seeded assignment, got %d: %s", len(resp.RoleAssignments), body) + } + a := resp.RoleAssignments[0] + if a.User.Name == "" || a.Scope.Project.Name == "" || a.Role.Name != roleName || + a.User.Domain.Name != "Default" || a.Scope.Project.Domain.Name != "Default" { + t.Errorf("include_names did not embed names: user=%+v scope=%+v role=%+v", a.User, a.Scope, a.Role) + } +} + +// TestDomainResponsesCarryLinks — POST and GET /v3/domains omitted `links`; +// the pinned OSC 7 formatter aborts with KeyError 'links' (SPEC-028). +func TestDomainResponsesCarryLinks(t *testing.T) { + requireDB(t) + r, _ := spec028Router() + + name := "spec028-links-" + uuid.NewString()[:8] + t.Cleanup(func() { testPool.Exec(context.Background(), "DELETE FROM domains WHERE name=$1", name) }) + + code, body := doReqBody(r, http.MethodPost, "/v3/domains", + `{"domain":{"name":"`+name+`","description":"spec028 links"}}`, adminHdr) + if code != http.StatusCreated { + t.Fatalf("create domain: code=%d body=%s", code, body) + } + if !strings.Contains(body, `"links":{"self":`) { + t.Errorf("create response lacks links.self: %s", body) + } + var id string + if err := testPool.QueryRow(context.Background(), + "SELECT id::text FROM domains WHERE name=$1", name).Scan(&id); err != nil { + t.Fatalf("resolve domain: %v", err) + } + + code, body = doReq(r, http.MethodGet, "/v3/domains/"+id, adminHdr) + if code != http.StatusOK { + t.Fatalf("show domain: code=%d body=%s", code, body) + } + if !strings.Contains(body, `"links":{"self":`) { + t.Errorf("show response lacks links.self: %s", body) + } + + // The list is how the CLI resolves a domain BY NAME (show and delete fall + // back to ?name= after the by-id GET 404s), so every item needs links + // too — the plain list answered without them, and 'domain show ' + // and 'domain delete ' aborted with KeyError 'links'. + code, body = doReq(r, http.MethodGet, "/v3/domains", adminHdr) + if code != http.StatusOK { + t.Fatalf("list domains: code=%d body=%s", code, body) + } + if !strings.Contains(body, `"links":{"self":`) { + t.Errorf("domain list items lack links.self: %s", body) + } + code, body = doReq(r, http.MethodGet, "/v3/domains?name="+name, adminHdr) + if code != http.StatusOK { + t.Fatalf("list domains by name: code=%d body=%s", code, body) + } + var filtered struct { + Domains []struct { + Links map[string]string `json:"links"` + } `json:"domains"` + } + if err := json.Unmarshal([]byte(body), &filtered); err != nil { + t.Fatalf("unmarshal name-filtered list: %v (body=%s)", err, body) + } + if len(filtered.Domains) != 1 || filtered.Domains[0].Links["self"] == "" { + t.Errorf("name-filtered list item lacks links.self: %s", body) + } +} + +// TestApplicationCredentialConformance — three defects in one surface +// (SPEC-028): +// - a role given by NAME ONLY panicked on a nil type assertion (recovered +// by gin as 500, after the credential row was already committed); +// - a duplicate (user_id, name) answered 500 instead of 409; +// - list/show omitted the `roles` key entirely when the roles query failed +// — which it always did, because no migration ever created +// application_credential_roles — and the pinned OSC 7 formatter crashes +// on the missing key. +func TestApplicationCredentialConformance(t *testing.T) { + requireDB(t) + r, _ := spec028Router() + ctx := context.Background() + + userID := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO users (id, name, password_hash, domain_id) VALUES ($1,$2,'x',$3)`, + userID, "spec028-ac-user-"+userID[:8], defaultDomainID); err != nil { + t.Fatalf("seed user: %v", err) + } + roleID := uuid.NewString() + roleName := "spec028-ac-role-" + roleID[:8] + if _, err := testPool.Exec(ctx, "INSERT INTO roles (id, name) VALUES ($1,$2)", roleID, roleName); err != nil { + t.Fatalf("seed role: %v", err) + } + t.Cleanup(func() { + testPool.Exec(ctx, "DELETE FROM application_credentials WHERE user_id=$1", userID) + testPool.Exec(ctx, "DELETE FROM users WHERE id=$1", userID) + testPool.Exec(ctx, "DELETE FROM roles WHERE id=$1", roleID) + }) + + // Role by name only: 201 (was a panic → 500), with the role resolved to + // its id and name, and actually persisted. + code, body := doReqBody(r, http.MethodPost, "/v3/users/"+userID+"/application_credentials", + `{"application_credential":{"name":"spec028-ac-byname","roles":[{"name":"`+roleName+`"}]}}`, adminHdr) + if code != http.StatusCreated { + t.Fatalf("create with role by name: code=%d body=%s; want 201 (was a panic/500)", code, body) + } + if !strings.Contains(body, `"roles":[{"id":"`+roleID+`","name":"`+roleName+`"}]`) { + t.Errorf("create response did not resolve the role: %s", body) + } + var n int + if err := testPool.QueryRow(ctx, + "SELECT count(*) FROM application_credential_roles WHERE role_id=$1", roleID).Scan(&n); err != nil { + t.Fatalf("count role rows: %v", err) + } + if n != 1 { + t.Errorf("role association persisted: count=%d; want 1", n) + } + + // Duplicate name for the same user: 409 (was 500). + code, body = doReqBody(r, http.MethodPost, "/v3/users/"+userID+"/application_credentials", + `{"application_credential":{"name":"spec028-ac-byname"}}`, adminHdr) + if code != http.StatusConflict { + t.Errorf("duplicate create: code=%d body=%s; want 409", code, body) + } + + // List and show always carry roles (empty array when none). + code, body = doReq(r, http.MethodGet, "/v3/users/"+userID+"/application_credentials", adminHdr) + if code != http.StatusOK { + t.Fatalf("list: code=%d body=%s", code, body) + } + if !strings.Contains(body, `"roles":[`) { + t.Errorf("list response lacks the roles key: %s", body) + } + var credID string + if err := testPool.QueryRow(ctx, + "SELECT id::text FROM application_credentials WHERE user_id=$1 AND name='spec028-ac-byname'", userID). + Scan(&credID); err != nil { + t.Fatalf("resolve credential: %v", err) + } + code, body = doReq(r, http.MethodGet, "/v3/users/"+userID+"/application_credentials/"+credID, adminHdr) + if code != http.StatusOK { + t.Fatalf("show: code=%d body=%s", code, body) + } + if !strings.Contains(body, `"roles":[{"id":"`+roleID+`","name":"`+roleName+`"}]`) { + t.Errorf("show response must carry the persisted role: %s", body) + } +} + +// TestTokenRevokeUnknownToken — DELETE /v3/auth/tokens answered 204 +// unconditionally; upstream validates the subject token and answers 404 when +// it names nothing revocable, so typos and expired ids were invisible to +// clients (SPEC-028). +func TestTokenRevokeUnknownToken(t *testing.T) { + requireDB(t) + r, authSvc := spec028Router() + + // Garbage subject token: 404, not 204. + code, body := doReq(r, http.MethodDelete, "/v3/auth/tokens", + map[string]string{"X-Subject-Token": "not-a-token"}) + if code != http.StatusNotFound { + t.Errorf("revoke garbage token: code=%d body=%s; want 404", code, body) + } + + // Missing subject token: 400. + code, _ = doReq(r, http.MethodDelete, "/v3/auth/tokens", map[string]string{}) + if code != http.StatusBadRequest { + t.Errorf("revoke without subject token: code=%d; want 400", code) + } + + // A well-formed, validly signed, unexpired token: revoked with 204. + tok, err := jwt.NewWithClaims(jwt.SigningMethodHS256, &TokenClaims{ + UserID: "00000000-0000-0000-0000-00000000a0a0", + RegisteredClaims: jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Hour)), + }, + }).SignedString([]byte("spec028-test-secret")) + if err != nil { + t.Fatalf("mint token: %v", err) + } + if _, err := authSvc.ValidateToken(tok); err != nil { + t.Fatalf("minted token must validate: %v", err) + } + code, body = doReq(r, http.MethodDelete, "/v3/auth/tokens", + map[string]string{"X-Subject-Token": tok}) + if code != http.StatusNoContent { + t.Errorf("revoke valid token: code=%d body=%s; want 204", code, body) + } +} + +// TestIdentityErrorEnvelopeIsKeystoneShaped — keystoneauth1's from_response +// parses only {"error": {...}} (or the API-SIG {"errors": [...]}) bodies; +// every other JSON shape degrades to "Unrecognized schema in response body.", +// throwing away the reason. Upstream Keystone renders code/message/title, so +// 'domain delete ' must say why it refused (SPEC-028). +func TestIdentityErrorEnvelopeIsKeystoneShaped(t *testing.T) { + requireDB(t) + r, _ := spec028Router() + + name := "spec028-envelope-" + uuid.NewString()[:8] + t.Cleanup(func() { testPool.Exec(context.Background(), "DELETE FROM domains WHERE name=$1", name) }) + + code, body := doReqBody(r, http.MethodPost, "/v3/domains", + `{"domain":{"name":"`+name+`"}}`, adminHdr) + if code != http.StatusCreated { + t.Fatalf("create domain: code=%d body=%s", code, body) + } + + // Deleting an ENABLED domain is upstream's 403 — in upstream's envelope. + var id string + if err := testPool.QueryRow(context.Background(), + "SELECT id::text FROM domains WHERE name=$1", name).Scan(&id); err != nil { + t.Fatalf("resolve domain: %v", err) + } + code, body = doReq(r, http.MethodDelete, "/v3/domains/"+id, adminHdr) + if code != http.StatusForbidden { + t.Fatalf("delete enabled domain: code=%d body=%s; want 403", code, body) + } + var env struct { + Error struct { + Code int `json:"code"` + Message string `json:"message"` + Title string `json:"title"` + } `json:"error"` + } + if err := json.Unmarshal([]byte(body), &env); err != nil { + t.Fatalf("error body is not keystone-shaped: %v (body=%s)", err, body) + } + if env.Error.Code != http.StatusForbidden || env.Error.Title != "Forbidden" { + t.Errorf("envelope code/title = %d/%q; want 403/Forbidden (body=%s)", env.Error.Code, env.Error.Title, body) + } + if env.Error.Message == "" { + t.Errorf("envelope message is empty; the refusal reason must reach keystoneauth clients (body=%s)", body) + } + + // And the 404 shape, the one every 'show ' hits. + code, body = doReq(r, http.MethodGet, "/v3/domains/"+uuid.NewString(), adminHdr) + if code != http.StatusNotFound { + t.Fatalf("show missing domain: code=%d body=%s; want 404", code, body) + } + if err := json.Unmarshal([]byte(body), &env); err != nil { + t.Fatalf("404 body is not keystone-shaped: %v (body=%s)", err, body) + } + if env.Error.Code != http.StatusNotFound || env.Error.Title != "Not Found" || env.Error.Message == "" { + t.Errorf("404 envelope = %+v; want code 404, title Not Found, non-empty message", env.Error) + } +} diff --git a/internal/neutron/floatingip.go b/internal/neutron/floatingip.go index 5eda7004..1271ac92 100644 --- a/internal/neutron/floatingip.go +++ b/internal/neutron/floatingip.go @@ -61,19 +61,76 @@ func (svc *Service) ListFloatingIPs(c *gin.Context) { } f := &common.SQLFilter{} - f.Eq("project_id", projectID) + // The caller's project — unless an administrator filters by tenant: + // upstream honours the filter for admin tokens, and it was silently + // ignored before, so `floating ip list --project X` presented the + // ADMIN's addresses as project X's (SPEC-028; mirrors + // buildNetworkListFilter's tenant bypass). ::text so a non-UUID value + // narrows to nothing instead of failing the cast. + fipTenant := c.Query("tenant_id") + if fipTenant == "" { + fipTenant = c.Query("project_id") + } + if fipTenant != "" && common.HasRole(c, common.RoleAdmin) { + f.Eq("project_id::text", fipTenant) + } else { + f.Eq("project_id", projectID) + } + + // Standard Neutron list filters (neutron-lib FloatingIPDbMixin): + // floating_network_id, port_id, router_id, status, fixed_ip_address, + // floating_ip_address, description. All exact matches. They were silently + // ignored before, so `openstack floating ip list --network ` returned + // every tenant's addresses (SPEC-028). The slice keeps the WHERE layout + // deterministic. + for _, flt := range []struct{ param, column string }{ + {"floating_network_id", "floating_network_id"}, + {"port_id", "port_id"}, + {"router_id", "router_id"}, + {"status", "status"}, + {"fixed_ip_address", "fixed_ip_address"}, + {"floating_ip_address", "floating_ip_address"}, + {"description", "description"}, + } { + v := c.Query(flt.param) + if v == "" { + continue + } + switch flt.column { + case "floating_network_id", "port_id", "router_id": + // UUID columns: a non-UUID value can only match nothing, but it + // would fail the cast and answer 500 — answer the empty list a + // well-formed-but-unknown value gets. The comparison itself is + // uuid-typed (not ::text), so an upper-case filter value matches + // like any client would expect. + if _, err := uuid.Parse(v); err != nil { + c.JSON(http.StatusOK, gin.H{"floatingips": []gin.H{}}) + return + } + f.Eq(flt.column, v) + default: + f.Eq(flt.column, v) + } + } // Marker paging over a TOTAL order. created_at alone is not unique, so the order was undefined // among ties AND `created_at < $n` excluded every row sharing the marker's timestamp rather than // only those before it — a page walk silently dropped them. id is the primary key, so - // (created_at, id) is total (as #549 established for compute). + // (created_at, id) is total (as #549 established for compute). The marker resolves under the + // listing's own scope (see the tenant bypass above): an unresolvable marker would silently + // drop the condition and make the walk restart from the first page forever. + fipAdminBypass := fipTenant != "" && common.HasRole(c, common.RoleAdmin) if marker := c.Query("marker"); marker != "" { var markerCreatedAt time.Time var markerID string - err := svc.pool.QueryRow(c.Request.Context(), - "SELECT created_at, id FROM floating_ips WHERE id = $1 AND project_id = $2", - marker, projectID, - ).Scan(&markerCreatedAt, &markerID) + markerQuery := "SELECT created_at, id FROM floating_ips WHERE id = $1" + markerArgs := []any{marker} + if !fipAdminBypass { + markerQuery += " AND project_id = $2" + markerArgs = append(markerArgs, projectID) + } + err := svc.pool.QueryRow(c.Request.Context(), markerQuery, markerArgs...). + Scan(&markerCreatedAt, &markerID) if err == nil { f.Cond(fmt.Sprintf("(created_at, id) < (%s, %s)", f.Arg(markerCreatedAt), f.Arg(markerID))) } @@ -346,16 +403,25 @@ func (svc *Service) GetFloatingIP(c *gin.Context) { // id::text comparison resolves by UUID or floating-IP address without // crashing on a non-UUID identifier — the CLI looks a FIP up by its address - // (e.g. `server add floating ip
`). - err := svc.pool.QueryRow(c.Request.Context(), ` + // (e.g. `server add floating ip
`). An administrator may + // show any floating IP (upstream's get_floatingip policy is + // admin_or_owner), so the tenant-filtered listing's rows do not dead-end + // their follow-up show with 404 — the same carve-out the other reads got + // (SPEC-028). A non-admin keeps the project scope. + fipQuery := ` SELECT id, project_id, floating_network_id, floating_ip_address, fixed_ip_address, port_id, router_id, status, description, created_at, updated_at FROM floating_ips - WHERE (id::text = $1 OR floating_ip_address = $1) AND project_id = $2 - LIMIT 1 - `, floatingIPID, projectID).Scan(&fip.ID, &fip.ProjectID, &fip.FloatingNetworkID, &fip.FloatingIPAddress, - &fixedIP, &portID, &routerID, &fip.Status, &description, &fip.CreatedAt, &fip.UpdatedAt) + WHERE (id::text = $1 OR floating_ip_address = $1)` + fipArgs := []any{floatingIPID} + if !common.HasRole(c, common.RoleAdmin) { + fipQuery += ` AND project_id = $2` + fipArgs = append(fipArgs, projectID) + } + err := svc.pool.QueryRow(c.Request.Context(), fipQuery+" LIMIT 1", fipArgs...). + Scan(&fip.ID, &fip.ProjectID, &fip.FloatingNetworkID, &fip.FloatingIPAddress, + &fixedIP, &portID, &routerID, &fip.Status, &description, &fip.CreatedAt, &fip.UpdatedAt) if errors.Is(err, database.ErrNoRows) { common.SendError(c, common.NewNotFoundError("floating IP")) diff --git a/internal/neutron/network.go b/internal/neutron/network.go index dae2cfcd..a251fe45 100644 --- a/internal/neutron/network.go +++ b/internal/neutron/network.go @@ -475,6 +475,9 @@ type CreateNetworkRequest struct { Name string `json:"name" binding:"required"` AdminStateUp *bool `json:"admin_state_up"` Shared *bool `json:"shared"` + // The standard resource annotation (migration 075). A pointer so an + // omitted key and an explicit empty string differ, as upstream does. + Description *string `json:"description"` // Both spellings of the owning project. Neither was declared before, so the value a client // sent was discarded by ShouldBindJSON and never reached the handler: the request was // answered 201 with the network in the CALLER's project (cloud-storage#546 B11). Nothing was @@ -610,9 +613,9 @@ func (svc *Service) CreateNetwork(c *gin.Context) { } _, err := svc.pool.Exec(c.Request.Context(), ` - INSERT INTO networks (id, name, project_id, admin_state_up, status, shared, external, network_type, mtu, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) - `, networkID, req.Network.Name, projectID, adminStateUp, "ACTIVE", shared, external, networkType, mtu, now, now) + INSERT INTO networks (id, name, project_id, admin_state_up, status, shared, external, network_type, mtu, description, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + `, networkID, req.Network.Name, projectID, adminStateUp, "ACTIVE", shared, external, networkType, mtu, req.Network.Description, now, now) if err != nil { log.Error().Err(err).Str("operation", "create_network").Msg("database error") @@ -648,6 +651,7 @@ func (svc *Service) CreateNetwork(c *gin.Context) { "shared": shared, "router:external": external, "mtu": mtu, + "description": derefOr(req.Network.Description, ""), "subnets": []string{}, "created_at": now.Format(time.RFC3339), "updated_at": now.Format(time.RFC3339), @@ -700,13 +704,17 @@ func networkListDecision(sharedParam, routerExternalParam string) (sf sharedFilt } // buildNetworkListFilter builds the WHERE conditions and positional args for the -// network list from the caller's project and the neutron list filters Horizon sends -// (F4). It honors `shared` (only/exclude shared), `router:external` (a real predicate since -// SPEC-024 F11), and `tenant_id`/`project_id` (narrowing within the -// caller's authorized scope), plus the id/name filters the Terraform provider uses. -// Placeholders are numbered from $1; `empty` means the query must return nothing. The -// caller continues numbering marker/pagination args from $(len(args)+1). Pure — no DB. -func buildNetworkListFilter(caller, sharedParam, routerExternalParam, tenantParam, idParam, nameParam string) (f *common.SQLFilter, empty bool) { +// networks listing. isAdmin matters only together with tenantParam: an +// administrator filtering by tenant gets that tenant's networks outright +// (upstream honours the filter for admin tokens), so the caller's read scope +// must not be ANDed in — it would leave only the shared/external subset and +// answer [] for another project's private networks (SPEC-028). A non-admin +// keeps the scope AND tenant intersection. The filter also honors `shared` +// (only/exclude shared), `router:external` (a real predicate since SPEC-024 +// F11), and the id/name filters the Terraform provider uses. Placeholders are +// numbered from $1; `empty` means the query must return nothing. The caller +// continues numbering marker/pagination args from $(len(args)+1). Pure — no DB. +func buildNetworkListFilter(caller, sharedParam, routerExternalParam, tenantParam, idParam, nameParam string, isAdmin bool) (f *common.SQLFilter, empty bool) { sf, empty := networkListDecision(sharedParam, routerExternalParam) if empty { return nil, true @@ -723,14 +731,21 @@ func buildNetworkListFilter(caller, sharedParam, routerExternalParam, tenantPara // select it as a router gateway; requiring `shared` as well would leave the tenant-facing // Set Gateway dropdown empty and fix F11 for administrators only (FR-033a). Read-only: // visibility confers no authority (FR-033b, and #545's rule for shared networks). - f.Cond(networkReadScope(f.Arg(caller))) + // + // Skipped when an admin explicitly filters by tenant: the tenant filter + // is then the authoritative owner scope. + if !isAdmin || tenantParam == "" { + f.Cond(networkReadScope(f.Arg(caller))) + } } if ext, ok := neutronBool(routerExternalParam); ok { f.Eq("external", ext) } if tenantParam != "" { - f.Eq("project_id", tenantParam) + // ::text — a non-UUID tenant value must narrow to nothing, not + // fail the uuid cast and answer 500 (SPEC-028). + f.Eq("project_id::text", tenantParam) } if idParam != "" { f.Eq("id::text", idParam) @@ -865,7 +880,7 @@ func (svc *Service) ListNetworks(c *gin.Context) { if tenant == "" { tenant = c.Query("project_id") } - f, empty := buildNetworkListFilter(projectID, c.Query("shared"), c.Query("router:external"), tenant, c.Query("id"), c.Query("name")) + f, empty := buildNetworkListFilter(projectID, c.Query("shared"), c.Query("router:external"), tenant, c.Query("id"), c.Query("name"), common.HasRole(c, common.RoleAdmin)) if empty { // No filter combination produces this today; kept so a future filter can short-circuit // without re-plumbing the caller. @@ -873,9 +888,20 @@ func (svc *Service) ListNetworks(c *gin.Context) { return } - // Marker-based pagination (see resolveListMarker). - m, ok := svc.resolveListMarker(c, limit, "network", - "SELECT created_at, id FROM networks WHERE id::text = $1 AND "+networkReadScope("$2")) + // Marker-based pagination (see resolveListMarker). The marker resolves + // under the SAME scope the listing used: an admin's tenant-filtered walk + // dropped the read scope above (buildNetworkListFilter), so resolving + // with it would 404 the walk mid-page on the very rows the listing just + // returned (SPEC-028). + var m *listMarker + var ok bool + if common.HasRole(c, common.RoleAdmin) && tenant != "" { + m, ok = svc.resolveListMarker(c, limit, "network", + "SELECT created_at, id FROM networks WHERE id::text = $1") + } else { + m, ok = svc.resolveListMarker(c, limit, "network", + "SELECT created_at, id FROM networks WHERE id::text = $1 AND "+networkReadScope("$2"), projectID) + } if !ok { return } @@ -889,7 +915,7 @@ func (svc *Service) ListNetworks(c *gin.Context) { where := f.Conds() limitOffset := fmt.Sprintf(" LIMIT %s OFFSET %s", f.Arg(limit), f.Arg(offset)) rows, err := svc.pool.Query(c.Request.Context(), ` - SELECT id, name, admin_state_up, status, shared, mtu, created_at, updated_at, project_id, external + SELECT id, name, admin_state_up, status, shared, mtu, created_at, updated_at, project_id, external, COALESCE(description, '') FROM networks WHERE `+where+` ORDER BY created_at DESC, id DESC`+limitOffset, f.Args()...) @@ -903,12 +929,12 @@ func (svc *Service) ListNetworks(c *gin.Context) { var networks []gin.H for rows.Next() { - var id, name, status, owner string + var id, name, status, owner, description string var adminStateUp, shared, external bool var mtu int var createdAt, updatedAt time.Time - if err := rows.Scan(&id, &name, &adminStateUp, &status, &shared, &mtu, &createdAt, &updatedAt, &owner, &external); err != nil { + if err := rows.Scan(&id, &name, &adminStateUp, &status, &shared, &mtu, &createdAt, &updatedAt, &owner, &external, &description); err != nil { log.Warn().Err(err).Str("network_id", id).Msg("skipping unreadable network row") continue } @@ -926,6 +952,7 @@ func (svc *Service) ListNetworks(c *gin.Context) { "shared": shared, "router:external": external, "mtu": mtu, + "description": description, "created_at": createdAt.Format(time.RFC3339), "updated_at": updatedAt.Format(time.RFC3339), }) @@ -965,6 +992,11 @@ func (svc *Service) ListNetworks(c *gin.Context) { // Scoping the key rather than re-checking on hit keeps a hit free of database work, at the cost of // one entry per (caller, network) pair — a shared or external network is cached once per project // that asks for it, which is the price of the lookup preceding the check. +// +// The invariant holds only because GetNetwork's WRITE side caches nothing the project is not +// entitled to by the read scope: since the admin carve-out made visibility role-dependent, a +// fetched row outside the read scope (an admin reading another project's private network) is +// served but never cached — see the TrySet guard in GetNetwork. func networkCacheKey(projectID, networkID string) string { return "network:" + projectID + ":" + networkID } @@ -997,17 +1029,30 @@ func (svc *Service) GetNetwork(c *gin.Context) { } // Cache miss - query database - var id, name, status, owner string + var id, name, status, owner, description string var adminStateUp, shared, external bool var mtu int var createdAt, updatedAt time.Time - err := svc.pool.QueryRow(ctx, ` - SELECT id, name, admin_state_up, status, shared, mtu, created_at, updated_at, project_id, external + // An administrator may show any network — upstream's get_network policy + // is "admin_or_owner or shared or external", and the LIST already honours + // that for the admin (buildNetworkListFilter bypasses the read scope for + // an admin's tenant filter). Without the matching carve-out here, an + // admin could list another project's private network with --project and + // then get a 404 showing the id it just listed (SPEC-028). + query := ` + SELECT id, name, admin_state_up, status, shared, mtu, created_at, updated_at, project_id, external, COALESCE(description, '') FROM networks - WHERE (id::text = $1 OR name = $1) AND `+networkReadScope("$2")+` - LIMIT 1 - `, networkID, projectID).Scan(&id, &name, &adminStateUp, &status, &shared, &mtu, &createdAt, &updatedAt, &owner, &external) + WHERE (id::text = $1 OR name = $1)` + args := []any{networkID} + if !common.HasRole(c, common.RoleAdmin) { + query += ` AND ` + networkReadScope("$2") + args = append(args, projectID) + } + query += ` LIMIT 1` + + err := svc.pool.QueryRow(ctx, query, args...). + Scan(&id, &name, &adminStateUp, &status, &shared, &mtu, &createdAt, &updatedAt, &owner, &external, &description) if errors.Is(err, database.ErrNoRows) { common.SendError(c, common.NewNotFoundError("network")) @@ -1031,6 +1076,7 @@ func (svc *Service) GetNetwork(c *gin.Context) { "shared": shared, "router:external": external, "mtu": mtu, + "description": description, "created_at": createdAt.Format(time.RFC3339), "updated_at": updatedAt.Format(time.RFC3339), } @@ -1038,8 +1084,12 @@ func (svc *Service) GetNetwork(c *gin.Context) { // Store in cache (30min TTL per config) before adding the dynamic subnet // list, so cached entries never carry a stale subnets field. Written under the SAME // caller-scoped key the read above uses, so an entry can only ever be served back to a caller - // entitled to it. - if svc.cache != nil { + // entitled to it. The admin carve-out above can exceed that entitlement — an admin may + // fetch another project's PRIVATE network — and caching such a row under the + // project's key would hand it to the project's plain members as a 200 where the + // uncached answer is 404 (the key names only the project, not the role). Rows + // outside the project's read scope are therefore simply not cached. + if svc.cache != nil && (owner == projectID || shared || external) { svc.cache.TrySet(ctx, networkCacheKey(projectID, id), network, 30*time.Minute) } @@ -1116,9 +1166,10 @@ func (svc *Service) DeleteNetwork(c *gin.Context) { // Invalidate cache. Entries are keyed per caller (networkCacheKey), so evicting this // network means evicting every project's view of it, not just the deleter's. + // (Only `network::` keys are ever written — networkCacheKey — + // so this one pattern is the complete eviction.) if svc.cache != nil { svc.cache.TryDeletePattern(ctx, "network:*:"+id) - svc.cache.TryDeletePattern(ctx, "networks:*") } c.Status(http.StatusNoContent) @@ -1197,6 +1248,10 @@ func (svc *Service) UpdateNetwork(c *gin.Context) { Name *string `json:"name"` AdminStateUp *bool `json:"admin_state_up"` External *bool `json:"router:external"` + // The standard resource annotation (migration 075): 'network set + // --description' PUTs it, and the handler had no column to + // persist it to (SPEC-028). + Description *string `json:"description"` } `json:"network"` } @@ -1227,6 +1282,10 @@ func (svc *Service) UpdateNetwork(c *gin.Context) { u.Set("admin_state_up", *req.Network.AdminStateUp) } + if req.Network.Description != nil { + u.Set("description", *req.Network.Description) + } + if u.Empty() { common.SendError(c, common.NewBadRequestError("no updates provided")) return @@ -1277,7 +1336,6 @@ func (svc *Service) UpdateNetwork(c *gin.Context) { // reason DeleteNetwork's does. if svc.cache != nil { svc.cache.TryDeletePattern(c.Request.Context(), "network:*:"+networkID) - svc.cache.TryDeletePattern(c.Request.Context(), "networks:*") } svc.writeNetworkResponse(c, networkID) } @@ -1285,17 +1343,17 @@ func (svc *Service) UpdateNetwork(c *gin.Context) { // writeNetworkResponse emits a network by id with no visibility predicate, for the caller that just // wrote it (see UpdateNetwork). Ownership was already decided by the write. func (svc *Service) writeNetworkResponse(c *gin.Context, networkID string) { - var id, name, status, owner string + var id, name, status, owner, description string var adminStateUp, shared, external bool var mtu int var createdAt, updatedAt time.Time err := svc.pool.QueryRow(c.Request.Context(), ` - SELECT id, name, admin_state_up, status, shared, mtu, created_at, updated_at, project_id, external + SELECT id, name, admin_state_up, status, shared, mtu, created_at, updated_at, project_id, external, COALESCE(description, '') FROM networks WHERE id::text = $1 LIMIT 1 - `, networkID).Scan(&id, &name, &adminStateUp, &status, &shared, &mtu, &createdAt, &updatedAt, &owner, &external) + `, networkID).Scan(&id, &name, &adminStateUp, &status, &shared, &mtu, &createdAt, &updatedAt, &owner, &external, &description) if errors.Is(err, database.ErrNoRows) { common.SendError(c, common.NewNotFoundError("network")) @@ -1324,6 +1382,7 @@ func (svc *Service) writeNetworkResponse(c *gin.Context, networkID string) { "shared": shared, "router:external": external, "mtu": mtu, + "description": description, "created_at": createdAt.Format(time.RFC3339), "updated_at": updatedAt.Format(time.RFC3339), "subnets": subnets, @@ -1583,13 +1642,20 @@ type listMarker struct { // The networks, subnets and ports listings used to read the marker with no visibility and ignore one // that did not resolve, so another project's private row narrowed the page where an absent id did // not — telling the caller it exists. -func (svc *Service) resolveListMarker(c *gin.Context, limit any, resource, query string) (*listMarker, bool) { +// resolveListMarker resolves a listing's marker as upstream Neutron does (neutron-lib +// _resolve_pagination_marker): the marker names a row of the SAME collection under the +// SAME visibility the listing itself used, and the page continues after it. Callers +// pass the scope predicates' arguments after the marker id (normally just the caller's +// project id); a listing that dropped its read scope — an admin's tenant-filtered +// walk — passes none, so the marker resolves under the listing's actual scope and an +// admin's walk cannot 404 mid-page on a row the listing itself just returned. +func (svc *Service) resolveListMarker(c *gin.Context, limit any, resource, query string, scopeArgs ...any) (*listMarker, bool) { marker := c.Query("marker") if marker == "" || limit == nil { return nil, true } var m listMarker - err := svc.pool.QueryRow(c.Request.Context(), query, marker, c.GetString("project_id")).Scan(&m.createdAt, &m.id) + err := svc.pool.QueryRow(c.Request.Context(), query, append([]any{marker}, scopeArgs...)...).Scan(&m.createdAt, &m.id) if errors.Is(err, database.ErrNoRows) { common.SendError(c, common.NewNotFoundError(resource)) return nil, false @@ -1624,17 +1690,26 @@ func (svc *Service) ListSubnets(c *gin.Context) { f := &common.SQLFilter{} - // Scope by the owning NETWORK's visibility, not only the subnet's own project_id (see - // subnetReadScope). One argument read by two placeholders, so it is appended once and the - // placeholder reused. - caller := f.Arg(projectID) - f.Cond(subnetReadScope(caller)) - // Client filters (B6). Both spellings of the project attribute map onto project_id. tenant := c.Query("tenant_id") if tenant == "" { tenant = c.Query("project_id") } + + // Scope by the owning NETWORK's visibility, not only the subnet's own project_id (see + // subnetReadScope). One argument read by two placeholders, so it is appended once and the + // placeholder reused. Dropped for an administrator who filters by tenant or by network — + // upstream honours both for admin tokens, and network_id in particular is a pointer that + // comes straight out of a network body the admin was allowed to read, so scoping it again + // would answer 200 for the network and [] for its subnets (SPEC-028; mirrors + // buildNetworkListFilter's tenant bypass). A non-admin keeps the scope AND filter + // intersection. + adminBypass := common.HasRole(c, common.RoleAdmin) && (tenant != "" || c.Query("network_id") != "") + if !adminBypass { + caller := f.Arg(projectID) + f.Cond(subnetReadScope(caller)) + } + addSubnetListFilter(f, subnetFilterParams{ NetworkID: c.Query("network_id"), ID: c.Query("id"), @@ -1648,9 +1723,16 @@ func (svc *Service) ListSubnets(c *gin.Context) { // Marker-based pagination on a TOTAL order. `created_at` alone is not unique, so ties made the // page boundary unstable and a walk could repeat or skip rows — the same defect #549 found in the // compute listings. ListNetworks already pages on (created_at, id); FR-025 requires the same here. - // The marker itself resolves as resolveListMarker describes. - m, ok := svc.resolveListMarker(c, limit, "subnet", - "SELECT s.created_at, s.id FROM subnets s JOIN networks n ON s.network_id = n.id WHERE s.id::text = $1 AND "+subnetReadScope("$2")) + // The marker itself resolves as resolveListMarker describes — under the listing's own scope. + var m *listMarker + var ok bool + if adminBypass { + m, ok = svc.resolveListMarker(c, limit, "subnet", + "SELECT s.created_at, s.id FROM subnets s JOIN networks n ON s.network_id = n.id WHERE s.id::text = $1") + } else { + m, ok = svc.resolveListMarker(c, limit, "subnet", + "SELECT s.created_at, s.id FROM subnets s JOIN networks n ON s.network_id = n.id WHERE s.id::text = $1 AND "+subnetReadScope("$2"), projectID) + } if !ok { return } @@ -1758,14 +1840,22 @@ func (svc *Service) GetSubnet(c *gin.Context) { // Consequently s.project_id is selected, not assumed: this handler can return a subnet // the caller does not own — one on their network, or on a shared network — so reporting // the caller as tenant_id would tell them a resource is theirs when it is not. - err := svc.pool.QueryRow(c.Request.Context(), ` + // An administrator may show any subnet — the same carve-out GetNetwork + // has (upstream's get_subnet policy includes ADMIN_OR_NET_OWNER_READER). + // Without it, an admin could show a network and then 404 on the subnet + // ids its own body lists (SPEC-028). + query := ` SELECT s.id, s.name, s.network_id, s.cidr, s.gateway_ip, s.ip_version, s.enable_dhcp, s.dns_nameservers, s.created_at, s.updated_at, s.project_id FROM subnets s JOIN networks n ON s.network_id = n.id - WHERE (s.id::text = $1 OR s.name = $1) - AND `+subnetReadScope("$2")+` - LIMIT 1 - `, subnetID, projectID).Scan(&id, &name, &networkID, &cidr, &gatewayIP, &ipVersion, &enableDHCP, &dnsNameservers, &createdAt, &updatedAt, &owner) + WHERE (s.id::text = $1 OR s.name = $1)` + args := []any{subnetID} + if !common.HasRole(c, common.RoleAdmin) { + query += ` AND ` + subnetReadScope("$2") + args = append(args, projectID) + } + err := svc.pool.QueryRow(c.Request.Context(), query+" LIMIT 1", args...). + Scan(&id, &name, &networkID, &cidr, &gatewayIP, &ipVersion, &enableDHCP, &dnsNameservers, &createdAt, &updatedAt, &owner) if errors.Is(err, database.ErrNoRows) { common.SendError(c, common.NewNotFoundError("subnet")) @@ -1833,11 +1923,82 @@ func (svc *Service) DeleteSubnet(c *gin.Context) { return } + ctx := c.Request.Context() + tx, err := svc.pool.Begin(ctx) + if err != nil { + common.SendError(c, common.NewInternalServerError("failed to begin subnet deletion")) + return + } + defer func() { _ = tx.Rollback(context.Background()) }() + if _, err := LockSubnet(ctx, tx, resolvedID); err != nil { + if errors.Is(err, database.ErrNoRows) { + common.SendError(c, common.NewNotFoundError("subnet")) + } else { + common.SendError(c, common.NewInternalServerError("failed to lock subnet")) + } + return + } + + // A subnet with allocations is upstream's SubnetInUse: 409, not a delete + // that orphans the holder. The router interface case mattered most — the + // port survived the subnet (no FK from fixed_ips), silently blocked the + // network's deletion with a 409, and 'router remove port' 404s because the + // interface row cascaded away with the subnet (SPEC-028). + var holder string + ports, err := tx.Query(c.Request.Context(), + "SELECT id, fixed_ips FROM ports WHERE network_id = $1", networkID) + if err != nil { + log.Error().Err(err).Str("operation", "delete_subnet_inuse").Str("subnet_id", resolvedID).Msg("database error") + common.SendError(c, common.NewInternalServerError("failed to delete subnet")) + return + } + for ports.Next() { + var portID string + var fixedIPsJSON []byte + if err := ports.Scan(&portID, &fixedIPsJSON); err != nil { + ports.Close() + log.Error().Err(err).Str("operation", "delete_subnet_inuse").Str("subnet_id", resolvedID).Msg("database error") + common.SendError(c, common.NewInternalServerError("failed to delete subnet")) + return + } + var fixedIPs []struct { + SubnetID string `json:"subnet_id"` + } + if err := json.Unmarshal(fixedIPsJSON, &fixedIPs); err != nil { + ports.Close() + log.Error().Err(err).Str("operation", "delete_subnet_inuse").Str("subnet_id", resolvedID).Msg("invalid fixed_ips JSON") + common.SendError(c, common.NewInternalServerError("failed to delete subnet")) + return + } + for _, fixedIP := range fixedIPs { + if fixedIP.SubnetID == resolvedID { + holder = portID + break + } + } + if holder != "" { + break + } + } + rowsErr := ports.Err() + ports.Close() + if rowsErr != nil { + log.Error().Err(rowsErr).Str("operation", "delete_subnet_inuse").Str("subnet_id", resolvedID).Msg("database error") + common.SendError(c, common.NewInternalServerError("failed to delete subnet")) + return + } + + if holder != "" { + common.SendError(c, common.NewResourceInUseError("subnet", "delete", + fmt.Sprintf("Unable to delete subnet: one or more ports have an IP allocation from this subnet (first: %s).", holder))) + return + } + // Delete by the id the entitlement check above already resolved. Re-filtering on // project_id here would silently no-op for a network owner removing another // project's subnet — 204 with the row still present — because the two predicates // would disagree. One decision, one place. - _, err = svc.pool.Exec(c.Request.Context(), + _, err = tx.Exec(c.Request.Context(), "DELETE FROM subnets WHERE id = $1", resolvedID, ) @@ -1847,6 +2008,11 @@ func (svc *Service) DeleteSubnet(c *gin.Context) { return } + if err := tx.Commit(ctx); err != nil { + common.SendError(c, common.NewInternalServerError("failed to commit subnet deletion")) + return + } + svc.reconcileNetworkDHCPAfterSubnetDelete(c.Request.Context(), networkID) c.Status(http.StatusNoContent) @@ -1964,6 +2130,9 @@ func (svc *Service) UpdateSubnet(c *gin.Context) { // Raw for the same reason as CreateSubnet: an explicit null clears the gateway, an // omitted key leaves it alone (FR-023, contract S14). GatewayIP json.RawMessage `json:"gateway_ip"` + // 'subnet set --dhcp/--no-dhcp' PUTs it; the handler had no + // binding for it, so the flag was a 200 no-op (SPEC-028). + EnableDHCP *bool `json:"enable_dhcp"` } `json:"subnet"` } if err := c.ShouldBindJSON(&req); err != nil { @@ -2039,32 +2208,40 @@ func (svc *Service) UpdateSubnet(c *gin.Context) { // the old name still stored, because the two predicates disagree (the same trap // DeleteSubnet had). // gateway_ip is only touched when the key was supplied; an explicit null clears it (S14). - switch { - case gatewayGiven: - _, err = svc.pool.Exec(c.Request.Context(), - "UPDATE subnets SET name = $1, gateway_ip = $2 WHERE id = $3", - currentName, newGateway, resolvedID, - ) - case req.Subnet.Name != nil: + // Every supplied column lands in ONE statement: two separate UPDATEs (name/gateway, + // then the DHCP flag) could leave a half-applied update behind if the second failed. + u := &common.SQLUpdate{} + if req.Subnet.Name != nil { + u.Set("name", *req.Subnet.Name) + } + if gatewayGiven { + u.Set("gateway_ip", newGateway) + } + // The DHCP flag is applied independently of name/gateway, so it works alone + // ('subnet set --no-dhcp') and together — but only when the flag actually + // flips. Host state follows the row: dnsmasq restarts for a survivor when + // this was the network's only DHCP subnet, and stops when none remain — + // the same convergence the post-delete reconciler performs. + dhcpToggled := req.Subnet.EnableDHCP != nil && *req.Subnet.EnableDHCP != subnetDHCP + if dhcpToggled { + u.Set("enable_dhcp", *req.Subnet.EnableDHCP) + } + // An empty update succeeds without a write, matching Neutron and clients that + // send unchanged resources. In particular, leave updated_at untouched. + if !u.Empty() { + u.Set("updated_at", time.Now()) _, err = svc.pool.Exec(c.Request.Context(), - "UPDATE subnets SET name = $1 WHERE id = $2", - currentName, resolvedID, - ) - default: - // Nothing was supplied, so nothing is written. This used to issue - // `UPDATE subnets SET name = ` — a statement that changed no column and - // did not even touch updated_at (second review, AGY-2). - // - // The request still SUCCEEDS: upstream Neutron accepts an empty update and answers 200 with - // the unchanged resource, and Terraform and Horizon both send updates that can reduce to - // nothing. Refusing with 400 — the second review's suggestion — would have been an API - // compatibility regression, which this project's first principle forbids. + "UPDATE subnets SET "+u.Assignments()+" WHERE id = "+u.Arg(resolvedID), + u.Args()...) } if err != nil { log.Error().Err(err).Str("operation", "update_subnet").Str("subnet_id", resolvedID).Msg("database error") common.SendError(c, common.NewInternalServerError("failed to update subnet")) return } + if dhcpToggled { + svc.reconcileNetworkDHCPAfterSubnetDelete(c.Request.Context(), subnetNetwork) + } if gatewayChanged && subnetDHCP { svc.reconcileNetworkDHCPAfterGatewayChange(c.Request.Context(), subnetNetwork, subnetCIDR, currentGateway, newGateway) } diff --git a/internal/neutron/network_test.go b/internal/neutron/network_test.go index 444acbb3..9d2e038c 100644 --- a/internal/neutron/network_test.go +++ b/internal/neutron/network_test.go @@ -107,7 +107,7 @@ func TestBuildNetworkListFilter(t *testing.T) { // SPEC-024 F11: router:external is a real predicate now, complementary rather than always-empty // (FR-032, contract X6). The old assertion — empty=true — encoded the defect. - f, empty := buildNetworkListFilter(caller, "", "true", "", "", "") + f, empty := buildNetworkListFilter(caller, "", "true", "", "", "", false) if empty { t.Fatal("router:external=true must no longer force an empty result") } @@ -119,7 +119,7 @@ func TestBuildNetworkListFilter(t *testing.T) { } // ...and its complement. - f, _ = buildNetworkListFilter(caller, "", "false", "", "", "") + f, _ = buildNetworkListFilter(caller, "", "false", "", "", "", false) if args := f.Args(); len(args) != 2 || args[1] != false { t.Errorf("router:external=false args = %v, want [caller false]", args) } @@ -127,7 +127,7 @@ func TestBuildNetworkListFilter(t *testing.T) { // Default (no filters): scope = caller's own OR shared OR external, one arg (the caller). // `external` is a THIRD INDEPENDENT disjunct — it must not require `shared`, or a tenant cannot // see the external network they need to pick as a router gateway (FR-033a). - f, empty = buildNetworkListFilter(caller, "", "", "", "", "") + f, empty = buildNetworkListFilter(caller, "", "", "", "", "", false) if empty { t.Fatal("default filter should not be empty") } @@ -139,7 +139,7 @@ func TestBuildNetworkListFilter(t *testing.T) { } // shared=true → only shared, and NO caller arg (so pagination starts at $1). - f, _ = buildNetworkListFilter(caller, "true", "", "", "", "") + f, _ = buildNetworkListFilter(caller, "true", "", "", "", "", false) if where := f.Conds(); where != "shared = true" { t.Errorf("shared=true where = %q", where) } @@ -148,20 +148,45 @@ func TestBuildNetworkListFilter(t *testing.T) { } // shared=false → exclude shared, scoped to the caller. - f, _ = buildNetworkListFilter(caller, "false", "", "", "", "") + f, _ = buildNetworkListFilter(caller, "false", "", "", "", "", false) if where := f.Conds(); !strings.Contains(where, "shared = false") { t.Errorf("shared=false where = %q", where) } // tenant + name append sequential placeholders after the scope arg ($1). - f, _ = buildNetworkListFilter(caller, "", "", "proj-A", "", "netx") + // The tenant filter compares as ::text (a non-UUID value narrows to + // nothing rather than failing the uuid cast; SPEC-028). + f, _ = buildNetworkListFilter(caller, "", "", "proj-A", "", "netx", false) where := f.Conds() - if !strings.Contains(where, "project_id = $2") || !strings.Contains(where, "name = $3") { + if !strings.Contains(where, "project_id::text = $2") || !strings.Contains(where, "name = $3") { t.Errorf("tenant+name where = %q, want $2/$3 placeholders", where) } if args := f.Args(); len(args) != 3 { t.Errorf("tenant+name args = %v, want 3", args) } + + // Admin + tenant: the tenant filter is the authoritative owner scope — the + // caller's read scope must NOT be ANDed in, or another project's private + // networks can never appear and `network list --project ` answers [] + // (SPEC-028). Upstream honours the filter for admin tokens. The filter + // compares as ::text: a non-UUID value must narrow to nothing, not fail + // the uuid cast and answer 500 (SPEC-028). + f, _ = buildNetworkListFilter(caller, "", "", "proj-B", "", "", true) + where = f.Conds() + if where != "project_id::text = $1" { + t.Errorf("admin+tenant where = %q, want exactly \"project_id::text = $1\"", where) + } + // Non-admin + tenant keeps the scope intersection (unchanged semantics). + f, _ = buildNetworkListFilter(caller, "", "", "proj-B", "", "", false) + where = f.Conds() + if !strings.Contains(where, "project_id = $1 OR shared = true OR external = true") || !strings.Contains(where, "project_id::text = $2") { + t.Errorf("non-admin+tenant where = %q, want scope AND tenant", where) + } + // Admin without a tenant filter keeps the ordinary scope. + f, _ = buildNetworkListFilter(caller, "", "", "", "", "", true) + if where := f.Conds(); where != "(project_id = $1 OR shared = true OR external = true)" { + t.Errorf("admin default where = %q, want the ordinary read scope", where) + } } // TestBuildSubnetListFilter — SPEC-024 (cloud-storage#546) B6, FR-019/FR-020. diff --git a/internal/neutron/ports.go b/internal/neutron/ports.go index 85d07dc4..8696cc4c 100644 --- a/internal/neutron/ports.go +++ b/internal/neutron/ports.go @@ -326,15 +326,42 @@ func (svc *Service) ListPorts(c *gin.Context) { } pf := &common.SQLFilter{} - pf.Cond(portReadScope(pf.Arg(projectID))) + + // The caller's read scope (see portReadScope) — dropped for an + // administrator who filters by tenant, network, or device. Each is an + // explicit cross-project pointer (the network or device id comes + // straight out of a body the admin was allowed to read), and upstream + // honours all of them for admin tokens; scoping them again would answer + // 200 for the network and [] for its ports. Mirrors + // buildNetworkListFilter's tenant bypass (SPEC-028). A non-admin keeps + // the scope AND filter intersection. + tenant := c.Query("tenant_id") + if tenant == "" { + tenant = c.Query("project_id") + } + adminBypass := common.HasRole(c, common.RoleAdmin) && + (tenant != "" || c.Query("network_id") != "" || c.Query("device_id") != "") + if !adminBypass { + pf.Cond(portReadScope(pf.Arg(projectID))) + } + if tenant != "" { + pf.Eq("p.project_id::text", tenant) + } // Marker paging over a TOTAL order. p.created_at alone is not unique, so the order was undefined // among ties AND `p.created_at < $n` excluded every port sharing the marker's timestamp rather // than only those before it — ports created by one boot share it (as #549 established for // compute). - // The marker itself resolves as resolveListMarker describes. - m, ok := svc.resolveListMarker(c, limit, "port", - "SELECT p.created_at, p.id FROM ports p JOIN networks n ON p.network_id = n.id WHERE p.id::text = $1 AND "+portReadScope("$2")) + // The marker itself resolves as resolveListMarker describes — under the listing's own scope. + var m *listMarker + var ok bool + if adminBypass { + m, ok = svc.resolveListMarker(c, limit, "port", + "SELECT p.created_at, p.id FROM ports p JOIN networks n ON p.network_id = n.id WHERE p.id::text = $1") + } else { + m, ok = svc.resolveListMarker(c, limit, "port", + "SELECT p.created_at, p.id FROM ports p JOIN networks n ON p.network_id = n.id WHERE p.id::text = $1 AND "+portReadScope("$2"), projectID) + } if !ok { return } @@ -345,13 +372,16 @@ func (svc *Service) ListPorts(c *gin.Context) { // Standard Neutron port filters. Notably ?device_id= is how // `openstack server add floating ip` finds a server's port — without it the // client gets every port on the network and may grab the wrong one (e.g. the - // router interface), associating the FIP to the gateway instead of the VM. + // router interface), associating the FIP to the VM instead of the gateway. + // The uuid-typed columns compare as ::text: a non-UUID value in the query + // string must narrow to nothing, not fail the cast and answer 500 — the + // same guard the floating-ip filter loop carries (SPEC-028). for _, filter := range []struct{ col, val string }{ - {"p.device_id", c.Query("device_id")}, - {"p.network_id", c.Query("network_id")}, + {"p.device_id::text", c.Query("device_id")}, + {"p.network_id::text", c.Query("network_id")}, {"p.device_owner", c.Query("device_owner")}, {"p.mac_address", c.Query("mac_address")}, - {"p.id", c.Query("id")}, + {"p.id::text", c.Query("id")}, } { if filter.val != "" { pf.Eq(filter.col, filter.val) @@ -361,7 +391,7 @@ func (svc *Service) ListPorts(c *gin.Context) { where := pf.Where() limitOffset := fmt.Sprintf(" LIMIT %s OFFSET %s", pf.Arg(limit), pf.Arg(offset)) rows, err := svc.pool.Query(c.Request.Context(), ` - SELECT p.id, p.name, p.network_id, p.device_id, p.device_owner, p.mac_address, p.admin_state_up, p.status, p.fixed_ips, p.created_at, p.updated_at + SELECT p.id, p.name, p.network_id, p.project_id, p.device_id, p.device_owner, p.mac_address, p.admin_state_up, p.status, p.fixed_ips, p.created_at, p.updated_at FROM ports p JOIN networks n ON p.network_id = n.id`+where+` ORDER BY p.created_at DESC, p.id DESC`+limitOffset, pf.Args()...) @@ -381,8 +411,12 @@ func (svc *Service) ListPorts(c *gin.Context) { var adminStateUp bool var fixedIPsJSON []byte var createdAt, updatedAt time.Time + // The row's OWN project, not the caller's: the admin cross-project + // listing serves foreign ports, and stamping them with the admin's + // project would tell a client a resource is theirs when it is not. + var ownerID string - if err := rows.Scan(&id, &name, &networkID, &deviceID, &deviceOwner, &macAddress, &adminStateUp, &status, &fixedIPsJSON, &createdAt, &updatedAt); err != nil { + if err := rows.Scan(&id, &name, &networkID, &ownerID, &deviceID, &deviceOwner, &macAddress, &adminStateUp, &status, &fixedIPsJSON, &createdAt, &updatedAt); err != nil { log.Warn().Err(err).Msg("failed to scan port row") continue } @@ -415,7 +449,7 @@ func (svc *Service) ListPorts(c *gin.Context) { "id": id, "name": name, "network_id": networkID, - "tenant_id": projectID, + "tenant_id": ownerID, "device_id": deviceID.String, "device_owner": deviceOwner.String, "mac_address": macAddress, @@ -445,6 +479,15 @@ func (svc *Service) GetPort(c *gin.Context) { portID := c.Param("id") projectID := c.GetString("project_id") + // A non-UUID reference (a port name — the CLI resolves names with a list + // fallback only after a 404) can only name nothing here; the uuid-typed + // lookup used to surface the cast error as 500 instead, which broke the + // SDK's fallback for 'port show/set/delete ' (SPEC-028). + if _, err := uuid.Parse(portID); err != nil { + common.SendError(c, common.NewNotFoundError("port")) + return + } + var id, name, networkID string var deviceID, deviceOwner sql.NullString var macAddress, status string @@ -452,12 +495,24 @@ func (svc *Service) GetPort(c *gin.Context) { var fixedIPsJSON []byte var createdAt, updatedAt time.Time - err := svc.pool.QueryRow(c.Request.Context(), ` - SELECT p.id, p.name, p.network_id, p.device_id, p.device_owner, p.mac_address, p.admin_state_up, p.status, p.fixed_ips, p.created_at, p.updated_at + // An administrator may show any port — the same carve-out GetNetwork and + // GetSubnet have (upstream's get_port policy is admin_or_owner; the + // admin's cross-project visibility the listings grant must not 404 on + // the follow-up show). Non-admins keep the read scope. The row's OWN + // project is selected and rendered as tenant_id — never the caller's. + portQuery := ` + SELECT p.id, p.name, p.network_id, p.project_id, p.device_id, p.device_owner, p.mac_address, p.admin_state_up, p.status, p.fixed_ips, p.created_at, p.updated_at FROM ports p JOIN networks n ON p.network_id = n.id - WHERE p.id = $1 AND `+portReadScope("$2")+` - `, portID, projectID).Scan(&id, &name, &networkID, &deviceID, &deviceOwner, &macAddress, &adminStateUp, &status, &fixedIPsJSON, &createdAt, &updatedAt) + WHERE p.id = $1` + portArgs := []any{portID} + if !common.HasRole(c, common.RoleAdmin) { + portQuery += ` AND ` + portReadScope("$2") + portArgs = append(portArgs, projectID) + } + var ownerID string + err := svc.pool.QueryRow(c.Request.Context(), portQuery, portArgs...). + Scan(&id, &name, &networkID, &ownerID, &deviceID, &deviceOwner, &macAddress, &adminStateUp, &status, &fixedIPsJSON, &createdAt, &updatedAt) if errors.Is(err, database.ErrNoRows) { common.SendError(c, common.NewNotFoundError("port")) @@ -498,7 +553,7 @@ func (svc *Service) GetPort(c *gin.Context) { "id": id, "name": name, "network_id": networkID, - "tenant_id": projectID, + "tenant_id": ownerID, "device_id": deviceID.String, "device_owner": deviceOwner.String, "mac_address": macAddress, @@ -564,6 +619,12 @@ func (svc *Service) UpdatePort(c *gin.Context) { portID := c.Param("id") projectID := c.GetString("project_id") + // See GetPort: a non-UUID reference must 404, not 500 (SPEC-028). + if _, err := uuid.Parse(portID); err != nil { + common.SendError(c, common.NewNotFoundError("port")) + return + } + var req struct { Port struct { Name *string `json:"name"` @@ -617,7 +678,12 @@ func (svc *Service) UpdatePort(c *gin.Context) { svc.GetPort(c) } -// allocateIP allocates an IP from a CIDR range using cryptographically random offset +// allocateIP allocates an IP from a CIDR range using a cryptographically +// random offset. It mirrors upstream Neutron's generate_pools(): /30, /31 and +// /32 (and every IPv6 subnet) are valid pools, network/broadcast are excluded +// where they exist, and only an exhausted/invalid pool yields "" — the +// callers' "no usable address". It must never panic: rand.Int panics on a +// non-positive bound, so every span is guarded. func allocateIP(cidr string) string { _, ipNet, err := net.ParseCIDR(cidr) if err != nil { @@ -841,8 +907,13 @@ func (svc *Service) CreateSecurityGroup(c *gin.Context) { "name": req.SecurityGroup.Name, "tenant_id": projectID, "description": req.SecurityGroup.Description, - "created_at": now.Format(time.RFC3339), - "updated_at": now.Format(time.RFC3339), + // The key must exist and be a list: upstream reports [] for a + // fresh group, and OSC's table formatter iterates it — a null + // crashed 'security group create' with "'NoneType' object is + // not iterable" (SPEC-028). + "security_group_rules": []gin.H{}, + "created_at": now.Format(time.RFC3339), + "updated_at": now.Format(time.RFC3339), }, }) } diff --git a/internal/neutron/pr107_review_test.go b/internal/neutron/pr107_review_test.go new file mode 100644 index 00000000..5fb74bbf --- /dev/null +++ b/internal/neutron/pr107_review_test.go @@ -0,0 +1,70 @@ +package neutron + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "testing" + + "github.com/cobaltcore-dev/lightstack/internal/database" + "github.com/google/uuid" +) + +type failInterfaceDB struct{ database.DB } +type failInterfaceTx struct{ database.Tx } + +func (d failInterfaceDB) Begin(ctx context.Context) (database.Tx, error) { + tx, err := d.DB.Begin(ctx) + return failInterfaceTx{tx}, err +} +func (tx failInterfaceTx) Exec(ctx context.Context, query string, args ...any) (database.Result, error) { + if strings.Contains(query, "INSERT INTO router_interfaces") { + return nil, errors.New("injected interface failure") + } + return tx.Tx.Exec(ctx, query, args...) +} + +func TestRouterAttachmentRollbackAndExhaustion(t *testing.T) { + requireDB(t) + ctx := context.Background() + netID, subnetID, _ := seedSpec028NetSubnet(t, ctx, "review") + t.Cleanup(func() { cleanupSpec028Net(t, ctx, netID) }) + routerID, portID := uuid.NewString(), uuid.NewString() + if _, err := testPool.Exec(ctx, "INSERT INTO routers (id,name,project_id) VALUES ($1,$2,$3)", routerID, routerID, seededProjectID); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { testPool.Exec(ctx, "DELETE FROM routers WHERE id=$1", routerID) }) + if _, err := testPool.Exec(ctx, "UPDATE subnets SET cidr='10.99.0.5/32', gateway_ip=NULL WHERE id=$1", subnetID); err != nil { + t.Fatal(err) + } + fixed := fmt.Sprintf(`[{"subnet_id":"%s","ip_address":"10.99.0.5"}]`, subnetID) + if _, err := testPool.Exec(ctx, "INSERT INTO ports (id,name,network_id,project_id,mac_address,fixed_ips) VALUES ($1,$2,$3,$4,$5,$6::jsonb)", portID, portID, netID, seededProjectID, "02:00:00:00:98:01", fixed); err != nil { + t.Fatal(err) + } + r, svc := newNeutronTestRouterWithService() + hdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "member"} + path := "/v2.0/routers/" + routerID + "/add_router_interface" + body := fmt.Sprintf(`{"subnet_id":"%s"}`, subnetID) + if code, resp := doReqBody(r, http.MethodPut, path, body, hdr); code != 409 { + t.Fatalf("exhausted subnet: %d %s", code, resp) + } + body = fmt.Sprintf(`{"port_id":"%s","subnet_id":"%s"}`, portID, subnetID) + if code, resp := doReqBody(r, http.MethodPut, path, body, hdr); code != 400 { + t.Fatalf("ambiguous selector: %d %s", code, resp) + } + svc.pool = failInterfaceDB{testPool} + body = fmt.Sprintf(`{"port_id":"%s"}`, portID) + if code, resp := doReqBody(r, http.MethodPut, path, body, hdr); code != 500 { + t.Fatalf("injected failure: %d %s", code, resp) + } + var free bool + if err := testPool.QueryRow(ctx, "SELECT device_id IS NULL AND device_owner IS NULL FROM ports WHERE id=$1", portID).Scan(&free); err != nil || !free { + t.Fatalf("port not rolled back: free=%v err=%v", free, err) + } + svc.pool = testPool + if code, resp := doReqBody(r, http.MethodPut, path, body, hdr); code != 200 { + t.Fatalf("existing port on full subnet: %d %s", code, resp) + } +} diff --git a/internal/neutron/router.go b/internal/neutron/router.go index cbccaac6..68bc756f 100644 --- a/internal/neutron/router.go +++ b/internal/neutron/router.go @@ -93,19 +93,40 @@ func (svc *Service) ListRouters(c *gin.Context) { } f := &common.SQLFilter{} - f.Eq("project_id", projectID) + // The caller's project — unless an administrator filters by tenant: + // upstream honours the filter for admin tokens, and it was silently + // ignored before, so `router list --project X` presented the admin's + // routers as project X's (SPEC-028; mirrors buildNetworkListFilter's + // tenant bypass). ::text so a non-UUID value narrows to nothing instead + // of failing the cast. + routerTenant := c.Query("tenant_id") + if routerTenant == "" { + routerTenant = c.Query("project_id") + } + routerAdminBypass := routerTenant != "" && common.HasRole(c, common.RoleAdmin) + if routerAdminBypass { + f.Eq("project_id::text", routerTenant) + } else { + f.Eq("project_id", projectID) + } // Marker paging over a TOTAL order. created_at alone is not unique, so the order was undefined // among ties AND `created_at < $n` excluded every row sharing the marker's timestamp rather than // only those before it — a page walk silently dropped them. id is the primary key, so - // (created_at, id) is total (as #549 established for compute). + // (created_at, id) is total (as #549 established for compute). The marker resolves under the + // listing's own scope: an unresolvable marker would silently drop the condition and make the + // walk restart from the first page forever. if marker := c.Query("marker"); marker != "" { var markerCreatedAt time.Time var markerID string - err := svc.pool.QueryRow(c.Request.Context(), - "SELECT created_at, id FROM routers WHERE id = $1 AND project_id = $2", - marker, projectID, - ).Scan(&markerCreatedAt, &markerID) + markerQuery := "SELECT created_at, id FROM routers WHERE id = $1" + markerArgs := []any{marker} + if !routerAdminBypass { + markerQuery += " AND project_id = $2" + markerArgs = append(markerArgs, projectID) + } + err := svc.pool.QueryRow(c.Request.Context(), markerQuery, markerArgs...). + Scan(&markerCreatedAt, &markerID) if err == nil { f.Cond(fmt.Sprintf("(created_at, id) < (%s, %s)", f.Arg(markerCreatedAt), f.Arg(markerID))) } @@ -268,15 +289,24 @@ func (svc *Service) GetRouter(c *gin.Context) { // id::text comparison resolves by UUID or name without crashing on a // non-UUID identifier — the CLI looks a router up by name this way - // (consistent with GetNetwork/GetSubnet). - err := svc.pool.QueryRow(c.Request.Context(), ` + // (consistent with GetNetwork/GetSubnet). An administrator may show any + // router (upstream's get_router policy is admin_or_owner), so the + // tenant-filtered listing's rows do not dead-end their follow-up show + // with 404 — the same carve-out the other reads got (SPEC-028). A + // non-admin keeps the project scope. + routerQuery := ` SELECT id, name, project_id, admin_state_up, status, external_gateway_info, distributed, ha, created_at, updated_at FROM routers - WHERE (id::text = $1 OR name = $1) AND project_id = $2 - LIMIT 1 - `, routerID, projectID).Scan(&r.ID, &r.Name, &r.ProjectID, &r.AdminStateUp, &r.Status, - &gatewayInfo, &r.Distributed, &r.HA, &r.CreatedAt, &r.UpdatedAt) + WHERE (id::text = $1 OR name = $1)` + routerArgs := []any{routerID} + if !common.HasRole(c, common.RoleAdmin) { + routerQuery += ` AND project_id = $2` + routerArgs = append(routerArgs, projectID) + } + err := svc.pool.QueryRow(c.Request.Context(), routerQuery+" LIMIT 1", routerArgs...). + Scan(&r.ID, &r.Name, &r.ProjectID, &r.AdminStateUp, &r.Status, + &gatewayInfo, &r.Distributed, &r.HA, &r.CreatedAt, &r.UpdatedAt) if errors.Is(err, database.ErrNoRows) { common.SendError(c, common.NewNotFoundError("router")) @@ -453,6 +483,7 @@ func (svc *Service) DeleteRouter(c *gin.Context) { func (svc *Service) AddRouterInterface(c *gin.Context) { routerID := c.Param("id") projectID := c.GetString("project_id") + ctx := c.Request.Context() var req struct { SubnetID string `json:"subnet_id"` @@ -463,80 +494,226 @@ func (svc *Service) AddRouterInterface(c *gin.Context) { common.SendError(c, common.NewBadRequestError("invalid request body")) return } + if (req.SubnetID == "") == (req.PortID == "") { + common.SendError(c, common.NewBadRequestError("exactly one of subnet_id or port_id must be specified")) + return + } - // Verify router exists - var routerExists bool - err := svc.pool.QueryRow(c.Request.Context(), - "SELECT EXISTS(SELECT 1 FROM routers WHERE id = $1 AND project_id = $2)", - routerID, projectID, - ).Scan(&routerExists) - - if err != nil || !routerExists { - common.SendError(c, common.NewNotFoundError("router")) + // Authorise through the same resolver RemoveRouterInterface uses: the + // raw EXISTS lookup here did the same job for a UUID, but went unwrapped + // for the ownership question this answer now settles the same way. + id, ok := svc.resolveOwnRouter(c, routerID, projectID) + if !ok { return } - // Get subnet information - var subnetID, networkID, cidr, gatewayIP string - err = svc.pool.QueryRow(c.Request.Context(), - "SELECT id, network_id, cidr, gateway_ip FROM subnets WHERE id = $1", - req.SubnetID, - ).Scan(&subnetID, &networkID, &cidr, &gatewayIP) + var subnetID, networkID, cidr, portID string + var gatewayIP *string + var interfaceIP string + if req.PortID != "" { + // port_id path — attach a port the caller already owns. The handler + // used to ignore the field entirely and read the (empty) subnet_id + // instead, so 'router add port' answered 500 "failed to get subnet" + // (SPEC-028). + if _, err := uuid.Parse(req.PortID); err != nil { + common.SendError(c, common.NewNotFoundError("port")) + return + } + var fixedIPsJSON []byte + var deviceID sql.NullString + err := svc.pool.QueryRow(ctx, ` + SELECT id::text, network_id::text, fixed_ips, device_id + FROM ports + WHERE id = $1 AND project_id = $2 + `, req.PortID, projectID).Scan(&portID, &networkID, &fixedIPsJSON, &deviceID) + if errors.Is(err, database.ErrNoRows) { + common.SendError(c, common.NewNotFoundError("port")) + return + } + if err != nil { + log.Error().Err(err).Str("operation", "add_router_interface_port_lookup").Str("port_id", req.PortID).Msg("database error") + common.SendError(c, common.NewInternalServerError("failed to get port")) + return + } + if deviceID.Valid && deviceID.String != "" { + common.SendError(c, common.NewResourceInUseError("port", "attach port to router", + "Port "+portID+" already has a device assigned and cannot be attached to a router.")) + return + } + var fixedIPs common.FixedIPs + if err := json.Unmarshal(fixedIPsJSON, &fixedIPs); err != nil || len(fixedIPs) != 1 || fixedIPs[0].SubnetID == "" || net.ParseIP(fixedIPs[0].IPAddress) == nil { + common.SendError(c, common.NewBadRequestError("router port must have exactly one valid fixed IP")) + return + } + subnetID = fixedIPs[0].SubnetID + interfaceIP = fixedIPs[0].IPAddress + } + // Subnet row — reached from either path (the caller's subnet_id, or the + // subnet the port's first fixed IP comes from). + if req.PortID == "" { + if _, err := uuid.Parse(req.SubnetID); err != nil { + common.SendError(c, common.NewNotFoundError("subnet")) + return + } + subnetID = req.SubnetID + } + // The subnet must be the caller's, or sit on a network the caller can + // legitimately use — their own (an admin may have created the subnet + // on it, which subnetReadScope allows) or a shared/external one, + // mirroring the port path's project scoping. + err := svc.pool.QueryRow(ctx, ` + SELECT s.id::text, s.network_id::text, s.cidr, s.gateway_ip + FROM subnets s + JOIN networks n ON n.id = s.network_id + WHERE s.id = $1 + AND (s.project_id = $2 OR n.project_id = $2 OR n.shared = true OR n.external = true) + `, subnetID, projectID).Scan(&subnetID, &networkID, &cidr, &gatewayIP) if errors.Is(err, database.ErrNoRows) { + // Not-found for both a bogus id and another project's private subnet: + // the port path above is project-scoped the same way, and answering + // 404 rather than 403 keeps the resource's existence private. common.SendError(c, common.NewNotFoundError("subnet")) return } if err != nil { - log.Error().Err(err).Str("operation", "add_router_interface_subnet_lookup").Str("subnet_id", req.SubnetID).Msg("database error") + log.Error().Err(err).Str("operation", "add_router_interface_subnet_lookup").Str("subnet_id", subnetID).Msg("database error") common.SendError(c, common.NewInternalServerError("failed to get subnet")) return } - // Create a port for the router interface - portID := uuid.New().String() - macAddress := generateMAC() + // Serialize allocation with port/VM creation. The port claim and interface + // record commit together, including rollback on request cancellation. + tx, err := svc.pool.Begin(ctx) + if err != nil { + common.SendError(c, common.NewInternalServerError("failed to begin router attachment")) + return + } + defer func() { _ = tx.Rollback(context.Background()) }() + if _, err := LockSubnet(ctx, tx, subnetID); err != nil { + common.SendError(c, common.NewInternalServerError("failed to lock subnet")) + return + } + if req.PortID == "" { + if gatewayIP != nil { + interfaceIP = *gatewayIP + } else { + interfaceIP, err = AllocateFreeIP(ctx, tx, subnetID, cidr) + if err != nil { + if errors.Is(err, ErrNoAvailableIP) { + common.SendError(c, common.NewConflictError("subnet has no available IP address")) + } else { + common.SendError(c, common.NewInternalServerError("failed to allocate router address")) + } + return + } + } + } - fixedIPs := []map[string]string{ - { - "subnet_id": subnetID, - "ip_address": gatewayIP, - }, + // A second interface for the same (router, subnet) is upstream's 409 — + // and it must be refused BEFORE anything is written: the unique + // constraint used to turn it into a 500 that left the freshly created + // interface port behind, which then blocked the network's deletion with + // a 409 nobody could 'router remove port' away (the interface row had + // never been inserted, so the remove lookup found nothing; SPEC-028). + var alreadyAttached bool + err = tx.QueryRow(ctx, + "SELECT EXISTS(SELECT 1 FROM router_interfaces WHERE router_id = $1 AND subnet_id = $2)", + id, subnetID).Scan(&alreadyAttached) + if err != nil { + log.Error().Err(err).Str("operation", "add_router_interface_duplicate_check").Str("router_id", id).Msg("database error") + common.SendError(c, common.NewInternalServerError("failed to check router interface")) + return + } + if alreadyAttached { + common.SendError(c, common.NewResourceInUseError("router", "add interface to router", + "Router already has an interface on subnet "+subnetID+".")) + return } - fixedIPsJSON, _ := json.Marshal(fixedIPs) now := time.Now() - _, err = svc.pool.Exec(c.Request.Context(), ` - INSERT INTO ports (id, name, network_id, project_id, device_id, device_owner, mac_address, admin_state_up, status, fixed_ips, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) - `, portID, "router-interface-"+routerID[:8], networkID, projectID, routerID, "network:router_interface", - macAddress, true, "ACTIVE", fixedIPsJSON, now, now) + if req.PortID == "" { + // Create a port for the router interface + portID = uuid.New().String() + macAddress := generateMAC() + + fixedIPs := []map[string]string{ + { + "subnet_id": subnetID, + "ip_address": interfaceIP, + }, + } + fixedIPsJSON, _ := json.Marshal(fixedIPs) - if err != nil { - log.Error().Err(err).Str("operation", "add_router_interface_port").Msg("database error") - common.SendError(c, common.NewInternalServerError("failed to create router interface port")) - return + _, err = tx.Exec(ctx, ` + INSERT INTO ports (id, name, network_id, project_id, device_id, device_owner, mac_address, admin_state_up, status, fixed_ips, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + `, portID, "router-interface-"+id[:8], networkID, projectID, id, "network:router_interface", + macAddress, true, "ACTIVE", fixedIPsJSON, now, now) + + if err != nil { + log.Error().Err(err).Str("operation", "add_router_interface_port").Msg("database error") + common.SendError(c, common.NewInternalServerError("failed to create router interface port")) + return + } + } else { + // Claim the caller's port for the router — CONDITIONALLY. The + // device_id read earlier is only a check, not a reservation: two + // concurrent attaches of the same port to different routers both + // pass it, and an unconditional UPDATE lets both claims win (last + // write leaves the port owned by a router whose interface row also + // points at it). The WHERE keeps the claim atomic in the row lock; + // zero rows affected means someone claimed it in between. (The + // column is UUID: an unclaimed port is NULL, never ''.) + ct, err := tx.Exec(ctx, ` + UPDATE ports SET device_id = $1, device_owner = 'network:router_interface', updated_at = $2 + WHERE id = $3 AND device_id IS NULL + `, id, now, portID) + if err != nil { + log.Error().Err(err).Str("operation", "add_router_interface_port_claim").Str("port_id", portID).Msg("database error") + common.SendError(c, common.NewInternalServerError("failed to attach port to router")) + return + } + if ct.RowsAffected() == 0 { + common.SendError(c, common.NewResourceInUseError("port", "attach port to router", + "Port "+portID+" already has a device assigned and cannot be attached to a router.")) + return + } } // Create router interface record - _, err = svc.pool.Exec(c.Request.Context(), ` + _, err = tx.Exec(ctx, ` INSERT INTO router_interfaces (id, router_id, port_id, subnet_id, created_at) VALUES ($1, $2, $3, $4, $5) - `, uuid.New().String(), routerID, portID, subnetID, now) + `, uuid.New().String(), id, portID, subnetID, now) if err != nil { + if database.IsUniqueViolation(err) { + // The constraint is the duplicate-interface backstop: the + // EXISTS check above can lose a race with a concurrent attach + // of the same (router, subnet). Answer the same 409 the check + // produces instead of surfacing the race as a 500. + common.SendError(c, common.NewResourceInUseError("router", "add interface to router", + "Router already has an interface on subnet "+subnetID+".")) + return + } log.Error().Err(err).Str("operation", "add_router_interface_record").Msg("database error") common.SendError(c, common.NewInternalServerError("failed to create router interface record")) return } + if err := tx.Commit(ctx); err != nil { + common.SendError(c, common.NewInternalServerError("failed to commit router attachment")) + return + } + // WS-3: if this router already has an external gateway, masquerade the // just-attached subnet's egress out the uplink now (real mode), so egress // works whether the gateway was set before or after this interface. if svc.mode != "stub" { var snatOn bool if err := svc.pool.QueryRow(c.Request.Context(), - "SELECT EXISTS (SELECT 1 FROM routers rt WHERE rt.id = $1 AND "+snatOnPredicate+")", routerID).Scan(&snatOn); err == nil && snatOn { + "SELECT EXISTS (SELECT 1 FROM routers rt WHERE rt.id = $1 AND "+snatOnPredicate+")", id).Scan(&snatOn); err == nil && snatOn { if uplink, uerr := networking.DefaultUplinkInterface(); uerr == nil { if err := svc.routerManager.EnsureEgressSNAT(cidr, uplink); err != nil { log.Warn().Err(err).Str("router_id", routerID).Str("cidr", cidr).Msg("WS-3: egress SNAT on interface attach failed") @@ -555,7 +732,7 @@ func (svc *Service) AddRouterInterface(c *gin.Context) { maskBits, _ := ipNet.Mask.Size() cidrSuffix := fmt.Sprintf("%d", maskBits) - if err := svc.routerManager.AttachInterfaceToRouter(routerID, interfaceName, gatewayIP, cidrSuffix); err != nil { + if err := svc.routerManager.AttachInterfaceToRouter(id, interfaceName, interfaceIP, cidrSuffix); err != nil { log.Warn().Err(err).Str("router_id", routerID).Str("interface", interfaceName).Msg("failed to attach interface to router") } diff --git a/internal/neutron/spec028_conformance_test.go b/internal/neutron/spec028_conformance_test.go new file mode 100644 index 00000000..7aba89e6 --- /dev/null +++ b/internal/neutron/spec028_conformance_test.go @@ -0,0 +1,1146 @@ +package neutron + +// SPEC-028 conformance fixes for the Neutron list surface, found by the CLI +// conformance suites (issue #105). + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/cobaltcore-dev/lightstack/pkg/cache" + "github.com/gin-gonic/gin" + "github.com/google/uuid" +) + +// TestListNetworksAdminTenantFilter_DB — an administrator filtering +// `GET /v2.0/networks?tenant_id=` used to get the caller's read +// scope ANDed in (`project_id = caller OR shared OR external`), so another +// project's private networks could never appear and the answer was []. +// Real Neutron honours the tenant filter for admin tokens — the CLI's +// `network list --project ` depends on it. +func TestListNetworksAdminTenantFilter_DB(t *testing.T) { + requireDB(t) + ctx := context.Background() + + otherProject := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO projects (id, name, domain_id, enabled) VALUES ($1,$2,NULL,true)`, + otherProject, "spec028-tenant-proj-"+otherProject[:8]); err != nil { + t.Fatalf("insert other project: %v", err) + } + netID := uuid.NewString() + netName := "spec028-tenant-net-" + netID[:8] + if _, err := testPool.Exec(ctx, + `INSERT INTO networks (id, name, project_id, shared, external) VALUES ($1,$2,$3,false,false)`, + netID, netName, otherProject); err != nil { + t.Fatalf("insert other-tenant net: %v", err) + } + t.Cleanup(func() { + testPool.Exec(ctx, "DELETE FROM networks WHERE id=$1", netID) + testPool.Exec(ctx, "DELETE FROM projects WHERE id=$1", otherProject) + }) + + r := newNeutronTestRouter() + + // Admin: the tenant filter is the authoritative owner scope. + hdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "admin"} + _, body := doReq(r, http.MethodGet, "/v2.0/networks?tenant_id="+otherProject, hdr) + if nets := networksFromBody(t, body); !hasNet(nets, netName) { + t.Errorf("admin ?tenant_id= did not return the other project's private network; body=%s", body) + } + + // project_id is an alias for tenant_id. + _, body = doReq(r, http.MethodGet, "/v2.0/networks?project_id="+otherProject, hdr) + if nets := networksFromBody(t, body); !hasNet(nets, netName) { + t.Errorf("admin ?project_id= did not return the other project's private network; body=%s", body) + } + + // Non-admin: the read scope still applies, so another project's private + // network stays invisible even with the filter. + hdr["X-Test-Roles"] = "member" + _, body = doReq(r, http.MethodGet, "/v2.0/networks?tenant_id="+otherProject, hdr) + if nets := networksFromBody(t, body); hasNet(nets, netName) { + t.Errorf("non-admin ?tenant_id= leaked another project's private network; body=%s", body) + } +} + +// TestListFloatingIPsFilters_DB — `GET /v2.0/floatingips` ignored every list +// filter, so `openstack floating ip list --network ` returned addresses +// from all networks (SPEC-028). The standard exact-match filters must hold. +func TestListFloatingIPsFilters_DB(t *testing.T) { + requireDB(t) + ctx := context.Background() + + netA, netB := uuid.NewString(), uuid.NewString() + for _, n := range []struct{ id, name string }{{netA, "spec028-fipnet-a-" + netA[:8]}, {netB, "spec028-fipnet-b-" + netB[:8]}} { + if _, err := testPool.Exec(ctx, + `INSERT INTO networks (id, name, project_id, shared, external) VALUES ($1,$2,$3,false,false)`, + n.id, n.name, seededProjectID); err != nil { + t.Fatalf("insert net %s: %v", n.name, err) + } + } + fipA, fipB := uuid.NewString(), uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO floating_ips (id, project_id, floating_network_id, floating_ip_address, status, description) + VALUES ($1,$2,$3,$4,'ACTIVE','spec028 fip a')`, + fipA, seededProjectID, netA, "198.19.254.1"); err != nil { + t.Fatalf("insert fip a: %v", err) + } + if _, err := testPool.Exec(ctx, + `INSERT INTO floating_ips (id, project_id, floating_network_id, floating_ip_address, status, description) + VALUES ($1,$2,$3,$4,'DOWN','spec028 fip b')`, + fipB, seededProjectID, netB, "198.19.254.2"); err != nil { + t.Fatalf("insert fip b: %v", err) + } + t.Cleanup(func() { + testPool.Exec(ctx, "DELETE FROM floating_ips WHERE id = ANY($1)", []string{fipA, fipB}) + testPool.Exec(ctx, "DELETE FROM networks WHERE id = ANY($1)", []string{netA, netB}) + }) + + r := newNeutronTestRouter() + hdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "member"} + fips := func(body string) []map[string]any { + t.Helper() + var resp struct { + FloatingIPs []map[string]any `json:"floatingips"` + } + if err := json.Unmarshal([]byte(body), &resp); err != nil { + t.Fatalf("unmarshal floatingips: %v (body=%s)", err, body) + } + return resp.FloatingIPs + } + has := func(list []map[string]any, id string) bool { + for _, f := range list { + if f["id"] == id { + return true + } + } + return false + } + + // floating_network_id keeps only the address on that network. + _, body := doReq(r, http.MethodGet, "/v2.0/floatingips?floating_network_id="+netA, hdr) + if got := fips(body); !has(got, fipA) || has(got, fipB) { + t.Errorf("?floating_network_id=netA returned a=%v b=%v; want a only", has(got, fipA), has(got, fipB)) + } + + // status filters too. + _, body = doReq(r, http.MethodGet, "/v2.0/floatingips?status=DOWN", hdr) + if got := fips(body); !has(got, fipB) || has(got, fipA) { + t.Errorf("?status=DOWN returned a=%v b=%v; want b only", has(got, fipA), has(got, fipB)) + } + + // A non-UUID value on a UUID column answers the same empty list a + // well-formed unknown id gets, not a 500. + code, body := doReq(r, http.MethodGet, "/v2.0/floatingips?port_id=not-a-uuid", hdr) + if code != http.StatusOK { + t.Errorf("?port_id=not-a-uuid: code=%d; want 200", code) + } + if got := fips(body); len(got) != 0 { + t.Errorf("?port_id=not-a-uuid returned %d fips; want 0", len(got)) + } +} + +// --------------------------------------------------------------------------- +// CLI conformance round two (SPEC-028): the network suite's findings. + +// seedSpec028NetSubnet creates a network + subnet owned by the seeded +// project, with a random third octet so concurrent runs cannot collide. +func seedSpec028NetSubnet(t *testing.T, ctx context.Context, label string) (netID, subnetID, cidr string) { + t.Helper() + b, _ := uuid.NewRandom() + octet := int(b[0])%200 + 10 + netID, subnetID = uuid.NewString(), uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO networks (id, name, project_id, shared, external) VALUES ($1,$2,$3,false,false)`, + netID, "spec028-"+label+"-net-"+netID[:8], seededProjectID); err != nil { + t.Fatalf("insert network: %v", err) + } + cidr = fmt.Sprintf("10.%d.0.0/24", octet) + if _, err := testPool.Exec(ctx, + `INSERT INTO subnets (id, name, network_id, project_id, cidr, gateway_ip, ip_version) VALUES ($1,$2,$3,$4,$5,$6,4)`, + subnetID, "spec028-"+label+"-sub-"+subnetID[:8], netID, seededProjectID, cidr, fmt.Sprintf("10.%d.0.1", octet)); err != nil { + t.Fatalf("insert subnet: %v", err) + } + return netID, subnetID, cidr +} + +// cleanupSpec028Net removes everything a seed + router-interface test can +// leave on a network, then the network itself. +func cleanupSpec028Net(t *testing.T, ctx context.Context, netID string) { + t.Helper() + testPool.Exec(ctx, `DELETE FROM ports WHERE network_id = $1`, netID) + testPool.Exec(ctx, `DELETE FROM subnets WHERE network_id = $1`, netID) + testPool.Exec(ctx, `DELETE FROM networks WHERE id = $1`, netID) +} + +// TestPortShowAndUpdateByName404_DB — GET/PUT /v2.0/ports/{name} used to +// surface the uuid cast error as a 500, which breaks the SDK's name-fallback +// for 'port show/set/delete ' (it only falls back on 404). Real +// Neutron answers 404; now so does this (SPEC-028). +func TestPortShowAndUpdateByName404_DB(t *testing.T) { + requireDB(t) + ctx := context.Background() + netID, _, _ := seedSpec028NetSubnet(t, ctx, "portname") + t.Cleanup(func() { cleanupSpec028Net(t, ctx, netID) }) + + portID := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO ports (id, name, network_id, project_id, mac_address, admin_state_up, status, fixed_ips) + VALUES ($1,$2,$3,$4,$5,true,'ACTIVE','[]'::jsonb)`, + portID, "spec028-byname-port", netID, seededProjectID, "02:00:00:00:00:01"); err != nil { + t.Fatalf("insert port: %v", err) + } + + r := newNeutronTestRouter() + hdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "member"} + + if code, body := doReq(r, http.MethodGet, "/v2.0/ports/spec028-byname-port", hdr); code != http.StatusNotFound { + t.Errorf("GET port by name: code=%d; want 404 (body=%s)", code, body) + } + if code, body := doReqBody(r, http.MethodPut, "/v2.0/ports/spec028-byname-port", + `{"port":{"name":"renamed"}}`, hdr); code != http.StatusNotFound { + t.Errorf("PUT port by name: code=%d; want 404 (body=%s)", code, body) + } + if code, body := doReq(r, http.MethodGet, "/v2.0/ports/"+portID, hdr); code != http.StatusOK { + t.Errorf("GET port by id: code=%d; want 200 (body=%s)", code, body) + } +} + +// TestSubnetPoolShowByName404_DB — same divergence on subnet pools: a +// by-name GET/PUT was a 500, and the CLI's name resolution needs the 404 +// (SPEC-028). +func TestSubnetPoolShowByName404_DB(t *testing.T) { + requireDB(t) + ctx := context.Background() + poolID := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO subnet_pools (id, project_id, name, prefixes, min_prefixlen, max_prefixlen, ip_version) + VALUES ($1,$2,$3,$4,8,32,4)`, + poolID, seededProjectID, "spec028-byname-pool", []string{"198.51.100.0/24"}); err != nil { + t.Fatalf("insert subnet pool: %v", err) + } + t.Cleanup(func() { testPool.Exec(ctx, "DELETE FROM subnet_pools WHERE id=$1", poolID) }) + + r := newNeutronTestRouter() + hdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "member"} + + if code, body := doReq(r, http.MethodGet, "/v2.0/subnetpools/spec028-byname-pool", hdr); code != http.StatusNotFound { + t.Errorf("GET subnet pool by name: code=%d; want 404 (body=%s)", code, body) + } + if code, body := doReqBody(r, http.MethodPut, "/v2.0/subnetpools/spec028-byname-pool", + `{"subnetpool":{"name":"renamed"}}`, hdr); code != http.StatusNotFound { + t.Errorf("PUT subnet pool by name: code=%d; want 404 (body=%s)", code, body) + } + if code, body := doReq(r, http.MethodGet, "/v2.0/subnetpools/"+poolID, hdr); code != http.StatusOK { + t.Errorf("GET subnet pool by id: code=%d; want 200 (body=%s)", code, body) + } +} + +// TestSubnetPoolReportsOwningProjectForSharedPool_DB — shared pools are +// visible to every project, so ListSubnetPools/GetSubnetPool serve rows whose +// project_id is not the caller's. Both used to render the CALLER's project as +// tenant_id; the response must name the row's owner (the same round-8 bug +// fixed for cinder volumes and neutron ports). Visibility semantics stay: +// another project's private pool remains invisible and 404s by id. +func TestSubnetPoolReportsOwningProjectForSharedPool_DB(t *testing.T) { + requireDB(t) + ctx := context.Background() + r := newNeutronTestRouter() + ensureTenantB(t) + + ownerHdr := hdr(tenantA, "member") + otherHdr := hdr(tenantB, "member") + + createPool := func(name string, shared bool) string { + t.Helper() + code, body := doReqBody(r, http.MethodPost, "/v2.0/subnetpools", + fmt.Sprintf(`{"subnetpool":{"name":%q,"prefixes":["198.51.100.0/24"],"shared":%t}}`, name, shared), ownerHdr) + if code != http.StatusCreated { + t.Fatalf("create pool %s: code=%d (body=%s)", name, code, body) + } + var created struct { + Pool struct { + ID string `json:"id"` + } `json:"subnetpool"` + } + if err := json.Unmarshal([]byte(body), &created); err != nil { + t.Fatalf("unmarshal pool: %v (body=%s)", err, body) + } + return created.Pool.ID + } + sharedID := createPool("spec028-shared-owner", true) + privateID := createPool("spec028-private-owner", false) + t.Cleanup(func() { + testPool.Exec(ctx, "DELETE FROM subnet_pools WHERE id = ANY($1)", []string{sharedID, privateID}) + }) + + // The other project's list sees the shared pool, attributed to its owner, + // and does not see the private one. + code, body := doReq(r, http.MethodGet, "/v2.0/subnetpools", otherHdr) + if code != http.StatusOK { + t.Fatalf("list subnet pools as the other project: code=%d (body=%s)", code, body) + } + var list struct { + Pools []struct { + ID string `json:"id"` + TenantID string `json:"tenant_id"` + } `json:"subnetpools"` + } + if err := json.Unmarshal([]byte(body), &list); err != nil { + t.Fatalf("unmarshal list: %v (body=%s)", err, body) + } + var sawShared, sawPrivate bool + for _, p := range list.Pools { + switch p.ID { + case sharedID: + sawShared = true + if p.TenantID != tenantA { + t.Errorf("shared pool list tenant_id = %q; want its owner %q, not the caller %q", p.TenantID, tenantA, tenantB) + } + case privateID: + sawPrivate = true + } + } + if !sawShared { + t.Errorf("other project's list does not contain the shared pool %s: %s", sharedID, body) + } + if sawPrivate { + t.Errorf("other project's list leaked the private pool %s: %s", privateID, body) + } + + // Show the shared pool from the other project: 200, owner attribution. + code, body = doReq(r, http.MethodGet, "/v2.0/subnetpools/"+sharedID, otherHdr) + if code != http.StatusOK { + t.Fatalf("show shared pool as the other project: code=%d (body=%s)", code, body) + } + var got struct { + Pool struct { + TenantID string `json:"tenant_id"` + } `json:"subnetpool"` + } + if err := json.Unmarshal([]byte(body), &got); err != nil { + t.Fatalf("unmarshal shared pool: %v (body=%s)", err, body) + } + if got.Pool.TenantID != tenantA { + t.Errorf("shared pool show tenant_id = %q; want its owner %q, not the caller %q", got.Pool.TenantID, tenantA, tenantB) + } + + // A foreign private pool keeps the existing 404 behaviour. + if code, body := doReq(r, http.MethodGet, "/v2.0/subnetpools/"+privateID, otherHdr); code != http.StatusNotFound { + t.Errorf("show foreign private pool: code=%d (body=%s); want 404", code, body) + } + + // The owner's own read is unchanged. + code, body = doReq(r, http.MethodGet, "/v2.0/subnetpools/"+sharedID, ownerHdr) + if code != http.StatusOK { + t.Fatalf("owner shows shared pool: code=%d (body=%s)", code, body) + } +} + +// TestRouterInterfaceByPortAndDuplicate409_DB — three coupled defects in +// add_router_interface (SPEC-028): +// - port_id was ignored: 'router add port' read the empty subnet_id and +// answered 500 "failed to get subnet". +// - a duplicate interface hit the (router_id, subnet_id) unique constraint +// as a 500 that LEAKED the port it had already inserted, which then +// blocked the network's deletion with a 409 nothing could remove. +// - a routed subnet could be deleted (upstream: 409 SubnetInUse), orphaning +// the interface port. +func TestRouterInterfaceByPortAndDuplicate409_DB(t *testing.T) { + requireDB(t) + ctx := context.Background() + netID, subnetID, _ := seedSpec028NetSubnet(t, ctx, "if") + t.Cleanup(func() { cleanupSpec028Net(t, ctx, netID) }) + + routerID := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO routers (id, name, project_id) VALUES ($1,$2,$3)`, + routerID, "spec028-if-router-"+routerID[:8], seededProjectID); err != nil { + t.Fatalf("insert router: %v", err) + } + t.Cleanup(func() { testPool.Exec(ctx, "DELETE FROM routers WHERE id=$1", routerID) }) + + portID := uuid.NewString() + fixedIPs := fmt.Sprintf(`[{"subnet_id":"%s","ip_address":"10.99.0.5"}]`, subnetID) + if _, err := testPool.Exec(ctx, + `INSERT INTO ports (id, name, network_id, project_id, mac_address, admin_state_up, status, fixed_ips) + VALUES ($1,$2,$3,$4,$5,true,'ACTIVE',$6::jsonb)`, + portID, "spec028-if-port", netID, seededProjectID, "02:00:00:00:00:02", fixedIPs); err != nil { + t.Fatalf("insert port: %v", err) + } + + r := newNeutronTestRouter() + hdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "member"} + add := func(body string) (int, string) { + return doReqBody(r, http.MethodPut, "/v2.0/routers/"+routerID+"/add_router_interface", body, hdr) + } + + // Neither selector is an error the caller cannot diagnose. + if code, body := add(`{}`); code != http.StatusBadRequest { + t.Errorf("add with no selector: code=%d; want 400 (body=%s)", code, body) + } + if code, body := add(`{"port_id":"not-a-uuid"}`); code != http.StatusNotFound { + t.Errorf("add with non-UUID port_id: code=%d; want 404 (body=%s)", code, body) + } + if code, body := add(fmt.Sprintf(`{"port_id":"%s"}`, uuid.NewString())); code != http.StatusNotFound { + t.Errorf("add with unknown port_id: code=%d; want 404 (body=%s)", code, body) + } + + // The port path: 200, and the answer names the port and its subnet. + code, body := add(fmt.Sprintf(`{"port_id":"%s"}`, portID)) + if code != http.StatusOK { + t.Fatalf("add by port_id: code=%d; want 200 (body=%s)", code, body) + } + var resp struct { + SubnetID string `json:"subnet_id"` + PortID string `json:"port_id"` + } + if err := json.Unmarshal([]byte(body), &resp); err != nil { + t.Fatalf("unmarshal add response: %v (body=%s)", err, body) + } + if resp.PortID != portID || resp.SubnetID != subnetID { + t.Errorf("add by port_id: port=%s subnet=%s; want port=%s subnet=%s", resp.PortID, resp.SubnetID, portID, subnetID) + } + + // The port is claimed by the router now. + var owner string + if err := testPool.QueryRow(ctx, + `SELECT device_owner FROM ports WHERE id=$1`, portID).Scan(&owner); err != nil || owner != "network:router_interface" { + t.Errorf("claimed port device_owner=%q err=%v; want network:router_interface", owner, err) + } + + // An already-attached port is a 409, not a second interface. + if code, body := add(fmt.Sprintf(`{"port_id":"%s"}`, portID)); code != http.StatusConflict { + t.Errorf("add attached port again: code=%d; want 409 (body=%s)", code, body) + } + + // A second interface for the same subnet is a 409 that leaks nothing. + if code, body := add(fmt.Sprintf(`{"subnet_id":"%s"}`, subnetID)); code != http.StatusConflict { + t.Errorf("duplicate interface: code=%d; want 409 (body=%s)", code, body) + } + var leaked int + if err := testPool.QueryRow(ctx, + `SELECT COUNT(*) FROM ports WHERE network_id=$1 AND device_owner='network:router_interface'`, + netID).Scan(&leaked); err != nil || leaked != 1 { + t.Errorf("router-interface ports on network after duplicate: count=%d err=%v; want 1", leaked, err) + } + + // The routed subnet cannot be deleted (upstream: 409 SubnetInUse). + if code, body := doReq(r, http.MethodDelete, "/v2.0/subnets/"+subnetID, hdr); code != http.StatusConflict { + t.Errorf("delete routed subnet: code=%d; want 409 (body=%s)", code, body) + } + + // Removing the interface frees the subnet for deletion. + if code, body := doReqBody(r, http.MethodPut, "/v2.0/routers/"+routerID+"/remove_router_interface", + fmt.Sprintf(`{"port_id":"%s"}`, portID), hdr); code != http.StatusOK { + t.Fatalf("remove interface by port: code=%d; want 200 (body=%s)", code, body) + } + if code, body := doReq(r, http.MethodDelete, "/v2.0/subnets/"+subnetID, hdr); code != http.StatusNoContent { + t.Errorf("delete subnet after interface removal: code=%d; want 204 (body=%s)", code, body) + } +} + +// TestNetworkDescriptionRoundTrip_DB — networks had no description column, +// so 'network set --description' was a 400 "no updates provided" +// (SPEC-028, migration 075). +func TestNetworkDescriptionRoundTrip_DB(t *testing.T) { + requireDB(t) + ctx := context.Background() + r := newNeutronTestRouter() + hdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "member"} + + code, body := doReqBody(r, http.MethodPost, "/v2.0/networks", + `{"network":{"name":"spec028-desc-net","description":"first"}}`, hdr) + if code != http.StatusCreated { + t.Fatalf("create network with description: code=%d (body=%s)", code, body) + } + var created struct { + Network struct { + ID string `json:"id"` + Description string `json:"description"` + } `json:"network"` + } + if err := json.Unmarshal([]byte(body), &created); err != nil { + t.Fatalf("unmarshal create: %v (body=%s)", err, body) + } + t.Cleanup(func() { + testPool.Exec(ctx, "DELETE FROM ports WHERE network_id=$1", created.Network.ID) + testPool.Exec(ctx, "DELETE FROM networks WHERE id=$1", created.Network.ID) + }) + if created.Network.Description != "first" { + t.Errorf("create: description=%q; want first", created.Network.Description) + } + + if code, body := doReqBody(r, http.MethodPut, "/v2.0/networks/"+created.Network.ID, + `{"network":{"description":"second"}}`, hdr); code != http.StatusOK { + t.Fatalf("set description: code=%d (body=%s)", code, body) + } else { + var updated struct { + Network struct { + Description string `json:"description"` + } `json:"network"` + } + json.Unmarshal([]byte(body), &updated) + if updated.Network.Description != "second" { + t.Errorf("set description response: %q; want second", updated.Network.Description) + } + } + + _, body = doReq(r, http.MethodGet, "/v2.0/networks/"+created.Network.ID, hdr) + var got struct { + Network struct { + Description string `json:"description"` + } `json:"network"` + } + json.Unmarshal([]byte(body), &got) + if got.Network.Description != "second" { + t.Errorf("show after set: description=%q; want second (body=%s)", got.Network.Description, body) + } +} + +// TestSubnetSetDHCP_DB — 'subnet set --no-dhcp' PUTs enable_dhcp, which the +// update handler had no binding for: the flag was a 200 no-op (SPEC-028). +func TestSubnetSetDHCP_DB(t *testing.T) { + requireDB(t) + ctx := context.Background() + netID, subnetID, _ := seedSpec028NetSubnet(t, ctx, "dhcp") + t.Cleanup(func() { cleanupSpec028Net(t, ctx, netID) }) + + r := newNeutronTestRouter() + hdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "member"} + + if code, body := doReqBody(r, http.MethodPut, "/v2.0/subnets/"+subnetID, + `{"subnet":{"enable_dhcp":false}}`, hdr); code != http.StatusOK { + t.Fatalf("subnet set --no-dhcp: code=%d (body=%s)", code, body) + } + + var enableDHCP bool + if err := testPool.QueryRow(ctx, `SELECT enable_dhcp FROM subnets WHERE id=$1`, subnetID).Scan(&enableDHCP); err != nil || enableDHCP { + t.Errorf("enable_dhcp after update: %v err=%v; want false", enableDHCP, err) + } + + // And back on, together with a rename, in one request. + if code, body := doReqBody(r, http.MethodPut, "/v2.0/subnets/"+subnetID, + `{"subnet":{"enable_dhcp":true,"name":"spec028-dhcp-renamed"}}`, hdr); code != http.StatusOK { + t.Fatalf("subnet set dhcp+name: code=%d (body=%s)", code, body) + } + if err := testPool.QueryRow(ctx, `SELECT enable_dhcp FROM subnets WHERE id=$1`, subnetID).Scan(&enableDHCP); err != nil || !enableDHCP { + t.Errorf("enable_dhcp after re-enable: %v err=%v; want true", enableDHCP, err) + } +} + +// TestSubnetPoolPrefixValidationAndDescription_DB — a malformed prefix was +// stored as-is (upstream 400s), and pools had no description column +// (SPEC-028, migration 075). +func TestSubnetPoolPrefixValidationAndDescription_DB(t *testing.T) { + requireDB(t) + ctx := context.Background() + r := newNeutronTestRouter() + hdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "member"} + + if code, body := doReqBody(r, http.MethodPost, "/v2.0/subnetpools", + `{"subnetpool":{"name":"spec028-badpool","prefixes":["not-a-cidr"]}}`, hdr); code != http.StatusBadRequest { + t.Errorf("create pool with malformed prefix: code=%d; want 400 (body=%s)", code, body) + } + + code, body := doReqBody(r, http.MethodPost, "/v2.0/subnetpools", + `{"subnetpool":{"name":"spec028-desc-pool","prefixes":["198.51.100.0/24"],"description":"first"}}`, hdr) + if code != http.StatusCreated { + t.Fatalf("create pool with description: code=%d (body=%s)", code, body) + } + var created struct { + Pool struct { + ID string `json:"id"` + Description string `json:"description"` + } `json:"subnetpool"` + } + if err := json.Unmarshal([]byte(body), &created); err != nil { + t.Fatalf("unmarshal create: %v (body=%s)", err, body) + } + t.Cleanup(func() { testPool.Exec(ctx, "DELETE FROM subnet_pools WHERE id=$1", created.Pool.ID) }) + if created.Pool.Description != "first" { + t.Errorf("create: description=%q; want first", created.Pool.Description) + } + + if code, body := doReqBody(r, http.MethodPut, "/v2.0/subnetpools/"+created.Pool.ID, + `{"subnetpool":{"description":"second"}}`, hdr); code != http.StatusOK { + t.Fatalf("pool set description: code=%d (body=%s)", code, body) + } + _, body = doReq(r, http.MethodGet, "/v2.0/subnetpools/"+created.Pool.ID, hdr) + var got struct { + Pool struct { + Description string `json:"description"` + } `json:"subnetpool"` + } + json.Unmarshal([]byte(body), &got) + if got.Pool.Description != "second" { + t.Errorf("show after set: description=%q; want second (body=%s)", got.Pool.Description, body) + } + + if code, body := doReqBody(r, http.MethodPut, "/v2.0/subnetpools/"+created.Pool.ID, + `{"subnetpool":{"prefixes":["still-not-a-cidr"]}}`, hdr); code != http.StatusBadRequest { + t.Errorf("pool set malformed prefix: code=%d; want 400 (body=%s)", code, body) + } +} + +// TestSecurityGroupCreateRulesEmpty_DB — the create response omitted +// security_group_rules, so OSC rendered it as null and its table formatter +// crashed ('NoneType' object is not iterable). Upstream reports [] for a +// fresh group (SPEC-028). +func TestSecurityGroupCreateRulesEmpty_DB(t *testing.T) { + requireDB(t) + r := newNeutronTestRouter() + hdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "member"} + + code, body := doReqBody(r, http.MethodPost, "/v2.0/security-groups", + `{"security_group":{"name":"spec028-rules-sg","description":"SPEC-028 rules key"}}`, hdr) + if code != http.StatusCreated { + t.Fatalf("create security group: code=%d (body=%s)", code, body) + } + var created struct { + SG struct { + ID string `json:"id"` + Rules []any `json:"security_group_rules"` + } `json:"security_group"` + } + if err := json.Unmarshal([]byte(body), &created); err != nil { + t.Fatalf("unmarshal create: %v (body=%s)", err, body) + } + t.Cleanup(func() { testPool.Exec(context.Background(), "DELETE FROM security_groups WHERE id=$1", created.SG.ID) }) + if created.SG.Rules == nil { + t.Errorf("create: security_group_rules is null; want [] (body=%s)", body) + } else if len(created.SG.Rules) != 0 { + t.Errorf("create: security_group_rules has %d entries; want 0", len(created.SG.Rules)) + } +} + +// TestGetNetworkAdminCarveOut_DB — the admin tenant_id LIST filter is +// honoured (see TestListNetworksAdminTenantFilter_DB), but GET +// /v2.0/networks/{id} kept the caller's read scope unconditionally, so an +// admin could list another project's private network and then get a 404 +// showing the id it just listed. Upstream's get_network policy is +// "admin_or_owner or shared or external" (SPEC-028). +func TestGetNetworkAdminCarveOut_DB(t *testing.T) { + requireDB(t) + ctx := context.Background() + + otherProject := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO projects (id, name, domain_id, enabled) VALUES ($1,$2,NULL,true)`, + otherProject, "spec028-show-proj-"+otherProject[:8]); err != nil { + t.Fatalf("insert other project: %v", err) + } + netID := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO networks (id, name, project_id, shared, external) VALUES ($1,$2,$3,false,false)`, + netID, "spec028-show-net-"+netID[:8], otherProject); err != nil { + t.Fatalf("insert network: %v", err) + } + t.Cleanup(func() { + testPool.Exec(ctx, "DELETE FROM networks WHERE id=$1", netID) + testPool.Exec(ctx, "DELETE FROM projects WHERE id=$1", otherProject) + }) + + r := newNeutronTestRouter() + adminHdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "admin"} + memberHdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "member"} + + if code, body := doReq(r, http.MethodGet, "/v2.0/networks/"+netID, adminHdr); code != http.StatusOK { + t.Errorf("admin shows other project's private network: code=%d; want 200 (body=%s)", code, body) + } + // A plain member of ANOTHER project still cannot: 404, not 403, so the + // network's existence stays private. + if code, body := doReq(r, http.MethodGet, "/v2.0/networks/"+netID, memberHdr); code != http.StatusNotFound { + t.Errorf("member shows other project's private network: code=%d; want 404 (body=%s)", code, body) + } +} + +// TestAddRouterInterfaceSubnetScope_DB — the subnet lookup behind +// add_router_interface was unscoped: a caller who owned a router could attach +// an interface to ANOTHER project's private subnet by id. The port path was +// already project-scoped; now the subnet row must be the caller's, or sit on +// a shared/external network (SPEC-028). +func TestAddRouterInterfaceSubnetScope_DB(t *testing.T) { + requireDB(t) + ctx := context.Background() + + otherProject := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO projects (id, name, domain_id, enabled) VALUES ($1,$2,NULL,true)`, + otherProject, "spec028-ifscope-proj-"+otherProject[:8]); err != nil { + t.Fatalf("insert other project: %v", err) + } + // A private and a shared network, both owned by the other project. + privateNet, sharedNet := uuid.NewString(), uuid.NewString() + for _, n := range []struct { + id string + shared bool + }{{privateNet, false}, {sharedNet, true}} { + if _, err := testPool.Exec(ctx, + `INSERT INTO networks (id, name, project_id, shared, external) VALUES ($1,$2,$3,$4,false)`, + n.id, "spec028-ifscope-net-"+n.id[:8], otherProject, n.shared); err != nil { + t.Fatalf("insert network: %v", err) + } + } + privateSub, sharedSub := uuid.NewString(), uuid.NewString() + for _, s := range []struct{ id, net string }{{privateSub, privateNet}, {sharedSub, sharedNet}} { + if _, err := testPool.Exec(ctx, + `INSERT INTO subnets (id, name, network_id, project_id, cidr, gateway_ip, ip_version) VALUES ($1,$2,$3,$4,$5,$6,4)`, + s.id, "spec028-ifscope-sub-"+s.id[:8], s.net, otherProject, "10.231.0.0/24", "10.231.0.1"); err != nil { + t.Fatalf("insert subnet: %v", err) + } + } + routerID := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO routers (id, name, project_id) VALUES ($1,$2,$3)`, + routerID, "spec028-ifscope-rt-"+routerID[:8], seededProjectID); err != nil { + t.Fatalf("insert router: %v", err) + } + t.Cleanup(func() { + testPool.Exec(ctx, "DELETE FROM router_interfaces WHERE router_id=$1", routerID) + testPool.Exec(ctx, "DELETE FROM ports WHERE network_id = ANY($1)", []string{privateNet, sharedNet}) + testPool.Exec(ctx, "DELETE FROM subnets WHERE id = ANY($1)", []string{privateSub, sharedSub}) + testPool.Exec(ctx, "DELETE FROM networks WHERE id = ANY($1)", []string{privateNet, sharedNet}) + testPool.Exec(ctx, "DELETE FROM routers WHERE id=$1", routerID) + testPool.Exec(ctx, "DELETE FROM projects WHERE id=$1", otherProject) + }) + + r := newNeutronTestRouter() + hdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "member"} + + // Another project's PRIVATE subnet: refused, and as a 404 so the subnet's + // existence stays private. + code, body := doReqBody(r, http.MethodPut, "/v2.0/routers/"+routerID+"/add_router_interface", + fmt.Sprintf(`{"subnet_id":"%s"}`, privateSub), hdr) + if code != http.StatusNotFound { + t.Errorf("attach other project's private subnet: code=%d; want 404 (body=%s)", code, body) + } + + // A subnet on a SHARED network is legitimately attachable. + code, body = doReqBody(r, http.MethodPut, "/v2.0/routers/"+routerID+"/add_router_interface", + fmt.Sprintf(`{"subnet_id":"%s"}`, sharedSub), hdr) + if code != http.StatusOK { + t.Errorf("attach shared-network subnet: code=%d; want 200 (body=%s)", code, body) + } + + // A subnet ANOTHER project created on a network the caller owns is + // attachable too — the same entitlement subnetReadScope grants, so the + // network owner is not locked out of their own network by an admin + // subnet (e.g. the seeded public net's subnets). + adminSub := uuid.NewString() + ownNet := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO networks (id, name, project_id, shared, external) VALUES ($1,$2,$3,false,false)`, + ownNet, "spec028-ifscope-own-"+ownNet[:8], seededProjectID); err != nil { + t.Fatalf("insert own network: %v", err) + } + if _, err := testPool.Exec(ctx, + `INSERT INTO subnets (id, name, network_id, project_id, cidr, gateway_ip, ip_version) VALUES ($1,$2,$3,$4,$5,$6,4)`, + adminSub, "spec028-ifscope-asub-"+adminSub[:8], ownNet, otherProject, "10.232.0.0/24", "10.232.0.1"); err != nil { + t.Fatalf("insert admin subnet on own network: %v", err) + } + t.Cleanup(func() { + testPool.Exec(ctx, "DELETE FROM subnets WHERE id=$1", adminSub) + testPool.Exec(ctx, "DELETE FROM networks WHERE id=$1", ownNet) + }) + code, body = doReqBody(r, http.MethodPut, "/v2.0/routers/"+routerID+"/add_router_interface", + fmt.Sprintf(`{"subnet_id":"%s"}`, adminSub), hdr) + if code != http.StatusOK { + t.Errorf("attach admin-created subnet on own network: code=%d; want 200 (body=%s)", code, body) + } +} + +// TestGetNetworkCacheDoesNotCrossRoles_DB — the admin carve-out (see +// TestGetNetworkAdminCarveOut_DB) meets the project-scoped network cache: +// the cache key names only (project, network), and the lookup precedes any +// visibility check, so an ADMIN's fetch of another project's private network +// must not leave an entry that a plain member of the admin's project then +// reads as a 200 where the uncached answer is 404. Only rows the project is +// entitled to by the read scope may be cached (SPEC-028). +func TestGetNetworkCacheDoesNotCrossRoles_DB(t *testing.T) { + requireDB(t) + + // The poisoning only exists with a real cache behind the service. + cacheInst, err := cache.NewCache(cache.Config{ + RedisURL: "redis://127.0.0.1:6379", Enabled: true, + KeyPrefix: "spec028-cachetest:", DefaultTTL: time.Minute, + }) + if err != nil { + t.Skipf("no local Redis for the cache regression: %v", err) + } + t.Cleanup(func() { + // DeletePattern prepends the instance's KeyPrefix itself, so the + // pattern is just "*" — a prefixed pattern would double the prefix + // and match nothing (round 9). + cacheInst.DeletePattern(context.Background(), "*") + }) + + ctx := context.Background() + otherProject := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO projects (id, name, domain_id, enabled) VALUES ($1,$2,NULL,true)`, + otherProject, "spec028-cache-proj-"+otherProject[:8]); err != nil { + t.Fatalf("insert other project: %v", err) + } + foreignNet := uuid.NewString() // other project's PRIVATE network + if _, err := testPool.Exec(ctx, + `INSERT INTO networks (id, name, project_id, shared, external) VALUES ($1,$2,$3,false,false)`, + foreignNet, "spec028-cache-net-"+foreignNet[:8], otherProject); err != nil { + t.Fatalf("insert foreign network: %v", err) + } + ownNet := uuid.NewString() // the caller project's own network + if _, err := testPool.Exec(ctx, + `INSERT INTO networks (id, name, project_id, shared, external) VALUES ($1,$2,$3,false,false)`, + ownNet, "spec028-cache-own-"+ownNet[:8], seededProjectID); err != nil { + t.Fatalf("insert own network: %v", err) + } + t.Cleanup(func() { + testPool.Exec(ctx, "DELETE FROM networks WHERE id = ANY($1)", []string{foreignNet, ownNet}) + testPool.Exec(ctx, "DELETE FROM projects WHERE id=$1", otherProject) + }) + + gin.SetMode(gin.TestMode) + r := gin.New() + r.Use(func(c *gin.Context) { + if p := c.GetHeader("X-Test-Project"); p != "" { + c.Set("project_id", p) + } + if roles := c.GetHeader("X-Test-Roles"); roles != "" { + c.Set("roles", strings.Split(roles, ",")) + } + }) + NewService(testPool, "stub", cacheInst).RegisterRoutes(&r.RouterGroup) + + adminHdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "admin"} + memberHdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "member"} + + // The admin may fetch the foreign private network... + if code, body := doReq(r, http.MethodGet, "/v2.0/networks/"+foreignNet, adminHdr); code != http.StatusOK { + t.Fatalf("admin fetches foreign private network: code=%d; want 200 (body=%s)", code, body) + } + // ...but that fetch must not poison the project's cache: a plain member + // of the same project still gets the uncached answer, 404. + if code, body := doReq(r, http.MethodGet, "/v2.0/networks/"+foreignNet, memberHdr); code != http.StatusNotFound { + t.Errorf("member reads foreign private network after admin fetch: code=%d; want 404 — cache poisoned (body=%s)", code, body) + } + + // The entitlement-positive path still caches: two member reads of the + // project's own network both succeed (the second is a cache hit), and + // the entry is provably present after the first — so the guard above + // suppresses only the foreign-private rows, not caching itself. + for i := 1; i <= 2; i++ { + if code, body := doReq(r, http.MethodGet, "/v2.0/networks/"+ownNet, memberHdr); code != http.StatusOK { + t.Errorf("member read #%d of own network: code=%d; want 200 (body=%s)", i, code, body) + } + } + var cached gin.H + if err := cacheInst.Get(ctx, "network:"+seededProjectID+":"+ownNet, &cached); err != nil { + t.Errorf("own network not cached after member read: %v", err) + } else if cached["id"] != ownNet { + t.Errorf("cached entry holds the wrong network: %v", cached["id"]) + } +} + +// TestAddRouterInterfaceNullGateway_DB — a subnet without a gateway (an +// explicit null clears it, S14) made add_router_interface 500: the lookup +// scanned gateway_ip into a plain string and NULL cannot land in one. +// Upstream attaches with an address allocated from the subnet; the fix +// falls back to the same allocator CreatePort uses (SPEC-028). +func TestAddRouterInterfaceNullGateway_DB(t *testing.T) { + requireDB(t) + ctx := context.Background() + + netID := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO networks (id, name, project_id, shared, external) VALUES ($1,$2,$3,false,false)`, + netID, "spec028-nogw-net-"+netID[:8], seededProjectID); err != nil { + t.Fatalf("insert network: %v", err) + } + // gateway_ip deliberately omitted: NULL. + subID := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO subnets (id, name, network_id, project_id, cidr, ip_version) VALUES ($1,$2,$3,$4,$5,4)`, + subID, "spec028-nogw-sub-"+subID[:8], netID, seededProjectID, "10.233.0.0/24"); err != nil { + t.Fatalf("insert subnet: %v", err) + } + routerID := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO routers (id, name, project_id) VALUES ($1,$2,$3)`, + routerID, "spec028-nogw-rt-"+routerID[:8], seededProjectID); err != nil { + t.Fatalf("insert router: %v", err) + } + t.Cleanup(func() { + testPool.Exec(ctx, "DELETE FROM router_interfaces WHERE router_id=$1", routerID) + testPool.Exec(ctx, "DELETE FROM ports WHERE network_id=$1", netID) + testPool.Exec(ctx, "DELETE FROM subnets WHERE id=$1", subID) + testPool.Exec(ctx, "DELETE FROM networks WHERE id=$1", netID) + testPool.Exec(ctx, "DELETE FROM routers WHERE id=$1", routerID) + }) + + r := newNeutronTestRouter() + hdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "member"} + code, body := doReqBody(r, http.MethodPut, "/v2.0/routers/"+routerID+"/add_router_interface", + fmt.Sprintf(`{"subnet_id":"%s"}`, subID), hdr) + if code != http.StatusOK { + t.Errorf("attach subnet without gateway: code=%d; want 200 (body=%s)", code, body) + } +} + +// TestSubnetPortAdminCarveOut_DB — GetNetwork's admin carve-out (see +// TestGetNetworkAdminCarveOut_DB) stopped at networks: an admin could show +// a foreign private NETWORK but 404 on the subnet ids its own body lists, +// and list [] for them. Upstream's get_subnet policy includes +// ADMIN_OR_NET_OWNER_READER; the carve-out now covers subnets and ports, +// their listings under an explicit network_id pointer, and a member of +// another project still gets 404 (SPEC-028). +func TestSubnetPortAdminCarveOut_DB(t *testing.T) { + requireDB(t) + ctx := context.Background() + + otherProject := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO projects (id, name, domain_id, enabled) VALUES ($1,$2,NULL,true)`, + otherProject, "spec028-sub-carve-"+otherProject[:8]); err != nil { + t.Fatalf("insert other project: %v", err) + } + netID := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO networks (id, name, project_id, shared, external) VALUES ($1,$2,$3,false,false)`, + netID, "spec028-sub-carve-net-"+netID[:8], otherProject); err != nil { + t.Fatalf("insert network: %v", err) + } + subID := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO subnets (id, name, network_id, project_id, cidr, gateway_ip, ip_version) VALUES ($1,$2,$3,$4,$5,$6,4)`, + subID, "spec028-sub-carve-sub-"+subID[:8], netID, otherProject, "10.234.0.0/24", "10.234.0.1"); err != nil { + t.Fatalf("insert subnet: %v", err) + } + portID := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO ports (id, name, network_id, project_id, mac_address, admin_state_up, status, fixed_ips, created_at, updated_at) + VALUES ($1,$2,$3,$4,$5,true,'ACTIVE','[]'::jsonb,now(),now())`, + portID, "spec028-sub-carve-port-"+portID[:8], netID, otherProject, "02:00:00:00:23:28"); err != nil { + t.Fatalf("insert port: %v", err) + } + t.Cleanup(func() { + testPool.Exec(ctx, "DELETE FROM ports WHERE id=$1", portID) + testPool.Exec(ctx, "DELETE FROM subnets WHERE id=$1", subID) + testPool.Exec(ctx, "DELETE FROM networks WHERE id=$1", netID) + testPool.Exec(ctx, "DELETE FROM projects WHERE id=$1", otherProject) + }) + + r := newNeutronTestRouter() + adminHdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "admin"} + memberHdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "member"} + + if code, body := doReq(r, http.MethodGet, "/v2.0/subnets/"+subID, adminHdr); code != http.StatusOK { + t.Errorf("admin shows foreign subnet: code=%d; want 200 (body=%s)", code, body) + } + if code, body := doReq(r, http.MethodGet, "/v2.0/subnets/"+subID, memberHdr); code != http.StatusNotFound { + t.Errorf("member shows foreign subnet: code=%d; want 404 (body=%s)", code, body) + } + if code, body := doReq(r, http.MethodGet, "/v2.0/subnets?network_id="+netID, adminHdr); code != http.StatusOK || !strings.Contains(body, subID) { + t.Errorf("admin lists foreign network's subnets: code=%d body=%s; want 200 containing the subnet", code, body) + } + if code, body := doReq(r, http.MethodGet, "/v2.0/ports/"+portID, adminHdr); code != http.StatusOK { + t.Errorf("admin shows foreign port: code=%d; want 200 (body=%s)", code, body) + } + // Round 8: the row's OWN project is rendered, never the caller's — + // stamping the admin's project on a foreign port tells a client a + // resource is theirs when it is not. + if code, body := doReq(r, http.MethodGet, "/v2.0/ports/"+portID, adminHdr); code != http.StatusOK || !strings.Contains(body, `"tenant_id":"`+otherProject+`"`) { + t.Errorf("admin shows foreign port: tenant_id must be the row's owner %s (code=%d body=%s)", otherProject, code, body) + } + if code, body := doReq(r, http.MethodGet, "/v2.0/ports/"+portID, memberHdr); code != http.StatusNotFound { + t.Errorf("member shows foreign port: code=%d; want 404 (body=%s)", code, body) + } + if code, body := doReq(r, http.MethodGet, "/v2.0/ports?network_id="+netID, adminHdr); code != http.StatusOK || !strings.Contains(body, portID) { + t.Errorf("admin lists foreign network's ports: code=%d body=%s; want 200 containing the port", code, body) + } + if code, body := doReq(r, http.MethodGet, "/v2.0/ports?network_id="+netID, adminHdr); code != http.StatusOK || !strings.Contains(body, `"tenant_id":"`+otherProject+`"`) { + t.Errorf("admin lists foreign network's ports: tenant_id must be the row's owner %s (code=%d body=%s)", otherProject, code, body) + } +} + +// TestListTenantFilterHonoredForAdmin_DB — the admin tenant_id filter was +// honored by the NETWORKS listing only; every other list silently ignored +// it and presented the ADMIN's rows as the target project's. Router here +// stands for the fixed pattern (subnets/ports/FIPs/cinder share it); +// upstream honours the filter for admin tokens (SPEC-028). +func TestListTenantFilterHonoredForAdmin_DB(t *testing.T) { + requireDB(t) + ctx := context.Background() + + otherProject := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO projects (id, name, domain_id, enabled) VALUES ($1,$2,NULL,true)`, + otherProject, "spec028-rt-tenant-"+otherProject[:8]); err != nil { + t.Fatalf("insert other project: %v", err) + } + otherRouter := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO routers (id, name, project_id) VALUES ($1,$2,$3)`, + otherRouter, "spec028-rt-tenant-rt-"+otherRouter[:8], otherProject); err != nil { + t.Fatalf("insert other router: %v", err) + } + ownRouter := uuid.NewString() + if _, err := testPool.Exec(ctx, + `INSERT INTO routers (id, name, project_id) VALUES ($1,$2,$3)`, + ownRouter, "spec028-rt-tenant-own-"+ownRouter[:8], seededProjectID); err != nil { + t.Fatalf("insert own router: %v", err) + } + t.Cleanup(func() { + testPool.Exec(ctx, "DELETE FROM routers WHERE id = ANY($1)", []string{otherRouter, ownRouter}) + testPool.Exec(ctx, "DELETE FROM projects WHERE id=$1", otherProject) + }) + + r := newNeutronTestRouter() + adminHdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "admin"} + memberHdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "member"} + + // Admin + tenant filter: the OTHER project's router appears, the + // admin's own does not. + code, body := doReq(r, http.MethodGet, "/v2.0/routers?tenant_id="+otherProject, adminHdr) + if code != http.StatusOK || !strings.Contains(body, otherRouter) || strings.Contains(body, ownRouter) { + t.Errorf("admin routers?tenant_id=: code=%d body=%s; want only the other project's router", code, body) + } + // A non-admin's tenant filter stays intersected with their scope: the + // foreign router never appears. + code, body = doReq(r, http.MethodGet, "/v2.0/routers?tenant_id="+otherProject, memberHdr) + if code != http.StatusOK || strings.Contains(body, otherRouter) { + t.Errorf("member routers?tenant_id=: code=%d body=%s; must not contain the foreign router", code, body) + } + + // Round 8: the show must not dead-end the listing's rows — an admin + // follow-up `router show` on the foreign router answers 200, a member's + // 404 (the same carve-out GetNetwork/GetSubnet/GetPort carry). + if code, body := doReq(r, http.MethodGet, "/v2.0/routers/"+otherRouter, adminHdr); code != http.StatusOK { + t.Errorf("admin shows foreign router: code=%d; want 200 (body=%s)", code, body) + } + if code, body := doReq(r, http.MethodGet, "/v2.0/routers/"+otherRouter, memberHdr); code != http.StatusNotFound { + t.Errorf("member shows foreign router: code=%d; want 404 (body=%s)", code, body) + } +} + +// TestAllocateIPHandlesTinyAndIPv6Pools — upstream Neutron's generate_pools() +// allocates from /30, /31, /32 and every IPv6 subnet, excluding only the +// network/broadcast addresses where they exist; it refuses nothing as "too +// small". The old allocator returned "" for maxOffset <= 10 (every /29–/30) +// and for IPv6 (int64 overflow), turning valid subnets into 400s. Also guards +// the old failure mode: rand.Int panics on a non-positive bound. +func TestAllocateIPHandlesTinyAndIPv6Pools(t *testing.T) { + for _, cidr := range []string{ + "10.0.0.0/30", + "10.0.0.0/29", + "10.0.0.0/31", + "10.0.0.0/32", + "fd00::/64", + "fd00::/126", + "10.0.0.0/24", // healthy pool, unchanged + } { + cidr := cidr + t.Run(cidr, func(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Fatalf("allocateIP(%q) panicked: %v", cidr, r) + } + }() + if got := allocateIP(cidr); got == "" { + t.Errorf("allocateIP(%q) = \"\"; want an address (upstream allocates from this pool)", cidr) + } + }) + } + + // Invalid input still fails closed rather than panicking. + if got := allocateIP("not-a-cidr"); got != "" { + t.Errorf("allocateIP(\"not-a-cidr\") = %q; want \"\"", got) + } +} + +// TestCreatePortAllocatesFromTinyPool — a /30 (two usable hosts, both +// candidates when the subnet has no gateway) is a valid upstream pool. The old +// allocator refused it with a 400; now the port must be created with a real +// fixed IP and nothing blank may be persisted. The same must hold for IPv6, +// which the old int64-overflow guard refused wholesale (SPEC-028, round 9). +func TestCreatePortAllocatesFromTinyPool(t *testing.T) { + requireDB(t) + ctx := context.Background() + r := newNeutronTestRouter() + hdr := map[string]string{"X-Test-Project": seededProjectID, "X-Test-Roles": "member"} + + // createNetSubnet creates a network plus one subnet and returns the network + // id. Cidr/ipVersion are the only varying inputs between the IPv4 and IPv6 + // halves of this test. + createNetSubnet := func(name, cidr string, ipVersion int) string { + t.Helper() + code, body := doReqBody(r, http.MethodPost, "/v2.0/networks", + fmt.Sprintf(`{"network":{"name":%q}}`, name+"-net"), hdr) + if code != http.StatusCreated { + t.Fatalf("create network %s: code=%d (body=%s)", name, code, body) + } + var net struct { + Network struct { + ID string `json:"id"` + } `json:"network"` + } + if err := json.Unmarshal([]byte(body), &net); err != nil { + t.Fatalf("unmarshal network: %v (body=%s)", err, body) + } + t.Cleanup(func() { + testPool.Exec(ctx, "DELETE FROM ports WHERE network_id=$1", net.Network.ID) + testPool.Exec(ctx, "DELETE FROM subnets WHERE network_id=$1", net.Network.ID) + testPool.Exec(ctx, "DELETE FROM networks WHERE id=$1", net.Network.ID) + }) + + // gateway_ip null makes BOTH /30 host addresses candidates, which is + // exactly the pool the old allocator used to refuse. + code, body = doReqBody(r, http.MethodPost, "/v2.0/subnets", + fmt.Sprintf(`{"subnet":{"name":%q,"network_id":%q,"cidr":%q,"gateway_ip":null,"ip_version":%d}}`, + name+"-sub", net.Network.ID, cidr, ipVersion), hdr) + if code != http.StatusCreated { + t.Fatalf("create %s subnet: code=%d (body=%s)", cidr, code, body) + } + return net.Network.ID + } + + assertAllocated := func(netID, label string) { + t.Helper() + code, body := doReqBody(r, http.MethodPost, "/v2.0/ports", + fmt.Sprintf(`{"port":{"network_id":%q,"name":%q}}`, netID, label), hdr) + if code != http.StatusCreated { + t.Fatalf("%s port create: code=%d body=%s; want 201 (the pool is valid upstream)", label, code, body) + } + var created struct { + Port struct { + FixedIPs []struct { + IPAddress string `json:"ip_address"` + } `json:"fixed_ips"` + } `json:"port"` + } + if err := json.Unmarshal([]byte(body), &created); err != nil { + t.Fatalf("%s: unmarshal port: %v (body=%s)", label, err, body) + } + if len(created.Port.FixedIPs) == 0 || strings.TrimSpace(created.Port.FixedIPs[0].IPAddress) == "" { + t.Fatalf("%s: fixed_ips = %+v; want a non-blank ip_address (body=%s)", label, created.Port.FixedIPs, body) + } + // The response is not enough: a blank address must never reach the table. + var stored sql.NullString + if err := testPool.QueryRow(ctx, + `SELECT fixed_ips->0->>'ip_address' FROM ports WHERE network_id=$1`, netID).Scan(&stored); err != nil { + t.Fatalf("%s: read back fixed_ips: %v", label, err) + } + if !stored.Valid || strings.TrimSpace(stored.String) == "" { + t.Fatalf("%s: persisted ip_address is %v; want a non-blank address", label, stored) + } + } + + assertAllocated(createNetSubnet("spec028-r9-tiny", "10.198.0.0/30", 4), "tiny-ipv4") + assertAllocated(createNetSubnet("spec028-r9-tiny6", "fd00:198::/64", 6), "tiny-ipv6") +} diff --git a/internal/neutron/subnet_pools.go b/internal/neutron/subnet_pools.go index df067983..fff2d6dc 100644 --- a/internal/neutron/subnet_pools.go +++ b/internal/neutron/subnet_pools.go @@ -1,7 +1,9 @@ package neutron import ( + "database/sql" "errors" + "net" "net/http" "time" @@ -18,8 +20,8 @@ func (svc *Service) ListSubnetPools(c *gin.Context) { projectID := c.GetString("project_id") rows, err := svc.pool.Query(c.Request.Context(), ` - SELECT id, name, prefixes, min_prefixlen, max_prefixlen, default_prefixlen, - shared, is_default, ip_version, created_at, updated_at + SELECT id, project_id, name, prefixes, min_prefixlen, max_prefixlen, default_prefixlen, + shared, is_default, ip_version, COALESCE(description, ''), created_at, updated_at FROM subnet_pools WHERE project_id = $1 OR shared = true ORDER BY created_at DESC @@ -34,15 +36,15 @@ func (svc *Service) ListSubnetPools(c *gin.Context) { pools := []gin.H{} for rows.Next() { - var id, name string + var id, ownerID, name, description string var prefixes []string var minPrefixlen, maxPrefixlen, ipVersion int var defaultPrefixlen *int var shared, isDefault bool var createdAt, updatedAt time.Time - err := rows.Scan(&id, &name, &prefixes, &minPrefixlen, &maxPrefixlen, &defaultPrefixlen, - &shared, &isDefault, &ipVersion, &createdAt, &updatedAt) + err := rows.Scan(&id, &ownerID, &name, &prefixes, &minPrefixlen, &maxPrefixlen, &defaultPrefixlen, + &shared, &isDefault, &ipVersion, &description, &createdAt, &updatedAt) if err != nil { continue } @@ -56,9 +58,13 @@ func (svc *Service) ListSubnetPools(c *gin.Context) { "shared": shared, "is_default": isDefault, "ip_version": ipVersion, - "tenant_id": projectID, - "created_at": createdAt.Format(time.RFC3339), - "updated_at": updatedAt.Format(time.RFC3339), + // The row's OWN project, not the caller's: the shared visibility + // clause above returns other projects' pools, and `tenant_id` must + // name the owner (same round-8 fix as cinder volumes and ports). + "tenant_id": ownerID, + "description": description, + "created_at": createdAt.Format(time.RFC3339), + "updated_at": updatedAt.Format(time.RFC3339), } if defaultPrefixlen != nil { @@ -88,6 +94,7 @@ func (svc *Service) CreateSubnetPool(c *gin.Context) { Shared *bool `json:"shared"` IsDefault *bool `json:"is_default"` IPVersion *int `json:"ip_version"` + Description *string `json:"description"` } `json:"subnetpool"` } @@ -96,6 +103,16 @@ func (svc *Service) CreateSubnetPool(c *gin.Context) { return } + // A pool's prefixes are CIDRs; a malformed one must be the caller's 400, + // not a 201 that stores garbage every subnet allocation then chokes on + // (subnets are validated this way already; SPEC-028). + for _, p := range req.SubnetPool.Prefixes { + if _, _, err := net.ParseCIDR(p); err != nil { + common.SendError(c, common.NewBadRequestError("invalid prefixes: '"+p+"' is not a valid CIDR")) + return + } + } + projectID := c.GetString("project_id") poolID := uuid.New().String() @@ -128,10 +145,10 @@ func (svc *Service) CreateSubnetPool(c *gin.Context) { _, err := svc.pool.Exec(c.Request.Context(), ` INSERT INTO subnet_pools (id, project_id, name, prefixes, min_prefixlen, max_prefixlen, - default_prefixlen, shared, is_default, ip_version, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + default_prefixlen, shared, is_default, ip_version, description, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) `, poolID, projectID, req.SubnetPool.Name, req.SubnetPool.Prefixes, minPrefixlen, maxPrefixlen, - req.SubnetPool.DefaultPrefixlen, shared, isDefault, ipVersion, now, now) + req.SubnetPool.DefaultPrefixlen, shared, isDefault, ipVersion, req.SubnetPool.Description, now, now) if err != nil { log.Error().Err(err).Str("operation", "create_subnet_pool").Msg("failed to create subnet pool") @@ -149,6 +166,7 @@ func (svc *Service) CreateSubnetPool(c *gin.Context) { "is_default": isDefault, "ip_version": ipVersion, "tenant_id": projectID, + "description": derefOr(req.SubnetPool.Description, ""), "created_at": now.Format(time.RFC3339), "updated_at": now.Format(time.RFC3339), } @@ -165,20 +183,29 @@ func (svc *Service) GetSubnetPool(c *gin.Context) { poolID := c.Param("id") projectID := c.GetString("project_id") - var id, name string + // A non-UUID reference (a pool name — the CLI resolves names with a list + // fallback only after a 404) can only name nothing here; the uuid-typed + // lookup used to surface the cast error as 500 instead (SPEC-028). + if _, err := uuid.Parse(poolID); err != nil { + common.SendError(c, common.NewNotFoundError("subnet pool")) + return + } + + var id, ownerID, name string var prefixes []string var minPrefixlen, maxPrefixlen, ipVersion int var defaultPrefixlen *int var shared, isDefault bool + var description sql.NullString var createdAt, updatedAt time.Time err := svc.pool.QueryRow(c.Request.Context(), ` - SELECT id, name, prefixes, min_prefixlen, max_prefixlen, default_prefixlen, - shared, is_default, ip_version, created_at, updated_at + SELECT id, project_id, name, prefixes, min_prefixlen, max_prefixlen, default_prefixlen, + shared, is_default, ip_version, description, created_at, updated_at FROM subnet_pools WHERE id = $1 AND (project_id = $2 OR shared = true) - `, poolID, projectID).Scan(&id, &name, &prefixes, &minPrefixlen, &maxPrefixlen, &defaultPrefixlen, - &shared, &isDefault, &ipVersion, &createdAt, &updatedAt) + `, poolID, projectID).Scan(&id, &ownerID, &name, &prefixes, &minPrefixlen, &maxPrefixlen, &defaultPrefixlen, + &shared, &isDefault, &ipVersion, &description, &createdAt, &updatedAt) if errors.Is(err, database.ErrNoRows) { common.SendError(c, common.NewNotFoundError("subnet pool")) @@ -199,9 +226,12 @@ func (svc *Service) GetSubnetPool(c *gin.Context) { "shared": shared, "is_default": isDefault, "ip_version": ipVersion, - "tenant_id": projectID, - "created_at": createdAt.Format(time.RFC3339), - "updated_at": updatedAt.Format(time.RFC3339), + // See ListSubnetPools: shared rows belong to another project, so + // rendering the caller's id here is false attribution. + "tenant_id": ownerID, + "description": description.String, + "created_at": createdAt.Format(time.RFC3339), + "updated_at": updatedAt.Format(time.RFC3339), } if defaultPrefixlen != nil { @@ -223,6 +253,7 @@ func (svc *Service) UpdateSubnetPool(c *gin.Context) { MinPrefixlen *int `json:"min_prefixlen"` MaxPrefixlen *int `json:"max_prefixlen"` DefaultPrefixlen *int `json:"default_prefixlen"` + Description *string `json:"description"` } `json:"subnetpool"` } @@ -231,6 +262,21 @@ func (svc *Service) UpdateSubnetPool(c *gin.Context) { return } + // See CreateSubnetPool: prefixes are CIDRs, and a malformed one is the + // caller's 400 (SPEC-028). + for _, p := range req.SubnetPool.Prefixes { + if _, _, err := net.ParseCIDR(p); err != nil { + common.SendError(c, common.NewBadRequestError("invalid prefixes: '"+p+"' is not a valid CIDR")) + return + } + } + + // See GetSubnetPool: a non-UUID reference must 404, not 500 (SPEC-028). + if _, err := uuid.Parse(poolID); err != nil { + common.SendError(c, common.NewNotFoundError("subnet pool")) + return + } + u := &common.SQLUpdate{} if req.SubnetPool.Name != nil { @@ -253,6 +299,10 @@ func (svc *Service) UpdateSubnetPool(c *gin.Context) { u.Set("default_prefixlen", *req.SubnetPool.DefaultPrefixlen) } + if req.SubnetPool.Description != nil { + u.Set("description", *req.SubnetPool.Description) + } + if u.Empty() { common.SendError(c, common.NewBadRequestError("no updates provided")) return diff --git a/internal/nova/advanced_actions.go b/internal/nova/advanced_actions.go index 919909d0..2f5b713a 100644 --- a/internal/nova/advanced_actions.go +++ b/internal/nova/advanced_actions.go @@ -7,10 +7,12 @@ import ( "net/http" "strings" "time" + "unicode/utf8" "github.com/cobaltcore-dev/lightstack/internal/database" "github.com/cobaltcore-dev/lightstack/internal/common" + "github.com/cobaltcore-dev/lightstack/pkg/hypervisor" "github.com/gin-gonic/gin" "github.com/google/uuid" "github.com/rs/zerolog/log" @@ -221,9 +223,23 @@ func (svc *Service) ShelveInstance(c *gin.Context, instanceID, projectID string) return } - // Create snapshot before shelving + // Create snapshot before shelving. instance_snapshots has + // UNIQUE(instance_id), so a stale row from a crashed shelving — or from + // an unshelve path that predates snapshot consumption — would fail this + // insert with 500. Reclaim any leftover first: at this point the instance + // is ACTIVE/SHUTOFF and about to be SHELVED, so no live snapshot of it + // can exist (SPEC-028: shelve → unshelve → shelve was a 500). snapshotName := fmt.Sprintf("shelved-%s-%d", instanceID[:8], time.Now().Unix()) + _, err = svc.pool.Exec(c.Request.Context(), ` + DELETE FROM instance_snapshots WHERE instance_id = $1 + `, instanceID) + if err != nil { + log.Error().Err(err).Str("operation", "reclaim_shelve_snapshot").Msg("database error") + common.SendError(c, common.NewInternalServerError("failed to create snapshot")) + return + } + _, err = svc.pool.Exec(c.Request.Context(), ` INSERT INTO instance_snapshots (id, instance_id, snapshot_name, flavor_id, created_at) SELECT gen_random_uuid(), $1, $2, flavor_id, $3 @@ -275,11 +291,13 @@ func (svc *Service) ShelveInstance(c *gin.Context, instanceID, projectID string) func (svc *Service) UnshelveInstance(c *gin.Context, instanceID, projectID string) { // Get instance + var domainScan *string var status string err := svc.pool.QueryRow(c.Request.Context(), - "SELECT status FROM instances WHERE id = $1 AND project_id = $2", + "SELECT COALESCE(libvirt_domain_id, ''), status FROM instances WHERE id = $1 AND project_id = $2", instanceID, projectID, - ).Scan(&status) + ).Scan(&domainScan, &status) + libvirtDomainID := guestDomain(domainScan) if errors.Is(err, database.ErrNoRows) { common.SendError(c, common.NewNotFoundError("instance")) @@ -297,12 +315,65 @@ func (svc *Service) UnshelveInstance(c *gin.Context, instanceID, projectID strin return } - // Update status to ACTIVE - _, err = svc.pool.Exec(c.Request.Context(), ` - UPDATE instances - SET status = $1, power_state = $2, task_state = $3, updated_at = $4 - WHERE id = $5 - `, "ACTIVE", 1, "", time.Now(), instanceID) + // Restart the guest shelve stopped — BEFORE the record claims ACTIVE. The + // state reconciler exists to correct records that disagree with the + // hypervisor, so a record written ACTIVE over a stopped guest was flipped + // back to SHUTOFF within one 30s sweep: `server unshelve` observably did + // nothing (SPEC-028, round 9). Starting first also wins the reconciler's + // compare-and-set by construction: a SHELVED row is never a reconcile + // candidate, and once the guest is up no verdict disagrees with ACTIVE. + // A domain that has gone missing since the shelve self-heals the way + // os-start does (FR-025) rather than answering 202 over nothing. + healedDomain := "" + if svc.vmManager != nil && libvirtDomainID != "" { + ctx, cancel := context.WithTimeout(c.Request.Context(), common.LifecycleResponseSlack) + defer cancel() + if err := svc.vmManager.StartVM(ctx, libvirtDomainID); err != nil { + if !errors.Is(err, hypervisor.ErrDomainNotFound) { + sendLifecycleError(c, "unshelve", err) + return + } + var herr error + healedDomain, herr = svc.reviveGuest(ctx, instanceID) + if herr != nil { + // The record must not go on claiming the server runs (FR-025c). + svc.persistRunState(c, instanceID, projectID, "ERROR", 0, "") + log.Error().Err(herr).Str("instance_id", instanceID). + Msg("unshelve self-heal failed; the server has no guest and could not be given one") + common.SendError(c, common.NewOperationConflictError("unshelve", + "the server's guest is absent and could not be re-created")) + return + } + svc.recordSelfHeal(c, instanceID, projectID, healedDomain) + } + } + + // Update status to ACTIVE. A self-healed guest runs under a NEW domain + // identifier — the row must name it, or the reconciler selects the row, + // asks about the dead domain, and flips ACTIVE back to SHUTOFF within one + // sweep: the very bug this restart exists to fix, reborn on the self-heal + // path. persistRunState does the same for os-start's heal (FR-025). + err = database.WithTx(c.Request.Context(), svc.pool, func(tx database.Tx) error { + if healedDomain != "" { + _, err = tx.Exec(c.Request.Context(), ` + UPDATE instances + SET status = $1, power_state = $2, task_state = $3, updated_at = $4, libvirt_domain_id = $5 + WHERE id = $6 + `, "ACTIVE", 1, "", time.Now(), healedDomain, instanceID) + } else { + _, err = tx.Exec(c.Request.Context(), ` + UPDATE instances + SET status = $1, power_state = $2, task_state = $3, updated_at = $4 + WHERE id = $5 + `, "ACTIVE", 1, "", time.Now(), instanceID) + } + + if err != nil { + return err + } + _, err = tx.Exec(c.Request.Context(), "DELETE FROM instance_snapshots WHERE instance_id = $1", instanceID) + return err + }) if err != nil { log.Error().Err(err).Str("operation", "unshelve_instance").Msg("database error") @@ -1037,6 +1108,13 @@ func (svc *Service) CreateBackupAction(c *gin.Context, instanceID, projectID str common.SendError(c, common.NewBadRequestError("missing backup name")) return } + // The backup name becomes the glance image's name, whose schema (and + // column) cap at 255 — over-long names answer 400 upstream, not a 500 + // from the VARCHAR overflow. + if utf8.RuneCountInString(backupName) > 255 { + common.SendError(c, common.NewBadRequestError("backup name too long (max 255 characters)")) + return + } backupType, _ := backupMap["backup_type"].(string) rotation, _ := backupMap["rotation"].(float64) @@ -1049,11 +1127,11 @@ func (svc *Service) CreateBackupAction(c *gin.Context, instanceID, projectID str } // Verify instance exists - var sourceImageID string + var existingInstanceID string err := svc.pool.QueryRow(c.Request.Context(), - "SELECT image_id FROM instances WHERE id = $1 AND project_id = $2", + "SELECT id FROM instances WHERE id = $1 AND project_id = $2", instanceID, projectID, - ).Scan(&sourceImageID) + ).Scan(&existingInstanceID) if errors.Is(err, database.ErrNoRows) { common.SendError(c, common.NewNotFoundError("instance")) @@ -1068,15 +1146,16 @@ func (svc *Service) CreateBackupAction(c *gin.Context, instanceID, projectID str // Generate new UUID for backup image backupImageID := uuid.New().String() - // Create backup image in images table - // Image name pattern: {backup_name}-{backup_type}-{timestamp} - timestamp := time.Now().Format("20060102-150405") - fullImageName := fmt.Sprintf("%s-%s-%s", backupName, backupType, timestamp) - + // Create backup image in the images table. The image is named EXACTLY + // what the caller asked for — upstream Nova passes `name` through + // verbatim — so clients can look the image up by name afterwards + // (OSC's `server backup create` polls for it; the previous + // `{name}-{type}-{timestamp}` suffix broke that lookup). Rotation finds + // its candidates through the image_tags below, not through the name. _, err = svc.pool.Exec(c.Request.Context(), ` INSERT INTO images (id, name, project_id, status, visibility, disk_format, container_format, created_at, updated_at) VALUES ($1, $2, $3, 'active', 'private', 'qcow2', 'bare', $4, $4) - `, backupImageID, fullImageName, projectID, time.Now()) + `, backupImageID, backupName, projectID, time.Now()) if err != nil { log.Error().Err(err).Str("operation", "create_backup_image").Msg("database error") diff --git a/internal/nova/aggregates.go b/internal/nova/aggregates.go index cd972f7a..82b550af 100644 --- a/internal/nova/aggregates.go +++ b/internal/nova/aggregates.go @@ -139,12 +139,29 @@ func (svc *Service) CreateAggregate(c *gin.Context) { }) } +// aggregateRefOr404 answers 404 for a path reference that is not a UUID. +// openstacksdk's find_aggregate probes by name first, so a name reaches the +// uuid-typed query and used to surface the cast error as 500 — the client +// aborts there instead of taking the 404 fallback it needs. A well-formed +// but unknown UUID was already a clean 404. +func aggregateRefOr404(c *gin.Context) (string, bool) { + ref := c.Param("id") + if _, err := uuid.Parse(ref); err != nil { + common.SendError(c, common.NewNotFoundError("aggregate")) + return "", false + } + return ref, true +} + // GetAggregate retrieves a specific host aggregate func (svc *Service) GetAggregate(c *gin.Context) { if !common.RequireAdmin(c, "os-aggregates:show") { return } - aggregateID := c.Param("id") + aggregateID, ok := aggregateRefOr404(c) + if !ok { + return + } var ( name string @@ -197,7 +214,6 @@ func (svc *Service) UpdateAggregate(c *gin.Context) { if !common.RequireAdmin(c, "os-aggregates:update") { return } - aggregateID := c.Param("id") var req struct { Aggregate struct { @@ -211,6 +227,11 @@ func (svc *Service) UpdateAggregate(c *gin.Context) { return } + aggregateID, ok := aggregateRefOr404(c) + if !ok { + return + } + // Check if aggregate exists var exists bool err := svc.pool.QueryRow(c.Request.Context(), @@ -259,7 +280,10 @@ func (svc *Service) DeleteAggregate(c *gin.Context) { if !common.RequireAdmin(c, "os-aggregates:delete") { return } - aggregateID := c.Param("id") + aggregateID, ok := aggregateRefOr404(c) + if !ok { + return + } result, err := svc.pool.Exec(c.Request.Context(), "DELETE FROM host_aggregates WHERE uuid = $1", @@ -289,7 +313,10 @@ func (svc *Service) AggregateAction(c *gin.Context) { return } - aggregateID := c.Param("id") + aggregateID, ok := aggregateRefOr404(c) + if !ok { + return + } var req map[string]any if err := c.ShouldBindJSON(&req); err != nil { diff --git a/internal/nova/handlers.go b/internal/nova/handlers.go index 1a5d878c..e5155f5c 100644 --- a/internal/nova/handlers.go +++ b/internal/nova/handlers.go @@ -1593,7 +1593,7 @@ func (svc *Service) ListServersDetail(c *gin.Context) { rows, err := svc.pool.Query(c.Request.Context(), fmt.Sprintf(` SELECT i.id, i.name, i.status, i.power_state, i.project_id, i.user_id, i.flavor_id, i.image_id, i.created_at, i.updated_at, i.launched_at, - i.key_name, + i.key_name, i.locked, f.vcpus, f.ram_mb, f.disk_gb, f.name as flavor_name FROM instances i LEFT JOIN flavors f ON i.flavor_id = f.id @@ -1616,10 +1616,11 @@ func (svc *Service) ListServersDetail(c *gin.Context) { var createdAt, updatedAt time.Time var launchedAt sql.NullTime var imageID, keyName sql.NullString + var locked bool if err := rows.Scan(&id, &name, &status, &powerState, &rowProjectID, &userID, &flavorID, &imageID, &createdAt, &updatedAt, &launchedAt, &keyName, - &vcpus, &ramMB, &diskGB, &flavorName); err != nil { + &locked, &vcpus, &ramMB, &diskGB, &flavorName); err != nil { log.Warn().Err(err).Msg("failed to scan server detail row") continue } @@ -1633,6 +1634,10 @@ func (svc *Service) ListServersDetail(c *gin.Context) { "created": createdAt.Format(time.RFC3339), "updated": updatedAt.Format(time.RFC3339), "OS-EXT-STS:power_state": powerState, + // Exposed since microversion 2.9 upstream; the lock/unlock actions + // maintain the column, but the detail responses never carried it, + // so `openstack server show` printed null (SPEC-028). + "locked": locked, "flavor": gin.H{ "id": flavorID, "vcpus": vcpus, @@ -1916,17 +1921,18 @@ func (svc *Service) GetServer(c *gin.Context) { var faultCode sql.NullInt32 var faultCreated sql.NullTime var powerState int + var locked bool var createdAt, updatedAt time.Time err := svc.pool.QueryRow(c.Request.Context(), ` SELECT id, name, status, power_state, project_id, user_id::text, flavor_id::text, image_id::text, - created_at, updated_at, key_name, + created_at, updated_at, key_name, locked, fault_code, fault_message, fault_created FROM instances WHERE id = $1 AND project_id = $2 `, instanceID, projectID).Scan(&id, &name, &status, &powerState, &projectID, &userID, &flavorID, &imageID, - &createdAt, &updatedAt, &keyName, &faultCode, &faultMessage, &faultCreated) + &createdAt, &updatedAt, &keyName, &locked, &faultCode, &faultMessage, &faultCreated) if errors.Is(err, database.ErrNoRows) { common.SendError(c, common.NewNotFoundError("instance")) @@ -1956,6 +1962,10 @@ func (svc *Service) GetServer(c *gin.Context) { "updated": updatedAt.Format(time.RFC3339), "addresses": addresses, "OS-EXT-STS:power_state": powerState, + // Exposed since microversion 2.9 upstream; the lock/unlock actions + // maintain the column, but this response never carried it, so + // `openstack server show` printed null (SPEC-028). + "locked": locked, } if userID.Valid { diff --git a/internal/nova/pr107_review_test.go b/internal/nova/pr107_review_test.go new file mode 100644 index 00000000..84481b46 --- /dev/null +++ b/internal/nova/pr107_review_test.go @@ -0,0 +1,105 @@ +package nova + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "strings" + "testing" + + "github.com/cobaltcore-dev/lightstack/internal/database" + "github.com/gin-gonic/gin" + "github.com/google/uuid" +) + +type failUnshelveDB struct{ database.DB } +type failUnshelveTx struct{ database.Tx } + +func (d failUnshelveDB) Begin(ctx context.Context) (database.Tx, error) { + tx, err := d.DB.Begin(ctx) + return failUnshelveTx{tx}, err +} +func (tx failUnshelveTx) Exec(ctx context.Context, q string, args ...any) (database.Result, error) { + if strings.Contains(q, "UPDATE instances") { + return nil, errors.New("injected state failure") + } + return tx.Tx.Exec(ctx, q, args...) +} + +func TestFailedUnshelveRetainsSnapshot(t *testing.T) { + requireDB(t) + id := seedSpec028Instance(t, "review-unshelve", false) + r := newNovaTestRouter() + payload, _ := json.Marshal(map[string]any{"shelve": nil}) + if code, body := doReq(r, http.MethodPost, "/v2.1/servers/"+id+"/action", string(payload), testAuthHdr(seededProjectID, seededUserID)); code != 202 { + t.Fatalf("shelve: %d %s", code, body) + } + svc := &Service{pool: failUnshelveDB{testPool}} + failing := gin.New() + failing.POST("/unshelve", func(c *gin.Context) { svc.UnshelveInstance(c, id, seededProjectID) }) + if code, body := doReq(failing, http.MethodPost, "/unshelve", "", nil); code != 500 { + t.Fatalf("unshelve: %d %s", code, body) + } + var count int + if err := testPool.QueryRow(t.Context(), "SELECT COUNT(*) FROM instance_snapshots WHERE instance_id=$1", id).Scan(&count); err != nil || count != 1 { + t.Fatalf("lost snapshot: count=%d err=%v", count, err) + } +} + +func TestServerGroupRulesPersistAndValidate(t *testing.T) { + requireDB(t) + r := newNovaTestRouter() + hdr := testAuthHdr(seededProjectID, seededUserID) + name := "review-group-" + uuid.NewString() + t.Cleanup(func() { testPool.Exec(context.Background(), "DELETE FROM server_groups WHERE name=$1", name) }) + for _, tc := range []struct { + policy string + rules map[string]int + code int + }{ + {"affinity", map[string]int{"max_server_per_host": 2}, 400}, + {"anti-affinity", map[string]int{"max_server_per_host": 0}, 400}, + {"anti-affinity", map[string]int{"unknown": 2}, 400}, + {"anti-affinity", map[string]int{"max_server_per_host": 2}, 200}, + } { + payload, _ := json.Marshal(map[string]any{"server_group": map[string]any{"name": name, "policy": tc.policy, "rules": tc.rules}}) + code, body := doReq(r, http.MethodPost, "/v2.1/os-server-groups", string(payload), hdr) + if code != tc.code { + t.Fatalf("%+v: %d %s", tc, code, body) + } + if code != 200 { + continue + } + var response struct { + ServerGroup map[string]any `json:"server_group"` + } + if err := json.Unmarshal([]byte(body), &response); err != nil { + t.Fatal(err) + } + id := response.ServerGroup["id"].(string) + for _, path := range []string{"/v2.1/os-server-groups/" + id, "/v2.1/os-server-groups"} { + code, body = doReq(r, http.MethodGet, path, "", hdr) + if code != 200 { + t.Fatalf("read: %d %s", code, body) + } + var doc map[string]any + if err := json.Unmarshal([]byte(body), &doc); err != nil { + t.Fatal(err) + } + group, ok := doc["server_group"].(map[string]any) + if !ok { + for _, item := range doc["server_groups"].([]any) { + candidate := item.(map[string]any) + if candidate["id"] == id { + group = candidate + } + } + } + rules, ok := group["rules"].(map[string]any) + if !ok || rules["max_server_per_host"] != float64(2) { + t.Fatalf("rules lost: %s", body) + } + } + } +} diff --git a/internal/nova/servergroups.go b/internal/nova/servergroups.go index 393665dd..2c176557 100644 --- a/internal/nova/servergroups.go +++ b/internal/nova/servergroups.go @@ -3,6 +3,7 @@ package nova import ( "net/http" "time" + "unicode/utf8" "github.com/cobaltcore-dev/lightstack/internal/common" "github.com/gin-gonic/gin" @@ -21,7 +22,7 @@ func (svc *Service) ListServerGroups(c *gin.Context) { } rows, err := svc.pool.Query(c.Request.Context(), - `SELECT id, name, policies, members, project_id, created_at, updated_at + `SELECT id, name, policies, members, project_id, rules, created_at, updated_at FROM server_groups WHERE project_id = $1`, projectID, ) @@ -37,21 +38,16 @@ func (svc *Service) ListServerGroups(c *gin.Context) { var id, name, projectID string var policies []string var members []string + var rules map[string]int var createdAt, updatedAt time.Time - err := rows.Scan(&id, &name, &policies, &members, &projectID, &createdAt, &updatedAt) + err := rows.Scan(&id, &name, &policies, &members, &projectID, &rules, &createdAt, &updatedAt) if err != nil { continue } - serverGroups = append(serverGroups, gin.H{ - "id": id, - "name": name, - "policies": policies, - "members": members, - "metadata": gin.H{}, - "project_id": projectID, - }) + serverGroups = append(serverGroups, + serverGroupResponse(id, name, policies, members, projectID, rules)) } if err := rows.Err(); err != nil { log.Error().Err(err).Msg("rows iteration error") @@ -62,21 +58,102 @@ func (svc *Service) ListServerGroups(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"server_groups": serverGroups}) } +// validServerGroupPolicies — the vocabulary upstream accepts for either the +// legacy `policies` array or the 2.64+ `policy` string. +var validServerGroupPolicies = map[string]bool{ + "affinity": true, + "anti-affinity": true, + "soft-affinity": true, + "soft-anti-affinity": true, +} + +// serverGroupPolicyFrom resolves the policy from either request shape: +// pre-2.63 clients send `policies: ["anti-affinity"]`, 2.64+ clients (OSC 7 +// among them, whenever the negotiated microversion is ≥2.64 — and the server +// advertises 2.90) send `policy: "anti-affinity"`. Returns ok=false when +// both or neither is present, which upstream answers with 400. +func serverGroupPolicyFrom(policies []string, policy string) (string, bool) { + if policy != "" && len(policies) > 0 { + return "", false + } + if policy != "" { + if !validServerGroupPolicies[policy] { + return "", false + } + return policy, true + } + if len(policies) == 1 { + if !validServerGroupPolicies[policies[0]] { + return "", false + } + return policies[0], true + } + // An empty policies array or more than one entry: upstream's array form + // takes exactly one policy. + return "", false +} + +// serverGroupResponse renders one server group for the API. It carries BOTH +// the legacy `policies` array and the 2.64+ `policy` string, so clients of +// any microversion read the field they negotiated for. +func serverGroupResponse(id, name string, policies []string, members []string, projectID string, rules map[string]int) gin.H { + if rules == nil { + rules = map[string]int{} + } + policy := "" + if len(policies) > 0 { + policy = policies[0] + } + return gin.H{ + "id": id, + "name": name, + "policies": policies, + "policy": policy, + "members": members, + "metadata": gin.H{}, + "rules": rules, + "project_id": projectID, + } +} + // CreateServerGroup handles POST /v2.1/os-server-groups func (svc *Service) CreateServerGroup(c *gin.Context) { projectID := c.GetString("project_id") var req struct { ServerGroup struct { - Name string `json:"name" binding:"required"` - Policies []string `json:"policies" binding:"required"` - } `json:"server_group" binding:"required"` + Name string `json:"name"` + Policies []string `json:"policies"` + Policy string `json:"policy"` + Rules map[string]int `json:"rules"` + } `json:"server_group"` } if err := c.ShouldBindJSON(&req); err != nil { common.SendError(c, common.NewBadRequestError("invalid request body")) return } + if req.ServerGroup.Name == "" || utf8.RuneCountInString(req.ServerGroup.Name) > 255 { + common.SendError(c, common.NewBadRequestError("name must contain 1 to 255 characters")) + return + } + + policy, ok := serverGroupPolicyFrom(req.ServerGroup.Policies, req.ServerGroup.Policy) + if !ok { + common.SendError(c, common.NewBadRequestError( + "server group needs exactly one policy (policy string or policies array)")) + return + } + + for key, value := range req.ServerGroup.Rules { + if key != "max_server_per_host" || value < 1 || policy != "anti-affinity" { + common.SendError(c, common.NewBadRequestError("rules require anti-affinity and a positive max_server_per_host")) + return + } + } + if req.ServerGroup.Rules == nil { + req.ServerGroup.Rules = map[string]int{} + } // An unscoped token has no project to create the group in. if projectID == "" { @@ -89,9 +166,9 @@ func (svc *Service) CreateServerGroup(c *gin.Context) { now := time.Now() _, err := svc.pool.Exec(c.Request.Context(), - `INSERT INTO server_groups (id, name, policies, members, project_id, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7)`, - groupID, req.ServerGroup.Name, req.ServerGroup.Policies, []string{}, projectID, now, now, + `INSERT INTO server_groups (id, name, policies, members, project_id, rules, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, + groupID, req.ServerGroup.Name, []string{policy}, []string{}, projectID, req.ServerGroup.Rules, now, now, ) if err != nil { log.Error().Err(err).Str("operation", "create_server_group").Msg("database error") @@ -100,14 +177,7 @@ func (svc *Service) CreateServerGroup(c *gin.Context) { } c.JSON(http.StatusOK, gin.H{ - "server_group": gin.H{ - "id": groupID, - "name": req.ServerGroup.Name, - "policies": req.ServerGroup.Policies, - "members": []string{}, - "metadata": gin.H{}, - "project_id": projectID, - }, + "server_group": serverGroupResponse(groupID, req.ServerGroup.Name, []string{policy}, []string{}, projectID, req.ServerGroup.Rules), }) } @@ -124,13 +194,14 @@ func (svc *Service) GetServerGroup(c *gin.Context) { var name, projectIDFromDB string var policies []string var members []string + var rules map[string]int var createdAt, updatedAt time.Time err := svc.pool.QueryRow(c.Request.Context(), - `SELECT name, policies, members, project_id, created_at, updated_at + `SELECT name, policies, members, project_id, rules, created_at, updated_at FROM server_groups WHERE id = $1 AND project_id = $2`, groupID, projectID, - ).Scan(&name, &policies, &members, &projectIDFromDB, &createdAt, &updatedAt) + ).Scan(&name, &policies, &members, &projectIDFromDB, &rules, &createdAt, &updatedAt) if err != nil { common.SendError(c, common.NewNotFoundError("server group")) @@ -138,14 +209,7 @@ func (svc *Service) GetServerGroup(c *gin.Context) { } c.JSON(http.StatusOK, gin.H{ - "server_group": gin.H{ - "id": groupID, - "name": name, - "policies": policies, - "members": members, - "metadata": gin.H{}, - "project_id": projectIDFromDB, - }, + "server_group": serverGroupResponse(groupID, name, policies, members, projectIDFromDB, rules), }) } diff --git a/internal/nova/spec028_conformance_test.go b/internal/nova/spec028_conformance_test.go new file mode 100644 index 00000000..a1593025 --- /dev/null +++ b/internal/nova/spec028_conformance_test.go @@ -0,0 +1,387 @@ +package nova + +// SPEC-028 conformance fixes for the compute surface, found by the CLI +// conformance suites (issue #105). Each test pins the upstream behaviour the +// corresponding bug diverged from. + +import ( + "context" + "net/http" + "strings" + "testing" + "time" + + "github.com/cobaltcore-dev/lightstack/pkg/hypervisor" + "github.com/google/uuid" +) + +// seedSpec028Instance inserts an ACTIVE instance owned by the seeded project +// and returns its id. The caller owns cleanup. +func seedSpec028Instance(t *testing.T, name string, withImage bool) string { + t.Helper() + id := uuid.NewString() + var err error + if withImage { + _, err = testPool.Exec(t.Context(), + `INSERT INTO instances (id, name, project_id, user_id, flavor_id, image_id, status) + VALUES ($1, $2, $3, $4, (SELECT id FROM flavors LIMIT 1), + (SELECT id FROM images LIMIT 1), 'ACTIVE')`, + id, name, seededProjectID, seededUserID) + } else { + _, err = testPool.Exec(t.Context(), + `INSERT INTO instances (id, name, project_id, user_id, flavor_id, status) + VALUES ($1, $2, $3, $4, (SELECT id FROM flavors LIMIT 1), 'ACTIVE')`, + id, name, seededProjectID, seededUserID) + } + if err != nil { + t.Fatalf("seed instance %s: %v", name, err) + } + t.Cleanup(func() { + _, _ = testPool.Exec(context.Background(), "DELETE FROM instances WHERE id=$1", id) + }) + return id +} + +// TestServerGroupCreatePolicy264 — microversion 2.64 replaced the `policies` +// array with a single `policy` string. The server advertises compute +// microversion 2.90, so every 2.64+ client (OSC 7 sends `policy` whenever the +// negotiated version is ≥2.64) was rejected with 400. The fix accepts both +// request shapes and answers with both fields so legacy and modern clients +// read what they expect. +func TestServerGroupCreatePolicy264(t *testing.T) { + requireDB(t) + r := newNovaTestRouter() + hdr := testAuthHdr(seededProjectID, seededUserID) + name := "spec028-sg-policy-" + uuid.NewString()[:8] + t.Cleanup(func() { + _, _ = testPool.Exec(context.Background(), + "DELETE FROM server_groups WHERE name=$1", name) + }) + + code, body := doReq(r, http.MethodPost, "/v2.1/os-server-groups", + `{"server_group":{"name":"`+name+`","policy":"anti-affinity"}}`, hdr) + if code != http.StatusOK { + t.Fatalf("create with 2.64 policy body: code=%d body=%s; want 200", code, body) + } + if !strings.Contains(body, `"policy":"anti-affinity"`) { + t.Errorf("create response lacks policy field: %s", body) + } + if !strings.Contains(body, `"policies":["anti-affinity"]`) { + t.Errorf("create response lacks legacy policies field: %s", body) + } + var id string + if err := testPool.QueryRow(t.Context(), + "SELECT id::text FROM server_groups WHERE name=$1", name).Scan(&id); err != nil { + t.Fatalf("group not persisted: %v", err) + } + + code, body = doReq(r, http.MethodGet, "/v2.1/os-server-groups/"+id, "", hdr) + if code != http.StatusOK { + t.Fatalf("show: code=%d body=%s; want 200", code, body) + } + if !strings.Contains(body, `"policy":"anti-affinity"`) { + t.Errorf("show response lacks policy field: %s", body) + } + + // Both shapes at once, and neither, are client errors (upstream 400s too). + code, _ = doReq(r, http.MethodPost, "/v2.1/os-server-groups", + `{"server_group":{"name":"x","policy":"affinity","policies":["affinity"]}}`, hdr) + if code != http.StatusBadRequest { + t.Errorf("policy and policies together: code=%d; want 400", code) + } + code, _ = doReq(r, http.MethodPost, "/v2.1/os-server-groups", + `{"server_group":{"name":"x"}}`, hdr) + if code != http.StatusBadRequest { + t.Errorf("neither policy nor policies: code=%d; want 400", code) + } + code, _ = doReq(r, http.MethodPost, "/v2.1/os-server-groups", + `{"server_group":{"name":"x","policy":"nonsense"}}`, hdr) + if code != http.StatusBadRequest { + t.Errorf("invalid policy value: code=%d; want 400", code) + } +} + +// TestAggregateNonUUIDIs404 — an aggregate addressed by a non-UUID path +// segment (openstacksdk's find_aggregate probes by name first) hit the +// `WHERE uuid = $1` cast error and answered 500, so the CLI aborted instead +// of falling back. Upstream answers 404 itemNotFound for an unknown +// reference of any shape. +func TestAggregateNonUUIDIs404(t *testing.T) { + requireDB(t) + r := newNovaTestRouter() + hdr := testAuthHdr(seededProjectID, seededUserID) + + code, body := doReq(r, http.MethodGet, "/v2.1/os-aggregates/my-aggregate-name", "", hdr) + if code != http.StatusNotFound { + t.Fatalf("get by non-uuid name: code=%d body=%s; want 404", code, body) + } + code, body = doReq(r, http.MethodDelete, "/v2.1/os-aggregates/my-aggregate-name", "", hdr) + if code != http.StatusNotFound { + t.Fatalf("delete by non-uuid name: code=%d body=%s; want 404", code, body) + } + // A well-formed but unknown UUID is 404 too (was already correct). + code, _ = doReq(r, http.MethodGet, "/v2.1/os-aggregates/"+uuid.NewString(), "", hdr) + if code != http.StatusNotFound { + t.Errorf("get by unknown uuid: code=%d; want 404", code) + } +} + +// TestCreateBackupImageNameExact — upstream Nova names the backup image +// exactly the requested `name`; the `--` suffix broke every +// client that looks the image up by name afterwards (OSC: "No Image found +// for "). +func TestCreateBackupImageNameExact(t *testing.T) { + requireDB(t) + r := newNovaTestRouter() + hdr := testAuthHdr(seededProjectID, seededUserID) + id := seedSpec028Instance(t, "spec028-bak-src", true) + imageName := "spec028-bak-img-" + uuid.NewString()[:8] + t.Cleanup(func() { + _, _ = testPool.Exec(context.Background(), "DELETE FROM images WHERE name=$1", imageName) + }) + + code, body := doReq(r, http.MethodPost, "/v2.1/servers/"+id+"/action", + `{"createBackup":{"name":"`+imageName+`","backup_type":"daily","rotation":1}}`, hdr) + if code != http.StatusAccepted { + t.Fatalf("createBackup: code=%d body=%s; want 202", code, body) + } + var n int + if err := testPool.QueryRow(t.Context(), + "SELECT count(*) FROM images WHERE name=$1", imageName).Scan(&n); err != nil { + t.Fatalf("count backup image: %v", err) + } + if n != 1 { + t.Errorf("backup image named exactly %q: count=%d; want 1", imageName, n) + } +} + +// TestShelveUnshelveReshelve — instance_snapshots has UNIQUE(instance_id), +// but the unshelve path never deleted the row, so shelve → unshelve → shelve +// failed with 500 "failed to create snapshot". Unshelve consumes the +// snapshot (upstream restores from it), and shelve reclaims any stale row a +// crashed run left behind. +func TestShelveUnshelveReshelve(t *testing.T) { + requireDB(t) + r := newNovaTestRouter() + hdr := testAuthHdr(seededProjectID, seededUserID) + id := seedSpec028Instance(t, "spec028-shelve-src", false) + action := func(name string) (int, string) { + return doReq(r, http.MethodPost, "/v2.1/servers/"+id+"/action", + `{"`+name+`":null}`, hdr) + } + + if code, body := action("shelve"); code != http.StatusAccepted { + t.Fatalf("first shelve: code=%d body=%s; want 202", code, body) + } + if code, body := action("unshelve"); code != http.StatusAccepted { + t.Fatalf("unshelve: code=%d body=%s; want 202", code, body) + } + var n int + if err := testPool.QueryRow(t.Context(), + "SELECT count(*) FROM instance_snapshots WHERE instance_id=$1", id).Scan(&n); err != nil { + t.Fatalf("count snapshots after unshelve: %v", err) + } + if n != 0 { + t.Errorf("unshelve left %d snapshot rows; want 0 (the snapshot is consumed)", n) + } + if code, body := action("shelve"); code != http.StatusAccepted { + t.Fatalf("re-shelve after unshelve: code=%d body=%s; want 202 (was 500: stale UNIQUE(instance_id) row)", code, body) + } +} + +// TestServerShowLockedField — `locked` was invisible in server show: the +// lock/unlock actions maintained the column, but the detail responses never +// carried it, so `openstack server show` printed null (upstream exposes the +// bool since microversion 2.9). +func TestServerShowLockedField(t *testing.T) { + requireDB(t) + r := newNovaTestRouter() + hdr := testAuthHdr(seededProjectID, seededUserID) + id := seedSpec028Instance(t, "spec028-locked-src", false) + + code, body := doReq(r, http.MethodGet, "/v2.1/servers/"+id, "", hdr) + if code != http.StatusOK { + t.Fatalf("show unlocked: code=%d body=%s; want 200", code, body) + } + if !strings.Contains(body, `"locked":false`) { + t.Errorf("show must carry locked:false, got: %s", body) + } + + if code, body := doReq(r, http.MethodPost, "/v2.1/servers/"+id+"/action", `{"lock":null}`, hdr); code != http.StatusAccepted { + t.Fatalf("lock: code=%d body=%s; want 202", code, body) + } + code, body = doReq(r, http.MethodGet, "/v2.1/servers/"+id, "", hdr) + if code != http.StatusOK { + t.Fatalf("show locked: code=%d body=%s; want 200", code, body) + } + if !strings.Contains(body, `"locked":true`) { + t.Errorf("show must carry locked:true after lock, got: %s", body) + } +} + +// Duplicate names are valid in Flamingo Nova; groups retain distinct IDs and policies. +func TestServerGroupDuplicateNamesAllowed(t *testing.T) { + requireDB(t) + r := newNovaTestRouter() + hdr := testAuthHdr(seededProjectID, seededUserID) + name := "spec028-sg-dup-" + uuid.NewString()[:8] + t.Cleanup(func() { + _, _ = testPool.Exec(context.Background(), + "DELETE FROM server_groups WHERE name=$1", name) + }) + + code, body := doReq(r, http.MethodPost, "/v2.1/os-server-groups", + `{"server_group":{"name":"`+name+`","policy":"anti-affinity"}}`, hdr) + if code != http.StatusOK { + t.Fatalf("first create: code=%d body=%s; want 200", code, body) + } + + code, body = doReq(r, http.MethodPost, "/v2.1/os-server-groups", + `{"server_group":{"name":"`+name+`","policy":"affinity"}}`, hdr) + if code != http.StatusOK { + t.Errorf("duplicate create: code=%d body=%s; want 200", code, body) + } + + var n int + if err := testPool.QueryRow(t.Context(), + "SELECT count(*) FROM server_groups WHERE project_id=$1 AND name=$2", + seededProjectID, name).Scan(&n); err != nil { + t.Fatalf("count: %v", err) + } + if n != 2 { + t.Errorf("rows for (%s, %s) = %d; want exactly 2", seededProjectID, name, n) + } +} + +// TestUnshelveRestartsGuest — shelve stops the guest; unshelve only wrote +// ACTIVE to the record and never started it again, so the state reconciler +// (whose whole job is correcting records that disagree with the hypervisor) +// flipped the unshelved server back to SHUTOFF within one 30s sweep: +// `server unshelve` observably did nothing. Upstream's unshelve boots the +// restored instance; the guest must be up before the record claims ACTIVE +// (SPEC-028, round 9 — caught by a full run losing the race). +func TestUnshelveRestartsGuest(t *testing.T) { + requireDB(t) + _, r, mgr := newLifecycleService(t) + id, domainID := seedLifecycleInstance(t, mgr, "spec028-r9-unshelve") + hdr := lifecycleHdr() + action := func(name string) (int, string) { + return doReq(r, http.MethodPost, "/v2.1/servers/"+id+"/action", + `{"`+name+`":null}`, hdr) + } + + if code, body := action("shelve"); code != http.StatusAccepted { + t.Fatalf("shelve: code=%d body=%s; want 202", code, body) + } + // The record flips synchronously; the guest stop is asynchronous — wait + // for the hypervisor to agree before unshelving, so the pin is about + // unshelve's start, not shelve's stop timing. + deadline := time.Now().Add(5 * time.Second) + for { + if status, _ := readInstanceState(t, id); status == "SHELVED" { + if run, err := mgr.GuestRunState(t.Context(), domainID); err == nil && run == hypervisor.RunStateDown { + break + } + } + if time.Now().After(deadline) { + t.Fatalf("shelve never stopped the guest (status=%s)", func() string { s, _ := readInstanceState(t, id); return s }()) + } + time.Sleep(50 * time.Millisecond) + } + + if code, body := action("unshelve"); code != http.StatusAccepted { + t.Fatalf("unshelve: code=%d body=%s; want 202", code, body) + } + + // The guest must be up — a record claiming ACTIVE over a stopped guest + // is exactly what the reconciler exists to correct. + if run, err := mgr.GuestRunState(t.Context(), domainID); err != nil || run != hypervisor.RunStateUp { + t.Errorf("guest run state after unshelve = %v (err=%v); want up — unshelve must restart the guest shelve stopped", run, err) + } + if status, power := readInstanceState(t, id); status != "ACTIVE" || power != 1 { + t.Errorf("instance state after unshelve = %s/%d; want ACTIVE/1", status, power) + } +} + +// TestUnshelveSelfHealPersistsRevivedDomain — the unshelve self-heal path +// (domain vanished while shelved) used to leave the row naming the DEAD +// domain while the revived guest ran under a new one: the reconciler then +// selected the row, asked about the stale id, and flipped ACTIVE back to +// SHUTOFF within one sweep — the exact bug the restart exists to fix, reborn +// on the self-heal path (round 10). +func TestUnshelveSelfHealPersistsRevivedDomain(t *testing.T) { + requireDB(t) + _, r, mgr := newLifecycleService(t) + id, domainID := seedLifecycleInstance(t, mgr, "spec028-r10-unshelve-heal") + hdr := lifecycleHdr() + action := func(name string) (int, string) { + return doReq(r, http.MethodPost, "/v2.1/servers/"+id+"/action", + `{"`+name+`":null}`, hdr) + } + + if code, body := action("shelve"); code != http.StatusAccepted { + t.Fatalf("shelve: code=%d body=%s; want 202", code, body) + } + deadline := time.Now().Add(5 * time.Second) + for { + if status, _ := readInstanceState(t, id); status == "SHELVED" { + if run, err := mgr.GuestRunState(t.Context(), domainID); err == nil && run == hypervisor.RunStateDown { + break + } + } + if time.Now().After(deadline) { + t.Fatalf("shelve never stopped the guest") + } + time.Sleep(50 * time.Millisecond) + } + + // The guest vanishes while shelved (node loss, cleanup — anything that + // undefines the domain). + if err := mgr.DeleteVM(t.Context(), domainID); err != nil { + t.Fatalf("delete the shelved guest's domain: %v", err) + } + + // The self-heal rebuilds the guest from STORED STATE, whose root disk is + // the attached volume at /dev/vda (a volume's path is computed, not + // probed — unlike the qcow2 overlay, which would need the node-local + // state root). + volID := uuid.NewString() + if _, err := testPool.Exec(t.Context(), + `INSERT INTO volumes (id, name, project_id, user_id, size_gb, status, bootable, volume_type, created_at, updated_at) + VALUES ($1, 'spec028-r10-heal-root', $2, $3, 1, 'in-use', true, '__DEFAULT__', now(), now())`, + volID, seededProjectID, seededUserID); err != nil { + t.Fatalf("seed root volume: %v", err) + } + if _, err := testPool.Exec(t.Context(), + `INSERT INTO volume_attachments (id, volume_id, instance_id, device, delete_on_termination, attached_at) + VALUES ($1, $2, $3, '/dev/vda', false, now())`, + uuid.NewString(), volID, id); err != nil { + t.Fatalf("seed root attachment: %v", err) + } + t.Cleanup(func() { + _, _ = testPool.Exec(context.Background(), `DELETE FROM volume_attachments WHERE volume_id = $1`, volID) + _, _ = testPool.Exec(context.Background(), `DELETE FROM volumes WHERE id = $1`, volID) + }) + + if code, body := action("unshelve"); code != http.StatusAccepted { + t.Fatalf("unshelve after guest vanished: code=%d body=%s; want 202 (self-heal)", code, body) + } + + // The row must name a RUNNING guest — a stale id is what the reconciler + // corrects into SHUTOFF. + var status, rowDomain string + if err := testPool.QueryRow(t.Context(), + `SELECT status, COALESCE(libvirt_domain_id, '') FROM instances WHERE id = $1`, id, + ).Scan(&status, &rowDomain); err != nil { + t.Fatalf("read instance row: %v", err) + } + if status != "ACTIVE" { + t.Errorf("status after self-healing unshelve = %s; want ACTIVE", status) + } + if rowDomain == "" || rowDomain == domainID { + t.Errorf("row's libvirt_domain_id = %q (was %q); want the REVIVED domain's id", rowDomain, domainID) + } + if run, err := mgr.GuestRunState(t.Context(), rowDomain); err != nil || run != hypervisor.RunStateUp { + t.Errorf("guest run state for the row's domain = %v (err=%v); want up — the row must name the revived guest", run, err) + } +} diff --git a/migrations/076_application_credential_roles.postgres.down.sql b/migrations/076_application_credential_roles.postgres.down.sql new file mode 100644 index 00000000..bc5aeca6 --- /dev/null +++ b/migrations/076_application_credential_roles.postgres.down.sql @@ -0,0 +1,2 @@ +-- Rollback application_credential_roles (SPEC-028). +DROP TABLE IF EXISTS application_credential_roles; diff --git a/migrations/076_application_credential_roles.postgres.up.sql b/migrations/076_application_credential_roles.postgres.up.sql new file mode 100644 index 00000000..b9c6bd81 --- /dev/null +++ b/migrations/076_application_credential_roles.postgres.up.sql @@ -0,0 +1,18 @@ +-- application_credential_roles — the roles an application credential narrows a token to. +-- +-- The handlers in internal/keystone/application_credentials.go have referenced this +-- table since migration 029 introduced application credentials, but 029 never created +-- it: every roles lookup failed with `relation "application_credential_roles" does not +-- exist`, the handlers swallowed the error, and the `roles` key was omitted from list +-- and show responses — upstream always includes it, and the pinned python-openstackclient +-- crashes rendering a credential without one (SPEC-028). +-- +-- Fresh installs get the table; installs migrated past 029 get it too, empty, which is +-- exactly what their credentials answer today (no roles were ever persisted). +CREATE TABLE IF NOT EXISTS application_credential_roles ( + application_credential_id UUID NOT NULL REFERENCES application_credentials(id) ON DELETE CASCADE, + role_id UUID NOT NULL REFERENCES roles(id) ON DELETE CASCADE, + PRIMARY KEY (application_credential_id, role_id) +); + +CREATE INDEX IF NOT EXISTS idx_application_credential_roles_role ON application_credential_roles(role_id); diff --git a/migrations/076_application_credential_roles.sqlite.down.sql b/migrations/076_application_credential_roles.sqlite.down.sql new file mode 100644 index 00000000..d633ffd8 --- /dev/null +++ b/migrations/076_application_credential_roles.sqlite.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS application_credential_roles; diff --git a/migrations/076_application_credential_roles.sqlite.up.sql b/migrations/076_application_credential_roles.sqlite.up.sql new file mode 100644 index 00000000..49fd84d7 --- /dev/null +++ b/migrations/076_application_credential_roles.sqlite.up.sql @@ -0,0 +1,10 @@ +-- SQLite stores all resource identifiers as TEXT. Keep the same relational +-- contract as the PostgreSQL variant without UUID column types (SPEC-028). +CREATE TABLE IF NOT EXISTS application_credential_roles ( + application_credential_id TEXT NOT NULL REFERENCES application_credentials(id) ON DELETE CASCADE, + role_id TEXT NOT NULL REFERENCES roles(id) ON DELETE CASCADE, + PRIMARY KEY (application_credential_id, role_id) +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_application_credential_roles_role + ON application_credential_roles(role_id); diff --git a/migrations/077_neutron_descriptions.down.sql b/migrations/077_neutron_descriptions.down.sql new file mode 100644 index 00000000..53daf918 --- /dev/null +++ b/migrations/077_neutron_descriptions.down.sql @@ -0,0 +1,3 @@ +-- Rollback the Neutron resource descriptions (SPEC-028). +ALTER TABLE networks DROP COLUMN description; +ALTER TABLE subnet_pools DROP COLUMN description; diff --git a/migrations/077_neutron_descriptions.up.sql b/migrations/077_neutron_descriptions.up.sql new file mode 100644 index 00000000..c52f0ad9 --- /dev/null +++ b/migrations/077_neutron_descriptions.up.sql @@ -0,0 +1,10 @@ +-- networks.description and subnet_pools.description — upstream Neutron's +-- standard resource annotation (SPEC-028). +-- +-- Neither table had the column, so `network set --description` answered +-- 400 "no updates provided" (the update handler binds name/admin_state_up/ +-- router:external only) and `subnet pool set --description` did the same. +-- Existing rows carry no description; NULL reads back as the empty string, +-- exactly what the CLI displayed for them before. +ALTER TABLE networks ADD COLUMN description TEXT; +ALTER TABLE subnet_pools ADD COLUMN description TEXT; diff --git a/migrations/078_server_groups_unique.postgres.down.sql b/migrations/078_server_groups_unique.postgres.down.sql new file mode 100644 index 00000000..461e35d0 --- /dev/null +++ b/migrations/078_server_groups_unique.postgres.down.sql @@ -0,0 +1,3 @@ +-- Reserved version: Nova permits server groups with duplicate names. +-- Never merge their policies or memberships, or delete these distinct groups. +SELECT 1; diff --git a/migrations/078_server_groups_unique.postgres.up.sql b/migrations/078_server_groups_unique.postgres.up.sql new file mode 100644 index 00000000..461e35d0 --- /dev/null +++ b/migrations/078_server_groups_unique.postgres.up.sql @@ -0,0 +1,3 @@ +-- Reserved version: Nova permits server groups with duplicate names. +-- Never merge their policies or memberships, or delete these distinct groups. +SELECT 1; diff --git a/migrations/078_server_groups_unique.sqlite.down.sql b/migrations/078_server_groups_unique.sqlite.down.sql new file mode 100644 index 00000000..461e35d0 --- /dev/null +++ b/migrations/078_server_groups_unique.sqlite.down.sql @@ -0,0 +1,3 @@ +-- Reserved version: Nova permits server groups with duplicate names. +-- Never merge their policies or memberships, or delete these distinct groups. +SELECT 1; diff --git a/migrations/078_server_groups_unique.sqlite.up.sql b/migrations/078_server_groups_unique.sqlite.up.sql new file mode 100644 index 00000000..461e35d0 --- /dev/null +++ b/migrations/078_server_groups_unique.sqlite.up.sql @@ -0,0 +1,3 @@ +-- Reserved version: Nova permits server groups with duplicate names. +-- Never merge their policies or memberships, or delete these distinct groups. +SELECT 1; diff --git a/migrations/079_owner_not_null.postgres.down.sql b/migrations/079_owner_not_null.postgres.down.sql new file mode 100644 index 00000000..871c3d5d --- /dev/null +++ b/migrations/079_owner_not_null.postgres.down.sql @@ -0,0 +1,5 @@ +-- Reverts 079: the ownership-label invariant goes back to conventional +-- (every writer binds the auth-context project) instead of structural. + +ALTER TABLE volumes ALTER COLUMN project_id DROP NOT NULL; +ALTER TABLE ports ALTER COLUMN project_id DROP NOT NULL; diff --git a/migrations/079_owner_not_null.postgres.up.sql b/migrations/079_owner_not_null.postgres.up.sql new file mode 100644 index 00000000..58d89223 --- /dev/null +++ b/migrations/079_owner_not_null.postgres.up.sql @@ -0,0 +1,10 @@ +-- Defense in depth for the ownership labels the cross-project listings and +-- shows render: volumes.project_id and ports.project_id are scanned into +-- plain strings, so a NULL would 500 the show and silently drop the row from +-- the listing. No writer can produce NULL today (every INSERT binds the +-- auth-context project, and an empty string fails the uuid cast), which is +-- exactly why the constraint belongs in the schema: it makes the invariant +-- structural instead of conventional (round 9). + +ALTER TABLE volumes ALTER COLUMN project_id SET NOT NULL; +ALTER TABLE ports ALTER COLUMN project_id SET NOT NULL; diff --git a/migrations/079_owner_not_null.sqlite.down.sql b/migrations/079_owner_not_null.sqlite.down.sql new file mode 100644 index 00000000..add40395 --- /dev/null +++ b/migrations/079_owner_not_null.sqlite.down.sql @@ -0,0 +1,4 @@ +DROP TRIGGER IF EXISTS volumes_project_id_not_null_insert; +DROP TRIGGER IF EXISTS volumes_project_id_not_null_update; +DROP TRIGGER IF EXISTS ports_project_id_not_null_insert; +DROP TRIGGER IF EXISTS ports_project_id_not_null_update; diff --git a/migrations/079_owner_not_null.sqlite.up.sql b/migrations/079_owner_not_null.sqlite.up.sql new file mode 100644 index 00000000..6731d9c8 --- /dev/null +++ b/migrations/079_owner_not_null.sqlite.up.sql @@ -0,0 +1,26 @@ +-- SQLite cannot add NOT NULL to these existing tables without rebuilding +-- referenced tables. Keep the PostgreSQL invariant with insert/update guards. +CREATE TRIGGER volumes_project_id_not_null_insert +BEFORE INSERT ON volumes +WHEN NEW.project_id IS NULL +BEGIN + SELECT RAISE(ABORT, 'volumes.project_id may not be null'); +END; +CREATE TRIGGER volumes_project_id_not_null_update +BEFORE UPDATE OF project_id ON volumes +WHEN NEW.project_id IS NULL +BEGIN + SELECT RAISE(ABORT, 'volumes.project_id may not be null'); +END; +CREATE TRIGGER ports_project_id_not_null_insert +BEFORE INSERT ON ports +WHEN NEW.project_id IS NULL +BEGIN + SELECT RAISE(ABORT, 'ports.project_id may not be null'); +END; +CREATE TRIGGER ports_project_id_not_null_update +BEFORE UPDATE OF project_id ON ports +WHEN NEW.project_id IS NULL +BEGIN + SELECT RAISE(ABORT, 'ports.project_id may not be null'); +END; diff --git a/migrations/080_owner_not_null_rest.postgres.down.sql b/migrations/080_owner_not_null_rest.postgres.down.sql new file mode 100644 index 00000000..4b5da829 --- /dev/null +++ b/migrations/080_owner_not_null_rest.postgres.down.sql @@ -0,0 +1,7 @@ +-- Reverts 080: the ownership-label invariant for networks, routers and +-- floating_ips goes back to conventional (every writer binds the auth-context +-- project) instead of structural. + +ALTER TABLE networks ALTER COLUMN project_id DROP NOT NULL; +ALTER TABLE routers ALTER COLUMN project_id DROP NOT NULL; +ALTER TABLE floating_ips ALTER COLUMN project_id DROP NOT NULL; diff --git a/migrations/080_owner_not_null_rest.postgres.up.sql b/migrations/080_owner_not_null_rest.postgres.up.sql new file mode 100644 index 00000000..8b64a3ef --- /dev/null +++ b/migrations/080_owner_not_null_rest.postgres.up.sql @@ -0,0 +1,10 @@ +-- Extends 079's hardening to the remaining tables whose project_id is scanned +-- into a plain string and rendered as the row's tenant_id by the cross-project +-- listings and shows (networks, routers, floating_ips — round 10). Every writer +-- binds the auth-context project (an empty string fails the uuid cast), so the +-- constraint makes an existing invariant structural, exactly as 079 did for +-- volumes and ports. + +ALTER TABLE networks ALTER COLUMN project_id SET NOT NULL; +ALTER TABLE routers ALTER COLUMN project_id SET NOT NULL; +ALTER TABLE floating_ips ALTER COLUMN project_id SET NOT NULL; diff --git a/migrations/080_owner_not_null_rest.sqlite.down.sql b/migrations/080_owner_not_null_rest.sqlite.down.sql new file mode 100644 index 00000000..4c9827f8 --- /dev/null +++ b/migrations/080_owner_not_null_rest.sqlite.down.sql @@ -0,0 +1,6 @@ +DROP TRIGGER IF EXISTS networks_project_id_not_null_insert; +DROP TRIGGER IF EXISTS networks_project_id_not_null_update; +DROP TRIGGER IF EXISTS routers_project_id_not_null_insert; +DROP TRIGGER IF EXISTS routers_project_id_not_null_update; +DROP TRIGGER IF EXISTS floating_ips_project_id_not_null_insert; +DROP TRIGGER IF EXISTS floating_ips_project_id_not_null_update; diff --git a/migrations/080_owner_not_null_rest.sqlite.up.sql b/migrations/080_owner_not_null_rest.sqlite.up.sql new file mode 100644 index 00000000..32d3c1ea --- /dev/null +++ b/migrations/080_owner_not_null_rest.sqlite.up.sql @@ -0,0 +1,38 @@ +-- SQLite cannot add NOT NULL to these existing tables without rebuilding +-- referenced tables. Keep the PostgreSQL invariant with insert/update guards. +CREATE TRIGGER networks_project_id_not_null_insert +BEFORE INSERT ON networks +WHEN NEW.project_id IS NULL +BEGIN + SELECT RAISE(ABORT, 'networks.project_id may not be null'); +END; +CREATE TRIGGER networks_project_id_not_null_update +BEFORE UPDATE OF project_id ON networks +WHEN NEW.project_id IS NULL +BEGIN + SELECT RAISE(ABORT, 'networks.project_id may not be null'); +END; +CREATE TRIGGER routers_project_id_not_null_insert +BEFORE INSERT ON routers +WHEN NEW.project_id IS NULL +BEGIN + SELECT RAISE(ABORT, 'routers.project_id may not be null'); +END; +CREATE TRIGGER routers_project_id_not_null_update +BEFORE UPDATE OF project_id ON routers +WHEN NEW.project_id IS NULL +BEGIN + SELECT RAISE(ABORT, 'routers.project_id may not be null'); +END; +CREATE TRIGGER floating_ips_project_id_not_null_insert +BEFORE INSERT ON floating_ips +WHEN NEW.project_id IS NULL +BEGIN + SELECT RAISE(ABORT, 'floating_ips.project_id may not be null'); +END; +CREATE TRIGGER floating_ips_project_id_not_null_update +BEFORE UPDATE OF project_id ON floating_ips +WHEN NEW.project_id IS NULL +BEGIN + SELECT RAISE(ABORT, 'floating_ips.project_id may not be null'); +END; diff --git a/migrations/081_server_group_names.postgres.down.sql b/migrations/081_server_group_names.postgres.down.sql new file mode 100644 index 00000000..01dded68 --- /dev/null +++ b/migrations/081_server_group_names.postgres.down.sql @@ -0,0 +1,2 @@ +-- Duplicate names are valid; rollback must not reintroduce a destructive rule. +SELECT 1; diff --git a/migrations/081_server_group_names.postgres.up.sql b/migrations/081_server_group_names.postgres.up.sql new file mode 100644 index 00000000..36d09e6e --- /dev/null +++ b/migrations/081_server_group_names.postgres.up.sql @@ -0,0 +1,3 @@ +-- Remove the uniqueness rule from pre-release SPEC-028 installations. +ALTER TABLE server_groups DROP CONSTRAINT IF EXISTS uq_server_groups_project_name; +DROP INDEX IF EXISTS uq_server_groups_project_name; diff --git a/migrations/081_server_group_names.sqlite.down.sql b/migrations/081_server_group_names.sqlite.down.sql new file mode 100644 index 00000000..01dded68 --- /dev/null +++ b/migrations/081_server_group_names.sqlite.down.sql @@ -0,0 +1,2 @@ +-- Duplicate names are valid; rollback must not reintroduce a destructive rule. +SELECT 1; diff --git a/migrations/081_server_group_names.sqlite.up.sql b/migrations/081_server_group_names.sqlite.up.sql new file mode 100644 index 00000000..b0038339 --- /dev/null +++ b/migrations/081_server_group_names.sqlite.up.sql @@ -0,0 +1,2 @@ +-- Remove the uniqueness rule from pre-release SPEC-028 installations. +DROP INDEX IF EXISTS uq_server_groups_project_name; diff --git a/migrations/082_server_group_rules.down.sql b/migrations/082_server_group_rules.down.sql new file mode 100644 index 00000000..14697a06 --- /dev/null +++ b/migrations/082_server_group_rules.down.sql @@ -0,0 +1 @@ +ALTER TABLE server_groups DROP COLUMN rules; diff --git a/migrations/082_server_group_rules.sqlite.down.sql b/migrations/082_server_group_rules.sqlite.down.sql new file mode 100644 index 00000000..14697a06 --- /dev/null +++ b/migrations/082_server_group_rules.sqlite.down.sql @@ -0,0 +1 @@ +ALTER TABLE server_groups DROP COLUMN rules; diff --git a/migrations/082_server_group_rules.sqlite.up.sql b/migrations/082_server_group_rules.sqlite.up.sql new file mode 100644 index 00000000..c52e76a7 --- /dev/null +++ b/migrations/082_server_group_rules.sqlite.up.sql @@ -0,0 +1 @@ +ALTER TABLE server_groups ADD COLUMN rules TEXT NOT NULL DEFAULT '{}'; diff --git a/migrations/082_server_group_rules.up.sql b/migrations/082_server_group_rules.up.sql new file mode 100644 index 00000000..f6026542 --- /dev/null +++ b/migrations/082_server_group_rules.up.sql @@ -0,0 +1 @@ +ALTER TABLE server_groups ADD COLUMN rules JSONB NOT NULL DEFAULT '{}'; diff --git a/pkg/networking/northsouth.go b/pkg/networking/northsouth.go index c9f2bde0..22fb34ed 100644 --- a/pkg/networking/northsouth.go +++ b/pkg/networking/northsouth.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" ) // DefaultUplinkInterface returns the host's primary uplink: the interface the @@ -29,7 +30,7 @@ func DefaultUplinkInterface() (string, error) { } } // Fallback: scan the route table for the default route (Dst nil or 0.0.0.0/0). - routes, err := netlink.RouteList(nil, netlink.FAMILY_V4) + routes, err := netlink.RouteList(nil, unix.AF_INET) if err != nil { return "", fmt.Errorf("list routes: %w", err) } diff --git a/test/cli/README.md b/test/cli/README.md new file mode 100644 index 00000000..a2a1f7f6 --- /dev/null +++ b/test/cli/README.md @@ -0,0 +1,163 @@ +# OpenStack CLI Conformance Suite (SPEC-028) + +Systematic verification that the **built lightstack binary** works against the +**python OpenStack CLI** (`openstack`) across all cases — every CLI-exposed +operation of the five services, happy path and negative path — enforced in CI. + +Implements [SPEC-028](../../docs/specs/028-cli-conformance-test-environment/plan.md) +for [issue #105](https://github.com/cobaltcore-dev/lightstack/issues/105). + +## Quick start + +```bash +make build # the suite tests THIS binary +make test-cli # runs against a running server, or starts one via docker +# or, with a server already running (stub mode, default creds): +./test/cli/bin/run.sh +./test/cli/bin/coverage.sh # the coverage gate — see below +``` + +Requirements: `bash`, `jq`, `curl`, `python3`, `md5sum` (coreutils — the +image suite checksums its uploads; macOS ships `md5` instead), and the pinned +CLI (`pip install -c test/cli/constraints.txt python-openstackclient`, or +point `OPENSTACK_CLI` at an existing binary). + +## How it works + +``` +bin/run.sh ── sources ──> lib/common.sh ──> lib/assert.sh, lib/report.sh + │ │ + │ fixtures_ensure/teardown (lib/fixtures.sh) │ results.json + results.junit.xml + ▼ ▼ +suites/.sh ── case records ──> results/results.jsonl ──> bin/coverage.sh +manifest/.json ──────────────────────────────────┘ (the gate) +``` + +- **Suites** (`suites/*.sh`) drive the real `openstack` CLI through the `osc` + wrapper and record one result per case. +- **Manifests** (`manifest/*.json`) enumerate the command surface per service — + the machine-checkable definition of "all cases". See + [manifest/README.md](manifest/README.md) for the schema. +- **The gate** (`bin/coverage.sh`) cross-references manifests with results and + fails on any tier A/B command that is untested, failing, or skipped. + `known-failing` cases are allowed only when they are in the checked-in + [baseline](baseline/known_failures.txt); anything else is a regression and + fails the gate. No pass-rate thresholds. + +## Known-failure baseline + +`baseline/known_failures.txt` is the ledger of cases that **cannot pass against +the pinned environment** — currently the stub-mode console (no guest to proxy, +so Nova answers 501) and the OSC 7.0.0 `volume backup show ` client bug +(fixed client-side in 7.3.0). One case id per line; blank lines and everything +after `#` are ignored, so every entry carries its reason. + +Both gates read the same file: + +- `coverage.sh` (the PR gate) fails when any recorded `known-failing` case is + **not** listed. A listed case that now passes produces a stale-baseline + warning, not a failure. +- `run.sh --strict` (the nightly) has the same semantics: skip and new + known-failing fail; baseline entries do not. + +When a genuine divergence is fixed, **remove its baseline entry** (and, if the +suite probe self-heals, convert it to a plain assertion). The stale warning will +remind you. A product divergence is not a baseline entry — it is a bug to fix or +a tracked issue. + +## Tiers + +| Tier | Meaning | Case id | Gated | +|------|---------|---------|-------| +| A | happy path + output shape (`-f json` fields present and typed) | `` | yes | +| B | negative paths (404/400/409 semantics via the CLI) | `.neg` | yes | +| C | cross-service workflows | scenario id, verbatim | nightly (`--strict`) | +| S | smoke — harness end-to-end proof | `smoke.*` | no | + +## The lib API (locked) + +Suites source `lib/common.sh` and use exactly these: + +```bash +osc # run the CLI (always -f json unless -f passed; retries + # once without -f for formatter-less mutation commands). + # Sets OSC_RC, OSC_OUT, OSC_ERR, LAST_CMD, LAST_CMD_MS. + +t_begin keystone.project.list # open a case (id = manifest id) +osc project list +assert_rc 0 # see lib/assert.sh for all assertions +assert_jq_type '.' array +assert_jq '[.[] | select((.Name // .name) == "default")] | length' 1 +t_end # close + record the case + +t_fail "reason" # mark the current case failed (repeatable) +t_known_failure "reason" # tracked; must also be in the baseline +t_skip "reason" # recorded as skipped (fails the gate) +``` + +## Conventions (mandatory) + +1. **Naming**: every resource a case creates is named `cli-conf--`. + The fixture sweep reclaims anything with the `cli-conf` prefix — floating + IPs excepted (nameless; suites that allocate them must release them). +2. **Rerun-safety**: a case must pass when run twice against the same server — + delete a possible leftover before creating, or derive the name from the + case id. +3. **One manifest file per service**, owned by that service's suite; ids in the + suite must match ids in the manifest exactly. +4. **Known failures** are recorded in the suite with `t_known_failure`, never + by deleting the case — and every one needs a reason string *and* a matching + entry in `baseline/known_failures.txt`. A known-failing case absent from the + baseline fails both gates; remove the entry when the divergence is fixed. +5. **Output shape matters**: assert the fields the CLI surfaces, not just the + exit code. Note OSC 7 emits display-cased keys for `list` commands + (`ID`, `Name`) but raw lowercase API fields for `create`/`show` — + `(.Name // .name)` covers both. + +## Running + +```bash +./test/cli/bin/run.sh # everything (fixtures + all suites + teardown) +./test/cli/bin/run.sh --suite keystone # one suite, no teardown (dev iteration) +./test/cli/bin/run.sh --fast # PR subset (everything except workflows) +./test/cli/bin/run.sh --strict # nightly: skip + new known-failing fail +./test/cli/bin/run.sh --no-cleanup # keep fixtures and created resources +bash suites/keystone.sh # standalone; results under results/standalone/ +./test/cli/bin/coverage.sh # gate against results/results.json +./test/cli/bin/coverage.sh --strict # also require tier C (nightly) +python3 -m unittest discover -s test/cli -p 'test_*.py' -v # harness regression tests; no server needed +``` + +Environment: the standard `OS_*` variables (defaults match a local stub-mode +deployment: `http://localhost:35357/v3`, `admin`/`secret`), plus `OPENSTACK_CLI`, +`OSC_TIMEOUT` (default 30s), `NO_COLOR`, and `CLI_BASELINE_FILE` (override the +known-failure ledger; used by the harness self-tests). + +If no server is at `OS_AUTH_URL` and docker is available, `run.sh` starts +`deployments/docker-compose.binary-test.yml` — the runtime image with this +checkout's `bin/lightstack` bind-mounted over the in-image binary, so the suite +always exercises the artifact `make build` produced, not a container rebuild. +CI runs the binary natively instead (artifact download + PostgreSQL service). + +## Results + +- `results/results.json` — aggregate + per-case records (latest full run) +- `results/results.junit.xml` — JUnit XML for CI tooling +- `results/standalone/` — per-run records for standalone suite invocations +- Exit code: 0 iff no case failed (known-failing and skipped do not fail the + run; the coverage gate and `--strict` evaluate known-failing against the + baseline) + +## Adding a case + +1. Add the command to your service's `manifest/.json` (tier A, and + tier B if a negative case makes sense). +2. Write the case in `suites/.sh` using the id from the manifest + (`` for tier A, `.neg` for tier B). +3. `./test/cli/bin/run.sh --suite && ./test/cli/bin/coverage.sh` + until green. + +## Pinning policy + +`constraints.txt` pins `python-openstackclient`. Upgrades are deliberate PRs +(batch the expected diffs), never ambient. Current pin: 7.0.0. diff --git a/test/cli/baseline/known_failures.txt b/test/cli/baseline/known_failures.txt new file mode 100644 index 00000000..3bbfc1d2 --- /dev/null +++ b/test/cli/baseline/known_failures.txt @@ -0,0 +1,28 @@ +# Known-failure baseline for the CLI conformance gates (SPEC-028). +# +# One case id per line. A case listed here may be recorded `known-failing` +# without failing bin/coverage.sh or bin/run.sh --strict. Any OTHER case +# recorded known-failing is a regression and fails both gates. Entries that +# start passing are reported as stale warnings (not failures) by both gates. +# +# This file holds only cases that cannot pass against the pinned environment. +# A product divergence belongs in a fix (or a tracked bug), not here. When a +# genuine divergence is fixed, remove its entry (or, for a self-healing probe, +# remove the entry once the probe asserts the real behaviour). +# +# Format: one per line; blank lines and everything after '#' ignored. + +# Stub-mode Nova has no guest to proxy: POST /v2.1/servers/{id}/remote-consoles +# deliberately answers 501 ("console unavailable: nova is running in stub +# mode"), so no console URL can exist. A live libvirt-backed compute node is +# required; the console suite probe asserts the 501 and self-heals when it +# stops being returned. +compute.console.url.show + +# Pinned python-openstackclient 7.0.0 client bug (constraints.txt): `volume +# backup show ` calls sdk get_backup() with a raw GET +# /v3/backups/{name-or-id} and no name-filtered list fallback; the server, like +# real Cinder, correctly answers 404 for a non-UUID reference. Fixed client-side +# in OSC 7.3.0; delete this entry when constraints.txt is raised (the suite +# probe already self-heals into an assertion of the real backup shape). +volume.backup.show.byname diff --git a/test/cli/bin/coverage.sh b/test/cli/bin/coverage.sh new file mode 100755 index 00000000..acba1863 --- /dev/null +++ b/test/cli/bin/coverage.sh @@ -0,0 +1,211 @@ +#!/bin/bash +# coverage.sh — the CLI conformance coverage gate (SPEC-028). +# +# Cross-references the command manifests (test/cli/manifest/*.json) with the +# recorded results (test/cli/results/results.json) and prints the per-service +# matrix. This is the machine-checkable definition of "all cases". +# +# Usage: +# coverage.sh [--results ] [--require-tiers A,B] [--strict] [--quiet] +# +# --results results.json to read (default test/cli/results/results.json) +# --require-tiers tiers that must be covered and passing (default: A,B) +# --strict additionally require tier C (workflows) — used by the +# nightly full run +# --quiet matrix only, no header/footer chatter +# +# Status per command×tier: +# pass a matching case recorded with status pass +# fail a matching case recorded as fail → gate fails +# known-failing a matching case recorded as known-failing → allowed if +# in the baseline, +# otherwise fails +# skip a matching case recorded as skip → gate fails +# untested no matching case at all → gate fails +# +# Known-failure baseline: test/cli/baseline/known_failures.txt lists the cases +# that are structurally unpassable against the pinned environment, each with a +# reason. A known-failing case missing from that ledger is a regression and a +# violation. A ledger entry that now passes is a stale-baseline warning, not a +# failure. Override the path with CLI_BASELINE_FILE. +# +# Tier→case-id mapping: tier A uses the manifest id as the case id; tier B uses +# ".neg". Tier C entries are workflow scenario ids used verbatim. + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CLI_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +MANIFEST_DIR="$CLI_ROOT/manifest" +DEFAULT_RESULTS="$CLI_ROOT/results/results.json" + +# shellcheck source=../lib/baseline.sh +source "$CLI_ROOT/lib/baseline.sh" + +RESULTS="$DEFAULT_RESULTS" +REQUIRE_TIERS="A,B" +QUIET=0 + +# A value-taking option must have its value: without this, a trailing bare +# `--results` under `set -u` dies with an unbound-variable error naming $2 +# instead of naming the option (run.sh guards the same way with _need_arg). +_need_arg() { + if (( $# < 2 )); then + echo "ERROR: option $1 requires a value" >&2 + exit 2 + fi +} + +while (( $# > 0 )); do + case "$1" in + --results) _need_arg "$@"; RESULTS="$2"; shift 2 ;; + --require-tiers) _need_arg "$@"; REQUIRE_TIERS="$2"; shift 2 ;; + --strict) REQUIRE_TIERS="A,B,C"; shift ;; + --quiet) QUIET=1; shift ;; + *) echo "unknown option: $1" >&2; exit 2 ;; + esac +done + +if ! ls "$MANIFEST_DIR"/*.json >/dev/null 2>&1; then + echo "ERROR: no manifests found in $MANIFEST_DIR" >&2 + exit 2 +fi +if [[ ! -f "$RESULTS" ]]; then + echo "ERROR: no results at $RESULTS — run ./test/cli/bin/run.sh first" >&2 + exit 2 +fi + +# Fail closed: a truncated, empty, or corrupt results file (e.g. a partial +# write from a killed runner) or an unparseable manifest must FAIL the gate, +# not iterate over nothing and print a vacuous pass. Every jq that feeds the +# gate is exit-checked, and a zero-row result is itself an error. +MANIFESTS_JSON=$(jq -s '.' "$MANIFEST_DIR"/*.json) || { + echo "ERROR: manifests in $MANIFEST_DIR are not parseable JSON" >&2 + exit 2 +} +# A parseable report is not necessarily valid evidence: an unknown status used +# to evade every failure predicate, and selecting the first duplicate ID could +# hide a later failed attempt. Require one recognized result per case. +CASES_JSON=$(jq -e ' + if type == "object" and (.cases | + type == "array" and length > 0 and + all(.[]; type == "object" and + (.id | type == "string" and length > 0) and + (.status == "pass" or .status == "fail" or + .status == "known-failing" or .status == "skip")) and + (map(.id) | length == (unique | length))) + then .cases + else error("expected nonempty cases with unique IDs and recognized statuses") + end +' "$RESULTS") || { + echo "ERROR: $RESULTS is not a valid results report — rerun ./test/cli/bin/run.sh" >&2 + exit 2 +} + +# Flatten: every manifest command × required tier, joined with its case status. +if ! ROWS=$(jq -n \ + --argjson manifests "$MANIFESTS_JSON" \ + --argjson in_cases "$CASES_JSON" \ + --arg required_tiers "$REQUIRE_TIERS" ' +[$manifests[] + | . as $m + | (.commands // [])[] as $c + | ($required_tiers | split(","))[] as $tier + | select(($c.tiers // ["A"]) | index($tier)) + | ($c.id + (if $tier == "B" then ".neg" else "" end)) as $case_id + | ([$in_cases[] | select(.id == $case_id)] | first | .status // "untested") as $status + | {service: ($m.service // "unknown"), id: $c.id, tier: $tier, + cmd: ($c.cmd // ""), case_id: $case_id, status: $status} +]'); then + echo "ERROR: failed to compute coverage rows from $RESULTS and the manifests" >&2 + exit 2 +fi +if [[ -z "$ROWS" ]] || [[ "$(printf '%s' "$ROWS" | jq 'length')" == "0" ]]; then + echo "ERROR: coverage computed zero rows — empty results or empty manifests; a gate over nothing proves nothing" >&2 + exit 2 +fi + +# Summary. +SUMMARY=$(printf '%s' "$ROWS" | jq '{ + total: length, + pass: [.[] | select(.status == "pass")] | length, + fail: [.[] | select(.status == "fail")] | length, + known_failing: [.[] | select(.status == "known-failing")] | length, + skipped: [.[] | select(.status == "skip")] | length, + untested: [.[] | select(.status == "untested")] | length, +}') + +# Per-service matrix. +(( QUIET )) || printf '%s\n' "$SUMMARY" | jq -r ' + "Coverage summary (required tiers: see exit status)", + " total: \(.total)", + " pass: \(.pass)", + " fail: \(.fail)", + " known-failing: \(.known_failing)", + " skipped: \(.skipped)", + " untested: \(.untested)" +' >&2 +printf '%s\n' "$ROWS" | jq -r ' + group_by(.service)[] + | .[0].service as $s + | "\($s): \([.[] | select(.status == "pass")] | length)/\(length) pass," + + " \([.[] | select(.status == "fail")] | length) fail," + + " \([.[] | select(.status == "known-failing")] | length) known-failing," + + " \([.[] | select(.status == "untested")] | length) untested" +' + +# --------------------------------------------------------------------------- +# Known-failure baseline. Both gates share the ledger in lib/baseline.sh: a +# known-failing case is tolerated only when it is listed there with a reason, +# so flipping a regressed case to known-failing cannot make either gate green. +# A ledger entry that now passes is stale — warn, do not fail: a fix must never +# be punished for making the baseline smaller. +# --------------------------------------------------------------------------- +BASELINE_IDS="$(cli_baseline_ids)" || exit 2 +declare -A BASELINE=() ROW_STATUS=() +while IFS= read -r id; do + [[ -n "$id" ]] && BASELINE["$id"]=1 +done <<<"$BASELINE_IDS" +while IFS=$'\t' read -r id status; do + [[ -n "$id" ]] && ROW_STATUS["$id"]="$status" +done < <(printf '%s' "$ROWS" | jq -r '.[] | [.case_id, .status] | @tsv') + +NEW_KNOWN="" +while IFS=$'\t' read -r tier id cmd; do + [[ -z "$id" ]] && continue + if [[ -z "${BASELINE[$id]:-}" ]]; then + NEW_KNOWN+=" [$tier] $id (known-failing, not in baseline): $cmd"$'\n' + fi +done < <(printf '%s' "$ROWS" | jq -r \ + '.[] | select(.status == "known-failing") | [.tier, .case_id, .cmd] | @tsv') + +STALE=() +while IFS= read -r id; do + [[ -z "$id" ]] && continue + if [[ "${ROW_STATUS[$id]:-}" == "pass" ]]; then + STALE+=("$id") + fi +done <<<"$BASELINE_IDS" +if (( ${#STALE[@]} > 0 )); then + echo "WARNING: stale known-failure baseline entries — these cases now pass:" >&2 + printf ' %s\n' "${STALE[@]}" >&2 + echo "Remove each entry from $(cli_baseline_file) (or convert its probe to an assertion)." >&2 +fi + +# The gate: fail on fail / untested / skip in the required tiers, and on any +# known-failing case the baseline does not explain. +VIOLATIONS=$(printf '%s' "$ROWS" | jq -r \ + '.[] | select(.status == "fail" or .status == "untested" or .status == "skip") + | " [\(.tier)] \(.id) (\(.status)): \(.cmd)"') +if [[ -n "$NEW_KNOWN" ]]; then + VIOLATIONS+="${VIOLATIONS:+$'\n'}$NEW_KNOWN" +fi +if [[ -n "$VIOLATIONS" ]]; then + echo + echo "Coverage gate FAILED — untested, failing, or skipped commands, or new known-failing cases:" + printf '%s\n' "$VIOLATIONS" + exit 1 +fi + +(( QUIET )) || echo "Coverage gate PASSED" +exit 0 diff --git a/test/cli/bin/run.sh b/test/cli/bin/run.sh new file mode 100755 index 00000000..bc913690 --- /dev/null +++ b/test/cli/bin/run.sh @@ -0,0 +1,277 @@ +#!/bin/bash +# run.sh — entrypoint for the lightstack OpenStack CLI conformance harness (SPEC-028). +# +# Usage: +# run.sh [--suite ]... [--fast] [--strict] [--no-cleanup] +# [--keep-server] [--results-dir ] [--help] +# +# --suite run only the named suite (repeatable). A single +# --suite implies --no-cleanup unless --teardown is +# also given, so a developer can iterate on one suite. +# --fast run the PR-time subset (all suites except workflows). +# --strict fail on skipped cases and on any known-failing case that +# is not in test/cli/baseline/known_failures.txt. The +# nightly runs this way: the PR gate tracks the debt, the +# nightly refuses to let it grow. (A baseline entry that +# now passes is a stale warning, not a failure.) +# --no-cleanup skip the fixture teardown sweep at the end. +# --teardown with --suite: still tear down afterwards. +# --keep-server do not stop a server this script started itself. +# --results-dir where results.json / results.junit.xml go +# (default: test/cli/results) +# +# Server selection: +# 1. If $OS_AUTH_URL (default http://localhost:35357/v3) answers, the +# already-running server is used as-is. +# 2. Otherwise, if docker is available, the binary-test compose stack is +# started (deployments/docker-compose.binary-test.yml — the binary built +# by `make build`, bind-mounted over the image copy) and stopped on exit. +# 3. Otherwise: error with instructions. +# +# Exit code: 0 iff every recorded case passed (known-failing and skipped do +# not fail the run; the coverage gate explains the known-failing baseline). +# With --strict, skipped cases and known-failing cases outside the baseline +# fail it. + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CLI_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +REPO_ROOT="$(cd "$CLI_ROOT/../.." && pwd)" + +# shellcheck source=../lib/common.sh +source "$CLI_ROOT/lib/common.sh" + +SUITES=() +FAST=0 +STRICT=0 +NO_CLEANUP=0 +TEARDOWN=0 +KEEP_SERVER=0 +RESULTS_DIR="$CLI_ROOT/results" + +usage() { + sed -n '2,34p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + exit "${1:-0}" +} + +# Fail with the usage hint instead of bash's unbound-variable error when an +# option that needs a value is given without one. +_need_arg() { + if (( $# < 2 )); then + log_error "option $1 requires a value" + usage 1 + fi +} + +while (( $# > 0 )); do + case "$1" in + --suite) _need_arg "$@"; SUITES+=("$2"); shift 2 ;; + --fast) FAST=1; shift ;; + --strict) STRICT=1; shift ;; + --no-cleanup) NO_CLEANUP=1; shift ;; + --teardown) TEARDOWN=1; shift ;; + --keep-server) KEEP_SERVER=1; shift ;; + --results-dir) _need_arg "$@"; RESULTS_DIR="$2"; shift 2 ;; + -h|--help) usage 0 ;; + *) log_error "unknown option: $1"; usage 1 ;; + esac +done + +# --------------------------------------------------------------------------- +# Preflight +# --------------------------------------------------------------------------- +for tool in jq curl python3; do + command -v "$tool" >/dev/null 2>&1 || { + log_error "required tool '$tool' not found in PATH" + exit 2 + } +done +_require_cli || exit 2 + +# --------------------------------------------------------------------------- +# Server selection +# --------------------------------------------------------------------------- +STARTED_STACK=0 + +_start_compose_stack() { + command -v docker >/dev/null 2>&1 || return 1 + docker compose version >/dev/null 2>&1 || return 1 + + if [[ ! -x "$REPO_ROOT/bin/lightstack" ]]; then + log_warn "bin/lightstack not found — building it first (make build)" + (cd "$REPO_ROOT" && make build) || return 1 + fi + + log_info "starting binary-test compose stack (uses bin/lightstack from this checkout)" + # -p pins the compose project: without it the project name defaults to the + # CWD ("lightstack") and `down -v` here would also tear down a developer's + # running docker-compose.yml stack. + # STARTED_STACK is set BEFORE `up -d`, not after: a partially-started + # project (up fails midway) still leaves containers and ports behind, and + # only a marked stack is torn down by the failed-startup exit path below — + # which runs before the EXIT trap exists. `down -v` on a project that never + # came up is a harmless no-op. + STARTED_STACK=1 + (cd "$REPO_ROOT" && docker compose -p lightstack-binary-test -f deployments/docker-compose.binary-test.yml up -d) || return 1 + + log_info "waiting for services to become ready..." + local deadline=$(( SECONDS + 180 )) + while (( SECONDS < deadline )); do + if _server_ready; then + log_info "all services ready" + return 0 + fi + sleep 3 + done + log_error "services did not become ready within 180s" + return 1 +} + +_server_ready() { + # Derive scheme://host (without the Keystone port) from OS_AUTH_URL so the + # probe works for non-default hosts too. The stub-mode stack is plain http. + local scheme="${OS_AUTH_URL%%://*}" + local host="${OS_AUTH_URL#*://}" + host="${host%%[:/]*}" + [[ -n "$host" ]] || host=localhost + local base="$scheme://$host" + curl -sf -o /dev/null --max-time 3 "$OS_AUTH_URL" || return 1 + curl -sf -o /dev/null --max-time 3 "$base:8774/v2.1" || return 1 + curl -sf -o /dev/null --max-time 3 "$base:9696/v2.0" || return 1 + curl -sf -o /dev/null --max-time 3 "$base:8776/v3" || return 1 + curl -sf -o /dev/null --max-time 3 "$base:9292/v2" || return 1 + return 0 +} + +# --------------------------------------------------------------------------- +# Suite selection — BEFORE the server is (possibly) started: the selection is +# pure flag logic, and an empty selection must die without having leased +# anything (a started compose stack would otherwise outlive the exit 2). +# --------------------------------------------------------------------------- +if (( ${#SUITES[@]} == 0 )); then + # Every suite in suites/, sorted for deterministic order. + for f in "$CLI_ROOT"/suites/*.sh; do + SUITES+=("$(basename "$f" .sh)") + done +fi +if (( FAST )); then + filtered=() + for s in "${SUITES[@]}"; do + [[ "$s" == "workflows" ]] && continue + filtered+=("$s") + done + SUITES=("${filtered[@]}") +fi + +# A selection that matches nothing must fail loudly, not run zero suites and +# exit 0 — the exact vacuous-green class this harness exists to prevent +# (e.g. `--fast --suite workflows`: the fast pool excludes workflows). The +# selection is necessarily empty here, so there is nothing to interpolate. +if (( ${#SUITES[@]} == 0 )); then + log_error "the --suite selection matches no suite under the current flags (--fast excludes workflows)" + exit 2 +fi + +# Strict mode is read by report_failed (lib/report.sh): known-failing and +# skipped cases fail the run as well. +if (( STRICT )); then + export CLI_STRICT=1 +fi + +# A single --suite implies no teardown unless explicitly requested. +if (( ${#SUITES[@]} == 1 )) && (( ! TEARDOWN )) && (( ! NO_CLEANUP )); then + NO_CLEANUP=1 +fi + +if ! curl -sf -o /dev/null --max-time 5 "$OS_AUTH_URL"; then + if ! _start_compose_stack; then + # The EXIT trap is not installed yet — anything the failed start left + # behind must be torn down HERE, or a half-started stack (up succeeded, + # readiness never came) would outlive the exit 2. + if (( STARTED_STACK )) && (( ! KEEP_SERVER )); then + (cd "$REPO_ROOT" && docker compose -p lightstack-binary-test -f deployments/docker-compose.binary-test.yml down -v) >/dev/null 2>&1 || true + fi + cat >&2 <"$RESULTS_FILE" +export CLI_RUNNER_ACTIVE=1 + +# shellcheck source=../lib/fixtures.sh +source "$CLI_ROOT/lib/fixtures.sh" + +cleanup() { + # failed is declared HERE, before report_write's check below — a later + # `local failed=0` would shadow an earlier assignment made while the + # variable was still global, silently disabling the check (round 9). + local rc=$? failed=0 + trap - EXIT INT TERM + if (( ! NO_CLEANUP )); then + fixtures_teardown || true + fi + if (( STARTED_STACK )) && (( ! KEEP_SERVER )); then + (cd "$REPO_ROOT" && docker compose -p lightstack-binary-test -f deployments/docker-compose.binary-test.yml down -v) >/dev/null 2>&1 || true + fi + report_write "$RESULTS_DIR" || failed=1 + report_summary + # report_failed must read the results BEFORE the run dir is removed — + # it used to run after `rm -rf` and always saw a missing file, which + # the old "missing means clean" reading silently absorbed (failure + # detection at exit was dead code). Now it genuinely gates the exit. + if (( rc == 0 )) && report_failed; then + failed=1 + fi + rm -rf "$RUN_DIR" + if (( failed )); then + exit 1 + fi + exit "$rc" +} +# INT/TERM re-exit with the conventional 128+signal status so the EXIT trap +# (which preserves and propagates it) runs once, with the right code — a trap +# on the signal itself would run cleanup with whatever $? happened to be, +# which can flatten a killed run into a successful exit. +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +fixtures_ensure || { + log_error "fixture setup failed — cannot run suites" + exit 2 +} + +FAILED_SUITES=0 +for suite in "${SUITES[@]}"; do + suite_file="$CLI_ROOT/suites/$suite.sh" + if [[ ! -f "$suite_file" ]]; then + log_error "suite not found: $suite (looked for $suite_file)" + FAILED_SUITES=1 + continue + fi + log_info "running suite: $suite" + if ! bash "$suite_file"; then + log_error "suite $suite exited non-zero" + FAILED_SUITES=1 + fi +done + +exit $FAILED_SUITES diff --git a/test/cli/constraints.txt b/test/cli/constraints.txt new file mode 100644 index 00000000..4962f248 --- /dev/null +++ b/test/cli/constraints.txt @@ -0,0 +1 @@ +python-openstackclient==7.0.0 diff --git a/test/cli/lib/assert.sh b/test/cli/lib/assert.sh new file mode 100644 index 00000000..63818584 --- /dev/null +++ b/test/cli/lib/assert.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# assert.sh — assertions on the OpenStack CLI's last output (SPEC-028). +# +# Locked API (suites depend on these names): +# assert_rc last osc() exit code +# assert_jq [expected] evaluate jq on OSC_OUT; +# without : pass when +# the expression is truthy +# (non-null, non-false) +# assert_jq_ne +# assert_jq_gt numeric greater-than +# assert_jq_type "array", "object", "string", ... +# assert_jq_len array/object length +# assert_contains OSC_OUT contains substring +# assert_err_contains OSC_ERR contains substring +# assert_regex OSC_OUT matches +# +# A failed assertion calls t_fail() (the case fails at t_end) and execution +# CONTINUES — later assertions in the same case still run, so one bad field +# produces one case failure with all the details, not an aborted suite. + +# Guard against double-sourcing. +[[ -n "${_CLI_ASSERT_SOURCED:-}" ]] && return 0 +_CLI_ASSERT_SOURCED=1 + +# Truncate values in error messages so a full JSON body does not flood the log. +_snip() { + local s="$1" + if (( ${#s} > 300 )); then + printf '%s...' "${s:0:300}" + else + printf '%s' "$s" + fi +} + +# Internal: the actual jq evaluation. Sets _JQ_RESULT / _JQ_ERROR. +_jq_eval() { + local expr="$1" out + if out=$(printf '%s' "$OSC_OUT" | jq -r "$expr" 2>&1); then + _JQ_RESULT="$out" + _JQ_ERROR="" + return 0 + fi + _JQ_RESULT="" + _JQ_ERROR="$out" + return 1 +} + +_jq_fail() { + t_fail "$1 (expr: $2, got: $3, stderr: ${4:-${_JQ_ERROR:-}}, output was: $(_snip "$OSC_OUT"))" +} + +assert_rc() { + local expected="$1" + if [[ "$OSC_RC" != "$expected" ]]; then + t_fail "exit code expected $expected, got $OSC_RC; stderr: $(_snip "$OSC_ERR")" + fi +} + +assert_jq() { + local expr="$1" expected="${2:-}" + if ! _jq_eval "$expr"; then + _jq_fail "jq failed on expression" "$expr" "$_JQ_ERROR" + return + fi + if (( $# >= 2 )); then + if [[ "$_JQ_RESULT" != "$expected" ]]; then + _jq_fail "unexpected jq result" "$expr" "want '$expected', got '$_JQ_RESULT'" + fi + else + # Truthiness: null and false are falsy, everything else passes. + if [[ -z "$_JQ_RESULT" || "$_JQ_RESULT" == "null" || "$_JQ_RESULT" == "false" ]]; then + _jq_fail "jq expression not truthy" "$expr" "'$_JQ_RESULT'" + fi + fi +} + +assert_jq_ne() { + local expr="$1" unexpected="$2" + if ! _jq_eval "$expr"; then + _jq_fail "jq failed on expression" "$expr" "$_JQ_ERROR" + return + fi + if [[ "$_JQ_RESULT" == "$unexpected" ]]; then + _jq_fail "jq result must not equal" "$expr" "'$unexpected'" + fi +} + +assert_jq_gt() { + local expr="$1" bound="$2" + if ! _jq_eval "$expr"; then + _jq_fail "jq failed on expression" "$expr" "$_JQ_ERROR" + return + fi + if ! [[ "$_JQ_RESULT" =~ ^-?[0-9]+(\.[0-9]+)?$ ]]; then + _jq_fail "jq result is not numeric" "$expr" "'$_JQ_RESULT'" + return + fi + if ! awk -v a="$_JQ_RESULT" -v b="$bound" 'BEGIN { exit !(a > b) }'; then + _jq_fail "jq result not greater than bound" "$expr" "$_JQ_RESULT <= $bound" + fi +} + +assert_jq_type() { + local expr="$1" want="$2" + local got + got=$(printf '%s' "$OSC_OUT" | jq -r "$expr | type" 2>/dev/null) + if [[ "$got" != "$want" ]]; then + t_fail "jq type of '$expr' expected $want, got ${got:-unset}; output was: $(_snip "$OSC_OUT")" + fi +} + +assert_jq_len() { + local expr="$1" want="$2" + local got + got=$(printf '%s' "$OSC_OUT" | jq -r "$expr | length" 2>/dev/null) + if [[ "$got" != "$want" ]]; then + t_fail "length of '$expr' expected $want, got ${got:-unset}; output was: $(_snip "$OSC_OUT")" + fi +} + +assert_contains() { + local needle="$1" + if [[ "$OSC_OUT" != *"$needle"* ]]; then + t_fail "output does not contain '$needle'; output: $(_snip "$OSC_OUT")" + fi +} + +assert_err_contains() { + local needle="$1" + if [[ "$OSC_ERR" != *"$needle"* ]]; then + t_fail "stderr does not contain '$needle'; stderr: $(_snip "$OSC_ERR")" + fi +} + +assert_regex() { + local pattern="$1" + if ! printf '%s' "$OSC_OUT" | grep -Eq "$pattern"; then + t_fail "output does not match /$pattern/; output: $(_snip "$OSC_OUT")" + fi +} diff --git a/test/cli/lib/baseline.sh b/test/cli/lib/baseline.sh new file mode 100644 index 00000000..ecdbb17c --- /dev/null +++ b/test/cli/lib/baseline.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# baseline.sh — checked-in known-failure baseline for the CLI conformance gates (SPEC-028). +# +# The baseline is the ledger of cases that are *structurally unpassable* against +# the current pinned environment (stub-mode server or a pinned-client bug). A +# case recorded `known-failing` must appear here, or the gate fails: otherwise a +# regression that flips a regular case to known-failing would pass both +# `coverage.sh` and `run.sh --strict`. +# +# File format (baseline/known_failures.txt): one case id per line; blank lines +# and everything after `#` are ignored, so each entry can carry its reason. +# Override the path with CLI_BASELINE_FILE (used by the harness self-tests). +# +# Locked API: +# cli_baseline_file print the baseline path +# cli_baseline_ids print the ids, one per line; returns 2 when the file is +# missing/unreadable (both gates fail closed on it) + +# Guard against double-sourcing. +[[ -n "${_CLI_BASELINE_SOURCED:-}" ]] && return 0 +_CLI_BASELINE_SOURCED=1 + +cli_baseline_file() { + printf '%s\n' "${CLI_BASELINE_FILE:-${CLI_ROOT:?CLI_ROOT must be set}/baseline/known_failures.txt}" +} + +cli_baseline_ids() { + local file + file="$(cli_baseline_file)" + if [[ ! -r "$file" ]]; then + echo "ERROR: known-failure baseline not readable: $file" >&2 + return 2 + fi + # Strip comments and all whitespace, then drop blank lines. Case ids never + # contain spaces; an accidental inline comment (the documented format) + # must not turn into part of an id. + sed -e 's/#.*//' "$file" | awk '{ gsub(/[[:space:]]/, ""); if (length > 0) print }' +} diff --git a/test/cli/lib/common.sh b/test/cli/lib/common.sh new file mode 100644 index 00000000..9b3d5808 --- /dev/null +++ b/test/cli/lib/common.sh @@ -0,0 +1,299 @@ +#!/bin/bash +# common.sh — shared foundation for the lightstack CLI conformance harness (SPEC-028). +# +# Every suite sources exactly this file; it in turn sources assert.sh and report.sh. +# The locked API (suites depend on these names — do not rename): +# +# osc run the OpenStack CLI (always -f json unless caller +# passed -f). Sets OSC_RC, OSC_OUT, OSC_ERR, LAST_CMD, +# LAST_CMD_MS. Never fails the case by itself. +# t_begin open a test case (id must be unique, e.g. keystone.project.list) +# t_fail mark the current case failed with a reason (repeatable) +# t_known_failure mark the current case known-failing (tracked, not +# fatal in non-strict mode; it must be listed in +# baseline/known_failures.txt or the strict gates fail) +# t_skip mark the current case skipped +# t_end close the current case and record it +# suite_begin / suite_end +# log_info / log_warn / log_error +# +# Environment (all optional): +# OS_AUTH_URL, OS_USERNAME, OS_PASSWORD, OS_PROJECT_NAME, +# OS_USER_DOMAIN_NAME, OS_PROJECT_DOMAIN_NAME — standard OpenStack client vars +# OPENSTACK_CLI — path to the openstack binary (default: whatever is in PATH) +# OSC_TIMEOUT — per-command timeout in seconds (default 30) +# RESULTS_FILE — JSONL sink for case records (set by bin/run.sh; standalone +# suite runs get one automatically) + +# Guard against double-sourcing. +[[ -n "${_CLI_COMMON_SOURCED:-}" ]] && return 0 +_CLI_COMMON_SOURCED=1 + +CLI_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CLI_ROOT="$(cd "$CLI_LIB_DIR/.." && pwd)" +# shellcheck source=assert.sh +source "$CLI_LIB_DIR/assert.sh" +# shellcheck source=report.sh +source "$CLI_LIB_DIR/report.sh" + +# --------------------------------------------------------------------------- +# Standalone mode: when a suite is run directly (bash suites/foo.sh) instead +# of via bin/run.sh, record results to results/standalone/ and exit non-zero +# if any case failed. run.sh sets CLI_RUNNER_ACTIVE=1 and owns reporting. +# Takes the exit status to preserve as $1 — the EXIT trap runs other commands +# before this, which would otherwise overwrite $?. +# --------------------------------------------------------------------------- +_standalone_exit() { + local rc="${1:-$?}" + trap - EXIT + if [[ "${CLI_RUNNER_ACTIVE:-0}" != "1" ]] && (( ${_CASES_RECORDED:-0} )); then + report_write "$CLI_ROOT/results/standalone" || rc=1 + report_summary + if report_failed; then + rc=1 + fi + fi + exit "$rc" +} + +# --------------------------------------------------------------------------- +# Environment +# --------------------------------------------------------------------------- + +# A clouds.yaml selection would silently override every OS_* variable below and +# point the suite at the wrong cloud. This harness always talks to the server +# named by OS_AUTH_URL. +unset OS_CLOUD 2>/dev/null || true + +: "${OS_AUTH_URL:=http://localhost:35357/v3}" +: "${OS_USERNAME:=admin}" +: "${OS_PASSWORD:=secret}" +: "${OS_PROJECT_NAME:=default}" +: "${OS_USER_DOMAIN_NAME:=Default}" +: "${OS_PROJECT_DOMAIN_NAME:=Default}" +export OS_AUTH_URL OS_USERNAME OS_PASSWORD OS_PROJECT_NAME \ + OS_USER_DOMAIN_NAME OS_PROJECT_DOMAIN_NAME +export OS_IDENTITY_API_VERSION=3 + +: "${OSC_TIMEOUT:=30}" +export OSC_TIMEOUT + +# Locate the CLI once, lazily. +_cli_find_binary() { + if [[ -n "${OPENSTACK_CLI:-}" ]]; then + command -v "$OPENSTACK_CLI" >/dev/null 2>&1 || return 1 + printf '%s\n' "$OPENSTACK_CLI" + return 0 + fi + command -v openstack 2>/dev/null || return 1 +} + +_require_cli() { + if [[ -n "${OSC_BIN:-}" ]]; then + return 0 + fi + if ! OSC_BIN="$(_cli_find_binary)"; then + cat >&2 <<'EOF' +[cli] ERROR: the OpenStack CLI (python-openstackclient) is not installed. + +Install the pinned version used by this harness: + + pip install -c test/cli/constraints.txt python-openstackclient + +or point OPENSTACK_CLI at an existing binary. +EOF + return 1 + fi + export OSC_BIN +} + +# Monotonic-ish wall clock in ms. Bash 5's EPOCHREALTIME avoids a fork; the +# python3 fallback keeps macOS/dev boxes working (python3 is guaranteed here — +# the CLI itself needs it). +_now_ms() { + if [[ -n "${EPOCHREALTIME:-}" ]]; then + local s frac + s=${EPOCHREALTIME%.*} + frac=${EPOCHREALTIME#*.} + printf '%s%03d\n' "$s" "$((10#${frac:0:3}))" + else + python3 -c 'import time; print(int(time.time() * 1000))' + fi +} + +# --------------------------------------------------------------------------- +# Logging (stderr only — stdout stays clean for summaries) +# --------------------------------------------------------------------------- + +_use_color() { [[ -t 2 && "${NO_COLOR:-}" == "" ]]; } +log_info() { if _use_color; then printf '\033[1;34m[cli]\033[0m %s\n' "$*" >&2; else printf '[cli] %s\n' "$*" >&2; fi; } +log_warn() { if _use_color; then printf '\033[1;33m[cli] WARN\033[0m %s\n' "$*" >&2; else printf '[cli] WARN %s\n' "$*" >&2; fi; } +log_error() { if _use_color; then printf '\033[0;31m[cli] ERROR\033[0m %s\n' "$*" >&2; else printf '[cli] ERROR %s\n' "$*" >&2; fi; } + +# --------------------------------------------------------------------------- +# The one way to invoke the CLI +# --------------------------------------------------------------------------- + +# Reusable stderr capture file (suites run sequentially). +_CLI_ERR_FILE="${TMPDIR:-/tmp}/lightstack-cli-err.$$" +_CLI_ERR_FILE_CREATED=0 + +# Inner executor: runs the CLI with exactly the given args. +_osc_exec() { + local args=("$@") + [[ $_CLI_ERR_FILE_CREATED -eq 0 ]] && : >"$_CLI_ERR_FILE" && _CLI_ERR_FILE_CREATED=1 + + local t0 t1 + t0=$(_now_ms) + # OSC_OUT/OSC_RC/OSC_ERR/LAST_CMD* are the harness's cross-file contract: + # every suite sources this library and reads them after each osc() call, + # which shellcheck cannot see from this file alone. + # shellcheck disable=SC2034 + OSC_OUT="$(timeout -k 10 "$OSC_TIMEOUT" "$OSC_BIN" "${args[@]}" 2>"$_CLI_ERR_FILE")" + OSC_RC=$? + t1=$(_now_ms) + OSC_ERR="$(<"$_CLI_ERR_FILE")" + LAST_CMD="$OSC_BIN ${args[*]}" + # shellcheck disable=SC2034 + LAST_CMD_MS=$(( t1 - t0 )) +} + +osc() { + _require_cli || return 1 + + local args=("$@") has_format=0 a + for a in "$@"; do + [[ "$a" == "-f" || "$a" == "--format" ]] && has_format=1 + done + # Machine-readable output unless the caller explicitly asked otherwise + # (e.g. -f value, -f csv, --help). + (( has_format )) || args+=(-f json) + + _osc_exec "${args[@]}" + + # Mutation commands with no formatter (set/delete/associate/...) reject + # -f json with argparse exit code 2. Retry once without the flag — this + # keeps suite code uniform and survives CLI version differences. + if (( OSC_RC == 2 )) && printf '%s' "$OSC_ERR" | grep -q -- "-f json"; then + _osc_exec "$@" + fi + + if (( OSC_RC == 124 )); then + log_warn "command timed out after ${OSC_TIMEOUT}s: $LAST_CMD" + elif (( OSC_RC != 0 )) && [[ -z "${CURRENT_CASE:-}" ]] && [[ "${OSC_QUIET:-0}" != "1" ]]; then + # A failing call outside any case is worth surfacing, but the caller + # decides whether it is a failure (negative tests expect rc != 0). + # OSC_QUIET=1 lets existence probes and best-effort sweeps silence + # the expected 404s. + log_warn "non-zero exit (${OSC_RC}) outside a case: $LAST_CMD" + fi + return 0 +} + +_cli_cleanup() { + # Capture the exit status BEFORE any other command — rm would otherwise + # reset it, flattening every early failure (unknown option, preflight + # errors) to a successful exit. + local rc=$? + rm -f "$_CLI_ERR_FILE" + # A suite can register extra cleanup by defining suite_cleanup() — for + # its own temp files, which would otherwise leak when a standalone run + # is killed mid-flight (compute.sh's public-key file is the first user). + if declare -f suite_cleanup >/dev/null 2>&1; then + suite_cleanup || true + fi + _standalone_exit "$rc" +} +# The single EXIT trap: always removes the stderr capture file, and in +# standalone mode (not under bin/run.sh) also writes results and converts +# case failures into a non-zero exit code. +trap _cli_cleanup EXIT + +# --------------------------------------------------------------------------- +# Case lifecycle +# --------------------------------------------------------------------------- + +CURRENT_CASE="" +CASE_FAILED=0 +CASE_KNOWN=0 +CASE_SKIPPED=0 +CASE_START=0 +CASE_ERRORS=() + +t_begin() { + if [[ -n "$CURRENT_CASE" ]]; then + # Defensive: a missing t_end would silently lose the case. + log_warn "t_begin called while case '$CURRENT_CASE' is open — auto-closing it" + t_end + fi + CURRENT_CASE="$1" + CASE_FAILED=0 + CASE_KNOWN=0 + CASE_SKIPPED=0 + CASE_START=$(_now_ms) +} + +t_fail() { + if [[ -z "${CURRENT_CASE:-}" ]]; then + log_error "t_fail called outside a case: $*" + return + fi + CASE_FAILED=1 + CASE_ERRORS+=("$*") +} + +t_known_failure() { + if [[ -z "${CURRENT_CASE:-}" ]]; then + log_error "t_known_failure called outside a case: $*" + return + fi + CASE_KNOWN=1 + CASE_ERRORS+=("known failure: $*") +} + +t_skip() { + if [[ -z "${CURRENT_CASE:-}" ]]; then + log_error "t_skip called outside a case: $*" + return + fi + CASE_SKIPPED=1 + CASE_ERRORS+=("skipped: $*") +} + +t_end() { + [[ -z "$CURRENT_CASE" ]] && return 0 + local dur=$(( $(_now_ms) - CASE_START )) + local status=pass + (( CASE_SKIPPED )) && status=skip + (( CASE_KNOWN )) && status=known-failing + (( CASE_FAILED )) && status=fail + + local detail + detail="$(printf '%s; ' "${CASE_ERRORS[@]}")" + + _record_case "${SUITE_NAME:-unknown}" "$CURRENT_CASE" "$status" "$dur" \ + "${LAST_CMD:-}" "${detail%; }" + + if [[ "$status" == pass ]]; then + if _use_color; then printf ' \033[0;32m✓\033[0m %s (%dms)\n' "$CURRENT_CASE" "$dur" >&2; else printf ' ok %s (%dms)\n' "$CURRENT_CASE" "$dur" >&2; fi + elif [[ "$status" == fail ]]; then + if _use_color; then printf ' \033[0;31m✗ FAIL\033[0m %s (%dms): %s\n' "$CURRENT_CASE" "$dur" "${detail%; }" >&2; else printf ' FAIL %s (%dms): %s\n' "$CURRENT_CASE" "$dur" "${detail%; }" >&2; fi + elif [[ "$status" == known-failing ]]; then + if _use_color; then printf ' \033[1;33m○ known-failing\033[0m %s: %s\n' "$CURRENT_CASE" "${detail%; }" >&2; else printf ' known-failing %s: %s\n' "$CURRENT_CASE" "${detail%; }" >&2; fi + else + printf ' - skip %s: %s\n' "$CURRENT_CASE" "${detail%; }" >&2 + fi + + CURRENT_CASE="" + CASE_ERRORS=() +} + +suite_begin() { + SUITE_NAME="$1" + printf '\n== suite: %s ==\n' "$SUITE_NAME" >&2 +} + +suite_end() { + t_end # close any dangling case + printf '== end suite: %s ==\n' "${SUITE_NAME:-?}" >&2 +} diff --git a/test/cli/lib/fixtures.sh b/test/cli/lib/fixtures.sh new file mode 100644 index 00000000..d6c2fae5 --- /dev/null +++ b/test/cli/lib/fixtures.sh @@ -0,0 +1,302 @@ +#!/bin/bash +# fixtures.sh — idempotent shared fixtures + crash-safe sweep (SPEC-028). +# +# Locked API: +# fixtures_ensure create-or-get the shared fixtures (project, image, +# network+subnet); sweeps leftovers from a crashed prior +# run first. Populates FIX_PROJECT_ID, FIX_IMAGE_ID, +# FIX_NETWORK_ID, FIX_SUBNET_ID. +# fixtures_require the standalone-suite entry point: verify the shared +# fixtures exist and only fall back to the full +# sweep-then-create ensure when one is missing, so a +# suite running under bin/run.sh (which already ensured +# them) pays four cheap show calls, not a re-sweep. +# fixtures_teardown delete the resource kinds the sweep covers — +# everything whose NAME starts with the fixture prefix +# across servers, snapshots, backups, volumes, ports, +# subnets, routers, networks, security groups, +# keypairs, server groups, images, flavors, users, +# groups and projects — best-effort. NOT covered: +# nameless or differently-keyed kinds (floating IPs, +# roles, domains, volume types, subnet pools, +# aggregates, app-credentials) and volumes stuck in a +# transfer; the suites' cases pre-clean their own +# residue of those, so the gap only matters after a +# crash mid-case. +# +# Conventions every suite MUST follow: +# - resources created by a case are named cli-conf-- +# so the sweep can always reclaim them; +# - the sweep deletes by NAME PREFIX ONLY — never delete by bare ID. + +# Guard against double-sourcing. +[[ -n "${_CLI_FIXTURES_SOURCED:-}" ]] && return 0 +_CLI_FIXTURES_SOURCED=1 + +# shellcheck source=common.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh" + +FIX_PREFIX="cli-conf" +FIX_PROJECT="$FIX_PREFIX" +FIX_IMAGE="$FIX_PREFIX-image" +FIX_NETWORK="$FIX_PREFIX-net" +FIX_SUBNET="$FIX_PREFIX-subnet" + +# --------------------------------------------------------------------------- +# Sweep helpers +# --------------------------------------------------------------------------- + +# _ids_by_name_prefix — read list JSON from stdin, print the IDs of entries +# whose Name/name starts with $FIX_PREFIX, one per line. Keypairs are the one +# swept kind with no id at all (OSC lists Name/Fingerprint only), so the name +# is the emitted handle — `keypair delete` takes the name anyway. +_ids_by_name_prefix() { + jq -r --arg p "$FIX_PREFIX" \ + '.[] | select((.Name // .name // "") | startswith($p)) | (.ID // .id // .Name // .name)' 2>/dev/null +} + +# _sweep_list