From 383cc89633e3f22ac708200fb3bf0327ad745fce Mon Sep 17 00:00:00 2001 From: Uwe Schwaeke Date: Wed, 14 Oct 2026 19:27:27 +0000 Subject: [PATCH 1/8] chore(contrib): GitHub issue and pull-request templates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First-ever intake templates for the tracker (issue #16): - bug.yml: structured bug form — service dropdown with ports, endpoint + method, microversion hints (Nova 2.1-2.90, Cinder 3.0-3.71), curl repro, and a REQUIRED expected-behavior field asking for the upstream docs.openstack.org/gophercloud citation (SPEC-000 compliance is priority zero, so the reference for 'correct' is always upstream). - feature.yml: problem statement, required upstream-parity target (including an explicit out for deliberately lightstack-specific proposals), proposed API surface, client-impact checkboxes (CLI, Terraform, Horizon, gophercloud — which drives contract-test prioritization). - config.yml: blank issues off; Discussions + private security advisory links, SBOM findings pointed at docs/security/SBOM_TRIAGE.md. - PULL_REQUEST_TEMPLATE.md: what/why with issue link, spec/ADR reference section, checklist (contract tests for API surface, make build, CHANGELOG [Unreleased], docs, per-commit Signed-off-by matching the actual 'SAP Sign-off Check' CI job, no secrets), testing-evidence block. Signed-off-by: Uwe Schwaeke --- .github/ISSUE_TEMPLATE/bug.yml | 108 +++++++++++++++++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 11 +++ .github/ISSUE_TEMPLATE/feature.yml | 64 +++++++++++++++++ .github/PULL_REQUEST_TEMPLATE.md | 30 ++++++++ 4 files changed, 213 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml new file mode 100644 index 00000000..21786871 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -0,0 +1,108 @@ +name: Bug report +description: Report a deviation from upstream OpenStack API behavior or a lightstack defect +labels: ["bug", "triage"] +body: + - type: markdown + attributes: + value: > + Thanks for taking the time to file a bug. Per [SPEC-000](docs/specs/000-api-compliance), + OpenStack API compliance is priority zero: every bug report should be anchored in what + upstream OpenStack actually does. If you can cite the upstream API reference (or the + source service) for the expected behavior, triage gets much faster. + - type: textarea + id: summary + attributes: + label: Bug summary + description: One or two sentences. What is broken, and which OpenStack API does it deviate from? + placeholder: > + POST /v2.1/{tenant_id}/servers returns 500 instead of 202 when block_device_mapping_v2 + references a non-existent volume; upstream Nova returns a 400 with a structured fault. + validations: + required: true + - type: dropdown + id: service + attributes: + label: Affected service + options: + - Keystone (35357) + - Nova (8774) + - Neutron (9696) + - Cinder (8776) + - Glance (9292) + - Placement + - Metadata + - Cross-service + - Other + validations: + required: true + - type: input + id: endpoint + attributes: + label: API endpoint and HTTP method + description: The exact route you called, including the version prefix. + placeholder: "POST /v2.1/{tenant_id}/servers" + validations: + required: true + - type: input + id: microversion + attributes: + label: Microversion (if applicable) + description: Nova 2.1–2.90, Cinder 3.0–3.71. Leave empty if the API is not versioned. + placeholder: "X-OpenStack-Nova-API-Version: 2.90" + - type: textarea + id: repro-steps + attributes: + label: Reproduction steps + description: What you did, in order. Include the client you used (openstack CLI, Terraform provider, Horizon, gophercloud, raw curl). + placeholder: | + 1. Authenticate as a non-admin tenant user + 2. Create a server with ... + 3. Observe ... + validations: + required: true + - type: textarea + id: curl-repro + attributes: + label: curl reproduction (optional) + description: A self-contained request we can replay against a running lightstack. Redact tokens and secrets. + render: shell + placeholder: | + curl -si -X POST http://localhost:8774/v2.1/$TENANT/servers \ + -H "X-Auth-Token: $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{...}' + - type: textarea + id: expected + attributes: + label: Expected behavior (upstream OpenStack reference) + description: > + What upstream OpenStack does here — status code, response body, error semantics. Cite the + docs.openstack.org API reference, the gophercloud contract, or the upstream service source. + Compliance with upstream is priority zero (SPEC-000), so this is the anchor for the fix. + placeholder: > + https://docs.openstack.org/api-ref/compute/... — upstream returns 202 with the server + representation and sets the OS-EXT-STS:task_state to 'scheduling'; a missing image ref yields + 400 itemNotFound fault. + validations: + required: true + - type: textarea + id: actual + attributes: + label: Actual behavior + description: Status code, response body, and/or panic log you got from lightstack. + validations: + required: true + - type: textarea + id: logs + attributes: + label: Logs (optional) + description: Relevant lightstack log output or panic stack trace. Wrap long output in a code block; redact tokens. + render: text + - type: input + id: environment + attributes: + label: Environment + description: > + lightstack version/release you deployed (see `CHANGELOG.md`), deployment mode (stub / real), storage mode, + networking mode — see docs/STORAGE_MODES.md and docs/NETWORKING_MODES.md. + placeholder: "lightstack v0.x.y, real mode, storage=ceph-rgw, networking=vxlan" diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..ef60bb92 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,11 @@ +blank_issues_enabled: false +contact_links: + - name: Discussions + url: https://github.com/cobaltcore-dev/lightstack/discussions + about: Questions, design ideas, and deployment help — start here before opening an issue. + - name: Security vulnerabilities + url: https://github.com/cobaltcore-dev/lightstack/security/advisories/new + about: > + Report vulnerabilities privately via a GitHub security advisory — do NOT open a public issue. + For triage of SBOM findings (dependency advisories), see docs/security/SBOM_TRIAGE.md; + known/not-applicable findings are tracked in docs/security/openvex.json. diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml new file mode 100644 index 00000000..407c8099 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -0,0 +1,64 @@ +name: Feature request +description: Propose new API surface or behavior — ideally to close a gap with upstream OpenStack +labels: ["enhancement", "triage"] +body: + - type: markdown + attributes: + value: > + Before proposing something new, check the [roadmap](ROADMAP.md) and + [STATUS.md](STATUS.md) — much of the plan is driven by upstream parity. If this request + closes an API gap with upstream OpenStack, say so up front; per [SPEC-000](docs/specs/000-api-compliance) + compliance is priority zero and parity work is prioritized accordingly. + - type: textarea + id: problem + attributes: + label: Problem statement + description: What can't you do today, and who is blocked by it? If a real client (openstack CLI, Terraform provider, Horizon, gophercloud) fails against lightstack, name it. + placeholder: > + The Terraform openstack provider calls GET /v2.1/{tenant_id}/servers/detail?changes-since=... + during refresh; lightstack ignores the changes-since filter so the provider sees stale state. + validations: + required: true + - type: textarea + id: upstream-parity + attributes: + label: Upstream OpenStack parity target + description: > + Is there an upstream API this should match? Cite the docs.openstack.org API reference + (or upstream service source). SPEC-000 makes upstream compliance priority zero, so + "upstream does X, we don't" is the strongest possible justification here. If this is + deliberately lightstack-specific, say so and why. + placeholder: > + https://docs.openstack.org/api-ref/compute/... — changes-since is a standard list filter + since microversion 2.1; upstream returns only servers updated after the timestamp. + validations: + required: true + - type: textarea + id: api-surface + attributes: + label: Proposed API surface (optional) + description: Endpoint(s), request/response shapes, microversion gating. Match upstream naming exactly — clients depend on it. + render: text + placeholder: | + GET /v2.1/{tenant_id}/servers/detail?changes-since= + → 200 {"servers": [...]} filtered server-side; 400 on malformed timestamp (upstream behavior). + - type: checkboxes + id: client-impact + attributes: + label: Client impact + description: Which OpenStack clients would exercise this? Helps us prioritize contract tests in test/contract/. + options: + - label: OpenStack CLI (python-openstackclient) + - label: Terraform provider (terraform-provider-openstack) + - label: Horizon dashboard + - label: gophercloud SDKs + - type: textarea + id: alternatives + attributes: + label: Alternatives considered (optional) + description: Other ways to solve this, including "wait for upstream parity work" — and why they fall short. + - type: textarea + id: context + attributes: + label: Additional context (optional) + description: Links, sketches, related specs in docs/specs/ or ADRs in docs/adr/. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..137f585f --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,30 @@ +## What & why + + +Fixes # + + + +## Spec / ADR references + + +- [ ] Spec/ADR referenced (e.g. `docs/specs/000-api-compliance`) or "none — bugfix/docs only" + +## Checklist + +- [ ] Tests added/updated — unit tests for logic, contract tests in `test/contract/` (gophercloud-based) for any API surface change +- [ ] `make build` passes +- [ ] Changelog entry added to `CHANGELOG.md` under `[Unreleased]` (user-visible changes only) +- [ ] Docs updated under `docs/` (see `docs/CONTRIBUTING.md`) where behavior or configuration changed +- [ ] Every commit carries `Signed-off-by: Name ` — DCO; the "SAP Sign-off Check" CI job enforces this. Use `git commit -s`. +- [ ] No secrets, tokens, or credentials in the diff + +## Testing evidence + + +``` +$ make build +... +$ go test ./test/contract/... -run TestNovaServers +... +``` From 56dd97ec791b0a3736c22fd27ca76704394edca5 Mon Sep 17 00:00:00 2001 From: Uwe Schwaeke Date: Wed, 14 Oct 2026 19:27:27 +0000 Subject: [PATCH 2/8] =?UTF-8?q?docs(contrib):=20contributor=20contract=20?= =?UTF-8?q?=E2=80=94=20DCO,=20protected=20branches,=20dev=20loop,=20review?= =?UTF-8?q?=20process?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The contributor definition issue #16 asks for (all facts verified against the repo): - Sign-off Requirement (DCO): the dco-check job ('SAP Sign-off Check') in ci.yml inspects EVERY commit in a PR for 'Signed-off-by: Name ' and the aggregate CI Status job fails the PR without it. Documents git commit -s / --amend -s / rebase --signoff and links the DCO. This was previously enforced but documented nowhere — a first-time contributor's PR failed CI with no pointer to why. - Protected Branches: main, stable/**, releases/** — branch + PR only. - Review Process: the multi-round comprehensive review culture (severity- rated findings, dedicated fix commits, loop until clean, reports in docs/review/), with a real fix commit as the example. - Testing sections rewritten to the real loop: LIGHTSTACK_TEST_DB_URL (harness migrates, does not create the DB), contract-test workflow with the 5 known docker-exec environmental failures, chart-test, make lint. - Stale content fixed: Go 1.27.2 (go.mod), make migrate (not the nonexistent migrate-up), manual docker run postgres replaced by make db-up, dead docs/MVP_V1_COMPLETE.md reference dropped, shipped items (Floating IPs, L3 forwarding) pruned from Areas Needing Help, Placement item re-anchored, lowercase 'default' domain in the verify snippet corrected to 'Default' (Keystone resolves domain names case-sensitively). Signed-off-by: Uwe Schwaeke --- docs/CONTRIBUTING.md | 305 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 258 insertions(+), 47 deletions(-) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index a6813ea9..a5fe3570 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -7,13 +7,16 @@ Thank you for considering contributing to lightstack! This document provides gui ## Table of Contents - [Code of Conduct](#code-of-conduct) +- [Sign-off Requirement (DCO)](#sign-off-requirement-dco) - [Getting Started](#getting-started) - [Development Setup](#development-setup) - [Making Changes](#making-changes) - [Testing](#testing) - [Submitting Changes](#submitting-changes) +- [Review Process](#review-process) - [Code Style](#code-style) - [Documentation](#documentation) +- [Areas Needing Help](#areas-needing-help) --- @@ -27,13 +30,43 @@ This project follows a standard code of conduct: --- +## Sign-off Requirement (DCO) + +Every commit in a pull request must carry a `Signed-off-by` trailer with an `@sap.com` address: + +``` +Signed-off-by: Name +``` + +This is enforced by CI. The `dco-check` job (named **SAP Sign-off Check** in [`.github/workflows/ci.yml`](../.github/workflows/ci.yml)) inspects **every commit** in the PR — not just the head — and rejects any commit lacking a valid trailer. The aggregate **CI Status** job then fails the PR. The `@sap.com` address is a project requirement (corporate sign-off). + +The easiest way to comply: + +```bash +# Sign every commit as you create it +git commit -s + +# Fix up an existing commit that is missing the trailer +git commit --amend -s + +# Re-sign every commit in a PR at once (then force-push your branch) +git rebase --signoff +``` + +Your git identity must be configured to the sap.com address (`git config user.email`). Sign-off certifies that you wrote the code and have the right to submit it under the project's license — the [Developer Certificate of Origin (DCO)](https://developercertificate.org/), the same certification used by the probot DCO app. In short: by signing off, you state that the contribution is yours (or properly attributed) and may be contributed to this project as open source. + +> **Note**: force-pushing your *own* feature branch to add missing sign-offs is fine. Never force-push `main`, `stable/**`, or `releases/**` — see [Protected Branches](#protected-branches). + +--- + ## Getting Started ### Prerequisites **Required:** -- Go 1.26 or higher +- Go 1.27.2 or higher (the version pinned in [`go.mod`](../go.mod)) - PostgreSQL 18 or higher +- Docker (for `make db-up` and the container-based test stacks) - Git **Optional (for real mode testing):** @@ -54,25 +87,26 @@ git remote add upstream https://github.com/cobaltcore-dev/lightstack.git ## Development Setup -### 1. Install Dependencies +### 1. Install Tools ```bash # Install Go dependencies make install-deps -# Install development tools +# Install development tools: golangci-lint (pinned, built into .tools/bin), +# air (hot reload), the golang-migrate CLI, and Helm + the helm-unittest +# plugin needed by 'make chart-test' make install-tools ``` ### 2. Start PostgreSQL ```bash -# Using Docker -docker run -d --name lightstack-postgres \ - -e POSTGRES_DB=lightstack \ - -e POSTGRES_USER=lightstack \ - -e POSTGRES_PASSWORD=secret \ - -p 5432:5432 postgres:18.3 +# Dockerized PostgreSQL 18 (container name: lightstack-postgres) +make db-up + +# Stop it again with +make db-down # Or use your local PostgreSQL instance ``` @@ -80,7 +114,7 @@ docker run -d --name lightstack-postgres \ ### 3. Run Migrations ```bash -make migrate-up +make migrate ``` ### 4. Build and Run @@ -89,13 +123,17 @@ make migrate-up # Build make build -# Run -./bin/lightstack --config config/lightstack.yaml +# Run (builds if needed, starts with config/lightstack.yaml) +make run -# Or run with hot reload +# Or run with hot reload (requires air) make dev ``` +`make run` applies any pending database migrations at startup (the binary +runs migrations on boot before serving), so `make migrate` is only needed +when you want to apply them ahead of time or against another database. + ### 5. Verify Installation ```bash @@ -104,8 +142,8 @@ export OS_AUTH_URL=http://localhost:35357/v3 export OS_USERNAME=admin export OS_PASSWORD=secret export OS_PROJECT_NAME=default -export OS_USER_DOMAIN_NAME=default -export OS_PROJECT_DOMAIN_NAME=default +export OS_USER_DOMAIN_NAME=Default +export OS_PROJECT_DOMAIN_NAME=Default openstack token issue ``` @@ -122,6 +160,13 @@ Use descriptive branch names: - `refactor/storage-backend` - Code refactoring - `docs/api-reference` - Documentation updates +### Protected Branches + +The `main`, `stable/**`, and `releases/**` branches are push-protected: + +- **Always** work in a feature branch and open a pull request — never commit directly to these branches. +- **Never** force-push to them. Force-pushing your own feature branch (e.g. to add a missing sign-off) is fine. + ### Commit Messages Follow conventional commits format: @@ -153,6 +198,10 @@ refactor(cinder): simplify volume attachment logic docs(readme): update installation instructions ``` +Every commit must also carry a `Signed-off-by: Name ` trailer — +use `git commit -s` and it is added automatically. CI rejects PRs containing +any commit without it; see [Sign-off Requirement (DCO)](#sign-off-requirement-dco). + ### Code Organization ``` @@ -184,7 +233,7 @@ lightstack/ ### Unit Tests ```bash -# Run all unit tests +# Run all unit tests (excludes test/contract — use the contract-test flow below for those) make test # Run tests for specific package @@ -194,10 +243,86 @@ go test ./internal/keystone/... go test -cover ./... ``` +**DB-backed unit tests.** Some unit tests (e.g. keystone admin bootstrap, nova +handler tests) run against a real PostgreSQL and **skip** when no test database +is configured. To run them, point `LIGHTSTACK_TEST_DB_URL` at a scratch +database — CI does exactly this: + +```bash +# One-time: create the scratch database (the test harness connects and +# migrates, but does not create the database itself) +docker exec lightstack-postgres psql -U lightstack -c 'CREATE DATABASE lightstack_test' + +LIGHTSTACK_TEST_DB_URL="postgres://lightstack:secret@localhost:5432/lightstack_test?sslmode=disable" \ + go test $(go list ./... | grep -v '/test/contract') +``` + +### Contract Tests + +Contract tests (`test/contract/`) verify OpenStack API compatibility through +the real gophercloud client libraries. They need a running Lightstack: + +```bash +# 1. Build and start the server (make db-up + make run also work) +make build +./bin/lightstack --config config/lightstack.yaml & + +# 2. Point the OpenStack environment at it +export OS_AUTH_URL=http://localhost:35357/v3 +export OS_USERNAME=admin +export OS_PASSWORD=secret +export OS_PROJECT_NAME=default +export OS_USER_DOMAIN_NAME=Default +export OS_PROJECT_DOMAIN_NAME=Default + +# 3. Run the suite (bypass the test cache so a stale PASS never survives an environment change) +go clean -testcache && go test ./test/contract/... -timeout 10m +``` + +`make test-contract` wraps the same suite (with a 5-minute timeout) and checks +the server is up first. + +**Known environmental failures — the baseline.** Five contract tests seed +their fixtures through `docker exec lightstack-postgres psql`: + +- `TestNeutronGetAgent_Contract` +- `TestNeutronAddL3AgentToRouter_Contract` +- `TestTrunkBasicCRUD` +- `TestTrunkSubports` +- `TestCinderUnmanageVolume_Contract` + +They need a running container named `lightstack-postgres` — which the compose +stack and `make db-up` both provide. On a bare host (PostgreSQL running +natively, no such container) these five fail identically on every run. That is +the known environmental baseline, not a product regression: compare your run +against it before investigating a failure. + +See [`test/contract/README.md`](../test/contract/README.md) for how to write +contract tests, and [`docs/testing/CONTRACT_TEST_RESULTS.md`](testing/CONTRACT_TEST_RESULTS.md) +for current results. + +### Helm Chart Tests + +```bash +# Lint + render + unit-test both charts (charts/lightstack, charts/horizon). +# No cluster and no running Lightstack needed. +make chart-test +``` + +Requires the Helm CLI and the helm-unittest plugin — both are installed by +`make install-tools`. This is the same check CI's "Helm Charts" job runs. + +### Linting + +```bash +# golangci-lint (pinned in the isolated tools module, built into .tools/bin) +make lint +``` + ### Integration Tests ```bash -# Run full integration test suite +# Run full integration test suite (requires a running Lightstack) ./test/quick_test.sh # Run Horizon compatibility tests @@ -245,29 +370,38 @@ openstack image delete test-image 2. **Run tests:** ```bash - make test - ./test/quick_test.sh + make test # unit tests + make lint # golangci-lint + make chart-test # Helm charts (if touched) + # contract tests against a running server — see Testing above + go clean -testcache && go test ./test/contract/... -timeout 10m ``` 3. **Format code:** ```bash make fmt - make lint ``` -4. **Push changes:** +4. **Check sign-off:** every commit must carry `Signed-off-by: Name ` + (`git commit -s`). Fix missing trailers with `git commit --amend -s` or + `git rebase --signoff`, then force-push your branch. + +5. **Push changes:** ```bash git push origin feature/your-feature-name ``` -5. **Create Pull Request:** +6. **Create Pull Request:** - Go to GitHub and create a PR from your fork - Fill out the PR template - Link any related issues ### Pull Request Checklist -- [ ] Tests pass (`make test` and `./test/quick_test.sh`) +- [ ] Every commit is signed off (`Signed-off-by: Name `) +- [ ] Unit tests pass (`make test`, DB-backed tests included via `LIGHTSTACK_TEST_DB_URL`) +- [ ] Contract tests pass against the known baseline (see [Testing](#testing)) +- [ ] Helm chart tests pass (`make chart-test`) if charts were touched - [ ] Code is formatted (`make fmt`) - [ ] Linter passes (`make lint`) - [ ] Documentation is updated @@ -277,6 +411,78 @@ openstack image delete test-image --- +## Review Process + +Expect **multiple review rounds** — this is the norm, not a red flag. Reviews +here are comprehensive: a round walks the whole diff, reports every finding +with a severity and a rationale, and the PR is not merged until a round comes +back with a clean verdict (no findings). Finding and fixing real defects over +two or three rounds is the process working as intended. + +How a round-trip looks: + +1. **Review round** — findings are reported, each with a severity + (critical / high / medium / low) and the reasoning behind it. +2. **Fix commits** — each round's findings are addressed in dedicated + commits (not squashed away), so the history shows what was found and how + it was resolved. +3. **Re-run** — the review loop re-runs on the updated branch and either + reports the next round's findings or returns a clean verdict. +4. **Archive** — the final review reports are kept under + [`docs/review/`](review/) for reference (see the existing reports there + for the format). + +### What a good fix commit looks like + +A fix commit references the finding it addresses, explains the defect (not +just the change), and says how the fix is pinned by a test. This is a real +fix commit from the round-2 loop of an open PR (`fix/authz-and-correctness`, commit +`0e14680` — not yet in `main` when this was written; lightly trimmed): + +``` +fix: round-2 review findings — os-interface port_id path, upload-cap 413 + +Round 2 of the review loop (0 High, 1 Medium, 4 Low — all fixed): + +MEDIUM (F1) — the port_id path of os-interface scanned the ports row's + nullable device_id into a plain string. Every free port has NULL + there, so the scan FAILED on every successful attach; the error was + dropped and the attach worked only because pgx fills earlier columns + before the failed one — and any real database error on that lookup + surfaced as a bogus 200 with an empty net_id and MAC. The lookup now + reads COALESCE(device_id::text, '') and handles its errors like the + instance lookup above it. Pinned by + TestAttachInterfacePortIDReportsRealState (attach a device_id-NULL + port, assert the response carries the port's real net_id, MAC and + fixed_ips). + +LOW (F4) — uploadImageLocal applied the 5-second metadata-op timeout + to the body copy, so an over-cap upload hit the deadline (503) long + before the 10 GiB MaxBytesReader cap could answer 413. The copy now + runs under the caller's context only (cancelled on client + disconnect), matching the S3 path's connect-only timeout policy + (FR-014). maxImageUploadBytes is a var now, and + TestOverCapUploadAnswers413 pins the 413 end-to-end (image left + queued, not stuck saving). + +Unit suite (incl. DB-backed): PASS. Contract suite: unchanged (only +the 5 docker-exec-dependent tests fail, same as baseline). + +Signed-off-by: Name +``` + +Points that matter: + +- The subject follows conventional commits and names the round. +- Each finding is referenced by its ID and severity, and its defect is + explained — the *why*, not just the *what*. +- The commit states which test pins each fix, so the finding cannot silently + regress. +- The closing lines report the verification runs (and note any known + environmental baseline failures rather than hiding them). + +--- + ## Code Style ### Go Style Guidelines @@ -392,7 +598,8 @@ When adding features, update: - `README.md` - If it affects quick start - `docs/STORAGE_MODES.md` - If it adds storage options - `docs/REAL_LIBVIRT_MODE.md` - If it affects VM lifecycle -- `docs/MVP_V1_COMPLETE.md` - If it completes a major feature +- `CHANGELOG.md` - For every notable change (keep-a-changelog format) +- `docs/WHATS_LEFT.md` - If it opens or closes a known gap ### Example Documentation @@ -426,51 +633,55 @@ func (svc *Service) CreateVolume(ctx context.Context, size int, name string) (st ## Areas Needing Help +Cross-check anything here against [`docs/WHATS_LEFT.md`](WHATS_LEFT.md) and +[`CHANGELOG.md`](../CHANGELOG.md) before starting — this list only carries +items that are still open. (Floating IPs and router L3 forwarding used to be +listed here; both have shipped — see `internal/neutron/floatingip.go` and +[`docs/L3_ROUTER_IMPLEMENTATION.md`](L3_ROUTER_IMPLEMENTATION.md).) + ### High Priority 1. **Multi-node Networking** - - VXLAN overlay networks - - Cross-node VM communication + - Cross-node VM communication (VXLAN overlay) + - Multi-node real mode is the open Part B of the single-node real-mode + work — see the SPEC-009 note in + [`docs/WHATS_LEFT.md`](WHATS_LEFT.md) and + [`docs/VXLAN_IMPLEMENTATION.md`](VXLAN_IMPLEMENTATION.md) - Distributed DHCP -2. **Floating IPs** - - External network access - - NAT configuration - - IP allocation/deallocation - -3. **Router L3 Forwarding** - - Inter-network routing - - Static routes - - Default gateway configuration - ### Medium Priority -4. **eBPF Security Groups** - - Replace iptables with eBPF - - Kernel-space filtering - - Performance improvement +2. **eBPF Security Groups** + - East-west XDP filtering: the eBPF object does not load in a stock + build, so eBPF mode currently adds no east-west filtering — see + [`docs/EBPF_STATUS.md`](EBPF_STATUS.md) for the exact state and what + remains + - Kernel-space filtering, performance improvement over iptables -5. **Live Migration** +3. **Live Migration** - VM migration between hosts - Storage migration - Downtime minimization -6. **High Availability** +4. **High Availability** - Multi-node control plane - Leader election - Failover handling ### Low Priority -7. **Heat Orchestration** +5. **Heat Orchestration** - Template parsing - Stack creation - Resource dependencies -8. **Placement API** - - Resource scheduling - - Affinity/anti-affinity - - Resource providers +6. **Placement API** + - Currently a stub — resource scheduling, affinity/anti-affinity, + resource providers (see [`STATUS.md`](../STATUS.md) — "Placement API" future work; current + shape in [`internal/placement/placement.go`](../internal/placement/placement.go)) + +The full roadmap, including items deliberately *not* being built, lives in +[`ROADMAP.md`](../ROADMAP.md) and [`docs/WHATS_LEFT.md`](WHATS_LEFT.md). --- From 6e41b998b16b006c27680d7aa8f00c2e71d1d3d4 Mon Sep 17 00:00:00 2001 From: Uwe Schwaeke Date: Wed, 14 Oct 2026 19:27:27 +0000 Subject: [PATCH 3/8] docs(agents): repair CLAUDE.md, add tool-agnostic AGENTS.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AI-agent alignment, the other half of issue #16: CLAUDE.md referenced an 'Ultimate Project System' whose files do not exist in the repository — .beastmode/BEASTMODE.md (L0 autoload), .beastmode/context/{PHASE}.md, memory/constitution.md — so an agent following it literally hit dead paths. The rewrite keeps every piece of verified repo knowledge (architecture, config keys, tenancy caveats, conventions; the Nine Articles now stated inline) and drops the dead framing (slash-commands, swarm mode, persona). Stale facts corrected: seven listeners (Placement :8778), 70+ tables, pkg/hypervisor/ instead of the nonexistent xml_template.go, rate_limiting marked dead config. DB-backed test guidance now matches the actual harness (connects and migrates an existing scratch database; example points at lightstack_test, not the shared 'postgres' database). AGENTS.md (new, root): the tool-agnostic entry point — repo map, ground- truth commands, hard rules (DCO sign-off with sap.com address, protected branches, contract tests for API changes, tenant isolation security- critical per SPEC-022/023), working agreements (spec pattern, review-fix loops, changelog entries). Points to CLAUDE.md for Claude-specific depth and docs/CONTRIBUTING.md for the contribution process. Signed-off-by: Uwe Schwaeke --- AGENTS.md | 102 +++++++++++++++++++++ CLAUDE.md | 268 ++++++++++++++++++++++++------------------------------ 2 files changed, 220 insertions(+), 150 deletions(-) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..9697fa40 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,102 @@ +# AGENTS.md + +Guidance for any coding agent (Claude Code, Codex, Cursor, …) working in this repository. +For deeper Claude Code-specific guidance see `CLAUDE.md`; contribution process: `docs/CONTRIBUTING.md`. + +## What this is + +Lightstack is a single Go binary that reimplements the five core OpenStack services +(Keystone, Nova, Neutron, Cinder, Glance) plus a Placement stub and an EC2-compatible +metadata service — inspired by how K3s simplified Kubernetes. One process, one PostgreSQL +database, zero message queues, synchronous APIs. + +**SPEC-000: OpenStack API compliance is priority zero.** Upstream OpenStack behavior is the +reference implementation — never invent semantics. If Lightstack and upstream disagree, +Lightstack is wrong. All other specifications must comply with SPEC-000 +(`docs/specs/SPEC-000-COMPLIANCE-ADDENDUM.md`). + +One process, seven HTTP listeners: Keystone `:35357`, Nova `:8774`, Neutron `:9696`, +Cinder `:8776`, Glance `:9292`, Placement `:8778`, Metadata `:8775` (unauthenticated). + +Each service supports **stub mode** (fake data, no external dependencies — default, works on +macOS) and **real mode** (libvirt, netlink, Ceph RBD, S3 — Linux only), selected per service in +`config/lightstack.yaml` (`nova.libvirt_mode`, `neutron.networking_mode`, +`cinder.storage_mode`, `glance.storage_mode`). Storage modes may be comma-separated hybrids +(`local,rbd`). Note: the `rate_limiting.*` config block is **dead config** — no code reads it. + +## Repo map + +| Path | Contents | +|------|----------| +| `cmd/lightstack/` | Main binary: server plus subcommands `cleanup` (SPEC-010 teardown), `console-tokens` (SPEC-021), `healthcheck` (SPEC-013) | +| `cmd/lightstack-migrate/` | Standalone migration binary | +| `internal//` | One package per OpenStack service: `keystone`, `nova`, `neutron`, `cinder`, `glance`, plus `placement`, `metadata` | +| `internal/common/` | Config loading, error helpers/pagination, `BaseURL`, endpoint `scheme`, SQL builder, roles | +| `internal/database/` | pgx pool usage, models, `WithTx` transactions, migration wiring (golang-migrate) | +| `pkg/paths/` | `state_path` resolution (node-local state root, default `/var/lib/lightstack`) | +| `pkg/hypervisor`, `pkg/networking`, `pkg/storage` | libvirt, netlink/VXLAN, and storage backends (local/RBD/S3) | +| `test/contract/` | gophercloud contract tests — the OpenStack API compliance gate | +| `test/*.sh`, `test/terraform/` | Bash integration suites (OpenStack CLI) and Terraform conformance | +| `docs/specs/NNN-*/` | Feature specs: `spec.md`, `plan.md`, `tasks.md`, `contracts/`, `checklists/` | +| `docs/adr/` | Architecture decision records | +| `docs/review/` | Comprehensive review reports (fix loops run until CLEAN) | +| `charts/lightstack/`, `charts/horizon/` | Helm charts | +| `deployments/` | Docker compose stacks, horizon config, systemd units | +| `migrations/` | Numbered `NNN_name.up.sql` / `.down.sql` migrations | + +## Ground truth commands + +```bash +make build # Build bin/lightstack +make lint # golangci-lint (pinned via tools/ module) +make test # Unit tests (excludes test/contract) +make test-contract # gophercloud contract tests — needs a RUNNING server + # (docker compose -f deployments/docker-compose.yml up -d, or make run) +make chart-test # Helm lint + template + unittest (needs helm + helm-unittest; make install-tools) +``` + +DB-backed unit tests need an existing scratch database they may **migrate**: the +harness connects to the database named in `LIGHTSTACK_TEST_DB_URL` and applies all +migrations — it does **not** create the database (see `internal/common/maindb_test.go`), +so never point it at anything shared: + +```bash +make db-up # start dev PostgreSQL in Docker +LIGHTSTACK_TEST_DB_URL=postgres://lightstack:secret@localhost:5432/postgres make test +make migrate # apply migrations (golang-migrate) +``` + +Without `LIGHTSTACK_TEST_DB_URL`, DB-backed unit tests skip rather than fail. + +## Hard rules + +- **DCO sign-off on every commit**: `git commit -s`, `Signed-off-by: Name `. + The "SAP Sign-off Check" CI job rejects any PR commit missing a valid `@sap.com` trailer. +- **Conventional commits**: `feat(nova): …`, `fix(cinder): …`, `refactor(glance): …`, + `docs`, `test`, `chore`. +- **Branches**: `main`, `stable*`, and `releases*` are protected — work on PR branches only. +- **API changes need contract tests**: any endpoint added or behavior changed gets a test in + `test/contract/` (gophercloud). This is the SPEC-000 gate CI runs. +- **Tenant isolation and admin-only surfaces are security-critical** — see + `docs/specs/022-project-tenant-isolation/` and `docs/specs/023-tenant-scoping-authz/`. + Resources are scoped by `project_id`; the `admin` role override and `?all_tenants` follow + upstream `admin_or_owner` semantics. Do not loosen scoping predicates. +- **Never commit secrets** (JWT secrets, passwords, keys). The seeded `admin`/`secret` + credentials are development-only defaults. + +## Working agreements + +- **Significant features follow the spec pattern** (`docs/specs/README.md`): a numbered + `docs/specs/NNN-name/` directory with `spec.md` → `plan.md` → `tasks.md` → implementation → + validation. Read the spec covering the code you touch before changing it. +- **Expect review-fix loops**: comprehensive reviews (`docs/review/`) produce findings that are + fixed and re-reviewed until the report is CLEAN. Budget for iteration. +- **Changelog entries** for user-visible changes (`CHANGELOG.md`). +- **Never edit an applied migration's meaning** — add a new numbered migration instead. +- Prefer the shared helpers over re-implementations: `common.BaseURL`, `common.HasRole`, + `nova.targetInstance`, `database.WithTx`. The recurring review finding is "the right shared + thing exists and one caller bypasses it." +- **Test-first**: write the failing test (unit, contract, or integration script) before the + implementation; confirm it fails, then make it pass. +- Errors: wrap with `fmt.Errorf("...: %w", err)` and surface HTTP errors via the helpers in + `internal/common/error_helpers.go` — never leak raw error strings to API clients. diff --git a/CLAUDE.md b/CLAUDE.md index 95724641..dc0971d1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,65 +2,47 @@ This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. -## Ultimate Project System +## Project Overview -**IMPORTANT**: This project uses the Ultimate Project System (Beastmode × Spec-Kit synthesis). +Lightstack is a lightweight OpenStack implementation in Go, inspired by K3s. It provides a single ~35MB binary that implements the five core OpenStack services (Keystone, Nova, Neutron, Cinder, Glance), a Placement stub, and an EC2-compatible metadata service, with 100% Terraform compatibility. -### Workflow & Knowledge Hierarchy -- **L0 (Autoload)**: Read `.beastmode/BEASTMODE.md` at the start of every session -- **L1 (Phase Context)**: `.beastmode/context/{PHASE}.md` - Load at phase start -- **L2 (Domain Details)**: `.beastmode/context/{phase}/{domain}.md` - On-demand -- **L3 (Specs & Artifacts)**: `docs/specs/NNN-*/` - Feature specifications +**Core Philosophy:** +- **Terraform Compatibility First**: Users can use existing Terraform OpenStack provider scripts unchanged +- **UI/CLI Compatibility**: OpenStack CLI works identically; Horizon's dashboard is **usable for resource management**, with its open UI gaps listed in `docs/compatibility/HORIZON_COMPATIBILITY_REPORT.md` +- **Drop-in Replacement**: Zero modifications needed to migrate from OpenStack to Lightstack +- **Synchronous Operations**: No message queues - operations complete before API returns +- **Fail-Fast Design**: External dependency failures return immediately (< 1 second timeouts) +- **Multi-mode Support**: Each service supports stub mode (development) and real mode (production) -### Development Workflow -``` -specify → plan → tasks → implement → validate → release -``` -Each phase: **prime → execute → validate → checkpoint → retro** +## Project Principles + +These principles govern how features are designed and reviewed. They are referenced throughout the specs (`docs/specs/README.md`) and ADRs (`docs/adr/`): -### The Nine Articles (Constitution) -Immutable principles in `memory/constitution.md`: 1. **Library-First** - Features begin as standalone libraries 2. **CLI Interface** - All libraries expose CLI functionality -3. **Test-First** - TDD mandatory (NON-NEGOTIABLE: tests → fail → implement) +3. **Test-First** - TDD mandatory: write tests, confirm they fail, then implement 4. **Integration Testing** - Real dependencies over mocks 5. **Observability** - Everything inspectable 6. **Versioning** - Semantic versioning -7. **Simplicity** - ≤3 projects, YAGNI principles +7. **Simplicity** - YAGNI principles 8. **Anti-Abstraction** - Use frameworks directly 9. **Integration-First** - Contract tests before implementation -### Available Commands -- `/constitution` - Establish project principles -- `/specify [desc]` - Create feature specification -- `/plan [stack]` - Create implementation plan -- `/tasks` - Generate task list -- `/implement [--parallel]` - Execute with parallel agents (swarm mode) -- `/validate` - Run validation gates -- `/release` - Release feature - -### Swarm Mode -Parallel execution enabled for: -- Research: Multiple topics simultaneously -- Implementation: Parallel-safe waves (file isolation checked) -- Retro: Context + Meta walkers together - -### Persona -When working on this project, adopt the **deadpan minimalist** persona: -- Short sentences, maximum understatement -- Competent, slightly annoyed at the work (never at the user) +**SPEC-000 is priority zero**: OpenStack API compliance. Upstream OpenStack behavior is the reference implementation — never invent semantics. Every specification must comply with SPEC-000 (see `docs/specs/SPEC-000-COMPLIANCE-ADDENDUM.md`). -## Project Overview +## Development Workflow -Lightstack is a lightweight OpenStack implementation in Go, inspired by K3s. It provides a single ~35MB binary that implements all five core OpenStack services (Keystone, Nova, Neutron, Cinder, Glance) with 100% Terraform compatibility. +Significant changes follow the spec pattern (see `docs/specs/README.md`): -**Core Philosophy:** -- **Terraform Compatibility First**: Users can use existing Terraform OpenStack provider scripts unchanged -- **UI/CLI Compatibility**: OpenStack CLI works identically; Horizon's dashboard is **usable for resource management**, with its open UI gaps listed in `docs/compatibility/HORIZON_COMPATIBILITY_REPORT.md` -- **Drop-in Replacement**: Zero modifications needed to migrate from OpenStack to Lightstack -- **Synchronous Operations**: No message queues - operations complete before API returns -- **Fail-Fast Design**: External dependency failures return immediately (< 1 second timeouts) -- **Multi-mode Support**: Each service supports stub mode (development) and real mode (production) +``` +specify → plan → tasks → implement → validate +``` + +- Each feature lives in `docs/specs/NNN-name/` containing `spec.md`, `plan.md`, `tasks.md`, and usually `contracts/` and `checklists/` subdirectories. Read the relevant spec before changing a feature it covers. +- **API-surface changes need contract tests** in `test/contract/` (gophercloud-based). If you add or change an endpoint's behavior, add or extend a contract test there. +- **Every commit needs a DCO sign-off**: `Signed-off-by: Name ` (use `git commit -s`). The "SAP Sign-off Check" CI job rejects PR commits missing a valid `@sap.com` trailer. +- **Comprehensive review-fix loops are the norm**. Reviews are documented under `docs/review/`; findings are worked until the report is CLEAN. Expect to iterate. +- Commit messages follow the conventional commits format (see below). ## Build and Development Commands @@ -70,8 +52,8 @@ Lightstack is a lightweight OpenStack implementation in Go, inspired by K3s. It # Start all services (PostgreSQL + lightstack) docker compose -f deployments/docker-compose.yml up -d -# Configure environment -source ~/.lightstack-env # Sets OS_AUTH_URL, OS_USERNAME, etc. +# Configure environment (OS_AUTH_URL, OS_USERNAME, etc.) +source ~/.lightstack-env # Test it works openstack token issue @@ -96,30 +78,40 @@ make clean ### Testing ```bash -# Unit tests -make test # Runs all Go unit tests -go test ./internal/nova/... # Test specific package - -# Integration tests -./scripts/test-all.sh # Comprehensive test suite -./scripts/test-keystone.sh # Keystone-specific tests -./test/quick_test.sh # Fast integration test suite -./test/integration_test.sh # Full integration test suite - -# Specific feature tests -./test/horizon_compat_test.sh # Horizon dashboard compatibility -./test/volume_attach_test.sh # Volume attachment workflow -./test/vxlan_multinode_test.sh # Multi-node VXLAN networking +# Unit tests (excludes test/contract — use 'make test-contract' for those) +make test + +# Test a specific package +go test ./internal/nova/... + +# DB-backed unit tests: set LIGHTSTACK_TEST_DB_URL to an existing scratch +# database the test may MIGRATE (the harness connects and migrates; it does +# not create the database — see internal/common/maindb_test.go; e.g. from +# 'make db-up', then create lightstack_test once). When unset, DB-backed +# tests skip instead of failing. +LIGHTSTACK_TEST_DB_URL=postgres://lightstack:secret@localhost:5432/lightstack_test make test + +# Contract tests (gophercloud; requires a running Lightstack — the target +# refuses to start otherwise) +make test-contract + +# Integration tests (bash suites driving the OpenStack CLI; requires a +# running Lightstack) +make test-integration + +# Helm chart tests (lint + template + unittest; no cluster, no running +# Lightstack; requires helm + the helm-unittest plugin — 'make install-tools') +make chart-test ``` ### Database Operations ```bash # Start PostgreSQL (Docker) -make db-up # Starts postgres:18.3 container +make db-up # Starts a postgres container (POSTGRES_VERSION, default 18) # Run migrations -make migrate # Applies all pending migrations +make migrate # Applies all pending migrations via golang-migrate # Stop PostgreSQL make db-down # Stops and removes container @@ -132,22 +124,23 @@ make db-down # Stops and removes container make fmt # Runs go fmt # Lint code -make lint # Runs golangci-lint +make lint # Runs pinned golangci-lint (built from the tools/ module) # Install development tools -make install-tools # Installs air, golangci-lint, migrate +make install-tools # Installs air, golangci-lint, migrate, helm, helm-unittest ``` ## Architecture Overview ### Service Structure -Lightstack runs as a single process (`cmd/lightstack/main.go`) that starts six HTTP servers on different ports: +Lightstack runs as a single process (`cmd/lightstack/main.go`) that starts seven HTTP listeners on different ports: - **Keystone** (35357): Identity service, JWT-based authentication - **Nova** (8774): Compute service, VM lifecycle via libvirt - **Neutron** (9696): Network service, namespace isolation via netlink - **Cinder** (8776): Block storage, multi-backend support (local/RBD/S3) - **Glance** (9292): Image service, multi-backend with hybrid failover +- **Placement** (8778): Stub of the placement service - **Metadata** (8775): EC2-compatible metadata service (no auth) Each service is initialized in `main.go` with its configuration and shares: @@ -155,25 +148,40 @@ Each service is initialized in `main.go` with its configuration and shares: - **Auth middleware**: Shared JWT token validation via `keystone.AuthService` - **Logging middleware**: Structured JSON logging +The binary also dispatches subcommands: `lightstack cleanup` (SPEC-010 teardown), `lightstack healthcheck` (node health probes), and `lightstack console-tokens list|purge` (SPEC-021 noVNC token inspection). A separate `cmd/lightstack-migrate` binary applies migrations. + ### Code Organization ``` +cmd/lightstack/ # Main binary (server + cleanup/console-tokens/healthcheck subcommands) +cmd/lightstack-migrate/ # Standalone migration binary internal/ # Private packages ├── keystone/ # Identity - JWT auth, service catalog ├── nova/ # Compute - VM lifecycle, flavors, keypairs ├── neutron/ # Network - networks, subnets, ports, security groups ├── cinder/ # Block storage - volumes, snapshots ├── glance/ # Images - metadata, multi-backend storage -├── database/ # DB models, migrations (models.go, migrate.go) +├── placement/ # Placement stub +├── metadata/ # EC2 metadata service +├── database/ # DB models, transactions (WithTx), migrate ├── middleware/ # Auth, logging, CORS, recovery -├── common/ # Config loading, error handling +├── common/ # Config, errors, BaseURL, scheme, pagination, SQL builder ├── compute/ # Node registry for multi-node coordination -└── metadata/ # EC2 metadata service +├── console/ # noVNC console tokens (SPEC-021) +├── health/ # Health/readiness endpoints +└── teardown/ # SPEC-010 release teardown pkg/ # Public/reusable packages ├── hypervisor/ # libvirt abstraction (stub + real modes) ├── networking/ # netlink, VXLAN, security groups -└── storage/ # Storage backends (Ceph RBD, S3, local) +├── storage/ # Storage backends (Ceph RBD, S3, local) +├── cache/ # Caching +└── paths/ # state_path resolution (node-local state root) + +test/ # Test suites +├── contract/ # gophercloud contract tests (per-service subdirs) +├── terraform/ # Terraform provider conformance +└── *.sh # Bash integration suites (OpenStack CLI + jq) ``` ### Key Architectural Patterns @@ -253,7 +261,7 @@ Requires Linux with appropriate dependencies. **Nova real mode** (`libvirt_mode: real`): - Requires libvirt + KVM on Linux - Creates actual VMs via `github.com/digitalocean/go-libvirt` -- XML templates in `pkg/hypervisor/xml_template.go` +- XML templates in `pkg/hypervisor/` **Neutron real mode** (`networking_mode: iptables` or `ebpf`): - Requires Linux with network namespaces @@ -261,37 +269,41 @@ Requires Linux with appropriate dependencies. - iptables via `github.com/coreos/go-iptables` for security groups **Cinder/Glance real mode** (`storage_mode: local`, `rbd`, `s3`, or hybrid): -- `local`: Host filesystem storage +- `local`: Host filesystem storage (node-local, under the `state_path` root) - `rbd`: Ceph RBD via `github.com/ceph/go-ceph` -- `s3`: AWS S3/MinIO via `github.com/aws/aws-sdk-go-v2` +- `s3`: AWS S3/MinIO/Ceph RGW via `github.com/aws/aws-sdk-go-v2` ## Database Schema -PostgreSQL with 15 tables. Key tables: +PostgreSQL, 70+ tables. Key tables: - **users**: User credentials (bcrypt hashed passwords) -- **projects**: Projects/tenants -- **roles**, **role_assignments**: RBAC +- **projects**, **domains**, **roles**, **role_assignments**: Identity and RBAC - **instances**: VM instances (linked to flavors, images) - **flavors**: VM flavor definitions -- **networks**, **subnets**, **ports**: Network topology -- **volumes**: Block storage volumes +- **networks**, **subnets**, **ports**, **security_groups**: Network topology +- **volumes**, **snapshots**: Block storage - **images**: Image metadata (data in storage backend) - **keypairs**: SSH public keys - **compute_nodes**: Multi-node registry for VXLAN coordination -Migrations in `migrations/` directory, applied via `golang-migrate/migrate`. +Migrations live in `migrations/` as numbered `NNN_name.up.sql` / `NNN_name.down.sql` pairs, applied via `golang-migrate/migrate`. Never edit an applied migration in a way that changes its meaning for deployed databases — add a new one. ## Testing Guidelines ### Unit Tests - Located alongside source files as `*_test.go` -- Test database operations with mock connections or test DB +- DB-backed unit tests connect to the database in `LIGHTSTACK_TEST_DB_URL` (see `internal/common/maindb_test.go` for the TestMain pattern); they skip when it is unset - Test mode detection logic (stub vs real) +### Contract Tests (`test/contract/`) +- Go tests using gophercloud (pinned v1.14.1) as a stand-in for any OpenStack client +- Require a running Lightstack (start the compose stack or `make run` first) +- **Every API-surface change needs a contract test here** — this is the SPEC-000 compliance gate CI runs + ### Integration Tests -All integration tests are bash scripts in `test/` directory that: -1. Start Lightstack in background +Bash scripts in `test/` that: +1. Start with a running Lightstack (compose stack or `make run`) 2. Use OpenStack CLI (`openstack` command) to test workflows 3. Verify responses with `jq` for JSON parsing 4. Clean up resources @@ -307,20 +319,12 @@ openstack server show test-vm -f json | jq -r '.status' openstack server delete test-vm ``` -### Running Tests -- Integration tests require Lightstack to be running: `docker compose up -d` or `make run` -- Tests use environment variables from `.env` or `~/.lightstack-env` -- Quick validation: `./test/quick_test.sh` (runs in ~30 seconds) - -### Test-First Development (Constitution Article III) +### Test-First Development When implementing new features: -1. **Write tests first** - Create `*_test.go` files -2. **Get approval** - Review test strategy -3. **Confirm RED** - Tests must fail initially -4. **Implement** - Write code to make tests pass (GREEN) -5. **Refactor** - Clean up while keeping tests green - -This is **NON-NEGOTIABLE** per the project constitution. +1. **Write tests first** - Create `*_test.go` files / contract tests +2. **Confirm RED** - Tests must fail initially +3. **Implement** - Write code to make tests pass (GREEN) +4. **Refactor** - Clean up while keeping tests green ## Configuration @@ -329,13 +333,14 @@ Configuration loaded from `config/lightstack.yaml` (or via `--config` flag). **Critical settings:** - `database.url`: PostgreSQL connection string - `keystone.jwt_secret`: MUST change in production (security critical) +- `keystone.admin_password`: overrides the seeded `admin`/`secret` default — set it for any exposed deployment - `nova.libvirt_mode`: `stub` (dev) or `real` (prod) - `neutron.networking_mode`: `stub`, `iptables`, or `ebpf` - `cinder.storage_mode`: `stub`, `local`, `rbd`, `s3`, or hybrid (comma-separated) - `glance.storage_mode`: `stub`, `local`, `rbd`, `s3`, or hybrid (comma-separated) -- `rate_limiting.enabled`: Enable rate limiting (default: false) -- `rate_limiting.auth_rate`: Requests/minute per IP for auth endpoints -- `rate_limiting.api_rate`: Requests/minute per project for API endpoints +- `state_path`: node-local state root (default `/var/lib/lightstack`; resolved via `pkg/paths`) + +**Dead config — do not rely on it**: the `rate_limiting.*` block (`enabled`, `auth_rate`, `api_rate`, …) appears in `config/lightstack.yaml` and in ADR-002's design, but **no code reads it** — there is no `RateLimiting` field in the `Config` struct (`internal/common/config.go`) and no rate-limiting middleware. Treat it as unimplemented until a spec lands it. **Multi-node VXLAN** (advanced): - Set `neutron.vxlan_enabled: true` @@ -348,7 +353,7 @@ Seed data creates: - **Password**: `secret` - **Project**: `default` -*Change these in production environments.* +*Change these in production environments.* (Setting `keystone.admin_password` neutralizes the seeded default.) ## Common Patterns and Idioms @@ -358,10 +363,14 @@ Follow conventional commits format: (): [optional body] + +Signed-off-by: Name ``` **Types**: `feat`, `fix`, `refactor`, `docs`, `test`, `chore` +**DCO**: every commit must carry the `Signed-off-by:` trailer (`git commit -s`); CI's "SAP Sign-off Check" job enforces an `@sap.com` address on every PR commit. + **Examples**: - `feat(glance): add S3 backend support for image storage` - `fix(nova): correct power_state in server detail response` @@ -380,7 +389,7 @@ return common.HandleError(c, err) ### Database Queries ```go -// Always use context, the service's injected pool, and prepared statements +// Always use context, the service's injected pool, and parameterized statements ctx := c.Request.Context() err := svc.pool.QueryRow(ctx, "SELECT id FROM instances WHERE project_id = $1", @@ -413,9 +422,10 @@ if requestedVersion == "" { ## Important Notes ### Security Considerations -- Never commit production JWT secrets +- Never commit production JWT secrets or any credentials - Database passwords should use environment variables in production - Token TTL default is 24h (configurable) +- Tenant isolation and admin-only surfaces are security-critical — see SPEC-022 (`docs/specs/022-project-tenant-isolation/`) and SPEC-023 (`docs/specs/023-tenant-scoping-authz/`) ### Backwards Compatibility - All API endpoints must maintain OpenStack API compatibility @@ -439,53 +449,11 @@ if requestedVersion == "" { **"Database connection failed"**: Check PostgreSQL is running and connection string is correct **"Token validation failed"**: Ensure `jwt_secret` matches between Keystone and other services -## Active Technologies -- Go 1.26 + Gin (HTTP framework), pgx (PostgreSQL driver), gophercloud (contract testing), go-libvirt (hypervisor), netlink (networking) (002-horizon-full-compatibility) -- PostgreSQL 18 (primary database) (002-horizon-full-compatibility) -- 007-helm-prereqs-bootstrap-catalog: admin-password bootstrap (bcrypt, `internal/keystone`) + `LIGHTSTACK_ENDPOINT_SCHEME` catalog scheme — no new dependencies, no schema change -- 009-single-node-real-mode: real-mode single node — per-instance qcow2 overlay over a raw base, startup+periodic network reconciler, metadata/cloud-init reachability + `user_data`, north-south (reuse router NAT), block storage (sync attach + boot-from-volume). No new Go deps; image adds `qemu-img`+`genisoimage`; migration 060 (`volumes.source_image_id`); node-local data under `/var/lib/lightstack`. -- 011-ensure-storage-subdirs: real-mode node-local storage subdirs guaranteed to exist — Glance creates its canonical `/var/lib/lightstack/images` (drops the `~/.lightstack` fallback; matches `ceph.go`) + chart initContainer pre-creates all subdirs on the `host-storage` hostPath. No new Go/chart deps, no schema change. -- Helm 3.12+ chart (Go `text/template`); JSON Schema **draft-07**; Markdown docs. No Go code changes. + Helm's built-in `values.schema.json` validation + release-status/pod-readiness reporting; the **helm-unittest** plugin (existing harness). No new Go modules, no chart dependencies. (012-integration-contract) -- N/A — the external PostgreSQL is the product's and is untouched; this feature adds no schema or migration. (012-integration-contract) -- Go 1.26 + Gin (HTTP), pgx/pgxpool (existing `database.HealthCheck`). **No new Go modules.** Chart: Helm 3.12+, JSON Schema draft-07, helm-unittest (existing harness) — no new chart deps. (013-readiness-liveness-probes) -- External PostgreSQL — **no schema, no migration, no persisted data** (readiness is a live `Ping`). (013-readiness-liveness-probes) -- 014-ceph-rgw-s3-backend: Ceph RGW (S3) as Glance backend (validate + implement) + tenant object storage. Existing aws-sdk-go-v2 with a custom `http.Client` (CA pool + connect/first-byte transport timeouts; body streams, no total cap); stdlib `crypto/hmac`+`sha256` for server-side S3 SigV4/SigV2 verification; OpenStack EC2-credentials API reusing the `credentials` table (+ `access_key` col); Nova lazy fetch-on-boot via `storage.EnsureLocalBase` (+ checksum-on-upload, node-local base reclaim). Migration 062 (`credentials.access_key` + `s3` catalog seed); RGW `implicit tenants=s3`. No new Go modules. -- Go 1.26 (module targets 1.21+) + gin-gonic/gin v1.12.0 (HTTP), google/uuid v1.6.0 (`uuid.Validate` — already imported), jackc/pgx/v5 v5.10.0 (`pgx.ErrNoRows`). No new modules. (015-identity-uuid-404) -- PostgreSQL — **no schema change, no migration** (identity id columns stay `uuid`). (015-identity-uuid-404) -- 026-rename-to-lightstack: product rename to `lightstack` across 6,749 occurrences / 482 files / 51 paths. Go 1.26.2, **no dependency change**. Module path becomes `github.com/cobaltcore-dev/lightstack`, matching the repository. Case by role: `Lightstack` prose, `lightstack` identifiers/paths, `LIGHTSTACK_` for the 77 env vars. Three hand-written exceptions: iptables chain prefixes → `LS-SG-`/`LS-NSI-` (the full name overflows the 28-char budget), the catalog host token (migration `052` + `auth.go:684,686` must move together), and the acceptance check written as a character class so it does not match itself. No new tests — verification is the acceptance check plus manual checks. -- PostgreSQL — **no schema change, no new migration**; migration `052` edited in place (five `endpoints.url` rows). Nothing is deployed. (026-rename-to-lightstack) -- Go 1.26 (module targets 1.21+) + gin-gonic/gin (HTTP), jackc/pgx/v5 + pgxpool (PostgreSQL); **no new modules** (017-nova-server-detail-readback) -- PostgreSQL — new migration `063` adds `instances.key_name` (nullable `VARCHAR(255)`); reuses `server_security_groups` (migration 049) and the Neutron `security_groups` table for name→id resolution (017-nova-server-detail-readback) -- No product code. Verification tooling only — Helm 3.12+ (new verification chart; existing `charts/lightstack`), Bash + `python-openstackclient` + `curl` (existing `test/horizon_*.sh`), and the unmodified upstream kolla Horizon image (Flamingo 2025.2, Python/Django — config-only, never patched). + docker-compose (stub pass); k3s (root/privileged) + `charts/lightstack` in real mode; minikube + Rook/Ceph exposing an RGW (S3) endpoint; a new verification Horizon chart (kolla image + `horizon-config` ConfigMap + memcached + `bonigarcia/novnc`). (018-horizon-dashboard-verification) -- No schema change, no migration. Real-mode backends: node-local `/var/lib/lightstack` (VM disks, volumes), Ceph **RGW/S3** for Glance images. External PostgreSQL unchanged. **RBD not used.** (018-horizon-dashboard-verification) -- Go 1.26 (module targets 1.21+) + gin-gonic/gin (HTTP), jackc/pgx/v5 + pgxpool (PostgreSQL); `google/uuid`. **No new modules.** (019-horizon-compat-api-fixes) -- External PostgreSQL. **One** new migration `064_fix_cinder_volume_alias_localhost` (reseeds the `volume` alias `…015` endpoint URL only; s3 `…016` is config-overridden by `BuildServiceCatalog` and deliberately untouched). F9 reuses the existing `volumes.volume_type` column (migration `016`); F4 and F6 add **no** columns. (019-horizon-compat-api-fixes) -- Go 1.26 (module targets 1.21+) + gin-gonic/gin (HTTP), jackc/pgx/v5 + pgxpool (PostgreSQL). **No new modules.** (020-horizon-create-ui-enablement) -- External PostgreSQL. **No migration, no schema change** — reuses `quotas` (migration `007`, row-per-resource, `project_id UUID NOT NULL REFERENCES projects(id)`) and `cinder_quotas` (migration `039`). Neutron has no quota table and does not gain one (limits stay code defaults; usage is counted live). (020-horizon-create-ui-enablement) -- Go 1.26 (module targets 1.21+) + jackc/pgx/v5 + pgxpool (already a dependency), gin-gonic/gin. **No new modules.** (027-database-pool-injection) -- PostgreSQL. **No schema change, no migration.** The pool configuration from `lightstack.yaml` is unchanged. (027-database-pool-injection) -- Go 1.26 (module targets 1.21+) + gin-gonic/gin, jackc/pgx/v5 + pgxpool, digitalocean/go-libvirt (`DomainGetXMLDesc`, `DomainLookupByUUID` → `Domain.ID`, `ConnectGetCapabilities`, all present in the pinned revision `v0.0.0-20260609165003-6254771e63a8`), `encoding/xml`. **No new Go modules.** (021-real-novnc-console) -- PostgreSQL, migration `065_console_auth_tokens`. It is edited in place: nothing is deployed, and test databases are recreated. This amendment adds `domain_id`. Node-local: `/console-tokens/` (the projection) and `/console-sockets/` (QEMU-owned, 0750, sockets only). (021-real-novnc-console) -- Go 1.26 (module targets 1.21+) + `gin-gonic/gin` (HTTP), `jackc/pgx/v5` + `pgxpool` (PostgreSQL). **No new modules.** (022-project-tenant-isolation) -- External PostgreSQL. **One migration**, `072_groups_domain_id_uuid`: `groups.domain_id` (a `VARCHAR` with no foreign key since `010`) becomes `UUID NOT NULL REFERENCES domains(id)`, alias rows normalised, anything ambiguous stops the migration. Every other fix is behavioural. (022-project-tenant-isolation) -- Go 1.26 (repo pins `go 1.26`; Constitution requires 1.21+) + Gin (HTTP), pgx/pgxpool (PostgreSQL), golang-jwt/v5. **No new modules.** Contract tests use the already-pinned `gophercloud v1.14.1`. (023-tenant-scoping-authz) -- PostgreSQL — **no schema change, no migration**. `instance_actions.project_id` is already `UUID NOT NULL`; the change is which value is written to it. (023-tenant-scoping-authz) -- Go 1.26 (module targets 1.21+) + `gin-gonic/gin` (HTTP), `jackc/pgx/v5` + `pgxpool` (PostgreSQL), `vishvananda/netlink` (real-mode bridge/TAP), `google/uuid`. **No new modules.** (024-neutron-api-fidelity-realmode) -- External PostgreSQL. **One** migration `066_neutron_external_networks` adds `networks.external BOOLEAN NOT NULL DEFAULT false` (reversible). The other five defects need no schema change — `subnets.gateway_ip` and `security_group_rules.port_range_min/max` are already nullable; the API just refuses to produce those states. (024-neutron-api-fidelity-realmode) -- Go 1.26 (module targets 1.21+) + `gin-gonic/gin` (HTTP), `jackc/pgx/v5` + `pgxpool` (PostgreSQL), `digitalocean/go-libvirt`, `google/uuid`. **No new modules.** (025-nova-gaps-client-discovery) -- External PostgreSQL. **Two** reversible migrations: `067_instance_fault` (`instances.fault_code`/`fault_message`/`fault_created`, nullable — makes a failed boot readable as upstream's `fault` object) and `068_instance_actions_time_index` (index on `instance_actions(start_time)` for the retention sweep). Two new `NovaConfig` settings: `state_reconcile_interval` (30s) and `instance_action_retention` (90d). (025-nova-gaps-client-discovery) -- Go 1.26 (module targets 1.21+) + `gin-gonic/gin` (HTTP), `jackc/pgx/v5` + `pgxpool` (PostgreSQL), `google/uuid`. **No new modules.** One new shared helper: a `406 Not Acceptable` constructor in `internal/common/errors.go` (the product has none). (025-followup) -- External PostgreSQL. **No schema change, no migration** — every fix is behavioural. One existing column gains a reader: `snapshots.size_gb`, so the `gigabytes` quota counts volume **and** snapshot size as upstream does. (025-followup) - -## Recent Changes -- 025-followup: plan — SPEC-025's seven follow-ups (F1–F4, F6–F8), a pseudo-ticket on the same branch, **no new issue**. Same dominant finding as its parent: the shared thing exists and a caller bypasses it (`common.SQLFilter` + identity's filter pattern, `nova.CheckQuota`'s tx+row-lock, `common.BaseURL`'s host precedence, one already-shared quota resolution behind three views). Planning re-derived FR-019 from the code and found **eight** quota call sites across four files where the spec named two — three `INSERT INTO volumes` outside the shared provisioning routine, plus snapshots/backups/extend. Research (Context7, available again) turned up three things clarify could not know: `host` is **role-split** upstream so F7's refusal diverges two-sidedly, image-patch refusals are **403** not 400, and a request with **no** microversion header resolves to upstream's *minimum* where Lightstack uses its *maximum* — a third Article I deviation, pre-existing, which caps US6 to clients that negotiate explicitly. No migration, no new setting; cost sits in negative tests (every defect answers 2xx today) (SPEC-025-followup plan) -- 025-nova-gaps-client-discovery: plan — ten defects in two families (discovery self-links + compute answers that look successful). Dominant finding: **the right shape already exists and one caller ignores it** — `common.BaseURL` (5 services bypass it), `nova.targetInstance` (the action table already passes the resolved id; 17 handlers discard it and re-read the raw param), two accepted reconcilers to copy, and both halves of boot-from-volume. So: broad but shallow, **no new Go modules**, 2 small reversible migrations. Genuinely new: the self-heal of a missing libvirt guest (rebuildable from stored state — the blocker is that the `VMSpec` assembly is inline in a goroutine) and an upstream `fault` field. **B12's root cause stays open on purpose** — the chosen fix is correct whatever it turns out to be (SPEC-025 plan) -- 024-neutron-api-fidelity-realmode: plan — six Neutron defects (B6/B9/B10/B11/F10/F11). B10 is **one** change, not 244: every error path already funnels through `common.SendError`, so marking the Neutron route group converts handlers, auth middleware, panic recovery, unrouted-404 and 405 together. F10 = TAP moves to the **root** namespace (bridge, dnsmasq and SNAT are already there) + stub attach stops trusting a process-local map. B11 = honour a named project for admins, `400` otherwise (upstream `neutron_lib` rule, deliberately unlike #549's compute refusal). F11 = migration `066` + external as a **third** visibility disjunct (not requiring `shared`), with pre-existing gateways grandfathered (SPEC-024 plan) -- 023-tenant-scoping-authz: plan — finish applying the admin override across compute's server operations (upstream `admin_or_owner`, creation excluded) + `all_tenants` opt-in; refuse domain-scoped auth with `400` instead of `201`-with-nothing; close the `os-simple-tenant-usage` cross-tenant leak; honour query filters on 6 identity collections. No new deps, no schema change, no migration (SPEC-023 plan) -- 027-database-pool-injection: plan — remove the process-wide database handle. Every database-using constructor takes the `*pgxpool.Pool` first, and the 20 package-level helpers take it as a parameter. `WithTx(ctx, pool, fn)`. One commit, no API or schema change (SPEC-027 plan) -- 026-rename-to-lightstack: plan — staged rename (8 stages, gates green per stage); spec dir stays **untracked** (it cannot shed its own 70 occurrences); real-mode host validation mandatory (stub mode never reaches the chain namer or the state root) (SPEC-026 plan) -- 014-ceph-rgw-s3-backend: plan — RGW/S3 Glance backend (validate + boot lazy-fetch / timeout rework / TLS-CA) + tenant object storage (EC2-creds API + public self-validating `/v3/s3tokens` + `implicit tenants=s3`); migration 062 (`credentials.access_key` + `s3` catalog) (SPEC-014 plan) -- 011-ensure-storage-subdirs: fix real-mode storage-subdir gap (cloud-storage#510) — Glance no longer falls back to `~/.lightstack`; chart initContainer pre-creates node-local subdirs (SPEC-011 plan) -- v0.7.0 planning: Database DI (ADR-001), rate limiting (ADR-002), Terraform conformance, CI linting, Cinder route redirect, pagination standardization (SPEC-006) -- 002-horizon-full-compatibility: Added Go 1.26 + Gin (HTTP framework), pgx (PostgreSQL driver), gophercloud (contract testing), go-libvirt (hypervisor), netlink (networking) -- 009-single-node-real-mode: real-mode VM disk/network/metadata/north-south + block storage on a single node (SPEC-009 plan) +## Where to Look Next + +- `README.md` / `STATUS.md` / `ROADMAP.md` — project status and direction +- `docs/CONTRIBUTING.md` — contribution process +- `docs/specs/README.md` and `docs/specs/NNN-*/` — feature specifications +- `docs/adr/` — architecture decision records +- `docs/review/` — comprehensive review reports +- `CHANGELOG.md` — user-visible changes per release From e31108e51f9d2a048552bdbaa0e765a519c371d5 Mon Sep 17 00:00:00 2001 From: Uwe Schwaeke Date: Wed, 14 Oct 2026 19:27:27 +0000 Subject: [PATCH 4/8] docs(specs): live spec registry, lifecycle and numbering rules, spec template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Specifications side of issue #16's alignment chain. docs/specs/README.md was a stale March snapshot with every spec 'Draft'; it is now a live registry: one row per spec with an evidence-backed status (draft / accepted / implemented / superseded), commit and code-reference evidence, numbering rules (one number per spec, monotonic; the duplicate SPEC-025 dirs and the untracked 026 rename are documented as historical anomalies; next free number is 028), the lifecycle semantics, and a how-to-write-a- spec guide backed by docs/specs/_template/ — a skeleton mirroring the real pattern (spec.md with mandatory FR-NNN ids, plan.md, tasks.md in Tnnn test-first format, contracts/ with Given/When/Then tables that become the contract tests), modeled on 009-single-node-real-mode. Signed-off-by: Uwe Schwaeke --- docs/specs/README.md | 365 ++++++++--------------- docs/specs/_template/README.md | 37 +++ docs/specs/_template/contracts/README.md | 25 ++ docs/specs/_template/plan.md | 36 +++ docs/specs/_template/spec.md | 95 ++++++ docs/specs/_template/tasks.md | 39 +++ 6 files changed, 352 insertions(+), 245 deletions(-) create mode 100644 docs/specs/_template/README.md create mode 100644 docs/specs/_template/contracts/README.md create mode 100644 docs/specs/_template/plan.md create mode 100644 docs/specs/_template/spec.md create mode 100644 docs/specs/_template/tasks.md diff --git a/docs/specs/README.md b/docs/specs/README.md index 7babea81..a6e2323e 100644 --- a/docs/specs/README.md +++ b/docs/specs/README.md @@ -1,245 +1,120 @@ -# Lightstack Specifications Index - -**Version**: 1.0 -**Last Updated**: 2026-03-09 - -## Overview - -This directory contains technical specifications for Lightstack development. Each specification follows the library-first, test-first approach defined in the project constitution. - -## Active Specifications - -### Compliance Framework (Priority Zero) - -#### [SPEC-000: OpenStack API Compliance Framework](./000-api-compliance/README.md) -**Status**: Draft | **Priority**: ZERO (Supersedes All) | **Timeline**: Continuous - -**THE FOUNDATIONAL REQUIREMENT** - Lightstack must be 100% OpenStack API compliant. This is non-negotiable. - -**Key Deliverables**: -- Contract tests with OpenStack clients -- Terraform provider compatibility -- OpenStack CLI full support -- All SDK compatibility (Python, Go, Java) -- Horizon 2025.2 unmodified support -- Schema validation framework -- Error response compatibility -- Microversion support - -**All other specifications must comply with SPEC-000.** - ---- - -### Core Architecture - -#### [SPEC-001: Modular Architecture Transformation](./001-modular-architecture/README.md) -**Status**: Draft | **Priority**: Critical | **Timeline**: 16 weeks - -Transform Lightstack from monolithic to modular architecture with independent services. - -**Key Deliverables**: -- Independent binaries per service -- Service-to-service API communication -- Docker Compose deployment -- Migration guide - ---- - -### Authentication & Security - -#### [SPEC-002: Enhanced Authentication System](./002-authentication-enhancement/README.md) -**Status**: Draft | **Priority**: Critical | **Timeline**: 8 weeks | **Depends**: SPEC-001 - -Enterprise authentication mechanisms for Keystone. - -**Key Deliverables**: -- OAuth2/OIDC support -- SAML integration -- LDAP/Active Directory -- Application credentials -- Token revocation list - ---- - -### New Services - -#### [SPEC-003: Barbican - Key Management Service](./003-barbican/README.md) -**Status**: Draft | **Priority**: High | **Timeline**: 6 weeks | **Depends**: SPEC-001 - -Encryption key and secret management service. - -**Key Deliverables**: -- Secret storage (encrypted at rest) -- SoftHSM integration -- Cinder volume encryption -- Certificate management -- HSM backend support - ---- - -#### [SPEC-004: Designate - DNS as a Service](./004-designate/README.md) -**Status**: Draft | **Priority**: Medium | **Timeline**: 4 weeks | **Depends**: SPEC-001 - -Multi-tenant DNS management with BIND9 backend. - -**Key Deliverables**: -- Zone management -- Record set CRUD -- BIND9 backend -- Neutron floating IP auto-DNS -- Zone transfers - ---- - -#### [SPEC-005: Octavia - Load Balancing as a Service](./005-octavia/README.md) -**Status**: Draft | **Priority**: Medium | **Timeline**: 6 weeks | **Depends**: SPEC-001, Nova, Neutron - -L4/L7 load balancing with HAProxy backend. - -**Key Deliverables**: -- Load balancer management -- HAProxy backend -- Health monitoring -- TLS termination (Barbican integration) -- L7 policies - ---- - -### Code Quality & Architecture - -#### [SPEC-006: v0.7.0 Foundation](./006-v070-foundation/README.md) -**Status**: Draft | **Priority**: High | **Timeline**: TBD | **Depends**: SPEC-000 - -Architecture and code quality improvements for production readiness. - -**Key Deliverables**: -- Database dependency injection (661 references refactored) -- Rate limiting middleware (IP + project dual mode) -- CI linting re-enabled (staged golangci-lint config) -- Terraform conformance test suite (15+ configs) -- Cinder route consolidation (301 redirects) -- Pagination standardization (24 handlers) - -**ADRs**: [ADR-001](../adr/ADR-001-database-dependency-injection.md), [ADR-002](../adr/ADR-002-rate-limiting-architecture.md) - ---- - -## Specification Status Legend - -- **Draft**: Initial design, not yet implemented -- **In Progress**: Active development -- **Implemented**: Code complete, testing in progress -- **Released**: Deployed to production -- **Deprecated**: No longer active - -## Priority Levels - -- **Critical**: Blocking other work, must complete first -- **High**: Important for near-term goals -- **Medium**: Useful but not blocking -- **Low**: Nice-to-have, future consideration - -## Development Workflow - -``` -1. Specification → Review → Approval -2. Plan → Task List → Implementation -3. Test (TDD) → Validate → Release -``` - -Each specification follows: -- **Library-First** (Constitution Article I) -- **Test-First** (Constitution Article III) -- **Integration-First** (Constitution Article IX) - -## Implementation Dependencies - -``` -SPEC-000 (API Compliance) ──┬─→ Continuous validation of all work - │ -SPEC-001 (Modular Architecture) ◄─── Must pass SPEC-000 tests - ↓ - ├─→ SPEC-002 (Auth Enhancement) ◄─── Must pass SPEC-000 tests - └─→ SPEC-003, 004, 005 (New Services) ◄─── Must pass SPEC-000 tests - -SPEC-006 (v0.7.0 Foundation) ◄─── Must maintain SPEC-000 API compatibility -``` - -**Critical**: Every feature, every endpoint, every change must pass SPEC-000 compliance tests. - -## Timeline Overview - -| Spec | Duration | Start | End | Status | -|------|----------|-------|-----|--------| -| SPEC-001 | 16 weeks | Week 1 | Week 16 | Draft | -| SPEC-002 | 8 weeks | Week 17 | Week 24 | Draft | -| SPEC-003 | 6 weeks | Week 25 | Week 30 | Draft | -| SPEC-004 | 4 weeks | Week 31 | Week 34 | Draft | -| SPEC-005 | 6 weeks | Week 35 | Week 40 | Draft | -| SPEC-006 | TBD | TBD | TBD | Draft | - -Total: ~40 weeks to complete core specifications - -## Validation Gates - -Each specification must pass: - -1. **Contract Tests**: OpenStack CLI compatibility -2. **Integration Tests**: Real dependency testing -3. **Performance Tests**: < 5% regression -4. **Security Audit**: No critical vulnerabilities -5. **Documentation**: Complete user/admin guides -6. **Horizon Compatibility**: Dashboard integration works - -## Related Documents - -- [Roadmap](../ROADMAP.md) - Overall project roadmap -- [CLAUDE.md](../CLAUDE.md) - Development instructions -- [Constitution](../memory/constitution.md) - Project principles -- [BEASTMODE.md](../.beastmode/BEASTMODE.md) - Development workflow - -## Contributing to Specifications - -### Creating a New Specification - -1. Copy template: `cp templates/spec-template.md specs/NNN-name/README.md` -2. Fill in all sections -3. Submit for review -4. Get approval from maintainers -5. Mark as "Draft" -6. Implement following TDD cycle - -### Specification Template Sections - -- **Overview**: What problem does this solve? -- **Goals**: What will be delivered? -- **Non-Goals**: What is explicitly out of scope? -- **Architecture**: How does it work? -- **Database Schema**: Data model changes -- **API Endpoints**: New/modified APIs -- **Implementation Strategy**: Step-by-step approach -- **Testing Strategy**: How to validate -- **Migration Path**: Phased implementation -- **Success Criteria**: Definition of done -- **References**: External resources - -## Specification Review Process - -1. **Author** writes specification -2. **Peer Review** (2+ developers) -3. **Architecture Review** (1 architect) -4. **Security Review** (if applicable) -5. **Approval** (project lead) -6. **Implementation** begins - -## Questions? - -- GitHub Discussions: [lightstack Discussions](https://github.com/your-org/lightstack/discussions) -- Slack: #lightstack-dev -- Email: lightstack-dev@example.com - ---- - -**Maintained by**: Lightstack Core Team -**Last Review**: 2026-03-09 -**Next Review**: 2026-04-09 +# Lightstack Specifications Registry + +This directory holds Lightstack's feature specifications. Each spec lives in its own +`NNN-name/` directory and follows the shared internal pattern: +`spec.md` → `plan.md` → `tasks.md` → implementation, supported by `research.md`, +`data-model.md`, `contracts/`, `checklists/`, and host-validation runbooks. + +This README is a **live registry**, not a snapshot: statuses below reflect the actual +state of the repository (merged commits, code references, contract tests), not the +`Status:` header inside each spec (those headers are written at authoring time and are +not updated after merge). + +## SPEC-000: Priority Zero — Supersedes All + +[SPEC-000: OpenStack API Compliance Framework](./000-api-compliance/README.md) is the +foundational, standing requirement: Lightstack must be indistinguishable from canonical +OpenStack to every unmodified client (OpenStack CLI, SDKs, Terraform provider, Horizon). +It is **priority zero and supersedes all other specifications** — every spec, every PR, +and every change must comply. It is continuous by nature and never "done"; it is +enforced through the contract suites under `test/contract/` and the compatibility +reports under `docs/compatibility/`. + +## Spec Registry + +Statuses: `draft`, `accepted`, `implemented`, `superseded`, `archived` (see +[Lifecycle](#lifecycle)). No spec is currently superseded or archived. + +| # | Spec | One-line description | Status | Notes / evidence | +|---|------|----------------------|--------|------------------| +| 000 | [API Compliance Framework](./000-api-compliance/README.md) | Priority-zero standing framework: 100% OpenStack API compatibility, enforced by contract tests | accepted | Standing/continuous; contract suites (`test/contract/`) and compatibility reports exist, but compliance never terminates | +| 001 | [Modular Architecture Transformation](./001-modular-architecture/README.md) | Split the monolith into independently deployable per-service binaries | draft | Never started — no commits, no code references; overtaken in practice by the accepted single-binary direction (see ADR-001/ADR-002) | +| 002 | [Enhanced Authentication System](./002-authentication-enhancement/README.md) | OAuth2/OIDC, SAML, LDAP, application credentials, token revocation | draft | Partially delivered (application credentials in `internal/keystone/application_credentials.go`; RBAC contract tests from its T048); federated auth unimplemented | +| 003 | [Barbican — Key Management](./003-barbican/README.md) | Encryption key / secret management service | draft | Future service; no `internal/barbican` package exists | +| 004 | [Designate — DNS as a Service](./004-designate/README.md) | Multi-tenant DNS management | draft | Future service; no `internal/designate` package exists | +| 005 | [Octavia — Load Balancing](./005-octavia/README.md) | L4/L7 load balancing service | draft | Future service; no `internal/octavia` package exists | +| 006 | [v0.7.0 Foundation](./006-v070-foundation/README.md) | Code-quality foundation: DB injection, rate limiting, CI linting, Terraform conformance, route consolidation, pagination | accepted | Partially implemented: DI landed via SPEC-027; linting in CI; Terraform conformance harness merged; rate limiting (ADR-002) and Cinder 301 route consolidation still outstanding | +| 007 | [Helm Prerequisites](./007-helm-prereqs-bootstrap-catalog/spec.md) | Admin-password bootstrap + configurable service-catalog scheme | implemented | Merged (#50); 6 code references cite SPEC-007 | +| 008 | [Helm Charts](./008-helm-charts/spec.md) | Kubernetes deployment chart, single-node and multi-node | implemented | Chart merged (#54); lives in `charts/lightstack/`; multi-node post-conditions tracked separately | +| 009 | [Single-Node Real Mode (v1)](./009-single-node-real-mode/spec.md) | Real VMs end-to-end on one node: boot, networking, cloud-init, egress, volumes | implemented | Merged (#64); 81 code references; a few host-validation runbook tasks remain open in `tasks.md` | +| 010 | [Helm Teardown](./010-helm-teardown/spec.md) | Pre-delete hook for clean uninstall, no state leakage | implemented | Merged (#67); `internal/teardown/` exists | +| 011 | [Ensure Storage Subdirs](./011-ensure-storage-subdirs/spec.md) | Pre-create node-local storage subdirectories on real-mode deploy | implemented | Merged (#66) | +| 012 | [Integration Contract](./012-integration-contract/spec.md) | Helm values schema + status conventions | implemented | Merged (#69) | +| 013 | [Readiness/Liveness Probes](./013-readiness-liveness-probes/spec.md) | Honest readiness/liveness endpoints for k8s-native heartbeating | implemented | Merged (#70); `internal/health/` exists; 10 code references | +| 014 | [Ceph RGW (S3) Backend](./014-ceph-rgw-s3-backend/spec.md) | Glance image backend + tenant object storage on Ceph RGW | implemented | Merged; 31 code references | +| 015 | [Identity UUID 404](./015-identity-uuid-404/spec.md) | Return 404 (not 500) for malformed identity resource ids | implemented | Merged (#515) | +| 016 | [Terraform Provider Verification](./016-terraform-openstack-provider-verification/README.md) | Verify unmodified terraform-provider-openstack apply/destroy | implemented | Harness + report merged (#481); `docs/compatibility/TERRAFORM_COMPATIBILITY_REPORT.md` | +| 017 | [Nova Server-Detail Read-back](./017-nova-server-detail-readback/spec.md) | Terraform-compat server-detail fields (`key_name`, `security_groups`) | implemented | Merged (#524); cited by `test/contract/server_readback_test.go` | +| 018 | [Horizon Dashboard Verification](./018-horizon-dashboard-verification/spec.md) | Verify unmodified Horizon against Lightstack; publish findings report | implemented | Merged (#482); `docs/compatibility/HORIZON_COMPATIBILITY_REPORT.md` | +| 019 | [Horizon Compat API Fixes](./019-horizon-compat-api-fixes/spec.md) | Backend/response fixes for findings F4/F5/F6/F9 | implemented | Merged (#532) | +| 020 | [Horizon Create/UI Enablement](./020-horizon-create-ui-enablement/spec.md) | Enable Horizon create/write workflows (F2/F3/F7/F8) | implemented | Merged (#533); 58 code references | +| 021 | [Real noVNC Console](./021-real-novnc-console/spec.md) | A real graphical console via noVNC (finding F1) | implemented | Merged (#534); 57 code references; host-pass validation tasks remain open in `tasks.md` | +| 022 | [Project/Tenant Isolation](./022-project-tenant-isolation/spec.md) | Tenant lifecycle & isolation defect fixes (B2/B5/B8) | implemented | Merged (#545); EVIDENCE.md in spec dir | +| 023 | [Tenant Scoping & Authz](./023-tenant-scoping-authz/spec.md) | Tenant scoping and authorization model (P2/P3) | implemented | Merged (#549); 57 code references | +| 024 | [Neutron API Fidelity](./024-neutron-api-fidelity-realmode/spec.md) | Neutron API fidelity & real-mode networking fixes (B6/B9–B11, F10/F11) | implemented | Merged (#546); 60 code references | +| 025 | [Nova Gaps & Client Discovery](./025-nova-gaps-client-discovery/spec.md) | Ten defects: discovery self-links + compute answers that look successful | implemented | Merged; commit `3f6633d` | +| 025 | [SPEC-025 Follow-ups](./025-followup/spec.md) | Pseudo-ticket for the follow-ups found while implementing SPEC-025 | implemented | Merged on the same branch; commit `fb6443a` | +| 027 | [Database Pool Injection](./027-database-pool-injection/spec.md) | Inject `*pgxpool.Pool` through service constructors (implements ADR-001) | implemented | Merged; ADR-001 records "Implemented by SPEC-027" | + +### Numbering anomalies + +- **Duplicate `025`.** Two directories carry number 025: `025-nova-gaps-client-discovery` + (the ticket proper) and `025-followup` (a pseudo-ticket for its follow-ups, tracked by + a comment rather than an issue). Both stay as-is for history. +- **Missing `026`.** SPEC-026 was the o3k → lightstack rename (cloud-storage#575, + commit `7ea8682`). Its spec directory was deliberately never tracked — the rename + could not be applied to a spec describing the rename — so no `026-*` directory exists. + +## Numbering Rules + +- One number per spec. Numbers are **monotonic**: each new spec takes the next free + number and never reuses or renumbers an existing one, even if earlier numbers are + missing or duplicated (see anomalies above — they stay as-is for history). +- The **next free number is 028.** +- A spec directory is named `NNN-short-slug/` (lowercase, hyphenated), and the spec is + cited as `SPEC-NNN` in code comments, commit messages, and ADRs. + +## Lifecycle + +A spec moves through at most these states; state changes are made by editing this +registry (the `Status:` header inside a spec records its state at authoring time): + +1. **draft** — being specified or implemented. The spec exists, work may be underway, + but nothing has shipped. Spec-local questions are resolved in the spec's + `Clarifications` section. +2. **accepted** — the design is ratified (reviews passed, decisions locked, any ADRs + recorded) but implementation is not complete. A spec whose scope is only partially + delivered stays here until the remainder lands or is explicitly deferred. +3. **implemented** — the implementing PR(s) are merged to `main`. **The PR that + implements a spec is what moves it to implemented**; the author updates this + registry in that PR, citing the merge as evidence (commit/PR reference, code + references to the spec's FR IDs, or contract-test citations). +4. **superseded** — replaced by another spec or ADR; the row names the successor. +5. **archived** — deliberately retired without a successor (e.g. abandoned direction); + the directory moves to `docs/archive/` and the row names the reason. + +SPEC-000 is a special case: as the standing priority-zero framework it remains +**accepted** permanently — compliance is continuously enforced, never "implemented once". + +## How to Write a Spec + +Copy [`_template/`](./_template/) into `docs/specs/NNN-short-slug/` and follow the +workflow described there (**specify → plan → tasks → implement → validate**). The +internal pattern, per directory: + +| File | Purpose | +|------|---------| +| `spec.md` | The feature specification: problem, goals/non-goals, requirements with **FR-NNN ids** (these ids are how code comments and commit messages cite the spec — they are mandatory), success criteria, clarifications | +| `plan.md` | Implementation plan: technical approach, phases, sequencing, risks | +| `tasks.md` | Checklist of concrete tasks (`Tnnn`), test tasks written first (test-first), updated as work proceeds | +| `research.md` | External research and codebase findings that ground the design | +| `data-model.md` | Schema/entity changes the spec requires | +| `contracts/` | The API contracts the spec must satisfy, as Given/When/Then tables — these become the contract tests | +| `checklists/` | Per-requirement validation checklists | +| `quickstart.md`, `HOST_TESTING*.md` | Operator runbooks and host-validation passes (real-mode tests cannot run in CI) | + +Related indexes: + +- [CONTRACT-INDEX.md](./CONTRACT-INDEX.md) — which contract tests pin which spec. +- [ADR index](../adr/README.md) — architecture decision records; cross-cutting + decisions that affect multiple specs live there, not in a spec. diff --git a/docs/specs/_template/README.md b/docs/specs/_template/README.md new file mode 100644 index 00000000..ce932c15 --- /dev/null +++ b/docs/specs/_template/README.md @@ -0,0 +1,37 @@ +# Spec Template + +Copy this directory to `docs/specs/NNN-short-slug/` (next free number — see the +[specs registry](../README.md#numbering-rules)) and fill in each file. Delete the +files you genuinely don't need, but keep the naming and the workflow. + +## Workflow: specify → plan → tasks → implement → validate + +1. **Specify** — write `spec.md`. Ground it in the codebase and in upstream OpenStack + behavior (`research.md` if external investigation is needed). Resolve open + questions in the spec's *Clarifications* section, not in the implementation. + Every requirement gets an **FR-NNN id** — code comments and commit messages cite + specs by these ids (e.g. `SPEC-021 FR-023c`), so they are mandatory. +2. **Plan** — write `plan.md`: the technical approach, phased sequencing, risks, and + a constitution check against the project principles. +3. **Tasks** — write `tasks.md`: concrete `Tnnn` tasks, grouped in phases. Test tasks + come first and must fail before implementation (test-first). +4. **Implement** — work the tasks in order, ticking them off in `tasks.md` as they + complete. Cite FR ids in code comments where a requirement shapes the code. +5. **Validate** — run the contract tests for your `contracts/`, the unit tests, and + (for real-mode features) the host passes recorded in `HOST_TESTING*.md` and + `checklists/`. The PR that merges the implementation moves the spec to + **implemented** in the registry. + +## Files + +| File | Purpose | Required | +|------|---------|----------| +| `spec.md` | The specification: problem, goals/non-goals, FR-NNN requirements, success criteria | yes | +| `contracts/` | The API contracts this spec must satisfy — Given/When/Then tables that become contract tests | yes | +| `plan.md` | Implementation plan: approach, phases, risks | yes | +| `tasks.md` | Task checklist, test tasks first | yes | +| `research.md` | External research / codebase findings | if research was needed | +| `data-model.md` | Schema and entity changes | if the schema changes | +| `checklists/` | Per-requirement validation checklists | recommended | +| `quickstart.md` | Operator quickstart for the feature | recommended | +| `HOST_TESTING.md` / `HOST_TESTING_REAL_MODE.md` | Host-validation runbooks (what cannot run in CI) | for real-mode features | diff --git a/docs/specs/_template/contracts/README.md b/docs/specs/_template/contracts/README.md new file mode 100644 index 00000000..68496959 --- /dev/null +++ b/docs/specs/_template/contracts/README.md @@ -0,0 +1,25 @@ +# Contracts: + +External interface = the exact OpenStack API surface this spec touches, exercised +via the unmodified clients that pin it (python-openstackclient, gophercloud, +Terraform provider, Horizon). All behavior must match upstream with no client +modification. Contract tests are authored **RED-first**, before implementation, +and live under `test/contract/` (CI) and `test/` (host runbooks). + +## Behavioral contracts + +<!-- One table per API area. IDs are cited by tasks.md and checklists/. --> + +| ID | Given | When | Then | +|----|-------|------|------| +| C-1 | ... | ... | ... | +| C-2 | ... | ... | ... | + +## Contract tests (to author RED-first) + +- `test/contract/<area>_test.go` (gophercloud): C-1, C-2 — what it asserts. +- `test/<area>_real_test.sh` (host, real mode): C-1 — what it asserts. + +## Notes + +<!-- Upstream quirks, deliberate divergences (and why), fixtures needed. --> diff --git a/docs/specs/_template/plan.md b/docs/specs/_template/plan.md new file mode 100644 index 00000000..2578fcd8 --- /dev/null +++ b/docs/specs/_template/plan.md @@ -0,0 +1,36 @@ +# Implementation Plan: <TITLE> + +**Branch**: `<branch>` | **Date**: <YYYY-MM-DD> | **Spec**: [spec.md](./spec.md) + +## Summary + +<!-- Two or three sentences: what is built and the technical approach. + Name the dominant finding from research (the shape that already exists + in the codebase vs. what must be created). --> + +## Technical Context + +<!-- Language/versions, primary dependencies, storage, testing strategy, + target platform, constraints (API compatibility, sync operations, + modes), scale/scope (packages touched, migrations, new deps). --> + +**Project Type**: Single Go binary — all services constructed in +`cmd/lightstack/main.go`; no microservices, no new services unless the spec says so. + +## Constitution Check + +<!-- Evaluate the plan against the project principles (OpenStack API compat, + synchronous operations, fail-fast, multi-mode, project isolation, ...). + One row per principle: PASS / PASS (with a fix) / FAIL + notes. --> + +## Phases / Task Sequencing + +<!-- Phase 1 (setup), Phase 2 (foundational blockers), then one phase per user + story in priority order. Each phase: purpose, what unblocks, independent + test. Task ids live in tasks.md. --> + +## Risks & Mitigations + +| Risk | Mitigation | +|------|------------| +| ... | ... | diff --git a/docs/specs/_template/spec.md b/docs/specs/_template/spec.md new file mode 100644 index 00000000..f357db39 --- /dev/null +++ b/docs/specs/_template/spec.md @@ -0,0 +1,95 @@ +# Feature Specification: <TITLE> + +**Feature Branch**: `<branch-name>` +**Created**: <YYYY-MM-DD> +**Status**: Draft +**Input**: <issue link / user description — the exact ask, unedited> + +## Clarifications + +<!-- Every question that changed the design, with the answer and the date. + Format: "Q: ... → A: ..." grouped per clarification session/round. --> + +### Session <YYYY-MM-DD> + +- Q: ... → A: ... + +## Problem + +<!-- What is broken or missing today, concretely. Cite code (file:line) and + upstream behavior. A reader who has not seen the issue must understand + the defect from this section alone. --> + +## Goals / Non-Goals + +<!-- Goals: what this spec delivers, each verifiable. + Non-Goals: what is explicitly out of scope — name adjacent work that + someone might mistake for this spec's job. --> + +**Goals**: + +1. ... + +**Non-Goals**: + +1. ... + +## User Scenarios & Testing *(mandatory)* + +<!-- How a user (CLI, SDK, Terraform, Horizon, operator) experiences the fix. + One user story per scenario, each with priority and the test that proves + it. Include edge cases. --> + +### User Story 1 - <title> (Priority: P1) + +... + +### Edge Cases + +- ... + +## Requirements *(mandatory)* + +### Functional Requirements + +<!-- Every requirement gets a stable FR id: FR-NNN, with sub-ids FR-NNNa when a + requirement refines another. These ids are how code comments, commit + messages, and contract tests cite this spec — do not renumber after review. + Use RFC 2119 language (MUST/SHOULD/MAY) and reference the exact upstream + OpenStack API behavior each requirement mirrors. --> + +- **FR-001**: The system MUST ... (matches upstream <service> behavior for + `<endpoint>` — see api-ref: <link>). +- **FR-001a**: ... MUST also ... . + +### OpenStack API References + +<!-- The upstream api-ref / client behavior each requirement is grounded in. + If a requirement deliberately deviates from upstream, say so here and why. --> + +- `<endpoint / field / error>` — <api-ref link or client source>, verified <date>. + +### Key Entities + +<!-- New or changed database entities/fields, if any. Full schema in data-model.md. --> + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +<!-- SC-NNN: observable, measurable pass conditions — what a reviewer runs to + declare the spec done. Map each back to the FRs and user stories. --> + +- **SC-001**: ... + +## Assumptions + +- ... + +## Dependencies + +<!-- Other specs, ADRs, migrations, or issues this work depends on or unblocks. --> + +## Out of Scope + +- ... diff --git a/docs/specs/_template/tasks.md b/docs/specs/_template/tasks.md new file mode 100644 index 00000000..018ef976 --- /dev/null +++ b/docs/specs/_template/tasks.md @@ -0,0 +1,39 @@ +# Tasks: <TITLE> + +**Input**: Design documents from `docs/specs/NNN-short-slug/` +**Prerequisites**: spec.md, plan.md, research.md, contracts/ + +**Tests**: REQUIRED. Write the test tasks first and confirm they FAIL before +implementing (test-first). Contract tests (gophercloud / OpenStack CLI) run in CI; +real-mode host tests are authored now and executed on the host per the +`HOST_TESTING*.md` runbooks. + +## Format: `[ID] [P?] [Story] Description with file path` + +- **[P]**: parallelizable (different files, no dependency on an incomplete task) +- **[Story]**: US1, US2, ... mapping to the user stories in spec.md + +--- + +## Phase 1: Setup (Shared Infrastructure) + +- [ ] T001 [P] ... (`path/to/file`) + +## Phase 2: Foundational (Blocking Prerequisites) + +- [ ] T002 ... (`path/to/file`) + +## Phase 3: User Story 1 — <title> 🎯 MVP + +### Tests (write first, MUST fail) ⚠️ + +- [ ] T003 [P] [US1] Contract test `test/contract/...` covering C-... from `contracts/` +- [ ] T004 [US1] Host test `test/...` per `HOST_TESTING.md` + +### Implementation + +- [ ] T005 [US1] ... (`path/to/file`) *(cite the FR id, e.g. FR-001, in the task)* + +<!-- Tick tasks off with [X] and append a short parenthetical of what was done + and how it was verified. Do not delete open items — leave host-gated + validation tasks open until the host pass runs. --> From af3967b53853f36156bb48cc63aa45b81b3c8048 Mon Sep 17 00:00:00 2001 From: Uwe Schwaeke <uwe.schwaeke@sap.com> Date: Wed, 14 Oct 2026 19:27:27 +0000 Subject: [PATCH 5/8] docs(adr): ADR-000 (when an ADR is required), index and template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Architectural-ADR side of the alignment chain. ADR-001 and ADR-002 existed by accident of history with no process defining when the form is required. ADR-000 sets the criteria — cross-cutting architectural decisions, hard-to-reverse choices, security/compliance posture changes, dependency changes — with ADR-001 (global DB pool to constructor injection) and ADR-002 (rate limiting architecture) as the worked examples, and clarifies that spec-local choices stay in the spec. The adr/README.md index gains the numbering convention (next free ADR-003), status values, and _TEMPLATE.md mirroring ADR-001's shape. Signed-off-by: Uwe Schwaeke <uwe.schwaeke@sap.com> --- docs/adr/ADR-000-when-to-write-an-adr.md | 50 ++++++++++++++++++++++++ docs/adr/README.md | 45 +++++++++++++++------ docs/adr/_TEMPLATE.md | 34 ++++++++++++++++ 3 files changed, 117 insertions(+), 12 deletions(-) create mode 100644 docs/adr/ADR-000-when-to-write-an-adr.md create mode 100644 docs/adr/_TEMPLATE.md diff --git a/docs/adr/ADR-000-when-to-write-an-adr.md b/docs/adr/ADR-000-when-to-write-an-adr.md new file mode 100644 index 00000000..d5c495cd --- /dev/null +++ b/docs/adr/ADR-000-when-to-write-an-adr.md @@ -0,0 +1,50 @@ +# ADR-000: When to Write an ADR + +**Status**: Accepted +**Date**: 2026-10-14 +**Deciders**: Lightstack Core Team + +## Context + +Lightstack records feature work in specs (`docs/specs/`, see the +[registry](../specs/README.md)) and cross-cutting decisions in ADRs. The boundary +between the two must be explicit, or every design choice inside a spec becomes an +unversioned argument in a tasks file, and the decisions that shaped the whole +binary are lost. + +## Decision + +Write an ADR when a decision is **any** of: + +1. **Architectural or cross-cutting** — it affects multiple services or the shape + of the binary itself (startup, wiring, process model, deployment topology). +2. **Hard to reverse** — changing it later costs a migration, a large refactor, or + an API break. +3. **A security or compliance posture change** — it alters how the system defends + itself or what it promises to clients and operators. +4. **Introduces or removes a dependency** — anything that changes what ships in + the single binary or what it requires at runtime. + +Decisions that are local to one spec — naming, internal structure, which handler +does what — stay in the spec's `spec.md` and its Clarifications. A spec whose +scope *contains* an ADR-worthy decision still needs the ADR; the spec implements +the decision, it does not own it (see ADR-001 ↔ SPEC-027). + +## Worked examples + +- **ADR-001 (database dependency injection)**: replacing the process-wide + `database.DB` global with constructor-injected pools touched 661 references + across 53 files and every service — cross-cutting and costly to reverse (criteria + 1 and 2). It surfaced in a code review (finding H-3), not inside any spec, and + was later implemented by a dedicated spec ([SPEC-027](../specs/027-database-pool-injection/spec.md)). +- **ADR-002 (rate limiting architecture)**: choosing two-tier, in-process, + standard-library rate limiting changed the system's abuse-resistance posture + (criterion 3) and ruled out whole design families (Redis-backed, per-endpoint) + for the single-binary architecture (criterion 1). + +## Consequences + +- Specs can evolve task-by-task while the decisions beneath them stay citable and + stable; code comments and specs link decisions via `ADR-NNN`. +- If a later change reverses an ADR, write a new ADR that supersedes it — never + edit the old decision in place. diff --git a/docs/adr/README.md b/docs/adr/README.md index 697eedcf..f6d10b34 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -1,20 +1,41 @@ # Architecture Decision Records -This directory contains Architecture Decision Records (ADRs) for the Lightstack project. +This directory holds Lightstack's Architecture Decision Records (ADRs) — short, +immutable documents that capture **why** a significant technical decision was made, +not just what it is. + +An ADR is required when a decision meets the criteria in +[ADR-000](ADR-000-when-to-write-an-adr.md) (architectural/cross-cutting, hard to +reverse, a security/compliance posture change, or a dependency change). Decisions +that are local to one spec stay in that spec's `spec.md`. ## Format -Each ADR follows this structure: -- **Title**: Short descriptive name -- **Status**: Proposed | Accepted | Deprecated | Superseded -- **Context**: What is the issue we're facing? -- **Decision**: What did we decide and why? -- **Alternatives Considered**: What else did we evaluate? -- **Consequences**: What are the positive and negative outcomes? +ADRs are numbered `ADR-NNN` (next free number: **003**), one file each: +`ADR-NNN-short-title.md`, created from [`_TEMPLATE.md`](_TEMPLATE.md). Once an ADR +is Accepted its Decision section does not get rewritten — later changes are +recorded as amendments inside the document, or as a new ADR that supersedes it. + +**Status** values: + +- **Proposed** — written, not yet ratified; open for review. +- **Accepted** — ratified; the recorded decision is the project's direction. +- **Superseded** — replaced by a later ADR; the header names the successor. ## Index -| ADR | Title | Status | Date | -|-----|-------|--------|------| -| [ADR-001](ADR-001-database-dependency-injection.md) | Database Dependency Injection | Accepted | 2026-04-20 | -| [ADR-002](ADR-002-rate-limiting-architecture.md) | Rate Limiting Architecture | Accepted | 2026-04-20 | +| ADR | Title | Status | Date | One line | +|-----|-------|--------|------|----------| +| [ADR-000](ADR-000-when-to-write-an-adr.md) | When to write an ADR | Accepted | 2026-10-14 | The process ADR: what makes a decision ADR-worthy | +| [ADR-001](ADR-001-database-dependency-injection.md) | Database Dependency Injection | Accepted | 2026-04-20 | `*pgxpool.Pool` through service constructors instead of a global — implemented by [SPEC-027](../specs/027-database-pool-injection/spec.md) | +| [ADR-002](ADR-002-rate-limiting-architecture.md) | Rate Limiting Architecture | Accepted | 2026-04-20 | Two-tier in-process rate limiting (`golang.org/x/time/rate`, IP + project) — not yet implemented | + +## Relationship to specs + +ADRs record the *decision*; the spec that implements it records the *how* and the +task breakdown. The link is bidirectional: the ADR's **Spec** header points at the +implementing spec (or specs), and the spec's Input/Context cites the ADR. Examples: +ADR-001 ↔ SPEC-006 / SPEC-027. + +See also the [specs registry](../specs/README.md) and the +[contract index](../specs/CONTRACT-INDEX.md). diff --git a/docs/adr/_TEMPLATE.md b/docs/adr/_TEMPLATE.md new file mode 100644 index 00000000..ff751bd0 --- /dev/null +++ b/docs/adr/_TEMPLATE.md @@ -0,0 +1,34 @@ +# ADR-NNN: <Short descriptive title> + +**Status**: Proposed | Accepted | Superseded (by ADR-NNN) +**Date**: YYYY-MM-DD +**Deciders**: <who ratified it> +**Spec**: <optional — link to the spec (or specs) that implement this decision, +e.g. [SPEC-006](../specs/006-v070-foundation/README.md)> + +## Context + +What is the issue we're facing? Cite evidence (review findings, code, metrics) +and the constraints that narrow the options. State which criteria from +[ADR-000](ADR-000-when-to-write-an-adr.md) make this an ADR. + +## Decision + +The decision, stated in one bold sentence up front, then the details: the shape +of the change, migration strategy, and configuration if any. + +## Alternatives Considered + +For each serious alternative: what it is, and the concrete reason it was +rejected. + +## Consequences + +### Positive +- ... + +### Negative +- ... + +### Neutral +- ... From faf9dd858cf19b8d75338bade513c0fba311eb36 Mon Sep 17 00:00:00 2001 From: Uwe Schwaeke <uwe.schwaeke@sap.com> Date: Wed, 14 Oct 2026 19:27:27 +0000 Subject: [PATCH 6/8] docs(specs): SPEC-to-contract-test cross-index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the Specifications -> Contracts link issue #16 asks for and doubles as the gap list for issue #105 (testing). Built from test-source evidence (grep for SPEC-/FR-/US- citations in test/contract/), not assumption: inferred mappings are marked as such. Findings it surfaces: 20 of 97 contract test files cite a spec (Cinder 0/18, Glance 0/7 — the largest #105 back-fill clusters); 19 of 28 spec dirs have no pinning tests (expected for future services and non-API-observable infra specs, genuine gaps for the implemented SPEC-015/019/022 and the four unpinned contract areas of SPEC-024); the duplicate SPEC-025 dirs share FR numbering, so future citations should use the SPEC-025-followup form. Includes the maintenance commands to regenerate the counts. Signed-off-by: Uwe Schwaeke <uwe.schwaeke@sap.com> --- docs/specs/CONTRACT-INDEX.md | 262 +++++++++++++++++++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 docs/specs/CONTRACT-INDEX.md diff --git a/docs/specs/CONTRACT-INDEX.md b/docs/specs/CONTRACT-INDEX.md new file mode 100644 index 00000000..d8db2384 --- /dev/null +++ b/docs/specs/CONTRACT-INDEX.md @@ -0,0 +1,262 @@ +# SPEC ↔ Contract-Test Cross-Index + +This index maps every specification in `docs/specs/NNN-*/` to the gophercloud-based +contract tests in `test/contract/` that pin it, and — in the other direction — +every contract test file to the spec it cites (if any). It exists to close the +"Specifications → Contracts" link required by issue #16, and to serve as the +scoping tool for issue #105 (testing gaps): every test file with a `—` in Table 2 +is unspec'd coverage, and every implemented, API-observable spec with no pinning +test in Table 1 is a genuine gap. + +Everything below is generated from test-source evidence (header-comment +citations of the form `SPEC-NNN FR-xxx`), not from memory. The exact commands +used are in [Maintenance](#maintenance) so this file can be regenerated and +kept current. Where a mapping is inferred from file/topic naming rather than an +explicit citation, it is marked **(inferred)**. + +> **Ambiguity note — the duplicate SPEC-025.** Two spec dirs share the number +> 025: `025-followup` and `025-nova-gaps-client-discovery`. Both were +> implemented on branch `feature/547` from cloud-storage#547, and both use +> overlapping FR numbering (FR-001…FR-056) and US1–US9 user stories, so a bare +> `SPEC-025` citation is ambiguous on its face. In practice every test +> citation disambiguates itself with a `cloud-storage#547` or finding-ID +> reference (B4, B7, B13, B14, B16, F16) that belongs to +> **025-nova-gaps-client-discovery**; all SPEC-025 rows below are attributed to +> that dir. `025-followup` has no citation that distinguishes it. + +--- + +## Table 1: Spec → contract tests + +| Spec (dir) | Name | `test/contract/` file(s) | Requirement coverage | Notes | +|---|---|---|---|---| +| SPEC-000 (`000-api-compliance`) | OpenStack API Compliance Framework | — (the suite as a whole) | — | Meta-spec: the entire `test/contract/` suite is its deliverable **(inferred)**; no test cites SPEC-000 directly. | +| SPEC-001 (`001-modular-architecture`) | Modular Architecture Transformation | — | — | Architecture spec, not API-observable via gophercloud. | +| SPEC-002 (`002-authentication-enhancement`) | Enhanced Authentication System | `nova/rbac_test.go` | T048 (task ref) | Historical origin only: the file header says the tests are "originally T048 of SPEC-002" and that the remaining SPEC-002 tests were deliberately **not** revived. Effectively superseded by SPEC-023. | +| SPEC-003 (`003-barbican`) | Barbican — Key Management | — | — | Future service (not implemented); expected empty. | +| SPEC-004 (`004-designate`) | Designate — DNS as a Service | — | — | Future service (not implemented); expected empty. | +| SPEC-005 (`005-octavia`) | Octavia — Load Balancing | — | — | Future service (not implemented); expected empty. | +| SPEC-006 (`006-v070-foundation`) | v0.7.0 Foundation | — | — | Architecture/quality spec, not API-observable. | +| SPEC-007 (`007-helm-prereqs-bootstrap-catalog`) | Helm Prerequisites — Bootstrap + Catalog | — | — | Helm-level; verified by unit/render tests, not gophercloud contract tests. | +| SPEC-008 (`008-helm-charts`) | Helm Charts for Kubernetes Deployment | — | — | Helm-level; verified by chart/unit tests. | +| SPEC-009 (`009-single-node-real-mode`) | Single-Node Real Mode (v1) | `ws0_disk_test.go` (WS-0), `ws4_userdata_test.go` (WS-4), `ws5_volume_test.go` (WS-5) | WS-0: FR-003, FR-004; WS-5: FR-021, FR-022; WS-4: (workstory ref only) | Verified in both directions: the spec's `contracts/compute-disk-lifecycle.md`, `contracts/metadata-cloudinit.md`, and `contracts/block-storage.md` each name the corresponding test file. | +| SPEC-010 (`010-helm-teardown`) | Teardown via Helm Pre-Delete Hooks | — | — | Helm/host-level; verified by unit + host e2e, not contract tests. | +| SPEC-011 (`011-ensure-storage-subdirs`) | Ensure Node-Local Storage Directories | — | — | Host-level, not API-observable. | +| SPEC-012 (`012-integration-contract`) | Integration Contract (Values Schema + Status) | — | — | Helm values/status conventions, not API-observable via gophercloud. | +| SPEC-013 (`013-readiness-liveness-probes`) | Readiness/liveness endpoints | — | — | k8s health endpoints; verified outside `test/contract/`. | +| SPEC-014 (`014-ceph-rgw-s3-backend`) | Ceph RGW (S3) — Glance backend & tenant object storage | `keystone/ec2_authz_test.go`, `keystone/ec2_credentials_test.go` | FR-005, FR-007 | Only the EC2-credentials (S3-auth) slice is contract-pinned; the RGW/Glance-backend slice is host-verified. | +| SPEC-015 (`015-identity-uuid-404`) | Identity resource not-found for malformed ids | — | — | **Genuine gap.** API-observable (Keystone 404 shape), implemented, but no contract test cites it. Conformance was verified once via host testing (`CONFORMANCE.md`, DevStack comparison #73), not pinned by a regression test. Candidate for #105. | +| SPEC-016 (`016-terraform-openstack-provider-verification`) | Verify Terraform OpenStack-provider compatibility | — | — | Verification campaign (manual + integration), not a behavior spec; no dedicated contract tests expected. `server_readback_test.go` grew out of this work **(inferred)** but cites only SPEC-017. | +| SPEC-017 (`017-nova-server-detail-readback`) | Nova server-detail read-back | `server_readback_test.go` | finding F1 (spec ref) | Spec's `contracts/nova-server-detail.md` also names `test/contract/` (gophercloud) as the verifying layer — mapping verified in both directions. | +| SPEC-018 (`018-horizon-dashboard-verification`) | Verify Horizon dashboard compatibility | — | — | Verification campaign (browser-level report); not contract-test-pinned by design. | +| SPEC-019 (`019-horizon-compat-api-fixes`) | Horizon compat — Cinder/Neutron/Glance API fixes (F4, F5, F6, F9) | — | — | **Genuine gap.** API-observable, implemented. Its four contracts (cinder-volume-response, glance-image-delete, keystone-catalog-endpoints, neutron-network-list-filters) are plausibly exercised incidentally by `cinder/volume_lifecycle_test.go`/`cinder/update_test.go`, `glance/image_lifecycle_test.go`, `keystone/service_catalog_test.go`, and `neutron/network_lifecycle_test.go` **(inferred)** — but nothing cites SPEC-019. Candidate for #105. | +| SPEC-020 (`020-horizon-create-ui-enablement`) | Horizon compat — create/write UI enablement (F2, F3, F7, F8) | `nova/image_fixture_test.go` | FR-026 | Only the Nova image-fixture slice is cited. The Cinder/Neutron quota-and-extensions contracts are plausibly covered by `cinder/quotas_test.go`, `neutron/quotas_test.go`, `neutron/extensions_test.go`, `nova/limits_test.go` **(inferred)**. | +| SPEC-021 (`021-real-novnc-console`) | Horizon compat — a real console / noVNC (F1) | `nova/console_test.go` | FR-005, FR-025, FR-026 | Header notes the tests were rewritten with SPEC-021; one behaviour (serial-log fallback) is deliberately unchanged. | +| SPEC-022 (`022-project-tenant-isolation`) | Project/Tenant Lifecycle & Isolation Fixes | — | — | **Gap (soft).** `contracts/keystone-domain-resolution.md` requires "test/contract/keystone/ (existing, must stay green)" — a reverse pin on the uncited keystone suite, but no test cites SPEC-022. Candidate for #105. | +| SPEC-023 (`023-tenant-scoping-authz`) | Tenant Scoping & Authorization Model | `nova/rbac_test.go` | FR-008a | Reverse pin too: `contracts/usage-and-filters.md` mandates that `nova/tenant_usage_test.go` "MUST continue to pass unchanged" — that file cites nothing itself. `keystone/identity_links_test.go` mentions SPEC-023 only as runbook context, not as a pin. | +| SPEC-024 (`024-neutron-api-fidelity-realmode`) | Neutron API Fidelity & Real-Mode Networking | `neutron/error_envelope_test.go`, `neutron/network_lifecycle_test.go` | US2/B10: FR-010; F11: FR-035 | Only 2 of the spec's ~6 contract areas (error envelope, external networks) are pinned; port/DHCP, subnet listing, security-group shape, ownership are unpinned. | +| SPEC-025 (`025-nova-gaps-client-discovery`) | Nova Gaps & Client Discovery (#547) | `nova/version_document_test.go`, `keystone/identity_links_test.go`, `nova/boot_network_failure_test.go`, `nova/action_by_name_test.go`, `nova/server_addresses_test.go`, `nova/limits_usage_test.go`, `nova/deleted_action_history_test.go`, `nova/boot_from_new_volume_test.go`, `nova/selflink_absolute_test.go` | US1: FR-002; US2: FR-007/010/011/012/013; US3: FR-016a, FR-017; US5: FR-029/030/032/033; US6: FR-035, FR-036; US7: FR-040, FR-041, FR-043/043a/043b; US9: FR-049/050/054/055/056; plus FR-007 (selflinks) | Best-pinned spec in the repo (9 test files). Spec's `contracts/` dir names the same test areas — mapping verified in both directions. | +| SPEC-025 (`025-followup`) | SPEC-025 Follow-ups (pseudo-ticket) | — | — | No citation distinguishes it from its parent spec (see ambiguity note above). If a follow-up FR is pinned anywhere, it is silently counted under `025-nova-gaps-client-discovery`. | +| SPEC-027 (`027-database-pool-injection`) | Inject the DB pool through service constructors (ADR-001) | — | — | Internal architecture; verified by unit tests, not API-observable. | + +--- + +## Table 2: Contract tests by service (issue #105 gap list) + +97 `*_test.go` files under `test/contract/` (plus two non-test `helpers.go`). +Files with `—` are unspec'd coverage — candidates for #105 back-filling. + +### Nova (35 files, 11 cited) + +| Test file | Spec citation | +|---|---| +| `nova/action_by_name_test.go` | SPEC-025 (US5, FR-029/030/032/033) | +| `nova/advanced_server_actions_test.go` | — | +| `nova/aggregates_test.go` | — | +| `nova/availability_zones_test.go` | — | +| `nova/boot_from_new_volume_test.go` | SPEC-025 (US9, FR-049/050/054/055/056) | +| `nova/boot_network_failure_test.go` | SPEC-025 (US3, FR-016a/017) | +| `nova/console_test.go` | SPEC-021 (FR-005/025/026) | +| `nova/deleted_action_history_test.go` | SPEC-025 (US7, FR-043/043a/043b) | +| `nova/diagnostics_test.go` | — | +| `nova/flavor_actions_test.go` | — | +| `nova/flavor_extra_specs_keys_test.go` | — | +| `nova/flavor_extra_specs_test.go` | — | +| `nova/flavors_negative_test.go` | — | +| `nova/flavors_test.go` | — | +| `nova/hypervisors_test.go` | — | +| `nova/image_fixture_test.go` | SPEC-020 (FR-026) | +| `nova/instance_actions_test.go` | — | +| `nova/limits_test.go` | — | +| `nova/limits_usage_test.go` | SPEC-025 (US7, FR-040/041) | +| `nova/metadata_test.go` | — | +| `nova/migration_test.go` | — | +| `nova/migrations_list_test.go` | — | +| `nova/rbac_test.go` | SPEC-002 (T048, historical) + SPEC-023 (FR-008a) | +| `nova/security_group_actions_test.go` | — | +| `nova/selflink_absolute_test.go` | SPEC-025 (FR-007) | +| `nova/server_actions_test.go` | — | +| `nova/server_addresses_test.go` | SPEC-025 (US6, FR-035/036) | +| `nova/server_groups_test.go` | — | +| `nova/server_lifecycle_test.go` | — | +| `nova/server_tags_test.go` | — | +| `nova/server_update_test.go` | — | +| `nova/servergroups_test.go` | — | +| `nova/services_test.go` | — | +| `nova/tenant_usage_test.go` | — (reverse-pinned by SPEC-023 `contracts/usage-and-filters.md`: must keep passing) | +| `nova/version_document_test.go` | SPEC-025 (US1, FR-002) | + +### Keystone (14 files, 3 cited) + +| Test file | Spec citation | +|---|---| +| `keystone/application_credentials_test.go` | — | +| `keystone/credentials_test.go` | — | +| `keystone/domains_test.go` | — | +| `keystone/ec2_authz_test.go` | SPEC-014 (FR-005/007) | +| `keystone/ec2_credentials_test.go` | SPEC-014 (FR-005/007) | +| `keystone/groups_test.go` | — | +| `keystone/identity_links_test.go` | SPEC-025 (US2, FR-007/010/011/012/013) | +| `keystone/password_test.go` | — | +| `keystone/projects_test.go` | — | +| `keystone/role_assignments_test.go` | — | +| `keystone/roles_test.go` | — | +| `keystone/service_catalog_test.go` | — (plausibly exercises SPEC-019 `keystone-catalog-endpoints` **(inferred)**) | +| `keystone/services_test.go` | — | +| `keystone/users_test.go` | — (reverse-pinned as a class by SPEC-022 `keystone-domain-resolution.md`: "existing, must stay green") | + +### Neutron (18 files, 2 cited) + +| Test file | Spec citation | +|---|---| +| `neutron/address_scopes_test.go` | — | +| `neutron/auto_allocated_topology_test.go` | — | +| `neutron/availability_zones_test.go` | — | +| `neutron/error_envelope_test.go` | SPEC-024 (US2/B10, FR-010) | +| `neutron/extensions_test.go` | — (plausibly exercises SPEC-020 quota/extensions contracts **(inferred)**) | +| `neutron/l3_agents_test.go` | — | +| `neutron/metering_test.go` | — | +| `neutron/network_ip_availability_test.go` | — | +| `neutron/network_lifecycle_test.go` | SPEC-024 (F11, FR-035); plausibly also SPEC-019 network-list-filters **(inferred)** | +| `neutron/port_forwarding_test.go` | — | +| `neutron/qos_test.go` | — | +| `neutron/quotas_test.go` | — (plausibly exercises SPEC-020 `neutron-quotas-and-extensions` **(inferred)**) | +| `neutron/rbac_test.go` | — (topic-adjacent to SPEC-023, but no citation) | +| `neutron/routers_test.go` | — | +| `neutron/service_providers_test.go` | — | +| `neutron/subnet_pools_test.go` | — | +| `neutron/topology_test.go` | — | +| `neutron/trunks_test.go` | — | + +### Cinder (18 files, 0 cited) + +| Test file | Spec citation | +|---|---| +| `cinder/availability_zones_test.go` | — | +| `cinder/backups_test.go` | — | +| `cinder/groups_test.go` | — | +| `cinder/limits_test.go` | — | +| `cinder/manage_test.go` | — | +| `cinder/qos_specs_test.go` | — | +| `cinder/quotas_test.go` | — (plausibly exercises SPEC-020 `cinder-extensions-and-quota-sets` **(inferred)**) | +| `cinder/services_test.go` | — | +| `cinder/snapshot_metadata_test.go` | — | +| `cinder/snapshot_update_test.go` | — | +| `cinder/update_test.go` | — (plausibly exercises SPEC-019 `cinder-volume-response` **(inferred)**) | +| `cinder/volume_actions_advanced_test.go` | — | +| `cinder/volume_lifecycle_test.go` | — (plausibly exercises SPEC-019 `cinder-volume-response` **(inferred)**) | +| `cinder/volume_metadata_test.go` | — | +| `cinder/volume_transfers_test.go` | — | +| `cinder/volume_type_access_test.go` | — | +| `cinder/volume_type_extra_specs_test.go` | — | +| `cinder/volume_types_test.go` | — | + +### Glance (7 files, 0 cited) + +| Test file | Spec citation | +|---|---| +| `glance/cache_test.go` | — | +| `glance/image_data_test.go` | — | +| `glance/image_lifecycle_test.go` | — (plausibly exercises SPEC-019 `glance-image-delete` **(inferred)**) | +| `glance/import_test.go` | — | +| `glance/members_test.go` | — | +| `glance/metadefs_test.go` | — | +| `glance/tasks_test.go` | — | + +### Top level (5 files, 4 cited) + +| Test file | Spec citation | +|---|---| +| `server_readback_test.go` | SPEC-017 (finding F1) | +| `template_test.go` | — (documentation template, not a live contract) | +| `ws0_disk_test.go` | SPEC-009 (WS-0, FR-003/004) | +| `ws4_userdata_test.go` | SPEC-009 (WS-4) | +| `ws5_volume_test.go` | SPEC-009 (WS-5, FR-021/022) | + +--- + +## Summary counts + +- **20 of 97** contract test files cite a spec (verified by + `grep -rlE 'SPEC-[0-9]' test/contract/ --include='*_test.go' | wc -l`). + By service: Nova 11/35, Keystone 3/14, Neutron 2/18, top level 4/5, + **Cinder 0/18, Glance 0/7**. +- **19 of 28** spec dirs have no pinning contract tests: + 000, 001, 003, 004, 005, 006, 007, 008, 010, 011, 012, 013, 015, 016, 018, + 019, 022, 025-followup, 027. + Of those, absence is **expected** for: + - future services: **003, 004, 005** (Barbican, Designate, Octavia — not implemented); + - non-API-observable / infra specs: **001, 006, 007, 008, 010, 011, 012, 013, 027**; + - verification campaigns rather than behavior specs: **016, 018**; + - the meta-spec **000**, whose deliverable is the suite itself. +- **Genuine gaps** (implemented, API-observable specs with no pinning contract + test — the #105 back-fill list on the spec side): + - **SPEC-015** (identity not-found shape) — verified once by hand, never pinned. + - **SPEC-019** (Horizon-compat Cinder/Neutron/Glance/Keystone fixes) — likely + incidentally covered, never cited. + - **SPEC-022** (project/tenant isolation) — reverse-pinned only ("existing + tests must stay green"), never cited. + - **SPEC-025-followup** — indistinguishable from its parent spec in citations. +- **Unspec'd test clusters** (the #105 back-fill list on the test side): + all of **Cinder (18 files)** and **Glance (7 files)**, 16 of 18 Neutron files, + 11 of 14 Keystone files, and 24 of 35 Nova files carry no spec citation. + +--- + +## Maintenance + +This index is generated from test-source evidence. To refresh it: + +```sh +# All spec citations with file:line (Table 1 / citation column source): +grep -rnoE 'SPEC-[0-9]{3}' test/contract/ --include='*.go' | sort | uniq -c | sort -rn + +# Requirement-level citations (FR coverage column): +grep -rnoE 'FR-[0-9]+[a-z]?' test/contract/ --include='*.go' | sort + +# Count of test files citing a spec vs total (Summary counts): +grep -rlE 'SPEC-[0-9]' test/contract/ --include='*_test.go' | wc -l +find test/contract -name '*_test.go' | wc -l + +# Per-service breakdown: +for s in cinder glance keystone neutron nova; do + echo "$s: $(ls test/contract/$s/*_test.go | wc -l) test files, $(grep -rlE 'SPEC-[0-9]' test/contract/$s/*_test.go 2>/dev/null | wc -l) cited" +done + +# Reverse direction — which spec contracts reference test files: +grep -rn 'test/contract' docs/specs/0*/contracts/*.md +``` + +Guidance: + +- **New contract tests must cite the spec they pin** in their header comment, + in the form `// SPEC-NNN FR-xxx` (see `nova/console_test.go` or + `keystone/ec2_credentials_test.go` for the pattern). A test without a + citation is invisible to this index and counts as unspec'd coverage. +- When adding the citation, update Table 1 (spec row), Table 2 (file row), and + the summary counts in the same change. +- Beware the duplicate **SPEC-025** number: cite follow-up work as + `SPEC-025-followup FR-xxx` (or reference its source finding) so it can be + distinguished from `025-nova-gaps-client-discovery`. +- Inferred mappings (marked **(inferred)** above) should be replaced with real + citations as they are confirmed — do not add new inferred rows. From b38fc63248351ecb0ae5a6567183f12a2470479e Mon Sep 17 00:00:00 2001 From: Uwe Schwaeke <uwe.schwaeke@sap.com> Date: Mon, 2 Nov 2026 10:26:42 +0000 Subject: [PATCH 7/8] docs: correct storage and Terraform claims; coordinate spec numbering Review findings (comprehensive review of the open senolcolak PRs): - MEDIUM: CLAUDE.md gave Cinder an S3 backend and listed rbd,s3 as valid; only Glance supports S3 and rbd,s3 is rejected at startup. - MEDIUM: CLAUDE.md asserted '100% Terraform compatibility' and 'zero modifications', contradicted by the repo's own compatibility report and blueprint. Hedge to the validated topology and link the report. - MEDIUM: 'next free spec number is 028' collides with in-flight #107 and #108/#109. Freeze 028, name the claimants, and keep the later-merge renumber rule (no renumbering here). Flag #107's plan.md-only spec. - LOW: state Go 1.27.x and defer to go.mod, so #107's 1.27.3 bump does not instantly stale the floor. - LOW: registry PR citations #515-#549 are cloud-storage PRs, not this repo's; prefix them. SPEC-019 cites commit 0a4dde0. - LOW: AGENTS.md pointed the DB tests at the shared postgres database; standardize on lightstack_test. Signed-off-by: Uwe Schwaeke <uwe.schwaeke@sap.com> --- AGENTS.md | 4 +++- CLAUDE.md | 31 +++++++++++++++------------ docs/CONTRIBUTING.md | 2 +- docs/specs/README.md | 39 ++++++++++++++++++++++++---------- docs/specs/_template/README.md | 5 +++-- 5 files changed, 53 insertions(+), 28 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9697fa40..550a3f50 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,7 +62,9 @@ so never point it at anything shared: ```bash make db-up # start dev PostgreSQL in Docker -LIGHTSTACK_TEST_DB_URL=postgres://lightstack:secret@localhost:5432/postgres make test +# create the scratch database once (the harness migrates it but does not create it) +docker exec lightstack-postgres psql -U lightstack -c 'CREATE DATABASE lightstack_test' +LIGHTSTACK_TEST_DB_URL=postgres://lightstack:secret@localhost:5432/lightstack_test make test make migrate # apply migrations (golang-migrate) ``` diff --git a/CLAUDE.md b/CLAUDE.md index dc0971d1..e36af1e0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,12 +4,12 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project Overview -Lightstack is a lightweight OpenStack implementation in Go, inspired by K3s. It provides a single ~35MB binary that implements the five core OpenStack services (Keystone, Nova, Neutron, Cinder, Glance), a Placement stub, and an EC2-compatible metadata service, with 100% Terraform compatibility. +Lightstack is a lightweight OpenStack implementation in Go, inspired by K3s. It provides a single ~35MB binary that implements the five core OpenStack services (Keystone, Nova, Neutron, Cinder, Glance), a Placement stub, and an EC2-compatible metadata service. Terraform compatibility is **validated for one topology**, not literally 100% — see the qualifiers below. **Core Philosophy:** -- **Terraform Compatibility First**: Users can use existing Terraform OpenStack provider scripts unchanged +- **Terraform Compatibility First**: The unmodified Terraform OpenStack provider is validated for compute, networks, security groups, and volumes on the sc007 topology; see `docs/compatibility/TERRAFORM_COMPATIBILITY_REPORT.md` and `docs/blueprint.md` §6 for the scope, the port-id workaround, and the object-storage exclusion - **UI/CLI Compatibility**: OpenStack CLI works identically; Horizon's dashboard is **usable for resource management**, with its open UI gaps listed in `docs/compatibility/HORIZON_COMPATIBILITY_REPORT.md` -- **Drop-in Replacement**: Zero modifications needed to migrate from OpenStack to Lightstack +- **Drop-in Replacement**: Migrating from OpenStack is designed to be low-friction, but the Terraform path is not literally zero-modification — it needs a one-line port-id workaround, is validated on one real-mode (non-CI) topology, and does not serve Terraform's Swift-based object storage; see `docs/blueprint.md` §6 and the [Terraform compatibility report](docs/compatibility/TERRAFORM_COMPATIBILITY_REPORT.md) - **Synchronous Operations**: No message queues - operations complete before API returns - **Fail-Fast Design**: External dependency failures return immediately (< 1 second timeouts) - **Multi-mode Support**: Each service supports stub mode (development) and real mode (production) @@ -138,7 +138,7 @@ Lightstack runs as a single process (`cmd/lightstack/main.go`) that starts seven - **Keystone** (35357): Identity service, JWT-based authentication - **Nova** (8774): Compute service, VM lifecycle via libvirt - **Neutron** (9696): Network service, namespace isolation via netlink -- **Cinder** (8776): Block storage, multi-backend support (local/RBD/S3) +- **Cinder** (8776): Block storage, multi-backend support (local/RBD) - **Glance** (9292): Image service, multi-backend with hybrid failover - **Placement** (8778): Stub of the placement service - **Metadata** (8775): EC2-compatible metadata service (no auth) @@ -193,9 +193,10 @@ test/ # Test suites - Check mode via `libvirtMode` or `networkingMode` fields in service structs **Multi-backend Storage Pattern** (Cinder/Glance): -- Storage modes specified as comma-separated strings: `"local"`, `"rbd"`, `"s3"`, `"local,rbd"`, `"local,s3"`, `"rbd,s3"` -- First backend is primary, subsequent are fallback -- Implemented in `pkg/storage/` with `ImageStore` and backend-specific implementations +- **Cinder**: `stub`, `local`, `rbd`, `local,rbd` only. S3 is **not** a Cinder backend. +- **Glance**: `stub`, `local`, `rbd`, `s3`, `local,rbd`, `local,s3` only. `rbd,s3` is **rejected at startup** (Glance has no RBD backend, so the S3 replica would never be written). +- Storage modes are specified as comma-separated strings; the first backend is primary, subsequent ones are fallback +- Implemented in `pkg/storage/` with `ImageStore` and backend-specific implementations; see `docs/STORAGE_MODES.md` for the authoritative matrix **Synchronous Operations**: - All API operations complete before returning (no async state machines) @@ -268,10 +269,14 @@ Requires Linux with appropriate dependencies. - Uses `github.com/vishvananda/netlink` for namespace/bridge creation - iptables via `github.com/coreos/go-iptables` for security groups -**Cinder/Glance real mode** (`storage_mode: local`, `rbd`, `s3`, or hybrid): -- `local`: Host filesystem storage (node-local, under the `state_path` root) -- `rbd`: Ceph RBD via `github.com/ceph/go-ceph` -- `s3`: AWS S3/MinIO/Ceph RGW via `github.com/aws/aws-sdk-go-v2` +**Cinder/Glance real mode** (`storage_mode` varies by service): +- **Cinder** (`local`, `rbd`, or `local,rbd`): + - `local`: Host filesystem storage (node-local, under the `state_path` root) + - `rbd`: Ceph RBD via `github.com/ceph/go-ceph` +- **Glance** (`local`, `rbd`, `s3`, `local,rbd`, or `local,s3`): + - `local`: Host filesystem storage (node-local, under the `state_path` root) + - `rbd`: Ceph RBD via `github.com/ceph/go-ceph` + - `s3`: AWS S3/MinIO/Ceph RGW via `github.com/aws/aws-sdk-go-v2` (Glance only; `rbd,s3` is rejected at startup) ## Database Schema @@ -336,8 +341,8 @@ Configuration loaded from `config/lightstack.yaml` (or via `--config` flag). - `keystone.admin_password`: overrides the seeded `admin`/`secret` default — set it for any exposed deployment - `nova.libvirt_mode`: `stub` (dev) or `real` (prod) - `neutron.networking_mode`: `stub`, `iptables`, or `ebpf` -- `cinder.storage_mode`: `stub`, `local`, `rbd`, `s3`, or hybrid (comma-separated) -- `glance.storage_mode`: `stub`, `local`, `rbd`, `s3`, or hybrid (comma-separated) +- `cinder.storage_mode`: `stub`, `local`, `rbd`, or `local,rbd` (Cinder has no S3 backend) +- `glance.storage_mode`: `stub`, `local`, `rbd`, `s3`, `local,rbd`, or `local,s3` (`rbd,s3` is rejected at startup) - `state_path`: node-local state root (default `/var/lib/lightstack`; resolved via `pkg/paths`) **Dead config — do not rely on it**: the `rate_limiting.*` block (`enabled`, `auth_rate`, `api_rate`, …) appears in `config/lightstack.yaml` and in ADR-002's design, but **no code reads it** — there is no `RateLimiting` field in the `Config` struct (`internal/common/config.go`) and no rate-limiting middleware. Treat it as unimplemented until a spec lands it. diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index a5fe3570..a032158e 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -64,7 +64,7 @@ Your git identity must be configured to the sap.com address (`git config user.em ### Prerequisites **Required:** -- Go 1.27.2 or higher (the version pinned in [`go.mod`](../go.mod)) +- Go 1.27.x — use the exact version pinned in [`go.mod`](../go.mod) - PostgreSQL 18 or higher - Docker (for `make db-up` and the container-based test stacks) - Git diff --git a/docs/specs/README.md b/docs/specs/README.md index a6e2323e..012eba9b 100644 --- a/docs/specs/README.md +++ b/docs/specs/README.md @@ -42,16 +42,16 @@ Statuses: `draft`, `accepted`, `implemented`, `superseded`, `archived` (see | 012 | [Integration Contract](./012-integration-contract/spec.md) | Helm values schema + status conventions | implemented | Merged (#69) | | 013 | [Readiness/Liveness Probes](./013-readiness-liveness-probes/spec.md) | Honest readiness/liveness endpoints for k8s-native heartbeating | implemented | Merged (#70); `internal/health/` exists; 10 code references | | 014 | [Ceph RGW (S3) Backend](./014-ceph-rgw-s3-backend/spec.md) | Glance image backend + tenant object storage on Ceph RGW | implemented | Merged; 31 code references | -| 015 | [Identity UUID 404](./015-identity-uuid-404/spec.md) | Return 404 (not 500) for malformed identity resource ids | implemented | Merged (#515) | -| 016 | [Terraform Provider Verification](./016-terraform-openstack-provider-verification/README.md) | Verify unmodified terraform-provider-openstack apply/destroy | implemented | Harness + report merged (#481); `docs/compatibility/TERRAFORM_COMPATIBILITY_REPORT.md` | -| 017 | [Nova Server-Detail Read-back](./017-nova-server-detail-readback/spec.md) | Terraform-compat server-detail fields (`key_name`, `security_groups`) | implemented | Merged (#524); cited by `test/contract/server_readback_test.go` | -| 018 | [Horizon Dashboard Verification](./018-horizon-dashboard-verification/spec.md) | Verify unmodified Horizon against Lightstack; publish findings report | implemented | Merged (#482); `docs/compatibility/HORIZON_COMPATIBILITY_REPORT.md` | -| 019 | [Horizon Compat API Fixes](./019-horizon-compat-api-fixes/spec.md) | Backend/response fixes for findings F4/F5/F6/F9 | implemented | Merged (#532) | -| 020 | [Horizon Create/UI Enablement](./020-horizon-create-ui-enablement/spec.md) | Enable Horizon create/write workflows (F2/F3/F7/F8) | implemented | Merged (#533); 58 code references | -| 021 | [Real noVNC Console](./021-real-novnc-console/spec.md) | A real graphical console via noVNC (finding F1) | implemented | Merged (#534); 57 code references; host-pass validation tasks remain open in `tasks.md` | -| 022 | [Project/Tenant Isolation](./022-project-tenant-isolation/spec.md) | Tenant lifecycle & isolation defect fixes (B2/B5/B8) | implemented | Merged (#545); EVIDENCE.md in spec dir | -| 023 | [Tenant Scoping & Authz](./023-tenant-scoping-authz/spec.md) | Tenant scoping and authorization model (P2/P3) | implemented | Merged (#549); 57 code references | -| 024 | [Neutron API Fidelity](./024-neutron-api-fidelity-realmode/spec.md) | Neutron API fidelity & real-mode networking fixes (B6/B9–B11, F10/F11) | implemented | Merged (#546); 60 code references | +| 015 | [Identity UUID 404](./015-identity-uuid-404/spec.md) | Return 404 (not 500) for malformed identity resource ids | implemented | Merged (cloud-storage#515) | +| 016 | [Terraform Provider Verification](./016-terraform-openstack-provider-verification/README.md) | Verify unmodified terraform-provider-openstack apply/destroy | implemented | Harness + report merged (cloud-storage#481); `docs/compatibility/TERRAFORM_COMPATIBILITY_REPORT.md` | +| 017 | [Nova Server-Detail Read-back](./017-nova-server-detail-readback/spec.md) | Terraform-compat server-detail fields (`key_name`, `security_groups`) | implemented | Merged (cloud-storage#524); cited by `test/contract/server_readback_test.go` | +| 018 | [Horizon Dashboard Verification](./018-horizon-dashboard-verification/spec.md) | Verify unmodified Horizon against Lightstack; publish findings report | implemented | Merged (cloud-storage#482); `docs/compatibility/HORIZON_COMPATIBILITY_REPORT.md` | +| 019 | [Horizon Compat API Fixes](./019-horizon-compat-api-fixes/spec.md) | Backend/response fixes for findings F4/F5/F6/F9 | implemented | Merged; commit `0a4dde0` (cloud-storage#532) | +| 020 | [Horizon Create/UI Enablement](./020-horizon-create-ui-enablement/spec.md) | Enable Horizon create/write workflows (F2/F3/F7/F8) | implemented | Merged (cloud-storage#533); 58 code references | +| 021 | [Real noVNC Console](./021-real-novnc-console/spec.md) | A real graphical console via noVNC (finding F1) | implemented | Merged (cloud-storage#534); 57 code references; host-pass validation tasks remain open in `tasks.md` | +| 022 | [Project/Tenant Isolation](./022-project-tenant-isolation/spec.md) | Tenant lifecycle & isolation defect fixes (B2/B5/B8) | implemented | Merged (cloud-storage#545); EVIDENCE.md in spec dir | +| 023 | [Tenant Scoping & Authz](./023-tenant-scoping-authz/spec.md) | Tenant scoping and authorization model (P2/P3) | implemented | Merged (cloud-storage#549); 57 code references | +| 024 | [Neutron API Fidelity](./024-neutron-api-fidelity-realmode/spec.md) | Neutron API fidelity & real-mode networking fixes (B6/B9–B11, F10/F11) | implemented | Merged (cloud-storage#546); 60 code references | | 025 | [Nova Gaps & Client Discovery](./025-nova-gaps-client-discovery/spec.md) | Ten defects: discovery self-links + compute answers that look successful | implemented | Merged; commit `3f6633d` | | 025 | [SPEC-025 Follow-ups](./025-followup/spec.md) | Pseudo-ticket for the follow-ups found while implementing SPEC-025 | implemented | Merged on the same branch; commit `fb6443a` | | 027 | [Database Pool Injection](./027-database-pool-injection/spec.md) | Inject `*pgxpool.Pool` through service constructors (implements ADR-001) | implemented | Merged; ADR-001 records "Implemented by SPEC-027" | @@ -65,12 +65,29 @@ Statuses: `draft`, `accepted`, `implemented`, `superseded`, `archived` (see commit `7ea8682`). Its spec directory was deliberately never tracked — the rename could not be applied to a spec describing the rename — so no `026-*` directory exists. +**Merge dependencies.** Open PRs overlap this registry and the contributor docs: + +- **#107** adds `028-cli-conformance-test-environment` as `plan.md` only; it needs + `spec.md`/`tasks.md` or a documented plan-first carve-out before it can land. +- **#108/#109** claim 028/029 for SQLite. The later-merged spec PR must renumber as + described under [Numbering Rules](#numbering-rules). +- **#103**'s secret guard (not current behavior on `main`) will require the + contract-test startup instruction in `docs/CONTRIBUTING.md` to use `make run`/env + vars, and CLAUDE.md's credential section to mention `LIGHTSTACK_JWT_SECRET` / + `LIGHTSTACK_ADMIN_PASSWORD`. +- **#101** conflicts in `CLAUDE.md`/`docs/CONTRIBUTING.md`; its Terraform + compatibility hedges must win over any stronger "unchanged"/"zero-modification" claim. + ## Numbering Rules - One number per spec. Numbers are **monotonic**: each new spec takes the next free number and never reuses or renumbers an existing one, even if earlier numbers are missing or duplicated (see anomalies above — they stay as-is for history). -- The **next free number is 028.** +- The **next number to mint is 028**, but it is currently claimed by two in-flight PRs: + #107 (`028-cli-conformance-test-environment`, `plan.md` only) and #108/#109 + (SQLite, claiming 028/029). The number is **frozen until one lands**; the + later-merged PR must renumber its spec per these rules. Do not mint 028 from + `main` while both are open. - A spec directory is named `NNN-short-slug/` (lowercase, hyphenated), and the spec is cited as `SPEC-NNN` in code comments, commit messages, and ADRs. diff --git a/docs/specs/_template/README.md b/docs/specs/_template/README.md index ce932c15..c264d86d 100644 --- a/docs/specs/_template/README.md +++ b/docs/specs/_template/README.md @@ -1,7 +1,8 @@ # Spec Template -Copy this directory to `docs/specs/NNN-short-slug/` (next free number — see the -[specs registry](../README.md#numbering-rules)) and fill in each file. Delete the +Copy this directory to `docs/specs/NNN-short-slug/` (pick the next number to mint per +the [numbering rules](../README.md#numbering-rules) — 028 is currently frozen by +in-flight PRs) and fill in each file. Delete the files you genuinely don't need, but keep the naming and the workflow. ## Workflow: specify → plan → tasks → implement → validate From 3d76b0eea7ca466130038b001039571a42787596 Mon Sep 17 00:00:00 2001 From: Uwe Schwaeke <uwe.schwaeke@sap.com> Date: Mon, 2 Nov 2026 10:43:11 +0000 Subject: [PATCH 8/8] ci: move to Go 1.27.3 (GO-2026-6809) and pin Helm 3.22.1 Shared CI unblocker for the open PRs (independent of this PR's content): - govulncheck now reports GO-2026-6809 in crypto/x509 (reachable through the HTTP server), fixed in Go 1.27.3. Main pins 1.27.2 everywhere, so every PR's Runtime Vulnerabilities and SBOM job fails. Pin 1.27.3 in go.mod, tools/go.mod, CI workflows, the builder image, the test container and the single-node deploy script (version + sha256). - The unpinned Helm install now pulls Helm 4, whose 'helm plugin install' rejects the helm-unittest source without --verify=false, failing the Helm Charts job with 'plugin source does not support verification'. Install Helm 3.22.1 and helm-unittest 1.2.1 in an isolated plugin dir, matching PR #107, so the gate is version-stable. Signed-off-by: Uwe Schwaeke <uwe.schwaeke@sap.com> --- .github/workflows/ci.yml | 27 +++++++++++++-------------- .github/workflows/security.yml | 2 +- build/package/Dockerfile | 2 +- deployments/docker-compose.test.yml | 2 +- docs/MULTIARCH.md | 2 +- docs/security/SBOM_TRIAGE.md | 3 ++- go.mod | 2 +- scripts/deploy-single-node.sh | 4 ++-- tools/go.mod | 2 +- 9 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8dd960fe..9e0b6806 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v6 with: - go-version: '1.27.2' + go-version: '1.27.3' - name: Cache Go modules uses: actions/cache@v5 @@ -83,7 +83,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v6 with: - go-version: '1.27.2' + go-version: '1.27.3' - name: Cache Go modules uses: actions/cache@v5 @@ -118,7 +118,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v6 with: - go-version: '1.27.2' + go-version: '1.27.3' - name: Cache Go modules uses: actions/cache@v5 @@ -322,6 +322,9 @@ jobs: name: Helm Charts runs-on: ubuntu-latest timeout-minutes: 10 + env: + HELM_VERSION: v3.22.1 + HELM_UNITTEST_VERSION: v1.2.1 steps: - name: Checkout code @@ -329,18 +332,14 @@ jobs: # Same mechanism as `make install-tools`: the official get-helm-3 script plus the # helm-unittest plugin, which `make chart-test` hard-requires (Makefile:103-104). - - name: Install Helm and the helm-unittest plugin + - name: Install pinned Helm and helm-unittest run: | - if ! command -v helm >/dev/null 2>&1; then - # Downloaded, not piped: the default run shell is `bash -e` without pipefail, - # so `curl ... | bash` would swallow a failed download. - curl -fsSL -o "$RUNNER_TEMP/get-helm-3" \ - https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 - bash "$RUNNER_TEMP/get-helm-3" - fi - if ! helm plugin list 2>/dev/null | grep -q '^unittest'; then - helm plugin install https://github.com/helm-unittest/helm-unittest - fi + export HELM_PLUGINS="$RUNNER_TEMP/helm-plugins" + echo "HELM_PLUGINS=$HELM_PLUGINS" >> "$GITHUB_ENV" + curl -fsSL -o "$RUNNER_TEMP/get-helm-3" \ + https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 + bash "$RUNNER_TEMP/get-helm-3" --version "$HELM_VERSION" + helm plugin install https://github.com/helm-unittest/helm-unittest --version "$HELM_UNITTEST_VERSION" helm version --short # Lints + renders + unit-tests both product charts: charts/lightstack, then diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 2aaaaf58..5bafc671 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -24,7 +24,7 @@ jobs: - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: '1.27.2' + go-version: '1.27.3' cache: true - name: Scan Go code and pinned development tool diff --git a/build/package/Dockerfile b/build/package/Dockerfile index 969e9c9a..baabed86 100644 --- a/build/package/Dockerfile +++ b/build/package/Dockerfile @@ -1,5 +1,5 @@ # Multi-stage build for Lightstack - Multi-architecture support -FROM --platform=$BUILDPLATFORM golang:1.27.2-alpine3.24 AS builder +FROM --platform=$BUILDPLATFORM golang:1.27.3-alpine3.24 AS builder # Build arguments for cross-compilation ARG TARGETOS diff --git a/deployments/docker-compose.test.yml b/deployments/docker-compose.test.yml index 78f78bae..85aa9795 100644 --- a/deployments/docker-compose.test.yml +++ b/deployments/docker-compose.test.yml @@ -2,7 +2,7 @@ version: '3.8' services: test-runner: - image: golang:1.27.2 + image: golang:1.27.3 container_name: lightstack-test-runner working_dir: /workspace volumes: diff --git a/docs/MULTIARCH.md b/docs/MULTIARCH.md index e648a281..6155ed73 100644 --- a/docs/MULTIARCH.md +++ b/docs/MULTIARCH.md @@ -76,7 +76,7 @@ The Dockerfile uses Docker's built-in multi-architecture support: ```dockerfile # Builder stage - runs on native architecture -FROM --platform=$BUILDPLATFORM golang:1.27.2-alpine3.24 AS builder +FROM --platform=$BUILDPLATFORM golang:1.27.3-alpine3.24 AS builder # Build arguments automatically set by Docker ARG TARGETOS # Target OS (linux) diff --git a/docs/security/SBOM_TRIAGE.md b/docs/security/SBOM_TRIAGE.md index 9cd4d3be..797b46d9 100644 --- a/docs/security/SBOM_TRIAGE.md +++ b/docs/security/SBOM_TRIAGE.md @@ -46,7 +46,8 @@ The generated runtime SPDX artifact replaces this export as the release inventor | Go 1.26.2 standard-library advisories | Toolchain affects every binary | Pin Go 1.26.8 in modules, CI, builder image, and test container | | GPL and other linter-only dependencies in the root module | Development-only, not shipped | Move the pinned linter graph to `tools/go.mod` | | Vulnerable transitive modules in the pinned linter graph | Development-only but executed in CI | Upgrade the affected modules and scan the linter command with `govulncheck` | -| GO-2026-6603, -6605, -6607, -6608, -6610, -6611, -6612, -6613, -6617 in the Go 1.26.8 standard library, published 2026-10-08 | Reachable; toolchain affects every binary | Pin Go 1.27.2, the current release, in modules, CI, builder image, test container, and the single-node deploy script | +| GO-2026-6603, -6605, -6607, -6608, -6610, -6611, -6612, -6613, -6617 in the Go 1.26.8 standard library, published 2026-10-08 | Reachable; toolchain affects every binary | Upgrade the toolchain in modules, CI, builder image, test container, and the single-node deploy script; current pin is Go 1.27.3 | +| GO-2026-6809 in `crypto/x509`, detected by CI on 2026-10-30 | Reachable through the HTTP server; Go 1.27.2 trusts compromised intermediate certificates | Pin Go 1.27.3 throughout the build and test toolchain; keep vulnerability scanning enabled | | GO-2026-6603, -6610, -6611, -6612, -6617 in `x/net`, published 2026-10-08 | Reachable through gin's HTTP/2 support | Upgrade transitively from 0.57.0 to 0.60.0; `x/crypto`, `x/sys`, `x/sync`, and `x/text` move to its minimums | ## Triage procedure diff --git a/go.mod b/go.mod index 62a3d15b..11b695b1 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/cobaltcore-dev/lightstack -go 1.27.2 +go 1.27.3 require ( github.com/aws/aws-sdk-go-v2 v1.42.0 diff --git a/scripts/deploy-single-node.sh b/scripts/deploy-single-node.sh index c61fa981..bde0f8d6 100755 --- a/scripts/deploy-single-node.sh +++ b/scripts/deploy-single-node.sh @@ -470,8 +470,8 @@ build_lightstack() { log_info "Building Lightstack binary..." # Use the same patched toolchain as CI and the release image. - local go_version="1.27.2" - local go_sha256="ecbadb99091a3f46e31f5f934b068b1864eafa7995211b39eaddf76996045fe5" + local go_version="1.27.3" + local go_sha256="65efbc80fdc1fa2cd2631cad5b481777a68e8725f34e89f50fc6ac714e379b40" local installed_go="" if command -v go &> /dev/null; then installed_go=$(go env GOVERSION 2>/dev/null || true) diff --git a/tools/go.mod b/tools/go.mod index fe6322a3..095810b4 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -1,6 +1,6 @@ module github.com/cobaltcore-dev/lightstack/tools -go 1.27.2 +go 1.27.3 tool github.com/golangci/golangci-lint/v2/cmd/golangci-lint