diff --git a/.github/renovate.json b/.github/renovate.json index 7a238cb48..22b6dcc94 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -10,7 +10,6 @@ ], "commitMessageAction": "Renovate: Update", "constraints": { - "go": "1.26", "python": "3.13" }, "dependencyDashboardOSVVulnerabilitySummary": "all", @@ -73,12 +72,6 @@ } ], "packageRules": [ - { - "matchPackageNames": [ - "golang" - ], - "allowedVersions": "1.26.x" - }, { "matchPackageNames": [ "postgresql-18" diff --git a/CHANGELOG.md b/CHANGELOG.md index 97d962776..8e96cd11e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,72 @@ # Changelog +## 2026-08-31 — [#1177](https://github.com/cobaltcore-dev/cortex/pull/1177) + +### cortex v0.4.0 (sha-23f975db) + +New features: +- Pending cache overlay — a transparent, in-process write-through cache on `client.Client` that eliminates informer lag for configurable GVKs; after a successful write the overlay keeps the object in memory so subsequent reads return the fresh version immediately instead of the stale informer snapshot; includes per-object keyed mutex, tombstone tracking, field selector support, and fallback TTL expiry ([#1042](https://github.com/cobaltcore-dev/cortex/pull/1042)) +- Cache monitoring — new `cortex_cache_overlay_entries` (current count) and `cortex_cache_overlay_entries_max` (high-watermark) Prometheus gauge metrics per GVK/host ([#1160](https://github.com/cobaltcore-dev/cortex/pull/1160)) +- Forced destination behavior in Nova external scheduler — replicates Nova's native `force_hosts`/`force_nodes` behavior; when a scheduling request specifies forced destinations and no `_nova_check_type` hint is set, Cortex skips the entire filter/weigher pipeline and returns only the forced hosts directly; configurable via `forcedDestinationEnabled` ([#1163](https://github.com/cobaltcore-dev/cortex/pull/1163)) +- Forced destination evaluated before `canRunScheduler` — critical because Nova may not send weights for forced requests ([#1176](https://github.com/cobaltcore-dev/cortex/pull/1176)) +- Physical host size label — adds a `physical_host_size` label derived from memory inventory (e.g. `"4TiB"`, `"512GiB"`) to VMware host capacity metrics ([#1165](https://github.com/cobaltcore-dev/cortex/pull/1165)) + +Breaking changes: +- Knowledge resource renamed from `host-details` to `vmware-host-details` — existing Knowledge CRDs with the old name will be orphaned and must be deleted manually; extractor renamed from `sap_host_details_extractor` to `vmware_host_details_extractor`; SQL rewritten to filter at the query level, eliminating Go-side hypervisor type/family filtering ([#1166](https://github.com/cobaltcore-dev/cortex/pull/1166)) + +Non-breaking changes: +- Document forced destination behavior in Nova external scheduler ([#1174](https://github.com/cobaltcore-dev/cortex/pull/1174)) +- Document pending cache overlay architecture ([#1175](https://github.com/cobaltcore-dev/cortex/pull/1175)) +- Document host oversubscription detection and remediation ([#1153](https://github.com/cobaltcore-dev/cortex/pull/1153)) +- Update Go version to 1.27, remove Go version constraint from Renovate ([#1170](https://github.com/cobaltcore-dev/cortex/pull/1170)) +- Update `k8s.io/api`, `k8s.io/apimachinery`, `k8s.io/client-go` to v0.37.0 ([#1169](https://github.com/cobaltcore-dev/cortex/pull/1169)) +- Update `github.com/sapcc/go-bits` ([#1168](https://github.com/cobaltcore-dev/cortex/pull/1168), [#1171](https://github.com/cobaltcore-dev/cortex/pull/1171)) +- Update `kube-prometheus-stack` to v88.6.0 ([#1152](https://github.com/cobaltcore-dev/cortex/pull/1152)) + +### cortex-shim v0.1.15 (sha-23f975db) + +New features: +- Ingress resource template — new `ingress.yaml` template with configurable className, annotations, rules, and TLS support ([#1158](https://github.com/cobaltcore-dev/cortex/pull/1158)) + +Includes updated dependencies (Go 1.27, k8s.io v0.37.0). + +### cortex-postgres v0.6.15 (sha-23f975db) + +- Rebuild image to resolve CVEs ([#1164](https://github.com/cobaltcore-dev/cortex/pull/1164)) +- Update `debian:trixie-slim` base image digest ([#1159](https://github.com/cobaltcore-dev/cortex/pull/1159)) + +### cortex-nova v0.0.89 + +- New alert `CortexNovaCacheOverlayNotDraining` (5m window, warning severity) +- New values: `cache.enabled` (true), `cache.gvks`, `forcedDestinationEnabled` (true) +- Knowledge renamed: `host-details` → `vmware-host-details` + +Includes updated charts cortex v0.4.0 and cortex-postgres v0.6.15. + +### cortex-cinder v0.0.89 + +Includes updated charts cortex v0.4.0 and cortex-postgres v0.6.15. + +### cortex-manila v0.0.89 + +Includes updated charts cortex v0.4.0 and cortex-postgres v0.6.15. + +### cortex-crds v0.0.89 + +Includes updated chart cortex v0.4.0. + +### cortex-ironcore v0.0.89 + +Includes updated chart cortex v0.4.0. + +### cortex-pods v0.0.89 + +Includes updated chart cortex v0.4.0. + +### cortex-placement-shim v0.1.15 + +Includes updated chart cortex-shim v0.1.15. + ## 2026-08-24 — [#1154](https://github.com/cobaltcore-dev/cortex/pull/1154) ### cortex v0.3.8 (sha-1742e3ee) diff --git a/Dockerfile b/Dockerfile index 2580e9637..0cc33f1ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.26 AS builder +FROM golang:1.27 AS builder ARG TARGETOS ARG TARGETARCH # Path of our go.mod diff --git a/Makefile b/Makefile index 5c58fa58f..378f15ff9 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ GOLANGCI_LINT = $(LOCALBIN)/golangci-lint GOTESTSUM = $(LOCALBIN)/gotestsum CONTROLLER_TOOLS_VERSION ?= v0.21.0 -GOLANGCI_LINT_VERSION ?= v2.13.1 +GOLANGCI_LINT_VERSION ?= v2.13.2 GOTESTSUM_VERSION ?= v1.13.0 .PHONY: controller-gen diff --git a/api/external/nova/messages.go b/api/external/nova/messages.go index fba3a4cee..d3021ffd3 100644 --- a/api/external/nova/messages.go +++ b/api/external/nova/messages.go @@ -88,6 +88,75 @@ func (r ExternalSchedulerRequest) Filter(includedHosts map[string]float64) lib.F return r } +// IsForcedDestination reports whether Nova forced this request onto specific +// hosts/nodes and the scheduler filters should be skipped entirely. +// +// This replicates Nova's native behavior: when force_hosts or force_nodes is +// set and the scheduler hint "_nova_check_type" is not set, Nova skips the +// filters and returns only the forced destinations. +// See: https://github.com/sapcc/nova/blob/05f384a938e3d6a8740a8f404d79d767d6ebdbd7/nova/scheduler/host_manager.py#L610-L620 +func (r ExternalSchedulerRequest) IsForcedDestination() bool { + forceHosts := r.Spec.Data.ForceHosts != nil && len(*r.Spec.Data.ForceHosts) > 0 + forceNodes := r.Spec.Data.ForceNodes != nil && len(*r.Spec.Data.ForceNodes) > 0 + if !forceHosts && !forceNodes { + return false + } + // If _nova_check_type is set (e.g. rebuild/evacuate/resize), the host is + // forced but must still be validated by the filters, so don't skip. + checkType, err := r.Spec.Data.GetSchedulerHintStr("_nova_check_type") + if err == nil && checkType != "" { + return false + } + return true +} + +// ForcedHosts returns the subset of request hosts that match the forced +// destinations (force_hosts / force_nodes). Matching mirrors Nova: +// - force_hosts matches on the compute host name, case-insensitive. +// - force_nodes matches on the hypervisor hostname, case-sensitive. +// - When both are set, a host must match both (intersection). +// +// The returned slice preserves the order of the request hosts. If nothing +// matches, an empty slice is returned (Nova would then raise NoValidHost). +func (r ExternalSchedulerRequest) ForcedHosts() []string { + var forceHosts []string + if r.Spec.Data.ForceHosts != nil { + forceHosts = *r.Spec.Data.ForceHosts + } + var forceNodes []string + if r.Spec.Data.ForceNodes != nil { + forceNodes = *r.Spec.Data.ForceNodes + } + forceHostSet := make(map[string]bool, len(forceHosts)) + for _, h := range forceHosts { + forceHostSet[strings.ToLower(h)] = true + } + forceNodeSet := make(map[string]bool, len(forceNodes)) + for _, n := range forceNodes { + forceNodeSet[n] = true + } + matched := make([]string, 0, len(r.Hosts)) + // Candidates are (host, node) pairs, so a single compute host may appear + // multiple times (once per node). Deduplicate by compute host name, since + // the response only carries host names. + seen := make(map[string]bool, len(r.Hosts)) + for _, host := range r.Hosts { + if len(forceHostSet) > 0 && !forceHostSet[strings.ToLower(host.ComputeHost)] { + continue + } + if len(forceNodeSet) > 0 && !forceNodeSet[host.HypervisorHostname] { + continue + } + key := strings.ToLower(host.ComputeHost) + if seen[key] { + continue + } + seen[key] = true + matched = append(matched, host.ComputeHost) + } + return matched +} + type FlavorType string const ( diff --git a/api/external/nova/messages_test.go b/api/external/nova/messages_test.go index a9074bd3a..b76cddf03 100644 --- a/api/external/nova/messages_test.go +++ b/api/external/nova/messages_test.go @@ -569,3 +569,178 @@ func TestNovaImageMeta_GetHypervisorType(t *testing.T) { }) } } + +func strPtrSlice(s ...string) *[]string { + out := s + return &out +} + +func TestIsForcedDestination(t *testing.T) { + tests := []struct { + name string + forceHosts *[]string + forceNodes *[]string + hints map[string]any + expected bool + }{ + { + name: "no force", + expected: false, + }, + { + name: "empty force_hosts slice", + forceHosts: strPtrSlice(), + expected: false, + }, + { + name: "force_hosts set, no check_type", + forceHosts: strPtrSlice("node017-bb545"), + expected: true, + }, + { + name: "force_nodes set, no check_type", + forceNodes: strPtrSlice("domain-c123"), + expected: true, + }, + { + name: "force_hosts set with check_type rebuild", + forceHosts: strPtrSlice("node017-bb545"), + hints: map[string]any{"_nova_check_type": "rebuild"}, + expected: false, + }, + { + name: "force_hosts set with check_type as list", + forceHosts: strPtrSlice("node017-bb545"), + hints: map[string]any{"_nova_check_type": []any{"resize"}}, + expected: false, + }, + { + name: "force_hosts set with empty check_type", + forceHosts: strPtrSlice("node017-bb545"), + hints: map[string]any{"_nova_check_type": ""}, + expected: true, + }, + { + name: "force_hosts set, hints without check_type", + forceHosts: strPtrSlice("node017-bb545"), + hints: map[string]any{"some_other_hint": "value"}, + expected: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := ExternalSchedulerRequest{ + Spec: NovaObject[NovaSpec]{ + Data: NovaSpec{ + ForceHosts: tt.forceHosts, + ForceNodes: tt.forceNodes, + SchedulerHints: tt.hints, + }, + }, + } + if got := req.IsForcedDestination(); got != tt.expected { + t.Errorf("expected %v, got %v", tt.expected, got) + } + }) + } +} + +func TestForcedHosts(t *testing.T) { + hosts := []ExternalSchedulerHost{ + {ComputeHost: "node017-bb545", HypervisorHostname: "domain-c17"}, + {ComputeHost: "node018-bb545", HypervisorHostname: "domain-c18"}, + {ComputeHost: "NODE019-bb545", HypervisorHostname: "domain-c19"}, + } + tests := []struct { + name string + hosts []ExternalSchedulerHost + forceHosts *[]string + forceNodes *[]string + expected []string + }{ + { + name: "match single host", + forceHosts: strPtrSlice("node017-bb545"), + expected: []string{"node017-bb545"}, + }, + { + name: "match host case-insensitive", + forceHosts: strPtrSlice("node019-bb545"), + expected: []string{"NODE019-bb545"}, + }, + { + name: "match node case-sensitive", + forceNodes: strPtrSlice("domain-c18"), + expected: []string{"node018-bb545"}, + }, + { + name: "node case-sensitive no match", + forceNodes: strPtrSlice("DOMAIN-C18"), + expected: []string{}, + }, + { + name: "intersection of host and node", + forceHosts: strPtrSlice("node017-bb545", "node018-bb545"), + forceNodes: strPtrSlice("domain-c18"), + expected: []string{"node018-bb545"}, + }, + { + name: "intersection empty", + forceHosts: strPtrSlice("node017-bb545"), + forceNodes: strPtrSlice("domain-c18"), + expected: []string{}, + }, + { + name: "no match returns empty", + forceHosts: strPtrSlice("unknown-host"), + expected: []string{}, + }, + { + name: "multi-node host deduplicated by force_hosts", + hosts: []ExternalSchedulerHost{ + {ComputeHost: "hostA", HypervisorHostname: "node1"}, + {ComputeHost: "hostA", HypervisorHostname: "node2"}, + {ComputeHost: "hostB", HypervisorHostname: "node1"}, + }, + forceHosts: strPtrSlice("hostA"), + expected: []string{"hostA"}, + }, + { + name: "multi-node host intersection picks single node", + hosts: []ExternalSchedulerHost{ + {ComputeHost: "hostA", HypervisorHostname: "node1"}, + {ComputeHost: "hostA", HypervisorHostname: "node2"}, + {ComputeHost: "hostB", HypervisorHostname: "node1"}, + }, + forceHosts: strPtrSlice("hostA"), + forceNodes: strPtrSlice("node1"), + expected: []string{"hostA"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + candidates := tt.hosts + if candidates == nil { + candidates = hosts + } + req := ExternalSchedulerRequest{ + Hosts: candidates, + Spec: NovaObject[NovaSpec]{ + Data: NovaSpec{ + ForceHosts: tt.forceHosts, + ForceNodes: tt.forceNodes, + }, + }, + } + got := req.ForcedHosts() + if len(got) != len(tt.expected) { + t.Fatalf("expected %v, got %v", tt.expected, got) + } + for i, h := range tt.expected { + if got[i] != h { + t.Errorf("expected host[%d]=%s, got %s", i, h, got[i]) + } + } + }) + } +} diff --git a/cmd/manager/main.go b/cmd/manager/main.go index f69cc9005..130d3830b 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -65,6 +65,7 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/failover" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/inflight" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/quota" + "github.com/cobaltcore-dev/cortex/pkg/cache" "github.com/cobaltcore-dev/cortex/pkg/conf" "github.com/cobaltcore-dev/cortex/pkg/monitoring" "github.com/cobaltcore-dev/cortex/pkg/multicluster" @@ -391,6 +392,14 @@ func main() { }, } multiclusterClientConfig := conf.GetConfigOrDie[multicluster.ClientConfig]() + + var cacheMonitor *cache.Monitor + var cacheWrapper *cache.Wrapper + if c := conf.GetConfigOrDie[cache.RootConfig](); c.Cache.Enabled { + cacheMonitor = cache.NewMonitor("cortex_") + cacheWrapper = cache.NewWrapper(mgr, c.Cache, cacheMonitor) + multiclusterClient.Wrappers = append(multiclusterClient.Wrappers, cacheWrapper) + } if err := multiclusterClient.InitFromConf(ctx, mgr, multiclusterClientConfig); err != nil { setupLog.Error(err, "unable to initialize multicluster client") os.Exit(1) @@ -402,6 +411,9 @@ func main() { metrics.Registry = monitoring.WrapRegistry(metrics.Registry, metricsConfig) metrics.Registry.MustRegister(&logMetricsMonitor) metrics.Registry.MustRegister(multiclusterMonitor) + if cacheMonitor != nil { + metrics.Registry.MustRegister(cacheMonitor) + } // TODO: Remove me after scheduling pipeline steps don't require DB connections anymore. metrics.Registry.MustRegister(&db.Monitor) diff --git a/docs/architecture.md b/docs/architecture.md index 6c6ba83aa..d8a8eeaf4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -30,6 +30,12 @@ Cortex receives the list of possible hosts and their weights from Nova. It then > [!NOTE] > Since, by default, Nova does not support calling an external service, this functionality needs to be added like in [SAP's fork of Nova](https://github.com/sapcc/nova/blob/stable/2023.2-m3/nova/scheduler/external.py). +### Forced Destinations + +When Nova sets `force_hosts` or `force_nodes` on a request and the `_nova_check_type` scheduler hint is not set (or is empty), Cortex skips its entire filter/weigher pipeline and returns only the forced destinations. This replicates Nova's native behavior, where forced requests bypass the scheduler filters and land directly on the specified hosts. If `_nova_check_type` is set to a non-empty value (e.g. rebuild, evacuate, resize), the forced hosts still pass through the full pipeline because the filters need to validate the destination. + +This behavior is controlled by `forcedDestinationEnabled` in the Helm config (defaults to true). Set it to false as a kill-switch to route forced requests through the normal scheduling pipeline instead. See `api/external/nova/messages.go` for the detection logic (`IsForcedDestination`, `ForcedHosts`). + ## Placement API Shim [Placement](https://github.com/openstack/placement) is OpenStack's resource inventory service. It provides an API to query the inventory of resources in the OpenStack cloud, such as compute nodes, their available resources, and the current resource usage. In the OpenStack realm, Placement is used by [Nova](https://github.com/openstack/nova) to carry out virtual machine scheduling, as well as [Neutron](https://github.com/openstack/neutron) for network resource allocation. diff --git a/docs/pending-cache.md b/docs/pending-cache.md new file mode 100644 index 000000000..efbec0908 --- /dev/null +++ b/docs/pending-cache.md @@ -0,0 +1,82 @@ + + +# Pending Cache Overlay + +The pending cache is an in-process write-through overlay on `client.Client` that masks informer lag for configurable GVKs. After a successful write to the apiserver, the overlay keeps the written object in memory so that subsequent reads return the fresh version immediately, rather than the stale informer snapshot. Entries are evicted as soon as the informer observes the write, with a TTL fallback for safety. + +## Why It Exists + +The controller-runtime default client reads from informers. Between the moment a controller writes (or patches) an object and the moment the informer delivers the event, there is a window — typically tens to hundreds of milliseconds — during which any read returns the pre-write version. For scheduling workloads where multiple reconciles may touch the same objects in quick succession, this lag causes read-after-write inconsistencies: a controller creates a Reservation, then immediately lists Reservations and does not see the one it just created. + +The overlay closes this window transparently without switching to direct apiserver reads (which would defeat the purpose of informers at scale). + +## How It Works + +```mermaid +flowchart LR + Controller -- write --> Overlay + Overlay -- write --> APIServer + APIServer -- ok --> Overlay + Overlay -- upsert/tombstone --> Store["In-memory store
(per GVK × key)"] + Controller -- read --> Overlay + Overlay -- read --> Informer + Overlay -- merge --> Controller + Store -. merge .-> Overlay + InformerEvent["Informer event
(RV >= cached)"] -- evict --> Store + TTL["TTL ticker"] -- evict expired --> Store +``` + +Entry lifecycle: + +1. **Write** — The overlay forwards Create/Update/Patch/Delete to the inner client. On success it stores the returned object (or a tombstone for deletes) keyed by GVK + namespace/name, stamped with the object's UID and ResourceVersion. +2. **Read** — Get and List delegate to the inner (informer-backed) client, then merge the overlay: live entries override or supplement the informer result; tombstones suppress objects so deletes appear immediately. +3. **Eviction (primary)** — An informer event handler compares the observed object's UID and ResourceVersion against the cached entry. If the UID matches and the observed RV is >= the cached RV, the entry is removed. +4. **Eviction (fallback)** — A background ticker removes entries whose TTL has expired, guarding against entries that never appear in the informer (e.g. after a crash or a missed event). + +Writes to the same object are serialized by a per-object keyed mutex (`pkg/cache/lock.go`) so the inner write and the overlay mutation are atomic per key. The mutex map is reference-counted and entries are cleaned up when no writers are in flight, preventing unbounded growth over the manager's lifetime. + +For List queries with field selectors, the overlay re-evaluates overlay entries against the selector using registered `IndexerFunc`s (captured via `IndexField`). This ensures overlay-only entries that do not match the query are excluded, and entries whose queried fields changed are correctly filtered. + +## Integration with the Multicluster Client + +The overlay plugs in through the `multicluster.ClusterWrapper` interface. `cache.Wrapper` satisfies this interface structurally (no import of `pkg/multicluster`). When the cache is enabled, `main.go` creates a `cache.Wrapper` and appends it to `multicluster.Client.Wrappers`. During `InitFromConf`, every cluster (home and remotes) passes through the wrapper, which calls `cache.WrapCluster` to produce an `overlayCluster` that intercepts `GetClient()` and `GetFieldIndexer()`. The overlay's own lifecycle (eviction handlers + TTL cleanup) is registered as a `manager.Runnable` — it does not re-start the inner cluster. + +The wrapper also accepts a shared `Monitor` so a single Prometheus collector can distinguish entries across clusters by the rest config host. + +## Configuration + +Helm values under the `cache` key (see `helm/bundles/cortex-nova/values.yaml`): + +| Key | Type | Default | Description | +|---|---|---|---| +| `cache.enabled` | bool | `false` | Turn the overlay on. When false, clusters are used unwrapped with zero overhead. | +| `cache.gvks` | list of strings | `[]` | GVKs to overlay, formatted as `"//"`. Calls for unlisted GVKs pass through unchanged. | +| `cache.ttl` | duration | `2m` | Maximum lifetime of an overlay entry before TTL eviction. | + +Currently `cortex-nova` enables the cache for `cortex.cloud/v1alpha1/Reservation`. + +## Monitoring and Alerts + +The `Monitor` (`pkg/cache/monitor.go`) is a `prometheus.Collector` that exposes two gauge-family metrics per (GVK, host) pair: + +- **`cortex_cache_overlay_entries`** — current number of overlay entries (live + tombstone) at scrape time. +- **`cortex_cache_overlay_entries_max`** — high-watermark since the last scrape, reset on each collect. Because entries are extremely short-lived (evicted within milliseconds), a plain gauge would almost always read zero; the watermark captures transient spikes. + +A sustained non-zero current size is a strong signal that eviction is broken. The PrometheusRule **`CortexNovaCacheOverlayNotDraining`** fires (severity: warning) when `cortex_cache_overlay_entries` stays above the `alerts.thresholds.cacheOverlayMaxEntries` threshold (default 50) for 5 minutes. Investigate the informer for the affected GVK/cluster and the overlay eviction logs. + +## Code Pointers + +| What | Where | +|---|---| +| Overlay struct, CRUD methods, merge logic | `pkg/cache/client.go` | +| `overlayCluster` and `Wrapper` (ClusterWrapper integration) | `pkg/cache/cluster.go` | +| `Config` and `RootConfig` | `pkg/cache/config.go` | +| `Start`, eviction handlers, TTL cleanup | `pkg/cache/runnable.go` | +| Prometheus `Monitor` collector | `pkg/cache/monitor.go` | +| Per-object keyed mutex | `pkg/cache/lock.go` | +| Wiring into the manager and multicluster client | `cmd/manager/main.go` (search for `cache.`) | +| Helm values and alert rule | `helm/bundles/cortex-nova/values.yaml`, `helm/bundles/cortex-nova/templates/alerts.yaml` | diff --git a/docs/readme.md b/docs/readme.md index ec994f075..d1dbefb01 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -8,3 +8,4 @@ - For details on committed resource reservations, see [committed resource reservations](reservations/committed-resource-reservations.md). - For details on failover reservations, see [failover reservations](reservations/failover-reservations.md). - For multi-cluster scheduling, see [the multicluster guide](guides/multicluster/readme.md). +- For the pending cache overlay (informer lag mitigation), see [pending cache](pending-cache.md). diff --git a/docs/reservations/committed-resource-reservations.md b/docs/reservations/committed-resource-reservations.md index ae0b1f68c..25ce8a562 100644 --- a/docs/reservations/committed-resource-reservations.md +++ b/docs/reservations/committed-resource-reservations.md @@ -24,6 +24,7 @@ Implementation: `internal/scheduling/reservations/commitments/` - [Report-Capacity REST endpoint](#report-capacity-rest-endpoint) - [Syncer Task](#syncer-task) - [Placement Observability](#placement-observability) + - [Host Oversubscription Detection and Remediation](#host-oversubscription-detection-and-remediation) - [Configuration and Observability](#configuration-and-observability) ## Architecture Overview @@ -300,6 +301,45 @@ PAYG placements (flavor not in any configured group) are not counted. | `slot_missed` | CR has remaining capacity but no candidate host has a slot with remaining memory > 0 | | `slot_used` | CR has remaining capacity and at least one candidate host has a usable slot | +## Host Oversubscription Detection and Remediation + +Reservation slots can accumulate beyond a host's effective capacity due to concurrent slot creation with a stale informer cache, operator-driven VM migrations where the slot stays on the old host, or hardware changes that reduce `EffectiveCapacity`. The `HostOversubscriptionController` detects these violations and optionally evicts slots to restore capacity invariants. + +Implementation: `internal/scheduling/reservations/commitments/host_oversubscription_controller.go` + +### Operational Modes + +| Mode | Config | Behavior | +|---|---|---| +| Disabled | `enableOversubscriptionCheck: false` | Controller is not wired up; no detection runs | +| Detection only | `enableOversubscriptionCheck: true`, `enableOversubscriptionReservationEviction: false` | Violations are detected, metrics are emitted, but no slots are evicted | +| Active remediation | Both `true` | Violations trigger slot eviction after the grace period expires | + +### Grace Period and Rate Limiting + +When a violation is first detected, the controller waits `oversubscriptionGracePeriod` (default 3m) before evicting. This gives other controllers (e.g. failover) time to self-heal. If the violation resolves within the grace period, no eviction occurs. + +Checks for the same host are rate-limited to `oversubscriptionMinCheckInterval` (default 1m) to avoid thrashing under rapid reconcile events. + +### Eviction Target Selection + +The controller evicts one slot per grace-period cycle and re-evaluates: + +1. **Unallocated slots** (no running VMs) — smallest memory first, then smallest CPU. These are cheapest to re-place elsewhere. +2. **Allocated slots** (has VMs) — most idle first (highest unused-capacity ratio), then smallest total size. Only used when no unallocated candidates exist. + +### Metrics and Alerts + +| Metric | Type | Labels | Description | +|---|---|---|---| +| `cortex_committed_resource_host_oversubscribed` | Gauge | `host`, `az`, `resource` | Excess resource units beyond host capacity; non-zero = active violation | +| `cortex_committed_resource_host_oversubscribed_evicted_reservations_total` | Counter | `az` | Cumulative slots evicted by the controller | + +| Alert | Condition | Meaning | +|---|---|---| +| `CortexCommittedResourceHostOversubscribed` | gauge > 0 for 15m | Host remains oversubscribed despite eviction attempts — manual intervention needed | +| `CortexCommittedResourceHostOversubscriptionEvictionsHigh` | > 10 evictions/hour in an AZ | Recurring oversubscription — investigate capacity changes or slot creation races | + ## Configuration and Observability **Configuration**: `helm/bundles/cortex-nova/values.yaml` — API endpoint toggles, reconciliation intervals, scheduling pipeline selection, and per-flavor-group resource flags. @@ -308,3 +348,4 @@ PAYG placements (flavor not in any configured group) are not counted. - `cortex_committed_resource_change_api_*` - `cortex_committed_resource_usage_api_*` - `cortex_committed_resource_capacity_api_*` +- `cortex_committed_resource_host_oversubscribed*` diff --git a/go.mod b/go.mod index df4fbc723..f3a2e9312 100644 --- a/go.mod +++ b/go.mod @@ -1,43 +1,34 @@ module github.com/cobaltcore-dev/cortex -go 1.26.0 +go 1.27 require ( github.com/cobaltcore-dev/openstack-hypervisor-operator v1.2.3 github.com/go-gorp/gorp v2.2.0+incompatible + github.com/go-logr/logr v1.4.4 + github.com/google/uuid v1.6.0 github.com/gophercloud/gophercloud/v2 v2.14.0 - github.com/ironcore-dev/ironcore v0.5.0 + github.com/ironcore-dev/ironcore v0.6.1 + github.com/lib/pq v1.12.3 + github.com/mattn/go-sqlite3 v1.14.50 + github.com/ory/dockertest v3.3.5+incompatible github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.2 - github.com/sapcc/go-bits v0.0.0-20260818140528-75bdd20c7867 + github.com/sapcc/go-api-declarations v1.25.0 + github.com/sapcc/go-bits v0.0.0-20260827091731-7669cbdb53fb + go.uber.org/zap v1.28.0 go.xyrillian.de/gg v1.14.0 - k8s.io/api v0.36.4 - k8s.io/apimachinery v0.36.4 - k8s.io/client-go v0.36.4 + golang.org/x/sync v0.22.0 + golang.org/x/term v0.45.0 + gopkg.in/yaml.v3 v3.0.1 + k8s.io/api v0.37.0 + k8s.io/apimachinery v0.37.0 + k8s.io/client-go v0.37.0 sigs.k8s.io/controller-runtime v0.24.1 ) require ( - github.com/databus23/goslo.policy v0.0.0-20250326134918-4afc2c56a903 // indirect - github.com/go-openapi/swag/cmdutils v0.25.1 // indirect - github.com/go-openapi/swag/conv v0.25.1 // indirect - github.com/go-openapi/swag/fileutils v0.25.1 // indirect - github.com/go-openapi/swag/jsonutils v0.25.1 // indirect - github.com/go-openapi/swag/loading v0.25.1 // indirect - github.com/go-openapi/swag/mangling v0.25.1 // indirect - github.com/go-openapi/swag/netutils v0.25.1 // indirect - github.com/go-openapi/swag/stringutils v0.25.1 // indirect - github.com/go-openapi/swag/typeutils v0.25.1 // indirect - github.com/go-openapi/swag/yamlutils v0.25.1 // indirect - github.com/go-sql-driver/mysql v1.10.0 // indirect - github.com/gofrs/uuid/v5 v5.5.1 // indirect - github.com/gorilla/mux v1.8.1 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect - k8s.io/streaming v0.36.4 // indirect -) - -require ( - cel.dev/expr v0.25.1 // indirect + cel.dev/expr v0.25.2 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect @@ -48,6 +39,7 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/continuity v0.4.5 // indirect + github.com/databus23/goslo.policy v0.0.0-20250326134918-4afc2c56a903 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -55,25 +47,34 @@ require ( github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect - github.com/fxamacker/cbor/v2 v2.9.0 // indirect - github.com/go-logr/logr v1.4.4 + github.com/fxamacker/cbor/v2 v2.9.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect - github.com/go-openapi/jsonpointer v0.22.4 // indirect - github.com/go-openapi/jsonreference v0.21.4 // indirect - github.com/go-openapi/swag v0.25.1 // indirect - github.com/go-openapi/swag/jsonname v0.25.4 // indirect + github.com/go-openapi/jsonpointer v1.0.0 // indirect + github.com/go-openapi/jsonreference v1.0.0 // indirect + github.com/go-openapi/swag v0.27.1 // indirect + github.com/go-openapi/swag/cmdutils v0.27.1 // indirect + github.com/go-openapi/swag/conv v0.27.1 // indirect + github.com/go-openapi/swag/fileutils v0.27.1 // indirect + github.com/go-openapi/swag/jsonutils v0.27.1 // indirect + github.com/go-openapi/swag/loading v0.27.1 // indirect + github.com/go-openapi/swag/mangling v0.27.1 // indirect + github.com/go-openapi/swag/netutils v0.27.1 // indirect + github.com/go-openapi/swag/pools v0.27.1 // indirect + github.com/go-openapi/swag/stringutils v0.27.1 // indirect + github.com/go-openapi/swag/typeutils v0.27.1 // indirect + github.com/go-openapi/swag/yamlutils v0.27.1 // indirect + github.com/go-sql-driver/mysql v1.10.0 // indirect github.com/google/cel-go v0.29.0 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/uuid v1.6.0 + github.com/gorilla/mux v1.8.1 // indirect github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/lib/pq v1.12.3 - github.com/mattn/go-sqlite3 v1.14.50 github.com/moby/sys/user v0.4.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect @@ -81,13 +82,11 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opencontainers/runc v1.3.6 // indirect - github.com/ory/dockertest v3.3.5+incompatible github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/poy/onpar v0.3.5 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect - github.com/sapcc/go-api-declarations v1.25.0 github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/pflag v1.0.10 // indirect @@ -95,45 +94,42 @@ require ( github.com/ziutek/mymysql v1.5.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect - go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect - go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/otel/sdk v1.43.0 // indirect - go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/sdk v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.28.0 go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect golang.org/x/net v0.58.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 // indirect - golang.org/x/term v0.45.0 golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.15.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect - google.golang.org/grpc v1.82.1 // indirect - google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/grpc v1.83.1 // indirect + google.golang.org/protobuf v1.36.12 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v3 v3.0.1 gotest.tools v2.2.0+incompatible // indirect - k8s.io/apiextensions-apiserver v0.36.0 // indirect - k8s.io/apiserver v0.36.0 // indirect - k8s.io/component-base v0.36.0 // indirect + k8s.io/apiextensions-apiserver v0.36.3 // indirect + k8s.io/apiserver v0.36.3 // indirect + k8s.io/component-base v0.36.3 // indirect k8s.io/klog/v2 v2.140.0 // indirect - k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect - k8s.io/utils v0.0.0-20260507154919-ff6756f316d2 // indirect + k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect + k8s.io/streaming v0.37.0 // indirect + k8s.io/utils v0.0.0-20260626114624-be93311217bd // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.4.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index bf1971e7d..cabb61c0f 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= -cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= @@ -47,8 +47,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= -github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ= +github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-gorp/gorp v2.2.0+incompatible h1:xAUh4QgEeqPPhK3vxZN+bzrim1z5Av6q837gtjUlshc= github.com/go-gorp/gorp v2.2.0+incompatible/go.mod h1:7IfkAQnO7jfT/9IQ3R9wL1dFhukN6aQxzKTHnkxzA/E= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -58,44 +58,44 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-openapi/jsonpointer v0.22.4 h1:dZtK82WlNpVLDW2jlA1YCiVJFVqkED1MegOUy9kR5T4= -github.com/go-openapi/jsonpointer v0.22.4/go.mod h1:elX9+UgznpFhgBuaMQ7iu4lvvX1nvNsesQ3oxmYTw80= -github.com/go-openapi/jsonreference v0.21.4 h1:24qaE2y9bx/q3uRK/qN+TDwbok1NhbSmGjjySRCHtC8= -github.com/go-openapi/jsonreference v0.21.4/go.mod h1:rIENPTjDbLpzQmQWCj5kKj3ZlmEh+EFVbz3RTUh30/4= -github.com/go-openapi/swag v0.25.1 h1:6uwVsx+/OuvFVPqfQmOOPsqTcm5/GkBhNwLqIR916n8= -github.com/go-openapi/swag v0.25.1/go.mod h1:bzONdGlT0fkStgGPd3bhZf1MnuPkf2YAys6h+jZipOo= -github.com/go-openapi/swag/cmdutils v0.25.1 h1:nDke3nAFDArAa631aitksFGj2omusks88GF1VwdYqPY= -github.com/go-openapi/swag/cmdutils v0.25.1/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= -github.com/go-openapi/swag/conv v0.25.1 h1:+9o8YUg6QuqqBM5X6rYL/p1dpWeZRhoIt9x7CCP+he0= -github.com/go-openapi/swag/conv v0.25.1/go.mod h1:Z1mFEGPfyIKPu0806khI3zF+/EUXde+fdeksUl2NiDs= -github.com/go-openapi/swag/fileutils v0.25.1 h1:rSRXapjQequt7kqalKXdcpIegIShhTPXx7yw0kek2uU= -github.com/go-openapi/swag/fileutils v0.25.1/go.mod h1:+NXtt5xNZZqmpIpjqcujqojGFek9/w55b3ecmOdtg8M= -github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI= -github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= -github.com/go-openapi/swag/jsonutils v0.25.1 h1:AihLHaD0brrkJoMqEZOBNzTLnk81Kg9cWr+SPtxtgl8= -github.com/go-openapi/swag/jsonutils v0.25.1/go.mod h1:JpEkAjxQXpiaHmRO04N1zE4qbUEg3b7Udll7AMGTNOo= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.1 h1:DSQGcdB6G0N9c/KhtpYc71PzzGEIc/fZ1no35x4/XBY= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.1/go.mod h1:kjmweouyPwRUEYMSrbAidoLMGeJ5p6zdHi9BgZiqmsg= -github.com/go-openapi/swag/loading v0.25.1 h1:6OruqzjWoJyanZOim58iG2vj934TysYVptyaoXS24kw= -github.com/go-openapi/swag/loading v0.25.1/go.mod h1:xoIe2EG32NOYYbqxvXgPzne989bWvSNoWoyQVWEZicc= -github.com/go-openapi/swag/mangling v0.25.1 h1:XzILnLzhZPZNtmxKaz/2xIGPQsBsvmCjrJOWGNz/ync= -github.com/go-openapi/swag/mangling v0.25.1/go.mod h1:CdiMQ6pnfAgyQGSOIYnZkXvqhnnwOn997uXZMAd/7mQ= -github.com/go-openapi/swag/netutils v0.25.1 h1:2wFLYahe40tDUHfKT1GRC4rfa5T1B4GWZ+msEFA4Fl4= -github.com/go-openapi/swag/netutils v0.25.1/go.mod h1:CAkkvqnUJX8NV96tNhEQvKz8SQo2KF0f7LleiJwIeRE= -github.com/go-openapi/swag/stringutils v0.25.1 h1:Xasqgjvk30eUe8VKdmyzKtjkVjeiXx1Iz0zDfMNpPbw= -github.com/go-openapi/swag/stringutils v0.25.1/go.mod h1:JLdSAq5169HaiDUbTvArA2yQxmgn4D6h4A+4HqVvAYg= -github.com/go-openapi/swag/typeutils v0.25.1 h1:rD/9HsEQieewNt6/k+JBwkxuAHktFtH3I3ysiFZqukA= -github.com/go-openapi/swag/typeutils v0.25.1/go.mod h1:9McMC/oCdS4BKwk2shEB7x17P6HmMmA6dQRtAkSnNb8= -github.com/go-openapi/swag/yamlutils v0.25.1 h1:mry5ez8joJwzvMbaTGLhw8pXUnhDK91oSJLDPF1bmGk= -github.com/go-openapi/swag/yamlutils v0.25.1/go.mod h1:cm9ywbzncy3y6uPm/97ysW8+wZ09qsks+9RS8fLWKqg= -github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= -github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= +github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s= +github.com/go-openapi/jsonpointer v1.0.0/go.mod h1:Z3rw7dWu1p9IgitXCFamSlA5lmDiklEB6vkaxcNZW5Y= +github.com/go-openapi/jsonreference v1.0.0 h1:jlmTr6torcd1YgDQvSfNmRtKzYDO4FGBkrAdlAVWnpY= +github.com/go-openapi/jsonreference v1.0.0/go.mod h1:jtwdyGbJk0Xhe5Y+rwtglQP6Sb1WZST4rT32LWB+sv0= +github.com/go-openapi/swag v0.27.1 h1:VotvOLWW8q/EAxB0YdsBBGC8XYyeL1YwBj2ungAGPNg= +github.com/go-openapi/swag v0.27.1/go.mod h1:GTkJPwHfhJp6MWr4/rCh64HVI3Ofu+tcsbfjfHmTxpE= +github.com/go-openapi/swag/cmdutils v0.27.1 h1:I7sYqaWVl5mq0NEmNQkAmFDyNin9ufvMX/p2zwtQaOE= +github.com/go-openapi/swag/cmdutils v0.27.1/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM= +github.com/go-openapi/swag/conv v0.27.1 h1:8wi9ZG+olmY1wXphl93EWniPtbSPkXM/feH7FgjsvrU= +github.com/go-openapi/swag/conv v0.27.1/go.mod h1:QbqMivkpKhC3g1B1GGGOJ6ANewI3S62dbzYu3Duowqs= +github.com/go-openapi/swag/fileutils v0.27.1 h1:QQqBSoi5mW4XpU85nS0mLcA+zAE6vLzrb0QkmLKf9oM= +github.com/go-openapi/swag/fileutils v0.27.1/go.mod h1:VvJFZLTZS0AI854gEQz5tk7dBESdLjiNUMSZ/th2ry8= +github.com/go-openapi/swag/jsonutils v0.27.1 h1:SVgK3i4USzCU5mibOOS/l4ea2h9UQXy7J7RNLTjuXjU= +github.com/go-openapi/swag/jsonutils v0.27.1/go.mod h1:tdlEpZqdcQ17uj6J4YdK9vd8It5qWMwjWXOs0tjpRlk= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.27.1 h1:mJu3COL9WEaZVp/Kf2PRMi7tPszPEJfSr/OO75ynCs8= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.27.1/go.mod h1:mofwUWx70wvskwESqRJ//k/9kURmCgyJl5m5Ppoh5kY= +github.com/go-openapi/swag/loading v0.27.1 h1:/DxUgDXKbBX4bcn7r9uEXfJyzN5XpiJmZplzQTjrRCY= +github.com/go-openapi/swag/loading v0.27.1/go.mod h1:jvGh3iA2+zyUUycB5fgJWzeHnhrpvGnJJM0RVE9ZShE= +github.com/go-openapi/swag/mangling v0.27.1 h1:yC9D0HyUE8gbP+BfmGx9+AA89ikwZTMjESK3OnnoaqA= +github.com/go-openapi/swag/mangling v0.27.1/go.mod h1:jtBE2+V+3pILxOR7Vgce+Cwp6A2PgZbvVqfNntbVs0w= +github.com/go-openapi/swag/netutils v0.27.1 h1:mICMFoS82F5TZ4Zy3cqmcQk+BFeCp3Uyq3Np7GI0/qU= +github.com/go-openapi/swag/netutils v0.27.1/go.mod h1:J+WYyFMLtvtCGqa6jLv+YNUmIKI3ZRQRrvfNDMoQoEQ= +github.com/go-openapi/swag/pools v0.27.1 h1:9LeadcMyb2GJCbXX5hVQDbZ2Lq9TL4dCs/nx1j5DO0E= +github.com/go-openapi/swag/pools v0.27.1/go.mod h1:kVQefhSK5RWuRe7BXsL8htgBPAMpN7HDGpGEknqugeE= +github.com/go-openapi/swag/stringutils v0.27.1 h1:ZXePZ0r2p1qSjo8tD3Un4vFj8+FqlCkczxDrJIhYUp8= +github.com/go-openapi/swag/stringutils v0.27.1/go.mod h1:lzRN95CxXmA03XcDWHLOb6nOMcxCqR5rGY0lOgsfRoM= +github.com/go-openapi/swag/typeutils v0.27.1 h1:KSTdFlfnse4r6dP9IrEnwMldjE+zs71UeEB3//PtVXc= +github.com/go-openapi/swag/typeutils v0.27.1/go.mod h1:Srm0xFNRZ1Y+vCxJclo5qzx8aj+1pAKda/YfFPrG0dQ= +github.com/go-openapi/swag/yamlutils v0.27.1 h1:ftxv6xvXb1E3zohUc+okZ9nSqNb9StQX/FXnKZ98sQA= +github.com/go-openapi/swag/yamlutils v0.27.1/go.mod h1:bnxFIB1qewGRiZHypXGZ3fNgf13/0HfRgnS/iZBDrOo= +github.com/go-openapi/testify/enable/yaml/v2 v2.6.0 h1:gGHwAJ0R/5jU8BEGDbfRNR3hL68dAVi84WuOApp29B0= +github.com/go-openapi/testify/enable/yaml/v2 v2.6.0/go.mod h1:tY+St1SGq4NFl0QIqdTY4aEdbChAHxhyB77XQi9iJCo= +github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug= +github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw= github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/gofrs/uuid/v5 v5.5.1 h1:z1Ce19/JwNidXpy3tOQc3241lnJLKdKyq/xlNvlD4Ng= -github.com/gofrs/uuid/v5 v5.5.1/go.mod h1:bbAA98EoIlxyRHIVg6ektCSsZ5n8mSbwgEhvhMYlZgg= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/cel-go v0.29.0 h1:fEG+Ja3YRwNOqnQxTyJwoByAUAvTuxUGiro/jhrm4F4= @@ -123,8 +123,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/ironcore-dev/ironcore v0.5.0 h1:QAbXuqD8ZfS8wNP++EamrV7Xav3xfeZsC+28hddpkNE= -github.com/ironcore-dev/ironcore v0.5.0/go.mod h1:HPW1+WJGharyrX30vOm43cGpeBXQDP43r66S4ADnhZ0= +github.com/ironcore-dev/ironcore v0.6.1 h1:PbuLwUKof9Rn8VgVbvUgHSS6s6uKHnuOXqzRe1lGHgg= +github.com/ironcore-dev/ironcore v0.6.1/go.mod h1:pXt5mWICA5IuiaPJpGobWMhTEG9XnjtE3b8nnWOgu7E= github.com/itchyny/gojq v0.12.19 h1:ttXA0XCLEMoaLOz5lSeFOZ6u6Q3QxmG46vfgI4O0DEs= github.com/itchyny/gojq v0.12.19/go.mod h1:5galtVPDywX8SPSOrqjGxkBeDhSxEW1gSxoy7tn1iZY= github.com/itchyny/timefmt-go v0.1.8 h1:1YEo1JvfXeAHKdjelbYr/uCuhkybaHCeTkH8Bo791OI= @@ -153,8 +153,8 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.32.0 h1:Hw7s2pVrQo/8Yz5N77qdnpHaoc+c6cC9WIV1Jce+J6E= -github.com/onsi/ginkgo/v2 v2.32.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= +github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw= +github.com/onsi/ginkgo/v2 v2.32.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -185,8 +185,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sapcc/go-api-declarations v1.25.0 h1:vLkSVV8oaZExoBMEkwX31AqUjZIjwxKkxXr4q2sAAOg= github.com/sapcc/go-api-declarations v1.25.0/go.mod h1:7NrwidCCv/MxwBpb/qqYLbQb/eY6rlnYkXwWMGx/wlI= -github.com/sapcc/go-bits v0.0.0-20260818140528-75bdd20c7867 h1:5cyIp5P6NKhmmvL3sXbgdYKwWtwKbPDzE1rzMzZvaH4= -github.com/sapcc/go-bits v0.0.0-20260818140528-75bdd20c7867/go.mod h1:rF/e0K3R9qDS3xqDl4nMAGOoelSkUumR4BNwuhSMgZk= +github.com/sapcc/go-bits v0.0.0-20260827091731-7669cbdb53fb h1:jORKY0SgUAhXCq40zGSDRaBzDk2zUX+p2wsJeCYxITk= +github.com/sapcc/go-bits v0.0.0-20260827091731-7669cbdb53fb/go.mod h1:X2M3A28UnHSRGlRmiztvmBhdETrplwDFn7QLyxCOYtA= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= @@ -195,8 +195,8 @@ github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -209,20 +209,20 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -265,14 +265,14 @@ gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0 gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= -google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= -google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= -google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= -google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= +google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -285,26 +285,26 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -k8s.io/api v0.36.4 h1:RxrvqCL6vgH5/+UnTeu1IIFqYmGfy0hnyrod1rn35Oo= -k8s.io/api v0.36.4/go.mod h1:S2B3orCFBDhrgyWbLeuKcT2QdHIpQesBkCYSlWtwUOw= -k8s.io/apiextensions-apiserver v0.36.0 h1:Wt7E8J+VBCbj4FjiBfDTK/neXDDjyJVJc7xfuOHImZ0= -k8s.io/apiextensions-apiserver v0.36.0/go.mod h1:kGDjH0msuiIB3tgsYRV0kS9GqpMYMUsQ3GHv7TApyug= -k8s.io/apimachinery v0.36.4 h1:PT2UzkupGuAx/+xT5XjiMJ1WGpY3fn9/hdAvjweRet4= -k8s.io/apimachinery v0.36.4/go.mod h1:p2I2dipt7JHG+quVwQ1d02d28O4GdDi77RByQ13MTpk= -k8s.io/apiserver v0.36.0 h1:Jg5OFAENUACByUCg15CmhZAYrr5ZyJ+jodyA1mHl3YE= -k8s.io/apiserver v0.36.0/go.mod h1:mHvwdHf+qKEm+1/hYm756SV+oREOKSPnsjagOpx6Vho= -k8s.io/client-go v0.36.4 h1:MDvfDNvMSt0Br94SK8neviVlwL9qifw9B26hJCpD1K0= -k8s.io/client-go v0.36.4/go.mod h1:pNK4WKELbwlEDvtbE8l22lEZL5THYF61H5EealokZmA= -k8s.io/component-base v0.36.0 h1:hFjEktssxiJhrK1zfybkH4kJOi8iZuF+mIDCqS5+jRo= -k8s.io/component-base v0.36.0/go.mod h1:JZvIfcNHk+uck+8LhJzhSBtydWXaZNQwX2OdL+Mnwsk= +k8s.io/api v0.37.0 h1:Z//Vj9N7RA/yS2sDmxyeo7h+RR4zbUrd2vrd3Z0TbB4= +k8s.io/api v0.37.0/go.mod h1:LKXgcJWMc+f4OLbP5SFR8rulEg07zZhpi/zMULiBImk= +k8s.io/apiextensions-apiserver v0.36.3 h1:dPmOAPhwTtqb1bTxbFPsy18KHPhktQeO3WUPXunZIB0= +k8s.io/apiextensions-apiserver v0.36.3/go.mod h1:KTXFqgXiuw2pRoL+Wpmttqc+up9Xt/GohadPWeLLOa4= +k8s.io/apimachinery v0.37.0 h1:Np2AbDtf8x6RDHiD8T9LbKJ9gaegeVNa8yNm5FuGKm0= +k8s.io/apimachinery v0.37.0/go.mod h1:RN3nhprFSCxOi5Selxd7oMTXOe/c+ZbcE7Im+TS2zkE= +k8s.io/apiserver v0.36.3 h1:MGSg2SkdfuytiDEcRylT5mQFmmSsbx90XFUO67Y4bsQ= +k8s.io/apiserver v0.36.3/go.mod h1:fVH7zv9EUNUA7Fl7LtDKh8aB9W7u1VQPSGtWV5SjUxg= +k8s.io/client-go v0.37.0 h1:nsN31fy8wBySuZ+QRnKmrjRSQLOG2rvoGN0tKd12zhQ= +k8s.io/client-go v0.37.0/go.mod h1:FcGqw+Ll/gNQiq+nPGY1Oyt9y7SgDh1d3MW3RFDEbn0= +k8s.io/component-base v0.36.3 h1:vc/UFvPCkW0irPz84LAodAL1j3f4xktPM6dDJIEheAY= +k8s.io/component-base v0.36.3/go.mod h1:hZbNFG+gCMl9EbykDGEu73feKP9/Cq6JsV4pTo9GTO8= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= -k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= -k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= -k8s.io/streaming v0.36.4 h1:RS5YlhrdBN2pKGVjgygGntdu6SNdsduyjGWGe3cX0vo= -k8s.io/streaming v0.36.4/go.mod h1:tJ6S2bZa2HxIBauguBbCWSCYyd93Grfz1+z3tcOvlDE= -k8s.io/utils v0.0.0-20260507154919-ff6756f316d2 h1:wU4tMEhLGgIbLvXQb1cfN+EcM0wf7zC6CPF+C79jroc= -k8s.io/utils v0.0.0-20260507154919-ff6756f316d2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3ddJoY6OnyM225MXwGHPm0A= +k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I= +k8s.io/streaming v0.37.0 h1:iPBUZLZiKt5bV+lxJurASMOV07VuBhNpiwJt2//AWrM= +k8s.io/streaming v0.37.0/go.mod h1:APlJR26ZWRcVy5bIEj0QRrKUXROtBHPcxl2NT7EAzPU= +k8s.io/utils v0.0.0-20260626114624-be93311217bd h1:Ea7fgQ5we8Y9T0OX5o0dAHzQOBRI07D/dEYRaB9ZZEs= +k8s.io/utils v0.0.0-20260626114624-be93311217bd/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2Fol2CS0QHMNs/WI1MOSGzCm1KhM5ec= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.24.1 h1:miPEwrmirImAvgME1L9qebGHrOnGJoVmVdtOU9fRfo4= @@ -313,7 +313,7 @@ sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5E sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.4.0 h1:qmp2e3ZfFi1/jJbDGpD4mt3wyp6PE1NfKHCYLqgNQJo= -sigs.k8s.io/structured-merge-diff/v6 v6.4.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.4.2 h1:qdOxHwrl2Kaag1aQEarlYcOA9vSyGCp3CIki3aW8c4Q= +sigs.k8s.io/structured-merge-diff/v6 v6.4.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/helm/bundles/cortex-cinder/Chart.yaml b/helm/bundles/cortex-cinder/Chart.yaml index 92270d81f..bd7b362fd 100644 --- a/helm/bundles/cortex-cinder/Chart.yaml +++ b/helm/bundles/cortex-cinder/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-cinder description: A Helm chart deploying Cortex for Cinder. type: application -version: 0.0.88 +version: 0.0.89 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.14 + version: 0.6.15 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.8 + version: 0.4.0 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.8 + version: 0.4.0 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-crds/Chart.yaml b/helm/bundles/cortex-crds/Chart.yaml index 08bbdb859..5a4259d43 100644 --- a/helm/bundles/cortex-crds/Chart.yaml +++ b/helm/bundles/cortex-crds/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-crds description: A Helm chart deploying Cortex CRDs. type: application -version: 0.0.88 +version: 0.0.89 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.8 + version: 0.4.0 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-ironcore/Chart.yaml b/helm/bundles/cortex-ironcore/Chart.yaml index 934ca3a57..9897d7257 100644 --- a/helm/bundles/cortex-ironcore/Chart.yaml +++ b/helm/bundles/cortex-ironcore/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-ironcore description: A Helm chart deploying Cortex for IronCore. type: application -version: 0.0.88 +version: 0.0.89 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.8 + version: 0.4.0 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-manila/Chart.yaml b/helm/bundles/cortex-manila/Chart.yaml index 1c63ed8c4..12b1e3b07 100644 --- a/helm/bundles/cortex-manila/Chart.yaml +++ b/helm/bundles/cortex-manila/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-manila description: A Helm chart deploying Cortex for Manila. type: application -version: 0.0.88 +version: 0.0.89 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.14 + version: 0.6.15 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.8 + version: 0.4.0 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.8 + version: 0.4.0 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-nova/Chart.yaml b/helm/bundles/cortex-nova/Chart.yaml index 7d1a29f9d..aeee5b049 100644 --- a/helm/bundles/cortex-nova/Chart.yaml +++ b/helm/bundles/cortex-nova/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-nova description: A Helm chart deploying Cortex for Nova. type: application -version: 0.0.88 +version: 0.0.89 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.14 + version: 0.6.15 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.8 + version: 0.4.0 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.8 + version: 0.4.0 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-nova/templates/alerts.yaml b/helm/bundles/cortex-nova/templates/alerts.yaml index 6b26d749c..c3f61cc72 100644 --- a/helm/bundles/cortex-nova/templates/alerts.yaml +++ b/helm/bundles/cortex-nova/templates/alerts.yaml @@ -750,6 +750,34 @@ spec: object was created out-of-band on the wrong cluster. Investigate the affected resources and the routing configuration. + - alert: CortexNovaCacheOverlayNotDraining + # The cache overlay holds writes only until the informer observes + # them, which normally happens within milliseconds. If the current entry + # count stays above the threshold for the entire `for` window, entries are + # piling up rather than draining — i.e. eviction is broken (informer lag, + # UID mismatch, or TTL fallback firing). + expr: | + max by (gvk, host) (cortex_cache_overlay_entries{service="cortex-nova-metrics"}) > {{ .Values.alerts.thresholds.cacheOverlayMaxEntries }} + for: 5m + labels: + context: multicluster + dashboard: cortex-status-dashboard/cortex-status-dashboard + service: cortex + severity: warning + support_group: workload-management + annotations: + summary: "Cache overlay for `{{ "{{" }} $labels.gvk {{ "}}" }}` is not draining" + description: > + The in-process cache overlay for `{{ "{{" }} $labels.gvk {{ "}}" }}` + on host `{{ "{{" }} $labels.host {{ "}}" }}` has held entries above the + configured threshold across the whole alert window. The overlay is only + meant to bridge the short gap between a write and the informer observing + it, so it should drain back to zero within milliseconds. A sustained + non-zero current size indicates entries are not being evicted — + typically informer lag, a UID mismatch between the write and the observed + object, or the TTL fallback repeatedly firing. Investigate the informer + for this GVK/cluster and the overlay eviction logs. + {{- if .Values.kvm.enabled }} - alert: CortexNovaHostReservationsOversubscribed # Fires when the sum of running VM allocations + reservation blocks (committed + @@ -783,7 +811,7 @@ spec: failover) on host {{ "{{" }} $labels.compute_host {{ "}}" }} exceeds its effective capacity for {{ "{{" }} $labels.resource {{ "}}" }} by {{ "{{" }} $value | humanize {{ "}}" }}. This means the host is over-subscribed and committed resource or failover guarantees may not be - satisfied. Common causes: VM migrations with slot not yet reclaimed, outdated/stale reservations, or out of sync issues. + satisfied. Common causes: VM migrations with slot not yet reclaimed, outdated/stale reservations, or out of sync issues. If problem remains, inspect the reservations on this host and check the CR/failover controller logs. - alert: CortexCommittedResourceHostOversubscribed expr: max by (host, az, resource) (cortex_committed_resource_host_oversubscribed) > 0 diff --git a/helm/bundles/cortex-nova/templates/knowledges.yaml b/helm/bundles/cortex-nova/templates/knowledges.yaml index 223bc615f..8353052d2 100644 --- a/helm/bundles/cortex-nova/templates/knowledges.yaml +++ b/helm/bundles/cortex-nova/templates/knowledges.yaml @@ -178,15 +178,17 @@ spec: apiVersion: cortex.cloud/v1alpha1 kind: Knowledge metadata: - name: host-details + name: vmware-host-details spec: schedulingDomain: nova extractor: - name: sap_host_details_extractor + name: vmware_host_details_extractor description: | - This knowledge extracts SAP-specific host details information. + This knowledge extracts details of VMware compute hosts. KVM hosts are + served by a dedicated CRD and ironic hosts are excluded. recency: "60s" dependencies: datasources: - name: placement-resource-provider-traits + - name: placement-resource-provider-inventory-usages - name: nova-hypervisors diff --git a/helm/bundles/cortex-nova/templates/kpis.yaml b/helm/bundles/cortex-nova/templates/kpis.yaml index 418389e95..4c8dee11a 100644 --- a/helm/bundles/cortex-nova/templates/kpis.yaml +++ b/helm/bundles/cortex-nova/templates/kpis.yaml @@ -173,7 +173,7 @@ spec: - name: identity-projects - name: identity-domains knowledges: - - name: host-details + - name: vmware-host-details description: | This KPI tracks the resource utilization of projects running VMs on VMware hosts. --- @@ -203,7 +203,7 @@ spec: impl: vmware_host_capacity_kpi dependencies: knowledges: - - name: host-details + - name: vmware-host-details - name: host-utilization description: | This KPI tracks the capacity and utilization of VMware hosts in terms of CPU, RAM, and disk resources. \ No newline at end of file diff --git a/helm/bundles/cortex-nova/templates/kpis_kvm.yaml b/helm/bundles/cortex-nova/templates/kpis_kvm.yaml index 1ec5a2d2d..d9ad32d2e 100644 --- a/helm/bundles/cortex-nova/templates/kpis_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/kpis_kvm.yaml @@ -9,7 +9,6 @@ spec: impl: kvm_host_capacity_kpi dependencies: knowledges: - - name: host-details - name: host-utilization description: | This KPI tracks the total, utilized, reserved and failover capacity of KVM hosts. diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index cec441eaa..2624cc38a 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -22,6 +22,11 @@ alerts: highMemoryMiB: 6000 # Reconcile-duration threshold for CortexNovaReconcileDurationHigher10Min in seconds. reconcileDurationSeconds: 600 + # High-watermark of cache overlay entries above which + # CortexNovaCacheOverlayNotDraining fires. The overlay is expected to drain + # back to zero within milliseconds, so any sustained non-zero watermark + # indicates eviction is not keeping up. + cacheOverlayMaxEntries: 50 serviceMonitor: extraLabels: {} @@ -74,6 +79,12 @@ kvm: # Useful when rolling out a new KVM region before it is production-ready. criticalAlerts: true +# Cache configuration for the transparent in-process overlay cache. +cache: + enabled: true + gvks: + - cortex.cloud/v1alpha1/Reservation + cortex: &cortex crd: {enable: false} # Disable the default ServiceMonitor and metrics service from the kubebuilder stack. @@ -169,6 +180,11 @@ cortex-scheduling-controllers: # Number of top hosts to shuffle for evacuation requests. # Set to 0 or negative to disable shuffling. evacuationShuffleK: 3 + # If true (default), the external scheduler API replicates Nova's + # forced-destination behavior: requests with force_hosts/force_nodes skip + # the scheduling pipeline and filters. Set to false to disable this as a + # kill-switch and route forced requests through the normal pipeline. + forcedDestinationEnabled: true committedResourceReservationController: # Pipeline selection for CR reservation scheduling. The catch-all default covers # general-purpose flavors. For HANA flavor groups, add an explicit entry, e.g.: @@ -185,7 +201,7 @@ cortex-scheduling-controllers: # How long after a VM is allocated to a reservation before it is expected to appear # on the target host; allocations not confirmed within this window are removed allocationGracePeriod: "15m" - # How long to wait after detecting host over-subscription before evicting reservation slots. + # How long to wait after detecting host over-subscription before evicting reservation slots. # Gives other controllers (e.g. failover) time to self-heal. oversubscriptionGracePeriod: "3m" # Minimum time between consecutive over-subscription checks for the same host. diff --git a/helm/bundles/cortex-placement-shim/Chart.yaml b/helm/bundles/cortex-placement-shim/Chart.yaml index 25e5915c5..e2288b7bf 100644 --- a/helm/bundles/cortex-placement-shim/Chart.yaml +++ b/helm/bundles/cortex-placement-shim/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-placement-shim description: A Helm chart deploying the Cortex placement shim. type: application -version: 0.1.14 +version: 0.1.15 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-shim - name: cortex-shim repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.1.14 + version: 0.1.15 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case # of issues. See: https://github.com/sapcc/helm-charts/pkgs/container/helm-charts%2Fowner-info diff --git a/helm/bundles/cortex-pods/Chart.yaml b/helm/bundles/cortex-pods/Chart.yaml index 7fdee3167..e496f3387 100644 --- a/helm/bundles/cortex-pods/Chart.yaml +++ b/helm/bundles/cortex-pods/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-pods description: A Helm chart deploying Cortex for Pods. type: application -version: 0.0.88 +version: 0.0.89 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.8 + version: 0.4.0 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/dev/cortex-prometheus-operator/Chart.yaml b/helm/dev/cortex-prometheus-operator/Chart.yaml index 6f1aeaee1..9104a20a9 100644 --- a/helm/dev/cortex-prometheus-operator/Chart.yaml +++ b/helm/dev/cortex-prometheus-operator/Chart.yaml @@ -5,9 +5,9 @@ apiVersion: v2 name: cortex-prometheus-operator description: Prometheus operator setup for Cortex. type: application -version: 0.2.2 +version: 0.2.3 dependencies: # CRDs of the prometheus operator, such as PrometheusRule, ServiceMonitor, etc. - name: kube-prometheus-stack repository: oci://ghcr.io/prometheus-community/charts - version: 88.5.2 + version: 88.6.0 diff --git a/helm/library/cortex-postgres/Chart.yaml b/helm/library/cortex-postgres/Chart.yaml index 71bed9014..f3625ced4 100644 --- a/helm/library/cortex-postgres/Chart.yaml +++ b/helm/library/cortex-postgres/Chart.yaml @@ -5,5 +5,5 @@ apiVersion: v2 name: cortex-postgres description: Postgres setup for Cortex. type: application -version: 0.6.14 -appVersion: "sha-9f10115a" +version: 0.6.15 +appVersion: "sha-cf70e178" diff --git a/helm/library/cortex-shim/Chart.yaml b/helm/library/cortex-shim/Chart.yaml index 4bbe67390..f60d89720 100644 --- a/helm/library/cortex-shim/Chart.yaml +++ b/helm/library/cortex-shim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex-shim description: A Helm chart to distribute cortex shims. type: application -version: 0.1.14 -appVersion: "sha-1742e3ee" +version: 0.1.15 +appVersion: "sha-17e0a4fe" icon: "https://example.com/icon.png" dependencies: [] diff --git a/helm/library/cortex-shim/templates/ingress.yaml b/helm/library/cortex-shim/templates/ingress.yaml new file mode 100644 index 000000000..8608aee7d --- /dev/null +++ b/helm/library/cortex-shim/templates/ingress.yaml @@ -0,0 +1,24 @@ +# Copyright SAP SE +# SPDX-License-Identifier: Apache-2.0 + +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ .Values.namePrefix }}-shim-ingress + namespace: {{ .Release.Namespace }} + labels: + {{- include "chart.labels" . | nindent 4 }} + annotations: + {{- toYaml .Values.ingress.annotations | nindent 4 }} +spec: + ingressClassName: {{ .Values.ingress.className }} + rules: + {{- if .Values.ingress.rules }} + {{- toYaml .Values.ingress.rules | nindent 4 }} + {{- end }} + tls: + {{- if .Values.ingress.tls }} + {{- toYaml .Values.ingress.tls | nindent 4 }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/library/cortex-shim/values.yaml b/helm/library/cortex-shim/values.yaml index 2bf3c193c..3474a237f 100644 --- a/helm/library/cortex-shim/values.yaml +++ b/helm/library/cortex-shim/values.yaml @@ -41,6 +41,19 @@ deployment: terminationGracePeriodSeconds: 10 serviceAccountName: shim +# [INGRESS]: To enable an ingress resource for the placement shim set true +ingress: + # Enable or disable the ingress resource for the placement shim. + enabled: true + # Which ingress controller to use. + className: nginx + # Override this default set of annotations if needed. + annotations: + # Enables https://github.com/sapcc/disco + disco: "true" + # Enables automatic TLS certificate provisioning via cert-manager. + kubernetes.io/tls-acme: "true" + # [METRICS]: Set to true to generate manifests for exporting metrics. # To disable metrics export set false, and remove the container args # "--metrics-bind-address=:2112" and "--metrics-secure=false". diff --git a/helm/library/cortex/Chart.yaml b/helm/library/cortex/Chart.yaml index 50907159c..b49790ae6 100644 --- a/helm/library/cortex/Chart.yaml +++ b/helm/library/cortex/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex description: A Helm chart to distribute cortex. type: application -version: 0.3.8 -appVersion: "sha-77225628" +version: 0.4.0 +appVersion: "sha-91df27a0" icon: "https://example.com/icon.png" dependencies: [] diff --git a/internal/knowledge/extractor/controller_test.go b/internal/knowledge/extractor/controller_test.go index 8c5a40bf9..75ee4ee06 100644 --- a/internal/knowledge/extractor/controller_test.go +++ b/internal/knowledge/extractor/controller_test.go @@ -450,7 +450,7 @@ func TestKnowledgeReconciler_Reconcile_SupportedExtractors(t *testing.T) { "vm_life_span_histogram_extractor", "host_az_extractor", "host_pinned_projects_extractor", - "sap_host_details_extractor", + "vmware_host_details_extractor", } for _, extractorName := range supportedExtractors { diff --git a/internal/knowledge/extractor/plugins/compute/host_details.sql b/internal/knowledge/extractor/plugins/compute/host_details.sql deleted file mode 100644 index 6cdc24f9f..000000000 --- a/internal/knowledge/extractor/plugins/compute/host_details.sql +++ /dev/null @@ -1,53 +0,0 @@ -WITH host_traits AS ( - SELECT - h.service_host, - h.hypervisor_type, - h.running_vms, - h.state, - h.status, - h.service_disabled_reason, - STRING_AGG(t.name, ',') AS traits - FROM openstack_hypervisors h - LEFT JOIN openstack_resource_provider_traits t - ON h.id = t.resource_provider_uuid - GROUP BY h.service_host, h.hypervisor_type, h.running_vms, h.state, h.status, h.service_disabled_reason -) -SELECT - ht.service_host AS compute_host, - ht.running_vms AS running_vms, - -- CPU Architecture - CASE - WHEN ht.traits LIKE '%CUSTOM_HW_SAPPHIRE_RAPIDS%' THEN 'sapphire-rapids' - ELSE 'cascade-lake' - END AS cpu_architecture, - ht.hypervisor_type, - CASE - WHEN ht.service_host LIKE 'nova-compute-%' THEN 'vmware' - WHEN ht.service_host LIKE 'node%-bb%' THEN 'kvm' - ELSE 'unknown' - END AS hypervisor_family, - CASE - WHEN ht.traits LIKE '%CUSTOM_HANA_EXCLUSIVE_HOST%' THEN 'hana' - ELSE 'general-purpose' - END AS workload_type, - CASE - WHEN ht.traits LIKE '%CUSTOM_DECOMMISSIONING%' THEN true - ELSE false - END AS decommissioned, - CASE - WHEN ht.traits LIKE '%CUSTOM_EXTERNAL_CUSTOMER_EXCLUSIVE%' THEN true - ELSE false - END AS external_customer, - CASE - WHEN ht.traits LIKE '%COMPUTE_STATUS_DISABLED%' THEN false - WHEN ht.status != 'enabled' THEN false - WHEN ht.state != 'up' THEN false - ELSE true - END AS enabled, - CASE - WHEN ht.traits LIKE '%COMPUTE_STATUS_DISABLED%' THEN '[disabled] ' || COALESCE(ht.service_disabled_reason, '--') - WHEN ht.status != 'enabled' THEN '[disabled] ' || COALESCE(ht.service_disabled_reason, '--') - WHEN ht.state != 'up' THEN '[down] ' || COALESCE(ht.service_disabled_reason, '--') - ELSE NULL - END AS disabled_reason -FROM host_traits ht; \ No newline at end of file diff --git a/internal/knowledge/extractor/plugins/compute/host_details.go b/internal/knowledge/extractor/plugins/compute/vmware_host_details.go similarity index 75% rename from internal/knowledge/extractor/plugins/compute/host_details.go rename to internal/knowledge/extractor/plugins/compute/vmware_host_details.go index acec0fe4e..14bca07ba 100644 --- a/internal/knowledge/extractor/plugins/compute/host_details.go +++ b/internal/knowledge/extractor/plugins/compute/vmware_host_details.go @@ -14,7 +14,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -type HostDetails struct { +// VMwareHostDetails describes a single VMware compute host. Only VMware hosts +// are extracted here (KVM hosts are served by a dedicated CRD, and ironic hosts +// are excluded); the extractor SQL filters everything else out. +type VMwareHostDetails struct { // Name of the OpenStack compute host. ComputeHost string `db:"compute_host"` // Availability zone of the compute host. @@ -22,11 +25,6 @@ type HostDetails struct { // CPU Architecture of the compute host. // Can be "cascade-lake" or "sapphire-rapids" CPUArchitecture string `db:"cpu_architecture"` - // Hypervisor type of the compute host. - HypervisorType string `db:"hypervisor_type"` - // Hypervisor family of the compute host. - // Can be "kvm" or "vmware" - HypervisorFamily string `db:"hypervisor_family"` // Amount of VMs currently running on the compute host. RunningVMs int `db:"running_vms"` // Type of workload running on the compute host. @@ -42,26 +40,31 @@ type HostDetails struct { DisabledReason *string `db:"disabled_reason"` // Comma separated list of pinned projects of the ComputeHost. PinnedProjects *string `db:"pinned_projects"` + // Physical size category of a single host inside a VMware building block, + // derived from the memory inventory and rounded to the nearest TiB + // (e.g. "4TiB"), or to the nearest GiB (e.g. "512GiB") when smaller than + // 1 TiB. "unknown" when the memory inventory is missing. + PhysicalHostSize string `db:"physical_host_size"` } -type HostDetailsExtractor struct { +type VMwareHostDetailsExtractor struct { // Common base for all extractors that provides standard functionality. plugins.BaseExtractor[ - struct{}, // No options passed through yaml config - HostDetails, // Feature model + struct{}, // No options passed through yaml config + VMwareHostDetails, // Feature model ] } -//go:embed host_details.sql -var hostDetailsQuery string +//go:embed vmware_host_details.sql +var vmwareHostDetailsQuery string -// Extract the traits of a compute host from the database. -func (e *HostDetailsExtractor) Extract() ([]plugins.Feature, error) { +// Extract the details of the VMware compute hosts from the database. +func (e *VMwareHostDetailsExtractor) Extract() ([]plugins.Feature, error) { if e.DB == nil { return nil, errors.New("database connection is not initialized") } - var hostDetails []HostDetails - if _, err := e.DB.Select(&hostDetails, hostDetailsQuery); err != nil { + var hostDetails []VMwareHostDetails + if _, err := e.DB.Select(&hostDetails, vmwareHostDetailsQuery); err != nil { return nil, err } diff --git a/internal/knowledge/extractor/plugins/compute/vmware_host_details.sql b/internal/knowledge/extractor/plugins/compute/vmware_host_details.sql new file mode 100644 index 000000000..6be4fe1b4 --- /dev/null +++ b/internal/knowledge/extractor/plugins/compute/vmware_host_details.sql @@ -0,0 +1,82 @@ +-- Details of VMware compute hosts. Only VMware hosts are considered here: KVM +-- hosts are served by a dedicated CRD, and ironic hosts are excluded. The +-- WHERE clause keeps rows whose service host matches the VMware naming +-- convention (nova-compute-%) while dropping ironic ones (nova-compute-ironic-%). +WITH host_traits AS ( + SELECT + h.id AS hypervisor_id, + h.service_host, + h.running_vms, + h.state, + h.status, + h.service_disabled_reason, + STRING_AGG(t.name, ',') AS traits + FROM openstack_hypervisors h + LEFT JOIN openstack_resource_provider_traits t + ON h.id = t.resource_provider_uuid + WHERE h.service_host LIKE 'nova-compute-%' + AND h.service_host NOT LIKE 'nova-compute-ironic-%' + GROUP BY h.id, h.service_host, h.running_vms, h.state, h.status, h.service_disabled_reason +), +-- Physical memory size (in MiB) of a single host inside a VMware building block. +-- A building block pools multiple physical hosts into one resource provider, so +-- the per-host memory size is recovered from the pooled memory inventory: +-- amount_hosts = (total - reserved) / max_unit +-- physical_host_size = max_unit + reserved / amount_hosts (in MiB) +host_physical_memory AS ( + SELECT + i.resource_provider_uuid, + i.max_unit + + i.reserved / ((i.total - i.reserved) / CAST(i.max_unit AS FLOAT)) + AS physical_size_mb + FROM openstack_resource_provider_inventory_usages i + WHERE i.inventory_class_name = 'MEMORY_MB' + AND i.max_unit > 0 + AND (i.total - i.reserved) > 0 +) +SELECT + ht.service_host AS compute_host, + ht.running_vms AS running_vms, + -- CPU Architecture + CASE + WHEN ht.traits LIKE '%CUSTOM_HW_SAPPHIRE_RAPIDS%' THEN 'sapphire-rapids' + ELSE 'cascade-lake' + END AS cpu_architecture, + CASE + WHEN ht.traits LIKE '%CUSTOM_HANA_EXCLUSIVE_HOST%' THEN 'hana' + ELSE 'general-purpose' + END AS workload_type, + CASE + WHEN ht.traits LIKE '%CUSTOM_DECOMMISSIONING%' THEN true + ELSE false + END AS decommissioned, + CASE + WHEN ht.traits LIKE '%CUSTOM_EXTERNAL_CUSTOMER_EXCLUSIVE%' THEN true + ELSE false + END AS external_customer, + CASE + WHEN ht.traits LIKE '%COMPUTE_STATUS_DISABLED%' THEN false + WHEN ht.status != 'enabled' THEN false + WHEN ht.state != 'up' THEN false + ELSE true + END AS enabled, + CASE + WHEN ht.traits LIKE '%COMPUTE_STATUS_DISABLED%' THEN '[disabled] ' || COALESCE(ht.service_disabled_reason, '--') + WHEN ht.status != 'enabled' THEN '[disabled] ' || COALESCE(ht.service_disabled_reason, '--') + WHEN ht.state != 'up' THEN '[down] ' || COALESCE(ht.service_disabled_reason, '--') + ELSE NULL + END AS disabled_reason, + -- Physical host size category of a single host inside the building block. + -- The size is first rounded to whole GiB; if that is at least 1024 GiB it is + -- reported in TiB, otherwise in GiB. Rounding to GiB + -- first avoids a value like 1023.6 GiB being shown as "1024GiB" instead of + -- "1TiB". "unknown" when the memory inventory is missing. + CASE + WHEN hpm.physical_size_mb IS NULL THEN 'unknown' + WHEN ROUND(hpm.physical_size_mb / 1024.0) >= 1024 + THEN CAST(CAST(ROUND(ROUND(hpm.physical_size_mb / 1024.0) / 1024.0) AS INTEGER) AS TEXT) || 'TiB' + ELSE CAST(CAST(ROUND(hpm.physical_size_mb / 1024.0) AS INTEGER) AS TEXT) || 'GiB' + END AS physical_host_size +FROM host_traits ht +LEFT JOIN host_physical_memory hpm + ON ht.hypervisor_id = hpm.resource_provider_uuid; diff --git a/internal/knowledge/extractor/plugins/compute/host_details_test.go b/internal/knowledge/extractor/plugins/compute/vmware_host_details_test.go similarity index 57% rename from internal/knowledge/extractor/plugins/compute/host_details_test.go rename to internal/knowledge/extractor/plugins/compute/vmware_host_details_test.go index 68a3010d9..d24b1c0b1 100644 --- a/internal/knowledge/extractor/plugins/compute/host_details_test.go +++ b/internal/knowledge/extractor/plugins/compute/vmware_host_details_test.go @@ -16,15 +16,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" ) -func TestHostDetailsExtractor_Init(t *testing.T) { - extractor := &HostDetailsExtractor{} +func TestVMwareHostDetailsExtractor_Init(t *testing.T) { + extractor := &VMwareHostDetailsExtractor{} config := v1alpha1.KnowledgeSpec{} if err := extractor.Init(nil, nil, config); err != nil { t.Fatalf("expected no error, got %v", err) } } -func TestHostDetailsExtractor_Extract(t *testing.T) { +func TestVMwareHostDetailsExtractor_Extract(t *testing.T) { dbEnv := testlibDB.SetupDBEnv(t) testDB := db.DB{DbMap: dbEnv.DbMap} defer dbEnv.Close() @@ -38,6 +38,7 @@ func TestHostDetailsExtractor_Extract(t *testing.T) { if err := testDB.CreateTable( testDB.AddTable(nova.Hypervisor{}), testDB.AddTable(placement.Trait{}), + testDB.AddTable(placement.InventoryUsage{}), ); err != nil { t.Fatalf("expected no error, got %v", err) } @@ -45,30 +46,29 @@ func TestHostDetailsExtractor_Extract(t *testing.T) { hostPinnedProjects, err := v1alpha1.BoxFeatureList([]any{ &HostPinnedProjects{ComputeHost: new("nova-compute-bb01"), Label: new("project-123")}, &HostPinnedProjects{ComputeHost: new("nova-compute-bb01"), Label: new("project-456")}, - &HostPinnedProjects{ComputeHost: new("node001-bb02"), Label: nil}, - // No entry for ironic-host-1 since it is excluded in the feature host pinned projects - &HostPinnedProjects{ComputeHost: new("node002-bb03"), Label: nil}, - &HostPinnedProjects{ComputeHost: new("node003-bb03"), Label: nil}, - &HostPinnedProjects{ComputeHost: new("node004-bb03"), Label: nil}, }) if err != nil { t.Fatalf("expected no error, got %v", err) } - // Insert mock data into the hypervisors and traits tables + // Insert mock data into the hypervisors and traits tables. Only VMware hosts + // (nova-compute-% and not nova-compute-ironic-%) are expected to survive the + // extractor's WHERE filter; KVM and ironic hosts must be dropped entirely. hypervisors := []any{ // VMware host &nova.Hypervisor{ID: "uuid1", ServiceHost: "nova-compute-bb01", HypervisorType: "vcenter", RunningVMs: 5, State: "up", Status: "enabled"}, - // KVM host + // KVM host (should be filtered out) &nova.Hypervisor{ID: "uuid2", ServiceHost: "node001-bb02", HypervisorType: "qemu", RunningVMs: 3, State: "down", Status: "enabled"}, - // Ironic host (should be skipped) - &nova.Hypervisor{ID: "uuid3", ServiceHost: "ironic-host-01", HypervisorType: "ironic", RunningVMs: 0, State: "up", Status: "enabled"}, - // Host with no special traits - &nova.Hypervisor{ID: "uuid4", ServiceHost: "node002-bb03", HypervisorType: "test", RunningVMs: 2, State: "up", Status: "enabled"}, - // Host with disabled status, no entry in the resource providers - &nova.Hypervisor{ID: "uuid5", ServiceHost: "node003-bb03", HypervisorType: "test", RunningVMs: 2, State: "up", Status: "disabled", ServiceDisabledReason: new("example reason")}, - // Host with disabled trait - &nova.Hypervisor{ID: "uuid6", ServiceHost: "node004-bb03", HypervisorType: "test", RunningVMs: 2, State: "up", Status: "enabled", ServiceDisabledReason: new("example reason")}, + // Ironic host (should be filtered out) + &nova.Hypervisor{ID: "uuid3", ServiceHost: "nova-compute-ironic-01", HypervisorType: "ironic", RunningVMs: 0, State: "up", Status: "enabled"}, + // VMware host with a sub-TiB physical host size + &nova.Hypervisor{ID: "uuid7", ServiceHost: "nova-compute-bb04", HypervisorType: "vcenter", RunningVMs: 1, State: "up", Status: "enabled"}, + // VMware host just under 1 TiB that rounds up to a full TiB + &nova.Hypervisor{ID: "uuid8", ServiceHost: "nova-compute-bb05", HypervisorType: "vcenter", RunningVMs: 1, State: "up", Status: "enabled"}, + // Disabled VMware host via status + &nova.Hypervisor{ID: "uuid9", ServiceHost: "nova-compute-bb06", HypervisorType: "vcenter", RunningVMs: 2, State: "up", Status: "disabled", ServiceDisabledReason: new("example reason")}, + // Disabled VMware host via trait + &nova.Hypervisor{ID: "uuid10", ServiceHost: "nova-compute-bb07", HypervisorType: "vcenter", RunningVMs: 2, State: "up", Status: "enabled", ServiceDisabledReason: new("example reason")}, } if err := testDB.Insert(hypervisors...); err != nil { @@ -80,33 +80,66 @@ func TestHostDetailsExtractor_Extract(t *testing.T) { &placement.Trait{ResourceProviderUUID: "uuid1", Name: "CUSTOM_HW_SAPPHIRE_RAPIDS"}, &placement.Trait{ResourceProviderUUID: "uuid1", Name: "CUSTOM_HANA_EXCLUSIVE_HOST"}, &placement.Trait{ResourceProviderUUID: "uuid1", Name: "CUSTOM_EXTERNAL_CUSTOMER_EXCLUSIVE"}, - // KVM host traits + // KVM host traits (filtered out) &placement.Trait{ResourceProviderUUID: "uuid2", Name: "CUSTOM_NUMASIZE_C48_M729"}, - // Ironic host traits + // Ironic host traits (filtered out) &placement.Trait{ResourceProviderUUID: "uuid3", Name: "TRAIT_IGNORED"}, - // Disabled KVM host - &placement.Trait{ResourceProviderUUID: "uuid4", Name: "CUSTOM_DECOMMISSIONING"}, - // Host with disabled trait - &placement.Trait{ResourceProviderUUID: "uuid6", Name: "COMPUTE_STATUS_DISABLED"}, + // Disabled VMware host via trait + &placement.Trait{ResourceProviderUUID: "uuid10", Name: "COMPUTE_STATUS_DISABLED"}, } if err := testDB.Insert(traits...); err != nil { t.Fatalf("expected no error, got %v", err) } + // Memory inventory for the VMware building block host (uuid1). The building + // block pools 3 physical hosts of 4 TiB each (4 TiB = 4194304 MiB): + // amount_hosts = (total - reserved) / max_unit = 12582912 / 4194304 = 3 + // physical_host_size = max_unit + reserved / amount_hosts = 4194304 MiB = 4 TiB + inventories := []any{ + &placement.InventoryUsage{ + ResourceProviderUUID: "uuid1", + InventoryClassName: "MEMORY_MB", + Total: 12582912, + Reserved: 0, + MaxUnit: 4194304, + }, + // Building block of 2 physical hosts of 512 GiB each (512 GiB = 524288 MiB): + // amount_hosts = (1048576 - 0) / 524288 = 2 + // physical_host_size = 524288 MiB = 512 GiB (< 1 TiB, reported in GiB) + &placement.InventoryUsage{ + ResourceProviderUUID: "uuid7", + InventoryClassName: "MEMORY_MB", + Total: 1048576, + Reserved: 0, + MaxUnit: 524288, + }, + // Single host of 1048064 MiB (1023.5 GiB) which rounds to 1024 GiB and is + // therefore reported as "1TiB" rather than "1024GiB". + &placement.InventoryUsage{ + ResourceProviderUUID: "uuid8", + InventoryClassName: "MEMORY_MB", + Total: 1048064, + Reserved: 0, + MaxUnit: 1048064, + }, + } + if err := testDB.Insert(inventories...); err != nil { + t.Fatalf("expected no error, got %v", err) + } + hostAvailabilityZones, err := v1alpha1.BoxFeatureList([]any{ &HostAZ{AvailabilityZone: new("az1"), ComputeHost: "nova-compute-bb01"}, - &HostAZ{AvailabilityZone: nil, ComputeHost: "node001-bb02"}, - &HostAZ{AvailabilityZone: new("az2"), ComputeHost: "node002-bb03"}, - &HostAZ{AvailabilityZone: new("az2"), ComputeHost: "ironic-host-01"}, - &HostAZ{AvailabilityZone: new("az2"), ComputeHost: "node003-bb03"}, - &HostAZ{AvailabilityZone: new("az2"), ComputeHost: "node004-bb03"}, + &HostAZ{AvailabilityZone: new("az1"), ComputeHost: "nova-compute-bb04"}, + &HostAZ{AvailabilityZone: new("az1"), ComputeHost: "nova-compute-bb05"}, + &HostAZ{AvailabilityZone: new("az2"), ComputeHost: "nova-compute-bb06"}, + &HostAZ{AvailabilityZone: new("az2"), ComputeHost: "nova-compute-bb07"}, }) if err != nil { t.Fatalf("expected no error, got %v", err) } - extractor := &HostDetailsExtractor{} + extractor := &VMwareHostDetailsExtractor{} config := v1alpha1.KnowledgeSpec{} client := fake.NewClientBuilder(). WithScheme(scheme). @@ -127,55 +160,50 @@ func TestHostDetailsExtractor_Extract(t *testing.T) { t.Fatalf("expected no error, got %v", err) } - expected := []HostDetails{ + expected := []VMwareHostDetails{ { - ComputeHost: "ironic-host-01", - AvailabilityZone: "az2", - CPUArchitecture: "cascade-lake", - HypervisorType: "ironic", - HypervisorFamily: "unknown", - WorkloadType: "general-purpose", + ComputeHost: "nova-compute-bb01", + AvailabilityZone: "az1", + CPUArchitecture: "sapphire-rapids", + WorkloadType: "hana", Enabled: true, - ExternalCustomer: false, Decommissioned: false, + ExternalCustomer: true, DisabledReason: nil, - RunningVMs: 0, - PinnedProjects: nil, + RunningVMs: 5, + PinnedProjects: new("project-123,project-456"), + PhysicalHostSize: "4TiB", }, { - ComputeHost: "node001-bb02", - AvailabilityZone: "unknown", + ComputeHost: "nova-compute-bb04", + AvailabilityZone: "az1", CPUArchitecture: "cascade-lake", - HypervisorType: "qemu", - HypervisorFamily: "kvm", WorkloadType: "general-purpose", - Enabled: false, + Enabled: true, Decommissioned: false, ExternalCustomer: false, - DisabledReason: new("[down] --"), - RunningVMs: 3, + DisabledReason: nil, + RunningVMs: 1, PinnedProjects: nil, + PhysicalHostSize: "512GiB", }, { - ComputeHost: "node002-bb03", - AvailabilityZone: "az2", + ComputeHost: "nova-compute-bb05", + AvailabilityZone: "az1", CPUArchitecture: "cascade-lake", - HypervisorFamily: "kvm", - HypervisorType: "test", WorkloadType: "general-purpose", Enabled: true, - Decommissioned: true, + Decommissioned: false, ExternalCustomer: false, DisabledReason: nil, - RunningVMs: 2, + RunningVMs: 1, PinnedProjects: nil, + PhysicalHostSize: "1TiB", }, { - ComputeHost: "node003-bb03", + ComputeHost: "nova-compute-bb06", AvailabilityZone: "az2", CPUArchitecture: "cascade-lake", - HypervisorType: "test", - HypervisorFamily: "kvm", WorkloadType: "general-purpose", Enabled: false, Decommissioned: false, @@ -183,13 +211,12 @@ func TestHostDetailsExtractor_Extract(t *testing.T) { DisabledReason: new("[disabled] example reason"), RunningVMs: 2, PinnedProjects: nil, + PhysicalHostSize: "unknown", }, { - ComputeHost: "node004-bb03", + ComputeHost: "nova-compute-bb07", AvailabilityZone: "az2", CPUArchitecture: "cascade-lake", - HypervisorType: "test", - HypervisorFamily: "kvm", WorkloadType: "general-purpose", Enabled: false, Decommissioned: false, @@ -197,24 +224,11 @@ func TestHostDetailsExtractor_Extract(t *testing.T) { DisabledReason: new("[disabled] example reason"), RunningVMs: 2, PinnedProjects: nil, - }, - { - ComputeHost: "nova-compute-bb01", - AvailabilityZone: "az1", - CPUArchitecture: "sapphire-rapids", - HypervisorType: "vcenter", - HypervisorFamily: "vmware", - WorkloadType: "hana", - Enabled: true, - Decommissioned: false, - ExternalCustomer: true, - DisabledReason: nil, - RunningVMs: 5, - PinnedProjects: new("project-123,project-456"), + PhysicalHostSize: "unknown", }, } - // Check if the expected details match the extracted ones + // Only VMware hosts should be extracted; KVM and ironic hosts are filtered out. if len(features) != len(expected) { t.Fatalf("expected %d host details, got %d", len(expected), len(features)) } @@ -222,7 +236,7 @@ func TestHostDetailsExtractor_Extract(t *testing.T) { for _, expectedDetail := range expected { found := false for _, feature := range features { - extractedDetail := feature.(HostDetails) + extractedDetail := feature.(VMwareHostDetails) if extractedDetail.ComputeHost == expectedDetail.ComputeHost { found = true if !reflect.DeepEqual(extractedDetail, expectedDetail) { diff --git a/internal/knowledge/extractor/supported_extractors.go b/internal/knowledge/extractor/supported_extractors.go index 6f1cb2fd2..4cfcd006a 100644 --- a/internal/knowledge/extractor/supported_extractors.go +++ b/internal/knowledge/extractor/supported_extractors.go @@ -22,7 +22,7 @@ var supportedExtractors = map[string]plugins.FeatureExtractor{ "vm_life_span_histogram_extractor": &compute.VMLifeSpanHistogramExtractor{}, "host_az_extractor": &compute.HostAZExtractor{}, "host_pinned_projects_extractor": &compute.HostPinnedProjectsExtractor{}, - "sap_host_details_extractor": &compute.HostDetailsExtractor{}, + "vmware_host_details_extractor": &compute.VMwareHostDetailsExtractor{}, "flavor_groups": &compute.FlavorGroupExtractor{}, "netapp_storage_pool_cpu_usage_extractor": &storage.StoragePoolCPUUsageExtractor{}, diff --git a/internal/knowledge/kpis/plugins/infrastructure/shared.go b/internal/knowledge/kpis/plugins/infrastructure/shared.go index 4647cc28f..d39790521 100644 --- a/internal/knowledge/kpis/plugins/infrastructure/shared.go +++ b/internal/knowledge/kpis/plugins/infrastructure/shared.go @@ -15,18 +15,16 @@ import ( ) const ( - hostDetailsKnowledgeName = "host-details" + vmwareHostDetailsKnowledgeName = "vmware-host-details" hostUtilizationKnowledgeName = "host-utilization" - vmwareIronicHypervisorType = "ironic" - hypervisorFamilyVMware = "vmware" vmwareComputeHostPattern = "nova-compute-%" vmwareIronicComputeHostPattern = "nova-compute-ironic-%" kvmComputeHostPattern = "node%-bb%" ) -// vmwareHost wraps HostDetails with Prometheus metric helpers. +// vmwareHost wraps VMwareHostDetails with Prometheus metric helpers. type vmwareHost struct { - compute.HostDetails + compute.VMwareHostDetails } func (h vmwareHost) getHostLabels() []string { @@ -40,6 +38,10 @@ func (h vmwareHost) getHostLabels() []string { if h.DisabledReason != nil { disabledReason = *h.DisabledReason } + physicalHostSize := h.PhysicalHostSize + if physicalHostSize == "" { + physicalHostSize = "unknown" + } return []string{ h.AvailabilityZone, h.ComputeHost, @@ -51,6 +53,7 @@ func (h vmwareHost) getHostLabels() []string { disabledReason, strconv.FormatBool(pinnedProjects), pinnedProjectIds, + physicalHostSize, } } @@ -65,6 +68,7 @@ var vmwareHostLabels = []string{ "disabled_reason", "pinned_projects", "pinned_project_ids", + "physical_host_size", } var kvmHostLabels = []string{ diff --git a/internal/knowledge/kpis/plugins/infrastructure/shared_test.go b/internal/knowledge/kpis/plugins/infrastructure/shared_test.go index 68d5941f0..8fe8c9d7b 100644 --- a/internal/knowledge/kpis/plugins/infrastructure/shared_test.go +++ b/internal/knowledge/kpis/plugins/infrastructure/shared_test.go @@ -44,6 +44,7 @@ func mockVMwareHostLabels(computeHost, az string) map[string]string { "disabled_reason": "-", "pinned_projects": "false", "pinned_project_ids": "", + "physical_host_size": "unknown", } } @@ -57,7 +58,7 @@ func TestVMwareHost_GetHostLabels(t *testing.T) { }{ { name: "all optional fields nil", - host: vmwareHost{compute.HostDetails{ + host: vmwareHost{compute.VMwareHostDetails{ AvailabilityZone: "az1", ComputeHost: "nova-compute-1", CPUArchitecture: "cascade-lake", @@ -68,35 +69,44 @@ func TestVMwareHost_GetHostLabels(t *testing.T) { DisabledReason: nil, PinnedProjects: nil, }}, - want: []string{"az1", "nova-compute-1", "cascade-lake", "general-purpose", "true", "false", "false", "-", "false", ""}, + want: []string{"az1", "nova-compute-1", "cascade-lake", "general-purpose", "true", "false", "false", "-", "false", "", "unknown"}, }, { name: "disabled reason set", - host: vmwareHost{compute.HostDetails{ + host: vmwareHost{compute.VMwareHostDetails{ AvailabilityZone: "az2", ComputeHost: "nova-compute-2", DisabledReason: str("scheduled-maintenance"), }}, - want: []string{"az2", "nova-compute-2", "", "", "false", "false", "false", "scheduled-maintenance", "false", ""}, + want: []string{"az2", "nova-compute-2", "", "", "false", "false", "false", "scheduled-maintenance", "false", "", "unknown"}, }, { name: "pinned projects set", - host: vmwareHost{compute.HostDetails{ + host: vmwareHost{compute.VMwareHostDetails{ AvailabilityZone: "az1", ComputeHost: "nova-compute-3", PinnedProjects: str("proj-a,proj-b"), }}, - want: []string{"az1", "nova-compute-3", "", "", "false", "false", "false", "-", "true", "proj-a,proj-b"}, + want: []string{"az1", "nova-compute-3", "", "", "false", "false", "false", "-", "true", "proj-a,proj-b", "unknown"}, }, { name: "decommissioned and external customer", - host: vmwareHost{compute.HostDetails{ + host: vmwareHost{compute.VMwareHostDetails{ AvailabilityZone: "az3", ComputeHost: "nova-compute-4", Decommissioned: true, ExternalCustomer: true, }}, - want: []string{"az3", "nova-compute-4", "", "", "false", "true", "true", "-", "false", ""}, + want: []string{"az3", "nova-compute-4", "", "", "false", "true", "true", "-", "false", "", "unknown"}, + }, + { + name: "physical host size set", + host: vmwareHost{compute.VMwareHostDetails{ + AvailabilityZone: "az1", + ComputeHost: "nova-compute-5", + PhysicalHostSize: "4TiB", + }}, + want: []string{"az1", "nova-compute-5", "", "", "false", "false", "false", "-", "false", "", "4TiB"}, }, } diff --git a/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity.go b/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity.go index c7976db3a..45508636f 100644 --- a/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity.go +++ b/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity.go @@ -81,19 +81,16 @@ func (k *VMwareHostCapacityKPI) Collect(ch chan<- prometheus.Metric) { func (k *VMwareHostCapacityKPI) getVMwareHosts() ([]vmwareHost, error) { knowledge := &v1alpha1.Knowledge{} - if err := k.Client.Get(context.Background(), client.ObjectKey{Name: hostDetailsKnowledgeName}, knowledge); err != nil { + if err := k.Client.Get(context.Background(), client.ObjectKey{Name: vmwareHostDetailsKnowledgeName}, knowledge); err != nil { return nil, err } - details, err := v1alpha1.UnboxFeatureList[compute.HostDetails](knowledge.Status.Raw) + details, err := v1alpha1.UnboxFeatureList[compute.VMwareHostDetails](knowledge.Status.Raw) if err != nil { return nil, err } hosts := make([]vmwareHost, 0, len(details)) for _, d := range details { - if d.HypervisorType == vmwareIronicHypervisorType || d.HypervisorFamily != hypervisorFamilyVMware { - continue - } - hosts = append(hosts, vmwareHost{HostDetails: d}) + hosts = append(hosts, vmwareHost{VMwareHostDetails: d}) } return hosts, nil } diff --git a/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity_test.go b/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity_test.go index f0a025db4..dd2f50637 100644 --- a/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity_test.go +++ b/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity_test.go @@ -18,7 +18,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" ) -func buildHostCapacityClient(t *testing.T, hostDetails []compute.HostDetails, utilizations []compute.HostUtilization) *fake.ClientBuilder { +func buildHostCapacityClient(t *testing.T, hostDetails []compute.VMwareHostDetails, utilizations []compute.HostUtilization) *fake.ClientBuilder { t.Helper() scheme, err := v1alpha1.SchemeBuilder.Build() if err != nil { @@ -34,7 +34,7 @@ func buildHostCapacityClient(t *testing.T, hostDetails []compute.HostDetails, ut } return fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects( &v1alpha1.Knowledge{ - ObjectMeta: v1.ObjectMeta{Name: hostDetailsKnowledgeName}, + ObjectMeta: v1.ObjectMeta{Name: vmwareHostDetailsKnowledgeName}, Status: v1alpha1.KnowledgeStatus{Raw: rawDetails}, }, &v1alpha1.Knowledge{ @@ -55,11 +55,11 @@ func TestVMwareHostCapacityKPI_Init(t *testing.T) { } func TestVMwareHostCapacityKPI_getVMwareHosts(t *testing.T) { - hostDetails := []compute.HostDetails{ - {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware}, - {ComputeHost: "nova-compute-2", HypervisorFamily: hypervisorFamilyVMware}, - {ComputeHost: "nova-compute-ironic-1", HypervisorType: vmwareIronicHypervisorType, HypervisorFamily: hypervisorFamilyVMware}, - {ComputeHost: "nova-compute-3", HypervisorFamily: "other"}, + // The extractor SQL is responsible for filtering out KVM and ironic hosts, + // so the KPI returns every host present in the knowledge unchanged. + hostDetails := []compute.VMwareHostDetails{ + {ComputeHost: "nova-compute-1"}, + {ComputeHost: "nova-compute-2"}, } client := buildHostCapacityClient(t, hostDetails, nil) @@ -155,14 +155,14 @@ func TestVMwareHostCapacityKPI_getHostUtilizations(t *testing.T) { func TestVMwareHostCapacityKPI_Collect(t *testing.T) { tests := []struct { name string - hostDetails []compute.HostDetails + hostDetails []compute.VMwareHostDetails utilizations []compute.HostUtilization expectedMetrics []collectedVMwareMetric }{ { name: "single host emits usage and total metrics", - hostDetails: []compute.HostDetails{ - {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + hostDetails: []compute.VMwareHostDetails{ + {ComputeHost: "nova-compute-1", AvailabilityZone: "az1"}, }, utilizations: []compute.HostUtilization{ { @@ -186,9 +186,9 @@ func TestVMwareHostCapacityKPI_Collect(t *testing.T) { }, { name: "multiple hosts each emit their own metrics", - hostDetails: []compute.HostDetails{ - {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, - {ComputeHost: "nova-compute-2", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az2"}, + hostDetails: []compute.VMwareHostDetails{ + {ComputeHost: "nova-compute-1", AvailabilityZone: "az1"}, + {ComputeHost: "nova-compute-2", AvailabilityZone: "az2"}, }, utilizations: []compute.HostUtilization{ {ComputeHost: "nova-compute-1", VCPUsUsed: 2, TotalVCPUsAllocatable: 8, RAMUsedMB: 512, TotalRAMAllocatableMB: 2048, DiskUsedGB: 10, TotalDiskAllocatableGB: 100}, @@ -209,56 +209,18 @@ func TestVMwareHostCapacityKPI_Collect(t *testing.T) { {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-2", "az2", "disk"), Value: 200 * 1024 * 1024 * 1024}, }, }, - { - name: "ironic hosts are excluded", - hostDetails: []compute.HostDetails{ - {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, - {ComputeHost: "nova-compute-ironic-1", HypervisorType: vmwareIronicHypervisorType, HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, - }, - utilizations: []compute.HostUtilization{ - {ComputeHost: "nova-compute-1", VCPUsUsed: 2, TotalVCPUsAllocatable: 8, RAMUsedMB: 512, TotalRAMAllocatableMB: 2048, DiskUsedGB: 10, TotalDiskAllocatableGB: 100}, - {ComputeHost: "nova-compute-ironic-1", VCPUsUsed: 4, TotalVCPUsAllocatable: 16, RAMUsedMB: 1024, TotalRAMAllocatableMB: 4096, DiskUsedGB: 20, TotalDiskAllocatableGB: 200}, - }, - expectedMetrics: []collectedVMwareMetric{ - {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "cpu"), Value: 2}, - {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "ram"), Value: 512 * 1024 * 1024}, - {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "disk"), Value: 10 * 1024 * 1024 * 1024}, - {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "cpu"), Value: 8}, - {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "ram"), Value: 2048 * 1024 * 1024}, - {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "disk"), Value: 100 * 1024 * 1024 * 1024}, - }, - }, - { - name: "non-vmware hosts are excluded", - hostDetails: []compute.HostDetails{ - {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, - {ComputeHost: "nova-compute-2", HypervisorFamily: "kvm", AvailabilityZone: "az1"}, - }, - utilizations: []compute.HostUtilization{ - {ComputeHost: "nova-compute-1", VCPUsUsed: 2, TotalVCPUsAllocatable: 8, RAMUsedMB: 512, TotalRAMAllocatableMB: 2048, DiskUsedGB: 10, TotalDiskAllocatableGB: 100}, - {ComputeHost: "nova-compute-2", VCPUsUsed: 4, TotalVCPUsAllocatable: 16, RAMUsedMB: 1024, TotalRAMAllocatableMB: 4096, DiskUsedGB: 20, TotalDiskAllocatableGB: 200}, - }, - expectedMetrics: []collectedVMwareMetric{ - {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "cpu"), Value: 2}, - {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "ram"), Value: 512 * 1024 * 1024}, - {Name: "cortex_vmware_host_capacity_usage", Labels: hostCapacityLabels("nova-compute-1", "az1", "disk"), Value: 10 * 1024 * 1024 * 1024}, - {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "cpu"), Value: 8}, - {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "ram"), Value: 2048 * 1024 * 1024}, - {Name: "cortex_vmware_host_capacity_total", Labels: hostCapacityLabels("nova-compute-1", "az1", "disk"), Value: 100 * 1024 * 1024 * 1024}, - }, - }, { name: "host without matching utilization produces no metrics", - hostDetails: []compute.HostDetails{ - {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + hostDetails: []compute.VMwareHostDetails{ + {ComputeHost: "nova-compute-1", AvailabilityZone: "az1"}, }, utilizations: []compute.HostUtilization{}, expectedMetrics: []collectedVMwareMetric{}, }, { name: "utilization with zero allocatable resources is skipped", - hostDetails: []compute.HostDetails{ - {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + hostDetails: []compute.VMwareHostDetails{ + {ComputeHost: "nova-compute-1", AvailabilityZone: "az1"}, }, utilizations: []compute.HostUtilization{ {ComputeHost: "nova-compute-1", VCPUsUsed: 2, TotalVCPUsAllocatable: 0, RAMUsedMB: 512, TotalRAMAllocatableMB: 2048, DiskUsedGB: 10, TotalDiskAllocatableGB: 100}, @@ -267,7 +229,7 @@ func TestVMwareHostCapacityKPI_Collect(t *testing.T) { }, { name: "no hosts produces no metrics", - hostDetails: []compute.HostDetails{}, + hostDetails: []compute.VMwareHostDetails{}, utilizations: []compute.HostUtilization{}, expectedMetrics: []collectedVMwareMetric{}, }, diff --git a/internal/knowledge/kpis/plugins/infrastructure/vmware_project_utilization.go b/internal/knowledge/kpis/plugins/infrastructure/vmware_project_utilization.go index 6a85756ff..2459cb4c6 100644 --- a/internal/knowledge/kpis/plugins/infrastructure/vmware_project_utilization.go +++ b/internal/knowledge/kpis/plugins/infrastructure/vmware_project_utilization.go @@ -128,21 +128,18 @@ func (k *VMwareProjectUtilizationKPI) Collect(ch chan<- prometheus.Metric) { // getVMwareHosts retrieves the mapping of VMware hypervisors to their corresponding host information func (k *VMwareProjectUtilizationKPI) getVMwareHosts() (map[string]vmwareHost, error) { knowledge := &v1alpha1.Knowledge{} - if err := k.Client.Get(context.Background(), client.ObjectKey{Name: hostDetailsKnowledgeName}, knowledge); err != nil { + if err := k.Client.Get(context.Background(), client.ObjectKey{Name: vmwareHostDetailsKnowledgeName}, knowledge); err != nil { return nil, err } - hostDetails, err := v1alpha1.UnboxFeatureList[compute.HostDetails](knowledge.Status.Raw) + hostDetails, err := v1alpha1.UnboxFeatureList[compute.VMwareHostDetails](knowledge.Status.Raw) if err != nil { return nil, err } hostMapping := make(map[string]vmwareHost) for _, host := range hostDetails { - if host.HypervisorType == vmwareIronicHypervisorType || host.HypervisorFamily != hypervisorFamilyVMware { - continue - } - hostMapping[host.ComputeHost] = vmwareHost{HostDetails: host} + hostMapping[host.ComputeHost] = vmwareHost{VMwareHostDetails: host} } return hostMapping, nil diff --git a/internal/knowledge/kpis/plugins/infrastructure/vmware_project_utilization_test.go b/internal/knowledge/kpis/plugins/infrastructure/vmware_project_utilization_test.go index 15f18b894..6edd47e0c 100644 --- a/internal/knowledge/kpis/plugins/infrastructure/vmware_project_utilization_test.go +++ b/internal/knowledge/kpis/plugins/infrastructure/vmware_project_utilization_test.go @@ -53,7 +53,7 @@ func capacityMetric(computeHost, az, projectID, projectName, domainID, domainNam return collectedVMwareMetric{Name: "cortex_vmware_project_capacity_usage", Labels: labels, Value: value} } -func buildVMwareHostDetailsClient(t *testing.T, hostDetails []compute.HostDetails) *fake.ClientBuilder { +func buildVMwareHostDetailsClient(t *testing.T, hostDetails []compute.VMwareHostDetails) *fake.ClientBuilder { t.Helper() scheme, err := v1alpha1.SchemeBuilder.Build() if err != nil { @@ -65,7 +65,7 @@ func buildVMwareHostDetailsClient(t *testing.T, hostDetails []compute.HostDetail } return fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects( &v1alpha1.Knowledge{ - ObjectMeta: v1.ObjectMeta{Name: "host-details"}, + ObjectMeta: v1.ObjectMeta{Name: vmwareHostDetailsKnowledgeName}, Status: v1alpha1.KnowledgeStatus{Raw: raw}, }, ) @@ -82,24 +82,11 @@ func TestVMwareProjectUtilizationKPI_Init(t *testing.T) { } func TestVMwareProjectUtilizationKPI_getVMwareHosts(t *testing.T) { - hostDetails := []compute.HostDetails{ - { - ComputeHost: "nova-compute-1", - HypervisorFamily: hypervisorFamilyVMware, - }, - { - ComputeHost: "nova-compute-2", - HypervisorFamily: hypervisorFamilyVMware, - }, - { - ComputeHost: "nova-compute-ironic-1", - HypervisorType: vmwareIronicHypervisorType, - HypervisorFamily: hypervisorFamilyVMware, - }, - { - ComputeHost: "nova-compute-3", - HypervisorFamily: "other", - }, + // Filtering of KVM and ironic hosts happens in the extractor SQL, so the KPI + // maps every host present in the knowledge. + hostDetails := []compute.VMwareHostDetails{ + {ComputeHost: "nova-compute-1"}, + {ComputeHost: "nova-compute-2"}, } clientBuilder := buildVMwareHostDetailsClient(t, hostDetails) @@ -112,8 +99,8 @@ func TestVMwareProjectUtilizationKPI_getVMwareHosts(t *testing.T) { } expectedHosts := map[string]vmwareHost{ - "nova-compute-1": {HostDetails: hostDetails[0]}, - "nova-compute-2": {HostDetails: hostDetails[1]}, + "nova-compute-1": {VMwareHostDetails: hostDetails[0]}, + "nova-compute-2": {VMwareHostDetails: hostDetails[1]}, } if len(hostMapping) != len(expectedHosts) { @@ -125,7 +112,7 @@ func TestVMwareProjectUtilizationKPI_getVMwareHosts(t *testing.T) { if !ok { t.Fatalf("expected host %s not found in mapping", computeHost) } - if host.ComputeHost != expectedHost.ComputeHost || host.HypervisorFamily != expectedHost.HypervisorFamily { + if host.ComputeHost != expectedHost.ComputeHost { t.Errorf("host details mismatch for %s: expected %+v, got %+v", computeHost, expectedHost, host) } } @@ -272,7 +259,7 @@ func TestVMwareProjectUtilizationKPI_queryProjectInstanceCount(t *testing.T) { } } - client := buildVMwareHostDetailsClient(t, []compute.HostDetails{}) + client := buildVMwareHostDetailsClient(t, []compute.VMwareHostDetails{}) kpi := &VMwareProjectUtilizationKPI{} if err := kpi.Init(&testDB, client.Build(), conf.NewRawOpts("{}")); err != nil { t.Fatalf("expected no error on Init, got %v", err) @@ -468,7 +455,7 @@ func TestVMwareProjectUtilizationKPI_queryProjectCapacityUsage(t *testing.T) { } } - client := buildVMwareHostDetailsClient(t, []compute.HostDetails{}) + client := buildVMwareHostDetailsClient(t, []compute.VMwareHostDetails{}) kpi := &VMwareProjectUtilizationKPI{} if err := kpi.Init(&testDB, client.Build(), conf.NewRawOpts("{}")); err != nil { t.Fatalf("expected no error on Init, got %v", err) @@ -503,7 +490,7 @@ func TestVMwareProjectUtilizationKPI_Collect(t *testing.T) { projects []identity.Project domains []identity.Domain flavors []nova.Flavor - hostDetails []compute.HostDetails + hostDetails []compute.VMwareHostDetails expectedMetrics []collectedVMwareMetric }{ { @@ -514,8 +501,8 @@ func TestVMwareProjectUtilizationKPI_Collect(t *testing.T) { projects: []identity.Project{{ID: "project-1", Name: "Project One", DomainID: "domain-1"}}, domains: []identity.Domain{{ID: "domain-1", Name: "Domain One"}}, flavors: []nova.Flavor{{ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}}, - hostDetails: []compute.HostDetails{ - {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + hostDetails: []compute.VMwareHostDetails{ + {ComputeHost: "nova-compute-1", AvailabilityZone: "az1"}, }, expectedMetrics: []collectedVMwareMetric{ instanceMetric("nova-compute-1", "az1", "project-1", "Project One", "domain-1", "Domain One", "flavor-1", 1), @@ -540,9 +527,9 @@ func TestVMwareProjectUtilizationKPI_Collect(t *testing.T) { {ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}, {ID: "f2", Name: "flavor-2", VCPUs: 4, RAM: 8192, Disk: 2}, }, - hostDetails: []compute.HostDetails{ - {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, - {ComputeHost: "nova-compute-2", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az2"}, + hostDetails: []compute.VMwareHostDetails{ + {ComputeHost: "nova-compute-1", AvailabilityZone: "az1"}, + {ComputeHost: "nova-compute-2", AvailabilityZone: "az2"}, }, expectedMetrics: []collectedVMwareMetric{ instanceMetric("nova-compute-1", "az1", "project-1", "Project One", "domain-1", "Domain One", "flavor-1", 1), @@ -568,8 +555,8 @@ func TestVMwareProjectUtilizationKPI_Collect(t *testing.T) { projects: []identity.Project{{ID: "project-1", Name: "Project One", DomainID: "domain-1"}}, domains: []identity.Domain{{ID: "domain-1", Name: "Domain One"}}, flavors: []nova.Flavor{{ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}}, - hostDetails: []compute.HostDetails{ - {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + hostDetails: []compute.VMwareHostDetails{ + {ComputeHost: "nova-compute-1", AvailabilityZone: "az1"}, }, expectedMetrics: []collectedVMwareMetric{ instanceMetric("nova-compute-1", "az1", "project-1", "Project One", "domain-1", "Domain One", "flavor-1", 1), @@ -592,8 +579,8 @@ func TestVMwareProjectUtilizationKPI_Collect(t *testing.T) { {ID: "f2", Name: "flavor-2", VCPUs: 4, RAM: 8192, Disk: 2}, {ID: "f3", Name: "flavor-3", VCPUs: 8, RAM: 16384, Disk: 4}, }, - hostDetails: []compute.HostDetails{ - {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + hostDetails: []compute.VMwareHostDetails{ + {ComputeHost: "nova-compute-1", AvailabilityZone: "az1"}, }, expectedMetrics: []collectedVMwareMetric{ instanceMetric("nova-compute-1", "az1", "project-1", "Project One", "domain-1", "Domain One", "flavor-3", 1), @@ -613,9 +600,9 @@ func TestVMwareProjectUtilizationKPI_Collect(t *testing.T) { projects: []identity.Project{{ID: "project-1", Name: "Project One", DomainID: "domain-1"}}, domains: []identity.Domain{{ID: "domain-1", Name: "Domain One"}}, flavors: []nova.Flavor{{ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}}, - hostDetails: []compute.HostDetails{ - {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, - {ComputeHost: "nova-compute-2", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az2"}, + hostDetails: []compute.VMwareHostDetails{ + {ComputeHost: "nova-compute-1", AvailabilityZone: "az1"}, + {ComputeHost: "nova-compute-2", AvailabilityZone: "az2"}, }, expectedMetrics: []collectedVMwareMetric{ instanceMetric("nova-compute-1", "az1", "project-1", "Project One", "domain-1", "Domain One", "flavor-1", 2), @@ -636,8 +623,8 @@ func TestVMwareProjectUtilizationKPI_Collect(t *testing.T) { projects: []identity.Project{{ID: "project-1", Name: "Project One", DomainID: "domain-unknown"}}, domains: []identity.Domain{}, flavors: []nova.Flavor{{ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}}, - hostDetails: []compute.HostDetails{ - {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + hostDetails: []compute.VMwareHostDetails{ + {ComputeHost: "nova-compute-1", AvailabilityZone: "az1"}, }, expectedMetrics: []collectedVMwareMetric{ // The domain_id is extracted from the project record, so it should be "domain-unknown" even though there is no matching domain entry @@ -655,8 +642,8 @@ func TestVMwareProjectUtilizationKPI_Collect(t *testing.T) { projects: []identity.Project{}, domains: []identity.Domain{}, flavors: []nova.Flavor{{ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}}, - hostDetails: []compute.HostDetails{ - {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + hostDetails: []compute.VMwareHostDetails{ + {ComputeHost: "nova-compute-1", AvailabilityZone: "az1"}, }, expectedMetrics: []collectedVMwareMetric{ instanceMetric("nova-compute-1", "az1", "project-1", "", "", "", "flavor-1", 1), @@ -673,8 +660,8 @@ func TestVMwareProjectUtilizationKPI_Collect(t *testing.T) { projects: []identity.Project{{ID: "project-1", Name: "Project One", DomainID: "domain-1"}}, domains: []identity.Domain{{ID: "domain-1", Name: "Domain One"}}, flavors: []nova.Flavor{}, - hostDetails: []compute.HostDetails{ - {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + hostDetails: []compute.VMwareHostDetails{ + {ComputeHost: "nova-compute-1", AvailabilityZone: "az1"}, }, expectedMetrics: []collectedVMwareMetric{ instanceMetric("nova-compute-1", "az1", "project-1", "Project One", "domain-1", "Domain One", "flavor-missing", 1), @@ -693,8 +680,8 @@ func TestVMwareProjectUtilizationKPI_Collect(t *testing.T) { flavors: []nova.Flavor{ {ID: "f1", Name: "flavor-1", VCPUs: 2, RAM: 4096, Disk: 1}, }, - hostDetails: []compute.HostDetails{ - {ComputeHost: "nova-compute-1", HypervisorFamily: hypervisorFamilyVMware, AvailabilityZone: "az1"}, + hostDetails: []compute.VMwareHostDetails{ + {ComputeHost: "nova-compute-1", AvailabilityZone: "az1"}, }, expectedMetrics: []collectedVMwareMetric{}, }, diff --git a/internal/scheduling/nova/external_scheduler_api.go b/internal/scheduling/nova/external_scheduler_api.go index 4464c46a0..944ad9b1b 100644 --- a/internal/scheduling/nova/external_scheduler_api.go +++ b/internal/scheduling/nova/external_scheduler_api.go @@ -33,6 +33,17 @@ type HTTPAPIConfig struct { // NovaLimitHostsToRequest, if true, will filter the Nova scheduler response // to only include hosts that were in the original request. NovaLimitHostsToRequest bool `json:"novaLimitHostsToRequest,omitempty"` + // ForcedDestinationEnabled toggles replicating Nova's forced-destination + // behavior (force_hosts/force_nodes skip the scheduling pipeline and + // filters). Defaults to true when unset; set to false to disable and let + // forced requests flow through the normal pipeline instead. + ForcedDestinationEnabled *bool `json:"forcedDestinationEnabled,omitempty"` +} + +// forcedDestinationEnabled reports whether the forced-destination behavior is +// enabled. It defaults to true when the config value is unset. +func (c HTTPAPIConfig) forcedDestinationEnabled() bool { + return c.ForcedDestinationEnabled == nil || *c.ForcedDestinationEnabled } type HTTPAPIDelegate interface { @@ -208,6 +219,25 @@ func (httpAPI *httpAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req logger := slog.With(traceArgsAny...) logger.Info("handling POST request", "url", "/scheduler/nova/external", "body", string(body)) + // Replicate Nova's forced-destination behavior: when the request is forced + // onto specific hosts/nodes (force_hosts/force_nodes) and no _nova_check_type + // is set, Nova skips its filters entirely. We do the same here and return + // only the forced hosts, bypassing pipeline inference and execution. + // This must run before canRunScheduler because Nova may not send weights + // for forced-destination requests. + if httpAPI.config.forcedDestinationEnabled() && requestData.IsForcedDestination() { + hosts := requestData.ForcedHosts() + logger.Info("forced destination request, skipping filters", "hosts", hosts) + response := api.ExternalSchedulerResponse{Hosts: hosts} + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(response); err != nil { + c.Respond(logger, http.StatusInternalServerError, err, "failed to encode response") + return + } + c.Respond(logger, http.StatusOK, nil, "Success") + return + } + if ok, reason := httpAPI.canRunScheduler(requestData); !ok { internalErr := fmt.Errorf("cannot run scheduler: %s", reason) c.Respond(logger, http.StatusBadRequest, internalErr, reason) diff --git a/internal/scheduling/nova/external_scheduler_api_test.go b/internal/scheduling/nova/external_scheduler_api_test.go index 70ce3c334..da2eee5c0 100644 --- a/internal/scheduling/nova/external_scheduler_api_test.go +++ b/internal/scheduling/nova/external_scheduler_api_test.go @@ -383,6 +383,152 @@ func TestHTTPAPI_NovaExternalScheduler_DecisionCreation(t *testing.T) { } } +func TestHTTPAPI_NovaExternalScheduler_ForcedDestination(t *testing.T) { + forceHosts := []string{"host2"} + requestData := novaapi.ExternalSchedulerRequest{ + Spec: novaapi.NovaObject[novaapi.NovaSpec]{ + Data: novaapi.NovaSpec{ + InstanceUUID: "test-uuid", + ForceHosts: &forceHosts, + }, + }, + Hosts: []novaapi.ExternalSchedulerHost{ + {ComputeHost: "host1", HypervisorHostname: "domain-c1"}, + {ComputeHost: "host2", HypervisorHostname: "domain-c2"}, + }, + Weights: map[string]float64{ + "host1": 1.0, + "host2": 2.0, + }, + } + + delegateCalled := false + delegate := &mockHTTPAPIDelegate{ + processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { + delegateCalled = true + return nil + }, + } + api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI) + + body, err := json.Marshal(requestData) + if err != nil { + t.Fatalf("Failed to marshal request data: %v", err) + } + req := httptest.NewRequest(http.MethodPost, "/scheduler/nova/external", bytes.NewReader(body)) + w := httptest.NewRecorder() + + api.NovaExternalScheduler(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("Expected status %d, got %d", http.StatusOK, w.Code) + } + if delegateCalled { + t.Error("delegate should not be called for forced destination requests") + } + var response novaapi.ExternalSchedulerResponse + if err := json.NewDecoder(w.Body).Decode(&response); err != nil { + t.Fatalf("Failed to decode response: %v", err) + } + if len(response.Hosts) != 1 || response.Hosts[0] != "host2" { + t.Errorf("Expected hosts [host2], got %v", response.Hosts) + } +} + +func TestHTTPAPI_NovaExternalScheduler_ForcedDestinationNoWeights(t *testing.T) { + forceHosts := []string{"host2"} + requestData := novaapi.ExternalSchedulerRequest{ + Spec: novaapi.NovaObject[novaapi.NovaSpec]{ + Data: novaapi.NovaSpec{ + InstanceUUID: "test-uuid", + ForceHosts: &forceHosts, + }, + }, + Hosts: []novaapi.ExternalSchedulerHost{ + {ComputeHost: "host1", HypervisorHostname: "domain-c1"}, + {ComputeHost: "host2", HypervisorHostname: "domain-c2"}, + }, + // Weights intentionally omitted — Nova does not always send them + // for forced-destination requests. + } + + delegateCalled := false + delegate := &mockHTTPAPIDelegate{ + processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { + delegateCalled = true + return nil + }, + } + api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI) + + body, err := json.Marshal(requestData) + if err != nil { + t.Fatalf("Failed to marshal request data: %v", err) + } + req := httptest.NewRequest(http.MethodPost, "/scheduler/nova/external", bytes.NewReader(body)) + w := httptest.NewRecorder() + + api.NovaExternalScheduler(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("Expected status %d, got %d (forced destination with no weights must not be rejected)", http.StatusOK, w.Code) + } + if delegateCalled { + t.Error("delegate should not be called for forced destination requests") + } + var response novaapi.ExternalSchedulerResponse + if err := json.NewDecoder(w.Body).Decode(&response); err != nil { + t.Fatalf("Failed to decode response: %v", err) + } + if len(response.Hosts) != 1 || response.Hosts[0] != "host2" { + t.Errorf("Expected hosts [host2], got %v", response.Hosts) + } +} + +func TestHTTPAPI_NovaExternalScheduler_ForcedDestinationDisabled(t *testing.T) { + forceHosts := []string{"host2"} + requestData := novaapi.ExternalSchedulerRequest{ + Spec: novaapi.NovaObject[novaapi.NovaSpec]{ + Data: novaapi.NovaSpec{ + InstanceUUID: "test-uuid", + ForceHosts: &forceHosts, + }, + }, + Hosts: []novaapi.ExternalSchedulerHost{ + {ComputeHost: "host1", HypervisorHostname: "domain-c1"}, + {ComputeHost: "host2", HypervisorHostname: "domain-c2"}, + }, + Weights: map[string]float64{ + "host1": 1.0, + "host2": 2.0, + }, + Pipeline: "test-pipeline", + } + + delegateCalled := false + delegate := &mockHTTPAPIDelegate{ + processDecisionFunc: func(ctx context.Context, decision *v1alpha1.Decision) error { + delegateCalled = true + return nil + }, + } + disabled := false + api := NewAPI(HTTPAPIConfig{ForcedDestinationEnabled: &disabled}, delegate).(*httpAPI) + + body, err := json.Marshal(requestData) + if err != nil { + t.Fatalf("Failed to marshal request data: %v", err) + } + req := httptest.NewRequest(http.MethodPost, "/scheduler/nova/external", bytes.NewReader(body)) + w := httptest.NewRecorder() + + api.NovaExternalScheduler(w, req) + + if !delegateCalled { + t.Error("delegate should be called when forced destination is disabled") + } +} + func TestLimitHostsToRequest(t *testing.T) { tests := []struct { name string diff --git a/internal/shim/placement/field_index_test.go b/internal/shim/placement/field_index_test.go index 1f74ad9a3..a81ec2879 100644 --- a/internal/shim/placement/field_index_test.go +++ b/internal/shim/placement/field_index_test.go @@ -50,6 +50,10 @@ type stubCluster struct { func (s *stubCluster) GetClient() client.Client { return s.cl } func (s *stubCluster) GetCache() cache.Cache { return s.cache } +// GetFieldIndexer returns the cache, which implements client.FieldIndexer via +// its IndexField method — mirroring an unwrapped controller-runtime cluster. +func (s *stubCluster) GetFieldIndexer() client.FieldIndexer { return s.cache } + type stubManager struct { manager.Manager } diff --git a/pkg/cache/client.go b/pkg/cache/client.go new file mode 100644 index 000000000..d71064f49 --- /dev/null +++ b/pkg/cache/client.go @@ -0,0 +1,606 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package cache + +import ( + "context" + "errors" + "strconv" + "sync" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +// entry is a single overlaid object with the bookkeeping needed for eviction. +type entry struct { + obj client.Object + uid types.UID + resourceVersion string + deleted bool // tombstone: object was deleted through the caching client + expiresAt time.Time +} + +// resourceVersionAtLeast reports whether observed >= cached, treating +// ResourceVersions as opaque monotonically increasing integers (as the +// kubernetes apiserver guarantees per resource). Unparsable or empty values +// are treated conservatively: an empty cached RV means "evict on any sighting". +func resourceVersionAtLeast(observed, cached string) bool { + if cached == "" { + return true + } + if observed == "" { + return false + } + oi, oerr := strconv.ParseUint(observed, 10, 64) + ci, cerr := strconv.ParseUint(cached, 10, 64) + if oerr != nil || cerr != nil { + // Fall back to string comparison if not integers. + return observed >= cached + } + return oi >= ci +} + +// defaultTTL is used when Config.TTL is zero. +const defaultTTL = 2 * time.Minute + +// Overlay wraps an inner client.Client with a transparent in-process +// overlay. Writes populate the overlay; reads merge the overlay with the inner +// (informer-backed) result; entries are evicted once the real object appears in +// an informer (see runnable.go) or after TTL expiry. +// +// It embeds client.Client so all methods not overridden below are delegated to +// the inner client unchanged. +// +// The local overlay maps each cached GVK to a set of entries keyed by +// namespace/name. An entry is either live (a pending write not yet visible in +// the informer) or a tombstone (a Delete that has not yet propagated). Reads +// merge the inner result with these entries: tombstones suppress objects; +// live entries override or supplement the inner result. +type Overlay struct { + client.Client // inner client, used for delegation + + informerCache cache.Cache // used by Start to attach eviction event handlers + scheme *runtime.Scheme + ttl time.Duration + gvks map[schema.GroupVersionKind]bool + + // monitor observes the overlay size per GVK. It may be nil (metrics off); + // noteSizeLocked and Monitor.observeSize are nil-safe. host identifies this + // overlay's cluster (its rest config Host) so a single shared Monitor can + // distinguish home from remotes. + monitor *Monitor + host string + + mu sync.RWMutex + byGVK map[schema.GroupVersionKind]map[client.ObjectKey]*entry + indexers map[schema.GroupVersionKind]map[string]client.IndexerFunc + + // writeLocks serializes writes to the same object so the inner call and the + // overlay mutation are atomic per object, keeping the overlay from falling + // behind the apiserver under concurrent writes. + writeLocks *keyedMutex +} + +// WrapCluster wraps inner with a transparent in-process overlay cache and +// returns the wrapped cluster.Cluster together with the overlay's lifecycle +// Runnable. +// +// The returned cluster.Cluster delegates everything to inner except GetClient +// and GetFieldIndexer, which return the overlay client so per-cluster reads and +// writes flow through the cache. GVK strings in conf are formatted as +// "//" and resolved against inner.GetScheme(). +// +// The returned manager.Runnable is the overlay itself: it attaches informer +// eviction handlers (reading informers from inner.GetCache()) and runs the TTL +// cleanup loop. The caller MUST add it to the manager. It does NOT re-Start the +// inner cluster — the manager owns the inner cluster's lifecycle separately. +// +// monitor observes the overlay size per GVK and may be nil (metrics off). It is +// internal to this package (only Wrapper and tests call WrapCluster), so this +// signature does not touch the multicluster client. +func WrapCluster(inner cluster.Cluster, conf Config, monitor *Monitor) (cluster.Cluster, manager.Runnable, error) { + scheme := inner.GetScheme() + gvks, err := resolveGVKs(scheme, conf.GVKs) + if err != nil { + return nil, nil, err + } + ttl := conf.TTL.Duration + if ttl <= 0 { + ttl = defaultTTL + } + cc := &Overlay{ + Client: inner.GetClient(), + informerCache: inner.GetCache(), + scheme: scheme, + ttl: ttl, + gvks: gvks, + monitor: monitor, + host: inner.GetConfig().Host, + byGVK: make(map[schema.GroupVersionKind]map[client.ObjectKey]*entry), + indexers: make(map[schema.GroupVersionKind]map[string]client.IndexerFunc), + writeLocks: newKeyedMutex(), + } + return &overlayCluster{Cluster: inner, cache: cc}, cc, nil +} + +// resolveGVKs maps "//" strings to GVKs via the scheme's +// known types. Mirrors the resolution logic of multicluster.InitFromConf, but +// stays local to this package. +func resolveGVKs(scheme *runtime.Scheme, gvkStrs []string) (map[schema.GroupVersionKind]bool, error) { + byStr := make(map[string]schema.GroupVersionKind) + for gvk := range scheme.AllKnownTypes() { + byStr[gvk.GroupVersion().String()+"/"+gvk.Kind] = gvk + } + out := make(map[schema.GroupVersionKind]bool, len(gvkStrs)) + for _, s := range gvkStrs { + gvk, ok := byStr[s] + if !ok { + return nil, errors.New("cache: no gvk registered in scheme for " + s) + } + out[gvk] = true + } + return out, nil +} + +// gvkFor resolves the GVK of obj and reports whether it is cached. +func (c *Overlay) gvkFor(obj runtime.Object) (schema.GroupVersionKind, bool) { + gvks, _, err := c.scheme.ObjectKinds(obj) + if err != nil || len(gvks) != 1 { + return schema.GroupVersionKind{}, false + } + gvk := gvks[0] + return gvk, c.gvks[gvk] +} + +// itemGVKForList resolves the singular item GVK for a list object and reports +// whether that item GVK is cached. The list GVK's Kind ends with "List". +func (c *Overlay) itemGVKForList(list client.ObjectList) (schema.GroupVersionKind, bool) { + gvks, _, err := c.scheme.ObjectKinds(list) + if err != nil || len(gvks) != 1 { + return schema.GroupVersionKind{}, false + } + gvk := gvks[0] + if kind, ok := trimListSuffix(gvk.Kind); ok { + gvk.Kind = kind + } + return gvk, c.gvks[gvk] +} + +// trimListSuffix strips a trailing "List" from a Kind, reporting whether it did. +func trimListSuffix(kind string) (string, bool) { + const suffix = "List" + if len(kind) > len(suffix) && kind[len(kind)-len(suffix):] == suffix { + return kind[:len(kind)-len(suffix)], true + } + return kind, false +} + +// upsert stores a live (non-tombstone) entry for the object. +func (c *Overlay) upsert(gvk schema.GroupVersionKind, obj client.Object) { + c.mu.Lock() + defer c.mu.Unlock() + c.ensureGVK(gvk) + c.byGVK[gvk][client.ObjectKeyFromObject(obj)] = &entry{ + obj: obj.DeepCopyObject().(client.Object), + uid: obj.GetUID(), + resourceVersion: obj.GetResourceVersion(), + deleted: false, + expiresAt: time.Now().Add(c.ttl), + } + c.noteSizeLocked(gvk) +} + +// tombstone marks the object as deleted in the overlay so it is filtered out +// of reads until the deletion is observed in an informer. +func (c *Overlay) tombstone(gvk schema.GroupVersionKind, obj client.Object) { + c.mu.Lock() + defer c.mu.Unlock() + c.ensureGVK(gvk) + c.byGVK[gvk][client.ObjectKeyFromObject(obj)] = &entry{ + obj: obj.DeepCopyObject().(client.Object), + uid: obj.GetUID(), + resourceVersion: obj.GetResourceVersion(), + deleted: true, + expiresAt: time.Now().Add(c.ttl), + } + c.noteSizeLocked(gvk) +} + +// evictIfSeen removes the overlay entry for obj if the informer-observed object +// matches by UID and its ResourceVersion is at least as new as the cached one. +func (c *Overlay) evictIfSeen(gvk schema.GroupVersionKind, obj client.Object) { + c.mu.Lock() + defer c.mu.Unlock() + entries, ok := c.byGVK[gvk] + if !ok { + return + } + key := client.ObjectKeyFromObject(obj) + e, ok := entries[key] + if !ok { + return + } + // Only evict when the informer sees the same object generation (by UID) at + // a ResourceVersion >= the one we cached. Otherwise the informer might be + // showing an older revision than our pending write. + if e.uid != "" && obj.GetUID() != "" && e.uid != obj.GetUID() { + return + } + if !resourceVersionAtLeast(obj.GetResourceVersion(), e.resourceVersion) { + return + } + delete(entries, key) + c.noteSizeLocked(gvk) +} + +// getEntry returns the overlay entry for the key, if present. +func (c *Overlay) getEntry(gvk schema.GroupVersionKind, key client.ObjectKey) (*entry, bool) { + c.mu.RLock() + defer c.mu.RUnlock() + entries, ok := c.byGVK[gvk] + if !ok { + return nil, false + } + e, ok := entries[key] + return e, ok +} + +// cleanupExpired removes entries whose TTL has passed. +func (c *Overlay) cleanupExpired(now time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + for gvk, entries := range c.byGVK { + changed := false + for key, e := range entries { + if now.After(e.expiresAt) { + delete(entries, key) + changed = true + } + } + if changed { + c.noteSizeLocked(gvk) + } + } +} + +// registerIndex captures an IndexerFunc for a field so overlay entries can be +// matched against MatchingFields queries. +func (c *Overlay) registerIndex(gvk schema.GroupVersionKind, field string, fn client.IndexerFunc) { + c.mu.Lock() + defer c.mu.Unlock() + if c.indexers[gvk] == nil { + c.indexers[gvk] = make(map[string]client.IndexerFunc) + } + c.indexers[gvk][field] = fn +} + +// ensureGVK initialises the per-GVK entry map if absent. Callers must hold the write lock. +func (c *Overlay) ensureGVK(gvk schema.GroupVersionKind) { + if c.byGVK[gvk] == nil { + c.byGVK[gvk] = make(map[client.ObjectKey]*entry) + } +} + +// noteSizeLocked reports the current overlay size for the GVK to the monitor. +// Callers must hold c.mu. Nil-safe when there is no monitor. +func (c *Overlay) noteSizeLocked(gvk schema.GroupVersionKind) { + c.monitor.observeSize(c.host, gvk, len(c.byGVK[gvk])) +} + +// overlayList merges the overlay entries for the GVK into the informer result, +// deduplicating by client.ObjectKey (overlay wins), dropping tombstones, and filtering +// overlay-only entries against the list options' label and field selectors. +func (c *Overlay) overlayList(gvk schema.GroupVersionKind, existing []runtime.Object, lo *client.ListOptions) []runtime.Object { + c.mu.RLock() + defer c.mu.RUnlock() + entries := c.byGVK[gvk] + if len(entries) == 0 { + return existing + } + + result := make([]runtime.Object, 0, len(existing)+len(entries)) + // Track which overlay keys are handled so overlay-only entries can be added. + handled := make(map[client.ObjectKey]bool, len(entries)) + + for _, item := range existing { + obj, ok := item.(client.Object) + if !ok { + result = append(result, item) + continue + } + key := client.ObjectKeyFromObject(obj) + e, present := entries[key] + if !present { + result = append(result, item) + continue + } + handled[key] = true + // Overlay wins over the informer result for the same key. + if e.deleted { + // Tombstone: drop the object entirely. + continue + } + // The inner result matched the query against the informer's (old) field + // values. Re-check the overlay version: if a write changed a queried + // field, the overlay object no longer belongs in this result set. + if !c.matchesLocked(gvk, e.obj, lo) { + continue + } + result = append(result, e.obj.DeepCopyObject()) + } + + // Add overlay-only entries (not present in the informer result) that match + // the list options. + for key, e := range entries { + if handled[key] { + continue + } + if e.deleted { + continue + } + if !c.matchesLocked(gvk, e.obj, lo) { + continue + } + result = append(result, e.obj.DeepCopyObject()) + } + return result +} + +// matchesLocked reports whether obj satisfies the list options' namespace, +// label and field selectors. Callers must hold at least the read lock. +func (c *Overlay) matchesLocked(gvk schema.GroupVersionKind, obj client.Object, lo *client.ListOptions) bool { + if lo == nil { + return true + } + if lo.Namespace != "" && obj.GetNamespace() != lo.Namespace { + return false + } + if lo.LabelSelector != nil && !lo.LabelSelector.Matches(labels.Set(obj.GetLabels())) { + return false + } + if lo.FieldSelector != nil && !lo.FieldSelector.Empty() { + set := c.fieldSetLocked(gvk, obj) + if !lo.FieldSelector.Matches(set) { + return false + } + } + return true +} + +// fieldSetLocked builds a fields.Set for obj using the registered IndexerFuncs +// for the GVK. Callers must hold at least the read lock. +func (c *Overlay) fieldSetLocked(gvk schema.GroupVersionKind, obj client.Object) fields.Set { + set := fields.Set{} + for field, fn := range c.indexers[gvk] { + for _, v := range fn(obj) { + // A field selector matches a single value; take the first indexed + // value for the field (mirrors controller-runtime cache behaviour). + set[field] = v + break + } + } + return set +} + +// Create delegates to the inner client and, on success for a cached GVK, adds +// the object to the overlay. +func (c *Overlay) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + gvk, cached := c.gvkFor(obj) + if !cached { + return c.Client.Create(ctx, obj, opts...) + } + unlock := c.writeLocks.lock(lockKey{gvk: gvk, key: client.ObjectKeyFromObject(obj)}) + defer unlock() + if err := c.Client.Create(ctx, obj, opts...); err != nil { + return err + } + c.upsert(gvk, obj) + return nil +} + +// Update delegates to the inner client and, on success for a cached GVK, +// refreshes the overlay entry. +func (c *Overlay) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + gvk, cached := c.gvkFor(obj) + if !cached { + return c.Client.Update(ctx, obj, opts...) + } + unlock := c.writeLocks.lock(lockKey{gvk: gvk, key: client.ObjectKeyFromObject(obj)}) + defer unlock() + if err := c.Client.Update(ctx, obj, opts...); err != nil { + return err + } + c.upsert(gvk, obj) + return nil +} + +// Patch delegates to the inner client and, on success for a cached GVK, +// refreshes the overlay entry with the patched object. +func (c *Overlay) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + gvk, cached := c.gvkFor(obj) + if !cached { + return c.Client.Patch(ctx, obj, patch, opts...) + } + unlock := c.writeLocks.lock(lockKey{gvk: gvk, key: client.ObjectKeyFromObject(obj)}) + defer unlock() + if err := c.Client.Patch(ctx, obj, patch, opts...); err != nil { + return err + } + c.upsert(gvk, obj) + return nil +} + +// Delete delegates to the inner client and, on success for a cached GVK, stores +// a tombstone in the overlay. The object is NOT immediately removed from the +// local map; it stays as a deleted=true entry until the deletion propagates +// through the informer (which triggers eviction) or the TTL expires. This +// ensures that reads between the Delete call and the informer event correctly +// return NotFound rather than serving a stale object from the informer cache. +func (c *Overlay) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + gvk, cached := c.gvkFor(obj) + if !cached { + return c.Client.Delete(ctx, obj, opts...) + } + unlock := c.writeLocks.lock(lockKey{gvk: gvk, key: client.ObjectKeyFromObject(obj)}) + defer unlock() + if err := c.Client.Delete(ctx, obj, opts...); err != nil { + return err + } + c.tombstone(gvk, obj) + return nil +} + +// DeleteAllOf delegates to the inner client and, on success for a cached GVK, +// tombstones all overlay entries that match the delete options. Objects that +// live only in the informer cache (not in the overlay) will be evicted +// naturally once the deletion propagates through the informer. +func (c *Overlay) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + gvk, cached := c.gvkFor(obj) + if !cached { + return c.Client.DeleteAllOf(ctx, obj, opts...) + } + if err := c.Client.DeleteAllOf(ctx, obj, opts...); err != nil { + return err + } + dao := &client.DeleteAllOfOptions{} + dao.ApplyOptions(opts) + c.mu.Lock() + defer c.mu.Unlock() + for key, e := range c.byGVK[gvk] { + if !c.matchesLocked(gvk, e.obj, &dao.ListOptions) { + continue + } + c.byGVK[gvk][key] = &entry{ + obj: e.obj, + uid: e.uid, + resourceVersion: e.resourceVersion, + deleted: true, + expiresAt: time.Now().Add(c.ttl), + } + } + c.noteSizeLocked(gvk) + return nil +} + +// Get delegates to the inner client, then applies the overlay: a tombstone +// yields NotFound; a live overlay entry overrides the inner result; and an +// overlay entry can satisfy a Get that the inner client reports as NotFound. +func (c *Overlay) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + gvk, cached := c.gvkFor(obj) + if !cached { + return c.Client.Get(ctx, key, obj, opts...) + } + err := c.Client.Get(ctx, key, obj, opts...) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + e, ok := c.getEntry(gvk, key) + if !ok { + // No overlay entry: return the inner result (value or NotFound) as-is. + return err + } + if e.deleted { + return apierrors.NewNotFound(schema.GroupResource{Group: gvk.Group, Resource: gvk.Kind}, key.Name) + } + // Live overlay entry: copy it into obj, overriding the inner result. + // Deep-copy the cached object first so scheme.Convert cannot alias the + // overlay entry's maps, slices, or metadata into the caller's obj (which + // would let callers mutate the shared cache entry). + if cpErr := c.scheme.Convert(e.obj.DeepCopyObject(), obj, nil); cpErr != nil { + return cpErr + } + return nil +} + +// List delegates to the inner client, then merges the overlay into the result. +func (c *Overlay) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + itemGVK, cached := c.itemGVKForList(list) + if !cached { + return c.Client.List(ctx, list, opts...) + } + if err := c.Client.List(ctx, list, opts...); err != nil { + return err + } + items, err := meta.ExtractList(list) + if err != nil { + return err + } + lo := &client.ListOptions{} + lo.ApplyOptions(opts) + merged := c.overlayList(itemGVK, items, lo) + return meta.SetList(list, merged) +} + +// IndexField delegates to the inner client (if it is a FieldIndexer) and also +// registers the IndexerFunc with the overlay so overlay entries can be matched +// against MatchingFields. +func (c *Overlay) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { + if err := c.informerCache.IndexField(ctx, obj, field, extractValue); err != nil { + return err + } + if gvk, cached := c.gvkFor(obj); cached { + c.registerIndex(gvk, field, extractValue) + } + return nil +} + +// Status returns a status writer that mirrors status Update/Patch writes for +// cached GVKs into the overlay. +func (c *Overlay) Status() client.StatusWriter { + return &statusWriter{c: c, inner: c.Client.Status()} +} + +// statusWriter wraps the inner status writer and reflects status writes into +// the overlay for cached GVKs. +type statusWriter struct { + c *Overlay + inner client.StatusWriter +} + +func (s *statusWriter) Create(ctx context.Context, obj, subResource client.Object, opts ...client.SubResourceCreateOption) error { + return s.inner.Create(ctx, obj, subResource, opts...) +} + +func (s *statusWriter) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + gvk, cached := s.c.gvkFor(obj) + if !cached { + return s.inner.Update(ctx, obj, opts...) + } + unlock := s.c.writeLocks.lock(lockKey{gvk: gvk, key: client.ObjectKeyFromObject(obj)}) + defer unlock() + if err := s.inner.Update(ctx, obj, opts...); err != nil { + return err + } + s.c.upsert(gvk, obj) + return nil +} + +func (s *statusWriter) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + gvk, cached := s.c.gvkFor(obj) + if !cached { + return s.inner.Patch(ctx, obj, patch, opts...) + } + unlock := s.c.writeLocks.lock(lockKey{gvk: gvk, key: client.ObjectKeyFromObject(obj)}) + defer unlock() + if err := s.inner.Patch(ctx, obj, patch, opts...); err != nil { + return err + } + s.c.upsert(gvk, obj) + return nil +} + +func (s *statusWriter) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + return s.inner.Apply(ctx, obj, opts...) +} diff --git a/pkg/cache/client_test.go b/pkg/cache/client_test.go new file mode 100644 index 000000000..6b1d11eac --- /dev/null +++ b/pkg/cache/client_test.go @@ -0,0 +1,1009 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package cache + +import ( + "context" + "errors" + goruntime "runtime" + "strconv" + "sync" + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + toolscachek8s "k8s.io/client-go/tools/cache" + ccache "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" +) + +const azIndexField = "spec.availabilityZone" + +// --- shared test infrastructure --- + +func testScheme(t *testing.T) *runtime.Scheme { + t.Helper() + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("add to scheme: %v", err) + } + return scheme +} + +func newReservation(name, az, rv string) *v1alpha1.Reservation { + return &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID("uid-" + name), + ResourceVersion: rv, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + AvailabilityZone: az, + }, + } +} + +// fakeInformer is a controllable informer that records handlers and lets tests +// fire Add/Update events to trigger eviction. +type fakeInformer struct { + ccache.Informer + mu sync.Mutex + handlers []toolscachek8s.ResourceEventHandler +} + +func (f *fakeInformer) AddEventHandler(h toolscachek8s.ResourceEventHandler) (toolscachek8s.ResourceEventHandlerRegistration, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.handlers = append(f.handlers, h) + return nil, nil +} + +func (f *fakeInformer) fireAdd(obj any) { + f.mu.Lock() + handlers := append([]toolscachek8s.ResourceEventHandler(nil), f.handlers...) + f.mu.Unlock() + for _, h := range handlers { + h.OnAdd(obj, false) + } +} + +func (f *fakeInformer) fireUpdate(oldObj, newObj any) { + f.mu.Lock() + handlers := append([]toolscachek8s.ResourceEventHandler(nil), f.handlers...) + f.mu.Unlock() + for _, h := range handlers { + h.OnUpdate(oldObj, newObj) + } +} + +// fakeCache is a minimal cache.Cache that returns a single fakeInformer for any +// GetInformer call, so the Overlay runnable can wire eviction handlers. +type fakeCache struct { + ccache.Cache + inf *fakeInformer +} + +func (f *fakeCache) GetInformer(_ context.Context, _ client.Object, _ ...ccache.InformerGetOption) (ccache.Informer, error) { + return f.inf, nil +} + +func (f *fakeCache) IndexField(_ context.Context, _ client.Object, _ string, _ client.IndexerFunc) error { + return nil +} + +// fakeCluster composes a fake client.Client with a fakeCache to satisfy the +// cluster.Cluster interface consumed by cache.WrapCluster. +type fakeCluster struct { + cluster.Cluster + client client.Client + cache *fakeCache + scheme *runtime.Scheme + host string +} + +func (f *fakeCluster) GetClient() client.Client { return f.client } +func (f *fakeCluster) GetCache() ccache.Cache { return f.cache } +func (f *fakeCluster) GetScheme() *runtime.Scheme { return f.scheme } +func (f *fakeCluster) GetConfig() *rest.Config { + host := f.host + if host == "" { + host = "test-host" + } + return &rest.Config{Host: host} +} + +// fakeClient exposes the underlying informer so eviction tests can fire events, +// while itself acting as a cluster.Cluster wrapping the fake client.Client. +type fakeClient struct { + *fakeCluster + inf *fakeInformer +} + +func newTestClient(t *testing.T, objs ...client.Object) *fakeClient { + t.Helper() + scheme := testScheme(t) + inf := &fakeInformer{} + inner := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objs...). + WithStatusSubresource(&v1alpha1.Reservation{}). + WithIndex(&v1alpha1.Reservation{}, azIndexField, func(obj client.Object) []string { + res, ok := obj.(*v1alpha1.Reservation) + if !ok || res.Spec.AvailabilityZone == "" { + return nil + } + return []string{res.Spec.AvailabilityZone} + }). + Build() + return &fakeClient{ + fakeCluster: &fakeCluster{ + client: inner, + cache: &fakeCache{inf: inf}, + scheme: scheme, + }, + inf: inf, + } +} + +// errClient wraps a client.Client and injects configurable errors into +// mutating/read operations, so the error-propagation paths of Overlay +// (which must not touch the overlay on failure) can be exercised. +type errClient struct { + client.Client + createErr error + updateErr error + patchErr error + deleteErr error + deleteAllOfErr error + getErr error + listErr error +} + +func (e *errClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if e.createErr != nil { + return e.createErr + } + return e.Client.Create(ctx, obj, opts...) +} + +func (e *errClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if e.updateErr != nil { + return e.updateErr + } + return e.Client.Update(ctx, obj, opts...) +} + +func (e *errClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if e.patchErr != nil { + return e.patchErr + } + return e.Client.Patch(ctx, obj, patch, opts...) +} + +func (e *errClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + if e.deleteErr != nil { + return e.deleteErr + } + return e.Client.Delete(ctx, obj, opts...) +} + +func (e *errClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + if e.deleteAllOfErr != nil { + return e.deleteAllOfErr + } + return e.Client.DeleteAllOf(ctx, obj, opts...) +} + +func (e *errClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if e.getErr != nil { + return e.getErr + } + return e.Client.Get(ctx, key, obj, opts...) +} + +func (e *errClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if e.listErr != nil { + return e.listErr + } + return e.Client.List(ctx, list, opts...) +} + +// forceInnerAZ writes a divergent AvailabilityZone directly to the inner +// client, bypassing the caching wrapper. Used to prove reads are served from +// the overlay, not the inner client. +func forceInnerAZ(t *testing.T, inner client.Client, name, az string) { + t.Helper() + var cur v1alpha1.Reservation + if err := inner.Get(context.Background(), types.NamespacedName{Name: name}, &cur); err != nil { + t.Fatalf("forceInnerAZ get: %v", err) + } + cur.Spec.AvailabilityZone = az + if err := inner.Update(context.Background(), &cur); err != nil { + t.Fatalf("forceInnerAZ update: %v", err) + } +} + +// forceInnerStatusHost writes a divergent status Host directly to the inner client. +func forceInnerStatusHost(t *testing.T, inner client.Client, name, host string) { + t.Helper() + var cur v1alpha1.Reservation + if err := inner.Get(context.Background(), types.NamespacedName{Name: name}, &cur); err != nil { + t.Fatalf("forceInnerStatusHost get: %v", err) + } + cur.Status.Host = host + if err := inner.Status().Update(context.Background(), &cur); err != nil { + t.Fatalf("forceInnerStatusHost update: %v", err) + } +} + +// unknownObject is a client.Object whose type is not registered in the test +// scheme, used to exercise the "unresolvable GVK" branches. +type unknownObject struct { + metav1.TypeMeta + metav1.ObjectMeta +} + +func (u *unknownObject) DeepCopyObject() runtime.Object { return u } + +func reservationConfig() Config { + return Config{ + GVKs: []string{"cortex.cloud/v1alpha1/Reservation"}, + TTL: metav1.Duration{Duration: 2 * time.Minute}, + } +} + +// clusterFor wraps a client.Client as a cluster.Cluster so it can be passed to +// WrapCluster. Used by tests that inject a custom client.Client (e.g. errClient, +// orderingClient) below the overlay. +func clusterFor(t *testing.T, inner client.Client) cluster.Cluster { + t.Helper() + return &fakeCluster{ + client: inner, + cache: &fakeCache{inf: &fakeInformer{}}, + scheme: testScheme(t), + } +} + +// cachingFrom builds a *Overlay over the given cluster.Cluster. +func cachingFrom(t *testing.T, cl cluster.Cluster, conf Config) *Overlay { + t.Helper() + wrapped, runnable, err := WrapCluster(cl, conf, nil) + if err != nil { + t.Fatalf("New: %v", err) + } + cc, ok := wrapped.GetClient().(*Overlay) + if !ok { + t.Fatalf("WrapCluster did not return an Overlay") + } + if _, ok := runnable.(*Overlay); !ok { + t.Fatalf("WrapCluster did not return a *Overlay runnable") + } + return cc +} + +// newCaching builds a *Overlay over the given fake cluster using the +// default reservation config. +func newCaching(t *testing.T, inner *fakeClient) *Overlay { + t.Helper() + return cachingFrom(t, inner.fakeCluster, reservationConfig()) +} + +func listReservations(t *testing.T, c client.Client, opts ...client.ListOption) []v1alpha1.Reservation { + t.Helper() + var list v1alpha1.ReservationList + if err := c.List(context.Background(), &list, opts...); err != nil { + t.Fatalf("List: %v", err) + } + return list.Items +} + +func reservationGVK() schema.GroupVersionKind { + return v1alpha1.GroupVersion.WithKind("Reservation") +} + +func waitFor(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("condition not met within timeout") +} + +// --- constructor tests --- + +func TestNewUnknownGVKError(t *testing.T) { + _, _, err := WrapCluster(newTestClient(t).fakeCluster, Config{ + GVKs: []string{"cortex.cloud/v1alpha1/DoesNotExist"}, + }, nil) + if err == nil { + t.Fatalf("expected error for unknown GVK, got nil") + } +} + +func TestNewDefaultTTL(t *testing.T) { + c := cachingFrom(t, newTestClient(t).fakeCluster, Config{ + GVKs: []string{"cortex.cloud/v1alpha1/Reservation"}, + }) + if c.ttl != defaultTTL { + t.Fatalf("expected ttl %v, got %v", defaultTTL, c.ttl) + } +} + +func TestNewExplicitTTL(t *testing.T) { + c := cachingFrom(t, newTestClient(t).fakeCluster, Config{ + GVKs: []string{"cortex.cloud/v1alpha1/Reservation"}, + TTL: metav1.Duration{Duration: 90 * time.Second}, + }) + if c.ttl != 90*time.Second { + t.Fatalf("expected ttl 90s, got %v", c.ttl) + } +} + +func TestNewWrapsClusterClientAndIndexer(t *testing.T) { + inner := newTestClient(t) + wrapped, runnable, err := WrapCluster(inner.fakeCluster, reservationConfig(), nil) + if err != nil { + t.Fatalf("New: %v", err) + } + // GetClient and GetFieldIndexer must both return the overlay so per-cluster + // reads/writes and IndexField flow through the cache. + cc, ok := wrapped.GetClient().(*Overlay) + if !ok { + t.Fatalf("GetClient did not return the overlay") + } + if wrapped.GetFieldIndexer() != client.FieldIndexer(cc) { + t.Fatalf("GetFieldIndexer did not return the overlay") + } + // The returned Runnable is the same overlay. + if runnable != manager.Runnable(cc) { + t.Fatalf("returned Runnable is not the overlay") + } + // Everything else delegates to the inner cluster. + if wrapped.GetScheme() != inner.GetScheme() { + t.Fatalf("GetScheme did not delegate to inner") + } + if wrapped.GetCache() != inner.GetCache() { + t.Fatalf("GetCache did not delegate to inner") + } +} + +// --- overlay behaviour tests --- + +func TestCreateThenGetVisible(t *testing.T) { + c := newCaching(t, newTestClient(t)) + + r := newReservation("res-1", "az-1", "") + if err := c.Create(context.Background(), r); err != nil { + t.Fatalf("Create: %v", err) + } + + var got v1alpha1.Reservation + if err := c.Get(context.Background(), types.NamespacedName{Name: "res-1"}, &got); err != nil { + t.Fatalf("Get after create: %v", err) + } + if got.Spec.AvailabilityZone != "az-1" { + t.Fatalf("expected az-1, got %q", got.Spec.AvailabilityZone) + } +} + +func TestOverlayWhenInnerEmpty(t *testing.T) { + c := newCaching(t, newTestClient(t)) + c.upsert(reservationGVK(), newReservation("res-2", "az-1", "5")) + + if items := listReservations(t, c); len(items) != 1 || items[0].Name != "res-2" { + t.Fatalf("expected overlay entry res-2, got %+v", items) + } +} + +func TestEviction(t *testing.T) { + const cachedRV = "10" + cases := []struct { + name string + useUpdate bool + observedUID string + observedRV string + wantEvicted bool + }{ + {name: "add older RV keeps", observedRV: "9", wantEvicted: false}, + {name: "add equal RV evicts", observedRV: "10", wantEvicted: true}, + {name: "add newer RV evicts", observedRV: "11", wantEvicted: true}, + {name: "update newer RV evicts", useUpdate: true, observedRV: "11", wantEvicted: true}, + {name: "update older RV keeps", useUpdate: true, observedRV: "9", wantEvicted: false}, + {name: "uid mismatch keeps", observedUID: "uid-other", observedRV: "11", wantEvicted: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + inner := newTestClient(t) + c := newCaching(t, inner) + + ctx := t.Context() + go func() { + if err := c.Start(ctx); err != nil && ctx.Err() == nil { + t.Errorf("c.Start: %v", err) + } + }() + waitFor(t, func() bool { + inner.inf.mu.Lock() + defer inner.inf.mu.Unlock() + return len(inner.inf.handlers) > 0 + }) + + c.upsert(reservationGVK(), newReservation("res-3", "az-1", cachedRV)) + + observed := newReservation("res-3", "az-1", tc.observedRV) + if tc.observedUID != "" { + observed.UID = types.UID(tc.observedUID) + } + if tc.useUpdate { + inner.inf.fireUpdate(nil, observed) + } else { + inner.inf.fireAdd(observed) + } + + _, present := c.getEntry(reservationGVK(), client.ObjectKey{Name: "res-3"}) + if present == tc.wantEvicted { + t.Fatalf("evicted=%v, want evicted=%v", !present, tc.wantEvicted) + } + }) + } +} + +func TestEvictionIgnoresNonObject(t *testing.T) { + inner := newTestClient(t) + c := newCaching(t, inner) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + go func() { + if err := c.Start(ctx); err != nil && ctx.Err() == nil { + t.Errorf("c.Start: %v", err) + } + }() + waitFor(t, func() bool { + inner.inf.mu.Lock() + defer inner.inf.mu.Unlock() + return len(inner.inf.handlers) > 0 + }) + + c.upsert(reservationGVK(), newReservation("res-x", "az-1", "1")) + inner.inf.fireAdd("not-an-object") + if _, ok := c.getEntry(reservationGVK(), client.ObjectKey{Name: "res-x"}); !ok { + t.Fatalf("non-object event must not evict the entry") + } +} + +func TestTTLCleanup(t *testing.T) { + c := newCaching(t, newTestClient(t)) + c.upsert(reservationGVK(), newReservation("res-4", "az-1", "1")) + c.mu.Lock() + for _, entries := range c.byGVK { + for _, e := range entries { + e.expiresAt = time.Now().Add(-time.Second) + } + } + c.mu.Unlock() + c.cleanupExpired(time.Now()) + if _, ok := c.getEntry(reservationGVK(), client.ObjectKey{Name: "res-4"}); ok { + t.Fatalf("expired entry should be removed") + } +} + +func TestTombstone(t *testing.T) { + r := newReservation("res-5", "az-1", "") + inner := newTestClient(t, r) + c := newCaching(t, inner) + + if err := c.Delete(context.Background(), r); err != nil { + t.Fatalf("Delete: %v", err) + } + // Re-create directly in inner to simulate informer lag. + if err := inner.GetClient().Create(context.Background(), newReservation("res-5", "az-1", "")); err != nil { + t.Fatalf("re-create inner: %v", err) + } + + var got v1alpha1.Reservation + if err := c.Get(context.Background(), types.NamespacedName{Name: "res-5"}, &got); !apierrors.IsNotFound(err) { + t.Fatalf("expected NotFound for tombstoned object, got %v", err) + } + if items := listReservations(t, c); len(items) != 0 { + t.Fatalf("expected tombstone to filter from list, got %+v", items) + } +} + +func TestDeleteAllOf(t *testing.T) { + r1 := newReservation("res-dao-1", "az-1", "1") + r1.Labels = map[string]string{"zone": "a"} + r2 := newReservation("res-dao-2", "az-2", "1") + r2.Labels = map[string]string{"zone": "b"} + inner := newTestClient(t, r1, r2) + c := newCaching(t, inner) + + // Populate overlay for both so we can verify tombstoning. + c.upsert(reservationGVK(), r1) + c.upsert(reservationGVK(), r2) + + // DeleteAllOf with a label selector — only r1 should be tombstoned. + if err := c.DeleteAllOf(context.Background(), &v1alpha1.Reservation{}, client.MatchingLabels{"zone": "a"}); err != nil { + t.Fatalf("DeleteAllOf: %v", err) + } + + var got v1alpha1.Reservation + if err := c.Get(context.Background(), types.NamespacedName{Name: "res-dao-1"}, &got); !apierrors.IsNotFound(err) { + t.Fatalf("expected NotFound for tombstoned res-dao-1, got %v", err) + } + if err := c.Get(context.Background(), types.NamespacedName{Name: "res-dao-2"}, &got); err != nil { + t.Fatalf("res-dao-2 should still be visible, got %v", err) + } +} + +func TestUpdateOverridesInner(t *testing.T) { + r := newReservation("res-6", "az-old", "1") + inner := newTestClient(t, r) + c := newCaching(t, inner) + + var cur v1alpha1.Reservation + if err := inner.GetClient().Get(context.Background(), types.NamespacedName{Name: "res-6"}, &cur); err != nil { + t.Fatalf("inner get: %v", err) + } + cur.Spec.AvailabilityZone = "az-new" + if err := c.Update(context.Background(), &cur); err != nil { + t.Fatalf("Update: %v", err) + } + + var got v1alpha1.Reservation + if err := c.Get(context.Background(), types.NamespacedName{Name: "res-6"}, &got); err != nil { + t.Fatalf("Get: %v", err) + } + if got.Spec.AvailabilityZone != "az-new" { + t.Fatalf("expected az-new from overlay, got %q", got.Spec.AvailabilityZone) + } +} + +func TestLabelMatchingAfterOverlayChange(t *testing.T) { + // Regression: if a write changes a field that is part of the query, the + // overlay version must be re-matched against the list options. Before the + // fix, the inner List would include the object (matching the old value in + // the informer), and overlayList would silently swap in the new version, + // returning an object that does not satisfy the query. + r := newReservation("res-overlay-label", "az-1", "1") + r.Labels = map[string]string{"team": "a"} + inner := newTestClient(t, r) + c := newCaching(t, inner) + + // Update label in overlay only (bypass inner to simulate informer lag). + updated := r.DeepCopy() + updated.Labels = map[string]string{"team": "b"} + c.upsert(reservationGVK(), updated) + + // Query for the OLD label value — informer still returns the object, but + // the overlay version has team=b, so it must be excluded. + if got := listReservations(t, c, client.MatchingLabels{"team": "a"}); len(got) != 0 { + t.Fatalf("expected no results for team=a after overlay changed label to b, got %+v", got) + } + // Query for the NEW label value — overlay-only path must include it. + if got := listReservations(t, c, client.MatchingLabels{"team": "b"}); len(got) != 1 { + t.Fatalf("expected one result for team=b, got %+v", got) + } +} + +func TestLabelMatching(t *testing.T) { + c := newCaching(t, newTestClient(t)) + r := newReservation("res-7", "az-1", "1") + r.Labels = map[string]string{"team": "a"} + c.upsert(reservationGVK(), r) + + if match := listReservations(t, c, client.MatchingLabels{"team": "a"}); len(match) != 1 { + t.Fatalf("expected match for team=a, got %+v", match) + } + if noMatch := listReservations(t, c, client.MatchingLabels{"team": "b"}); len(noMatch) != 0 { + t.Fatalf("expected no match for team=b, got %+v", noMatch) + } +} + +func TestFieldMatching(t *testing.T) { + c := newCaching(t, newTestClient(t)) + if err := c.IndexField(context.Background(), &v1alpha1.Reservation{}, azIndexField, func(obj client.Object) []string { + res := obj.(*v1alpha1.Reservation) + if res.Spec.AvailabilityZone == "" { + return nil + } + return []string{res.Spec.AvailabilityZone} + }); err != nil { + t.Fatalf("IndexField: %v", err) + } + c.upsert(reservationGVK(), newReservation("res-8", "az-1", "1")) + + if match := listReservations(t, c, client.MatchingFields{azIndexField: "az-1"}); len(match) != 1 { + t.Fatalf("expected field match az-1, got %+v", match) + } + if noMatch := listReservations(t, c, client.MatchingFields{azIndexField: "az-2"}); len(noMatch) != 0 { + t.Fatalf("expected no field match az-2, got %+v", noMatch) + } +} + +func TestNonCachedGVKPassthrough(t *testing.T) { + inner := newTestClient(t) + c := cachingFrom(t, inner.fakeCluster, Config{}) + r := newReservation("res-9", "az-1", "") + if err := c.Create(context.Background(), r); err != nil { + t.Fatalf("Create: %v", err) + } + if _, ok := c.getEntry(reservationGVK(), client.ObjectKey{Name: "res-9"}); ok { + t.Fatalf("non-cached GVK should not populate overlay") + } + if err := c.Delete(context.Background(), r); err != nil { + t.Fatalf("Delete: %v", err) + } + var got v1alpha1.Reservation + if err := c.Get(context.Background(), types.NamespacedName{Name: "res-9"}, &got); !apierrors.IsNotFound(err) { + t.Fatalf("expected NotFound, got %v", err) + } +} + +func TestDedup(t *testing.T) { + r := newReservation("res-10", "az-1", "1") + c := newCaching(t, newTestClient(t, r)) + + newer := newReservation("res-10", "az-1", "2") + newer.Spec.TargetHost = "host-x" + c.upsert(reservationGVK(), newer) + + items := listReservations(t, c) + if len(items) != 1 { + t.Fatalf("expected exactly one item after dedup, got %d: %+v", len(items), items) + } + if items[0].Spec.TargetHost != "host-x" { + t.Fatalf("expected overlay version to win, got %+v", items[0]) + } +} + +// --- error-propagation tests --- + +func TestWriteErrorLeavesOverlayUntouched(t *testing.T) { + sentinel := errors.New("boom") + cases := []struct { + name string + rv string + seed bool + mkClient func(inner *fakeClient) client.Client + op func(c *Overlay, r *v1alpha1.Reservation) error + }{ + { + name: "create", + mkClient: func(inner *fakeClient) client.Client { + return &errClient{Client: inner.GetClient(), createErr: sentinel} + }, + op: func(c *Overlay, r *v1alpha1.Reservation) error { return c.Create(context.Background(), r) }, + }, + { + name: "update", + rv: "1", + mkClient: func(inner *fakeClient) client.Client { + return &errClient{Client: inner.GetClient(), updateErr: sentinel} + }, + op: func(c *Overlay, r *v1alpha1.Reservation) error { return c.Update(context.Background(), r) }, + }, + { + name: "patch", + rv: "1", + mkClient: func(inner *fakeClient) client.Client { + return &errClient{Client: inner.GetClient(), patchErr: sentinel} + }, + op: func(c *Overlay, r *v1alpha1.Reservation) error { + p := r.DeepCopy() + p.Spec.AvailabilityZone = "az-new" + return c.Patch(context.Background(), p, client.MergeFrom(r)) + }, + }, + { + name: "delete", + seed: true, + mkClient: func(inner *fakeClient) client.Client { + return &errClient{Client: inner.GetClient(), deleteErr: sentinel} + }, + op: func(c *Overlay, r *v1alpha1.Reservation) error { return c.Delete(context.Background(), r) }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := newReservation("res-"+tc.name, "az-1", tc.rv) + var base *fakeClient + if tc.seed { + base = newTestClient(t, r) + } else { + base = newTestClient(t) + } + c := cachingFrom(t, clusterFor(t, tc.mkClient(base)), reservationConfig()) + + if err := tc.op(c, r); !errors.Is(err, sentinel) { + t.Fatalf("expected sentinel error, got %v", err) + } + if _, ok := c.getEntry(reservationGVK(), client.ObjectKey{Name: r.Name}); ok { + t.Fatalf("overlay must not be touched on %s failure", tc.name) + } + }) + } +} + +func TestWriteServedFromOverlay(t *testing.T) { + cases := []struct { + name string + write func(t *testing.T, c *Overlay, cur *v1alpha1.Reservation) string + diverge func(t *testing.T, inner client.Client, name string) + read func(*v1alpha1.Reservation) string + }{ + { + name: "patch spec", + write: func(t *testing.T, c *Overlay, cur *v1alpha1.Reservation) string { + base := cur.DeepCopy() + cur.Spec.AvailabilityZone = "az-new" + if err := c.Patch(context.Background(), cur, client.MergeFrom(base)); err != nil { + t.Fatalf("Patch: %v", err) + } + return "az-new" + }, + diverge: func(t *testing.T, inner client.Client, name string) { forceInnerAZ(t, inner, name, "az-stale") }, + read: func(r *v1alpha1.Reservation) string { return r.Spec.AvailabilityZone }, + }, + { + name: "status update", + write: func(t *testing.T, c *Overlay, cur *v1alpha1.Reservation) string { + cur.Status.Host = "host-active" + if err := c.Status().Update(context.Background(), cur); err != nil { + t.Fatalf("Status().Update: %v", err) + } + return "host-active" + }, + diverge: func(t *testing.T, inner client.Client, name string) { + forceInnerStatusHost(t, inner, name, "host-stale") + }, + read: func(r *v1alpha1.Reservation) string { return r.Status.Host }, + }, + { + name: "status patch", + write: func(t *testing.T, c *Overlay, cur *v1alpha1.Reservation) string { + base := cur.DeepCopy() + cur.Status.Host = "host-patched" + if err := c.Status().Patch(context.Background(), cur, client.MergeFrom(base)); err != nil { + t.Fatalf("Status().Patch: %v", err) + } + return "host-patched" + }, + diverge: func(t *testing.T, inner client.Client, name string) { + forceInnerStatusHost(t, inner, name, "host-stale") + }, + read: func(r *v1alpha1.Reservation) string { return r.Status.Host }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := newReservation("res-served", "az-1", "") + inner := newTestClient(t, r) + c := newCaching(t, inner) + + var cur v1alpha1.Reservation + if err := inner.GetClient().Get(context.Background(), types.NamespacedName{Name: r.Name}, &cur); err != nil { + t.Fatalf("inner get: %v", err) + } + want := tc.write(t, c, &cur) + tc.diverge(t, inner.GetClient(), r.Name) + + var got v1alpha1.Reservation + if err := c.Get(context.Background(), types.NamespacedName{Name: r.Name}, &got); err != nil { + t.Fatalf("Get: %v", err) + } + if tc.read(&got) != want { + t.Fatalf("expected overlay value %q to be served, got %q", want, tc.read(&got)) + } + }) + } +} + +func TestGetPropagatesNonNotFoundError(t *testing.T) { + sentinel := errors.New("get boom") + c := cachingFrom(t, clusterFor(t, &errClient{Client: newTestClient(t).GetClient(), getErr: sentinel}), reservationConfig()) + c.upsert(reservationGVK(), newReservation("res-ge", "az-1", "1")) + + var got v1alpha1.Reservation + if err := c.Get(context.Background(), types.NamespacedName{Name: "res-ge"}, &got); !errors.Is(err, sentinel) { + t.Fatalf("expected sentinel error, got %v", err) + } +} + +func TestGetOverlayResurrectsNotFound(t *testing.T) { + c := newCaching(t, newTestClient(t)) + c.upsert(reservationGVK(), newReservation("res-gr", "az-z", "1")) + + var got v1alpha1.Reservation + if err := c.Get(context.Background(), types.NamespacedName{Name: "res-gr"}, &got); err != nil { + t.Fatalf("expected overlay to satisfy Get, got %v", err) + } + if got.Spec.AvailabilityZone != "az-z" { + t.Fatalf("expected az-z from overlay, got %q", got.Spec.AvailabilityZone) + } +} + +func TestGetNotFoundWithNoOverlay(t *testing.T) { + c := newCaching(t, newTestClient(t)) + + var got v1alpha1.Reservation + if err := c.Get(context.Background(), types.NamespacedName{Name: "missing"}, &got); !apierrors.IsNotFound(err) { + t.Fatalf("expected NotFound, got %v", err) + } +} + +func TestGetNonCachedPropagatesError(t *testing.T) { + sentinel := errors.New("get boom") + c := cachingFrom(t, clusterFor(t, &errClient{Client: newTestClient(t).GetClient(), getErr: sentinel}), Config{}) + var got v1alpha1.Reservation + if gerr := c.Get(context.Background(), types.NamespacedName{Name: "x"}, &got); !errors.Is(gerr, sentinel) { + t.Fatalf("expected sentinel error, got %v", gerr) + } +} + +func TestListPropagatesError(t *testing.T) { + sentinel := errors.New("list boom") + c := cachingFrom(t, clusterFor(t, &errClient{Client: newTestClient(t).GetClient(), listErr: sentinel}), reservationConfig()) + c.upsert(reservationGVK(), newReservation("res-le", "az-1", "1")) + + var list v1alpha1.ReservationList + if err := c.List(context.Background(), &list); !errors.Is(err, sentinel) { + t.Fatalf("expected sentinel error, got %v", err) + } +} + +func TestStatusUpdateErrorLeavesOverlayUntouched(t *testing.T) { + c := newCaching(t, newTestClient(t)) + + r := newReservation("res-se", "az-1", "1") + if err := c.Status().Update(context.Background(), r); err == nil { + t.Fatalf("expected status update to fail for missing object") + } + if _, ok := c.getEntry(reservationGVK(), client.ObjectKey{Name: "res-se"}); ok { + t.Fatalf("overlay must not be populated on status update failure") + } +} + +func TestStatusCreateDelegates(t *testing.T) { + r := newReservation("res-sc", "az-1", "") + c := newCaching(t, newTestClient(t, r)) + + if err := c.Status().Create(context.Background(), r, r); err == nil { + t.Fatalf("expected Status().Create to fail on fake client") + } + if _, ok := c.getEntry(reservationGVK(), client.ObjectKey{Name: "res-sc"}); ok { + t.Fatalf("Status().Create must not populate the overlay") + } +} + +func TestStatusUpdateNonCachedNoOverlay(t *testing.T) { + r := newReservation("res-sn", "az-1", "") + inner := newTestClient(t, r) + c := cachingFrom(t, inner.fakeCluster, Config{}) + var cur v1alpha1.Reservation + if err := inner.GetClient().Get(context.Background(), types.NamespacedName{Name: "res-sn"}, &cur); err != nil { + t.Fatalf("inner get: %v", err) + } + cur.Status.Host = "host-active" + if err := c.Status().Update(context.Background(), &cur); err != nil { + t.Fatalf("Status().Update: %v", err) + } + if _, ok := c.getEntry(reservationGVK(), client.ObjectKey{Name: "res-sn"}); ok { + t.Fatalf("non-cached GVK status update should not populate overlay") + } +} + +// --- concurrency regression test --- + +// orderingClient is a fake inner client whose Update records the committed +// ResourceVersion (in inner-commit order) and then yields, widening the window +// between the inner commit and the overlay upsert. This exposes the ordering +// race the per-object write lock is meant to prevent: without the lock two +// concurrent writers to the same object can commit in one order yet upsert in +// the reverse order, leaving the overlay behind the apiserver. +// +// The per-object write lock makes each inner call + upsert atomic, so the inner +// commit order and the overlay upsert order are identical: the overlay always +// reflects the last write that reached the inner client (lastRV). +type orderingClient struct { + client.Client + mu sync.Mutex + lastRV string // ResourceVersion of the most recent inner commit +} + +func (o *orderingClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + o.mu.Lock() + o.lastRV = obj.GetResourceVersion() + o.mu.Unlock() + // Yield after the inner commit but before the caller upserts, widening the + // commit-vs-overlay reorder window. + goruntime.Gosched() + return nil +} + +// TestConcurrentUpdatesOverlayNotBehind fires many concurrent Updates to the +// SAME object with distinct ResourceVersions. The per-object write lock must +// make each inner call + overlay update atomic, so once all writes settle the +// overlay entry reflects the last write that reached the inner client (overlay +// RV == last committed RV, never a reordered/stale one). Run under -race to +// also catch data races. +func TestConcurrentUpdatesOverlayNotBehind(t *testing.T) { + const ( + rounds = 50 + n = 8 + ) + for round := range rounds { + oc := &orderingClient{Client: newTestClient(t).GetClient()} + c := cachingFrom(t, clusterFor(t, oc), reservationConfig()) + + var wg sync.WaitGroup + for i := 1; i <= n; i++ { + wg.Add(1) + go func(rv int) { + defer wg.Done() + r := newReservation("res-conc", "az-1", strconv.Itoa(rv)) + if err := c.Update(context.Background(), r); err != nil { + t.Errorf("Update rv=%d: %v", rv, err) + } + }(i) + } + wg.Wait() + + oc.mu.Lock() + lastRV := oc.lastRV + oc.mu.Unlock() + + e, ok := c.getEntry(reservationGVK(), client.ObjectKey{Name: "res-conc"}) + if !ok { + t.Fatalf("round %d: expected overlay entry for res-conc", round) + } + if e.resourceVersion != lastRV { + t.Fatalf("round %d: overlay RV %q does not match last inner commit %q", + round, e.resourceVersion, lastRV) + } + } +} + +// --- helper / utility tests --- + +func TestGVKForUnknownType(t *testing.T) { + c := newCaching(t, newTestClient(t)) + if _, cached := c.gvkFor(&unknownObject{}); cached { + t.Fatalf("unknown type must not be reported as cached") + } +} + +func TestTrimListSuffix(t *testing.T) { + cases := []struct { + in string + wantKind string + wantOK bool + }{ + {"ReservationList", "Reservation", true}, + {"List", "List", false}, + {"Reservation", "Reservation", false}, + {"", "", false}, + } + for _, tc := range cases { + gotKind, gotOK := trimListSuffix(tc.in) + if gotKind != tc.wantKind || gotOK != tc.wantOK { + t.Errorf("trimListSuffix(%q) = (%q, %v), want (%q, %v)", tc.in, gotKind, gotOK, tc.wantKind, tc.wantOK) + } + } +} diff --git a/pkg/cache/cluster.go b/pkg/cache/cluster.go new file mode 100644 index 000000000..e0eefa950 --- /dev/null +++ b/pkg/cache/cluster.go @@ -0,0 +1,71 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package cache + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +// overlayCluster wraps a cluster.Cluster so its client and field indexer are +// served through the in-process overlay cache. Everything else (config, scheme, +// REST mapper, HTTP client, API reader, event recorder, and Start) is delegated +// to the embedded inner cluster unchanged. +// +// It deliberately does NOT override Start: the manager already owns the inner +// cluster's lifecycle (home cluster) or drives it via mgr.Add (remotes). The +// overlay's own lifecycle (eviction handlers + TTL cleanup) is registered with +// the manager directly by Wrapper.WrapCluster. +type overlayCluster struct { + cluster.Cluster + cache *Overlay +} + +// GetClient returns the overlay client so per-cluster reads and writes flow +// through the cache. +func (w *overlayCluster) GetClient() client.Client { return w.cache } + +// GetFieldIndexer returns the overlay so IndexField dual-registers the indexer +// with both the real cache (via Overlay.IndexField) and the overlay. +func (w *overlayCluster) GetFieldIndexer() client.FieldIndexer { return w.cache } + +// Wrapper implements the multicluster.ClusterWrapper interface for the overlay +// cache. It does not import the multicluster package; interface satisfaction is +// structural and checked at the call site (e.g. in main.go). It captures the +// manager at construction time because the ClusterWrapper interface does not +// pass a manager to WrapCluster. +type Wrapper struct { + mgr manager.Manager + conf Config + monitor *Monitor +} + +// NewWrapper returns a Wrapper that applies the overlay cache to any cluster +// passed to WrapCluster, registering the overlay's lifecycle Runnable with mgr. +// The monitor is shared across every wrapped cluster and is supplied by the +// caller (so this library package hardcodes no project-specific metric prefix); +// it may be nil to disable overlay size metrics. Register it on the metrics +// registry from the caller. +func NewWrapper(mgr manager.Manager, conf Config, monitor *Monitor) *Wrapper { + return &Wrapper{ + mgr: mgr, + conf: conf, + monitor: monitor, + } +} + +// WrapCluster applies the overlay cache to cl, registers the overlay's lifecycle +// Runnable with the captured manager, and returns the wrapped cluster. +// Satisfies multicluster.ClusterWrapper structurally. +func (w *Wrapper) WrapCluster(cl cluster.Cluster) (cluster.Cluster, error) { + wrapped, cleanUpRunnable, err := WrapCluster(cl, w.conf, w.monitor) + if err != nil { + return nil, err + } + if err := w.mgr.Add(cleanUpRunnable); err != nil { + return nil, err + } + return wrapped, nil +} diff --git a/pkg/cache/config.go b/pkg/cache/config.go new file mode 100644 index 000000000..08c58d573 --- /dev/null +++ b/pkg/cache/config.go @@ -0,0 +1,27 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package cache + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Config configures the transparent in-process overlay cache. +type Config struct { + // Enabled turns the overlay cache on. When false (the default), clusters are + // used unwrapped: no overlay, no eviction Runnable, zero overhead. + Enabled bool `json:"enabled"` + // GVKs the cache should overlay, formatted as "//". + // Calls for GVKs not listed here are passed through unchanged. + GVKs []string `json:"gvks"` + // TTL is the maximum lifetime of an overlay entry before it is evicted by + // the background cleanup goroutine, guarding against entries that never + // appear in the informer (e.g. after a crash). Defaults to 2m when zero. + TTL metav1.Duration `json:"ttl,omitzero"` +} + +type RootConfig struct { + // Cache configures the transparent in-process overlay cache. + Cache Config `json:"cache"` +} diff --git a/pkg/cache/lock.go b/pkg/cache/lock.go new file mode 100644 index 000000000..22d2a351a --- /dev/null +++ b/pkg/cache/lock.go @@ -0,0 +1,85 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package cache + +import ( + "sync" + + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// lockKey identifies the object whose write path a keyedMutex serializes. +type lockKey struct { + gvk schema.GroupVersionKind + key client.ObjectKey +} + +// refMutex is the actual per-object lock plus a reference count of how many +// callers currently hold or are waiting for it. The count lets keyedMutex know +// when the entry is unused so it can be deleted (see keyedMutex.lock). +type refMutex struct { + mu sync.Mutex + ref int +} + +// keyedMutex hands out one mutex per key, serializing operations that share a +// key while letting distinct keys proceed concurrently. +// +// A sync.Map (or a map we only ever insert into) would be simpler, but it would leak: +// this cache wraps a single client for the whole lifetime of the controller-manager process, +// and a controller reconciles a continuous stream of (often short-lived) objects. +// A map would therefore accumulate one mutex per distinct object ever written and +// grow without bound. To avoid that we reference count each entry and delete it +// once the last holder unlocks, so the map only ever holds locks for objects +// with writes currently in flight. +// +// The two mutexes have distinct, non-overlapping roles: +// - k.mu guards the locks map itself. It is only ever held for the tiny +// bookkeeping critical sections below (map lookup/insert/delete and the +// ref counter), never across the caller's I/O. +// - rm.mu is the real per-object lock, held by the caller across the inner +// client call and the overlay mutation. +// +// Because k.mu is never held while rm.mu is locked (we release k.mu before +// taking rm.mu), the two can never deadlock against each other. +type keyedMutex struct { + mu sync.Mutex + locks map[lockKey]*refMutex +} + +func newKeyedMutex() *keyedMutex { + return &keyedMutex{locks: make(map[lockKey]*refMutex)} +} + +// lock acquires the per-key mutex and returns a function that releases it. +func (k *keyedMutex) lock(lk lockKey) func() { + // Look up (or create) the entry for this key and register our interest by + // bumping ref, all under k.mu so the map stays consistent. We increment ref + // here, before taking rm.mu, so that a concurrent unlock cannot see ref==0 + // and delete the entry out from under us while we are blocked waiting on it. + k.mu.Lock() + rm := k.locks[lk] + if rm == nil { + rm = &refMutex{} + k.locks[lk] = rm + } + rm.ref++ + k.mu.Unlock() + + // Take the real per-object lock outside k.mu; this is where a second caller + // for the same key blocks (and where we may block across the caller's I/O). + rm.mu.Lock() + return func() { + rm.mu.Unlock() + // Drop our reference and, if we were the last holder, remove the entry + // so the map does not grow unbounded. + k.mu.Lock() + rm.ref-- + if rm.ref == 0 { + delete(k.locks, lk) + } + k.mu.Unlock() + } +} diff --git a/pkg/cache/monitor.go b/pkg/cache/monitor.go new file mode 100644 index 000000000..3ec1ef2e5 --- /dev/null +++ b/pkg/cache/monitor.go @@ -0,0 +1,111 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package cache + +import ( + "sync" + + "github.com/prometheus/client_golang/prometheus" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// gvkHostKey identifies one overlay-size series: a cached GVK on a specific +// cluster (identified by its rest config Host). One Monitor is shared across +// all overlays (home + remotes), so the host discriminates between them. +type gvkHostKey struct { + gvk schema.GroupVersionKind + host string +} + +// sizeStat holds the current and high-watermark entry counts for one series. +type sizeStat struct { + current int + max int +} + +// Monitor is a standalone prometheus.Collector that tracks the overlay's +// per-(gvk, host) entry count. It exposes both the current size and a +// high-watermark that is reset to the current size on every Collect, so +// transient spikes are captured even when the overlay is empty at scrape time. +// +// Overlay entries are extremely short-lived (evicted within milliseconds once +// the informer observes the write), so a plain gauge sampled at scrape time +// would almost always read zero. A consistently non-zero high-watermark is a +// strong signal that eviction is broken (informer lag, UID mismatch, TTL +// fallback firing). +type Monitor struct { + maxDesc *prometheus.Desc + currentDesc *prometheus.Desc + + mu sync.Mutex + stats map[gvkHostKey]*sizeStat +} + +// NewMonitor returns a Monitor whose metric names are prefixed with prefix +// (e.g. "cortex_" yields "cortex_cache_overlay_entries"). +func NewMonitor(prefix string) *Monitor { + labels := []string{"gvk", "host"} + return &Monitor{ + maxDesc: prometheus.NewDesc( + prefix+"cache_overlay_entries_max", + "High-watermark of pending-cache overlay entries (live + tombstone) "+ + "per gvk and host since the last scrape; reset on each scrape.", + labels, nil, + ), + currentDesc: prometheus.NewDesc( + prefix+"cache_overlay_entries", + "Current number of pending-cache overlay entries (live + tombstone) "+ + "per gvk and host at scrape time.", + labels, nil, + ), + stats: make(map[gvkHostKey]*sizeStat), + } +} + +// observeSize records that the overlay for (host, gvk) currently holds n +// entries, bumping the high-watermark if n exceeds it. It is called by the +// Overlay under its own lock after every entry-count change. Nil-safe: a nil +// receiver (cache running without a monitor) is a no-op. +func (m *Monitor) observeSize(host string, gvk schema.GroupVersionKind, n int) { + if m == nil { + return + } + key := gvkHostKey{gvk: gvk, host: host} + m.mu.Lock() + defer m.mu.Unlock() + s := m.stats[key] + if s == nil { + s = &sizeStat{} + m.stats[key] = s + } + s.current = n + if n > s.max { + s.max = n + } +} + +// Describe implements prometheus.Collector. +func (m *Monitor) Describe(ch chan<- *prometheus.Desc) { + ch <- m.maxDesc + ch <- m.currentDesc +} + +// Collect implements prometheus.Collector. It emits the current size and the +// high-watermark for every observed series, then resets each high-watermark to +// the current size so the next scrape measures a fresh window. +func (m *Monitor) Collect(ch chan<- prometheus.Metric) { + m.mu.Lock() + defer m.mu.Unlock() + for key, s := range m.stats { + gvkStr := key.gvk.GroupVersion().String() + "/" + key.gvk.Kind + ch <- prometheus.MustNewConstMetric( + m.currentDesc, prometheus.GaugeValue, float64(s.current), gvkStr, key.host, + ) + ch <- prometheus.MustNewConstMetric( + m.maxDesc, prometheus.GaugeValue, float64(s.max), gvkStr, key.host, + ) + // Reset the window: the next scrape measures the peak since now. + s.max = s.current + } +} diff --git a/pkg/cache/monitor_test.go b/pkg/cache/monitor_test.go new file mode 100644 index 000000000..92fb5d343 --- /dev/null +++ b/pkg/cache/monitor_test.go @@ -0,0 +1,177 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package cache + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// gvkString mirrors the label the Monitor emits for a GVK. +func gvkString(gvk schema.GroupVersionKind) string { + return gvk.GroupVersion().String() + "/" + gvk.Kind +} + +// gaugeValue gathers the monitor via a registry and returns the value of the +// named metric for the given gvk/host labels. It reports whether a series with +// those labels was found. +func gaugeValue(t *testing.T, m *Monitor, name, gvk, host string) (float64, bool) { + t.Helper() + reg := prometheus.NewRegistry() + if err := reg.Register(m); err != nil { + t.Fatalf("register monitor: %v", err) + } + families, err := reg.Gather() + if err != nil { + t.Fatalf("gather: %v", err) + } + for _, f := range families { + if f.GetName() != name { + continue + } + for _, met := range f.GetMetric() { + if labelMatches(met, gvk, host) { + return met.GetGauge().GetValue(), true + } + } + } + return 0, false +} + +func labelMatches(met *dto.Metric, gvk, host string) bool { + var gotGVK, gotHost string + for _, l := range met.GetLabel() { + switch l.GetName() { + case "gvk": + gotGVK = l.GetValue() + case "host": + gotHost = l.GetValue() + } + } + return gotGVK == gvk && gotHost == host +} + +const ( + maxMetric = "cortex_cache_overlay_entries_max" + currentMetric = "cortex_cache_overlay_entries" +) + +// TestMonitorWatermarkBeforeScrape checks the max reflects the peak on the very +// first scrape, even after entries are evicted below it. +func TestMonitorWatermarkBeforeScrape(t *testing.T) { + m := NewMonitor("cortex_") + gvk := reservationGVK() + host := "cluster-a" + gvkStr := gvkString(gvk) + + m.observeSize(host, gvk, 1) + m.observeSize(host, gvk, 2) + m.observeSize(host, gvk, 3) + m.observeSize(host, gvk, 1) // evicted back down + + if got, ok := gaugeValue(t, m, maxMetric, gvkStr, host); !ok || got != 3 { + t.Fatalf("max: got %v (found=%v), want 3", got, ok) + } +} + +// TestMonitorCollectResetsWindow verifies the high-watermark resets to the +// current size after a Collect, so the next scrape measures a fresh window. +func TestMonitorCollectResetsWindow(t *testing.T) { + m := NewMonitor("cortex_") + gvk := reservationGVK() + host := "cluster-a" + gvkStr := gvkString(gvk) + + m.observeSize(host, gvk, 3) + m.observeSize(host, gvk, 1) + + // First scrape sees the peak of 3 and resets max to current (1). + if got, _ := gaugeValue(t, m, maxMetric, gvkStr, host); got != 3 { + t.Fatalf("first scrape max: got %v, want 3", got) + } + // Second scrape (no further writes) reports max == current == 1. + maxOverTime, _ := gaugeValue(t, m, maxMetric, gvkStr, host) + cur, _ := gaugeValue(t, m, currentMetric, gvkStr, host) + if maxOverTime != cur || maxOverTime != 1 { + t.Fatalf("second scrape: max=%v current=%v, want both 1", maxOverTime, cur) + } +} + +// TestMonitorCurrentTracksLiveSize verifies the current gauge follows the real +// entry count as writes and evictions happen through the Overlay. +func TestMonitorCurrentTracksLiveSize(t *testing.T) { + m := NewMonitor("cortex_") + c := cachingFrom(t, newTestClient(t).fakeCluster, reservationConfig()) + c.monitor = m + c.host = "cluster-a" + gvk := reservationGVK() + gvkStr := gvkString(gvk) + + // Add two live entries. + c.upsert(gvk, newReservation("res-1", "az-1", "1")) + c.upsert(gvk, newReservation("res-2", "az-1", "1")) + if got, ok := gaugeValue(t, m, currentMetric, gvkStr, "cluster-a"); !ok || got != 2 { + t.Fatalf("after upserts: got %v (found=%v), want 2", got, ok) + } + + // Evict both via informer-observed objects at a >= ResourceVersion. + c.evictIfSeen(gvk, newReservation("res-1", "az-1", "1")) + c.evictIfSeen(gvk, newReservation("res-2", "az-1", "1")) + if got, _ := gaugeValue(t, m, currentMetric, gvkStr, "cluster-a"); got != 0 { + t.Fatalf("after evictions: got %v, want 0", got) + } +} + +// TestMonitorCountsTombstones verifies both live entries and tombstones are +// counted in the overlay size. +func TestMonitorCountsTombstones(t *testing.T) { + m := NewMonitor("cortex_") + c := cachingFrom(t, newTestClient(t).fakeCluster, reservationConfig()) + c.monitor = m + c.host = "cluster-a" + gvk := reservationGVK() + gvkStr := gvkString(gvk) + + c.upsert(gvk, newReservation("res-live", "az-1", "1")) + c.tombstone(gvk, newReservation("res-dead", "az-1", "1")) + if got, ok := gaugeValue(t, m, currentMetric, gvkStr, "cluster-a"); !ok || got != 2 { + t.Fatalf("live+tombstone: got %v (found=%v), want 2", got, ok) + } +} + +// TestMonitorPerHostSeparation verifies that two overlays sharing one Monitor +// but reporting different hosts produce distinct series. +func TestMonitorPerHostSeparation(t *testing.T) { + m := NewMonitor("cortex_") + gvk := reservationGVK() + gvkStr := gvkString(gvk) + + m.observeSize("cluster-a", gvk, 5) + m.observeSize("cluster-b", gvk, 2) + + if got, ok := gaugeValue(t, m, currentMetric, gvkStr, "cluster-a"); !ok || got != 5 { + t.Fatalf("cluster-a: got %v (found=%v), want 5", got, ok) + } + if got, ok := gaugeValue(t, m, currentMetric, gvkStr, "cluster-b"); !ok || got != 2 { + t.Fatalf("cluster-b: got %v (found=%v), want 2", got, ok) + } +} + +// TestMonitorNilSafe verifies that a nil *Monitor and an Overlay with no monitor +// do not panic on size observation. +func TestMonitorNilSafe(t *testing.T) { + var m *Monitor + m.observeSize("host", reservationGVK(), 3) // must not panic + + // An Overlay built with a nil monitor (via WrapCluster(..., nil)) must not + // panic when its entry count changes. + c := cachingFrom(t, newTestClient(t).fakeCluster, reservationConfig()) + if c.monitor != nil { + t.Fatalf("expected nil monitor on default overlay") + } + c.upsert(reservationGVK(), newReservation("res-nil", "az-1", "1")) +} diff --git a/pkg/cache/runnable.go b/pkg/cache/runnable.go new file mode 100644 index 000000000..a4d8a9a21 --- /dev/null +++ b/pkg/cache/runnable.go @@ -0,0 +1,96 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package cache + +import ( + "context" + "fmt" + "time" + + "k8s.io/apimachinery/pkg/runtime/schema" + toolscachek8s "k8s.io/client-go/tools/cache" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// minCleanupInterval bounds the TTL cleanup ticker from below. +const minCleanupInterval = 30 * time.Second + +// Start implements manager.Runnable. It attaches informer event handlers for +// eviction and runs the TTL cleanup loop until ctx is done. +func (c *Overlay) Start(ctx context.Context) error { + log := ctrl.LoggerFrom(ctx).WithName("cache") + + for gvk := range c.gvks { + obj, err := c.newObjectForGVK(gvk) + if err != nil { + log.Error(err, "failed to build object for gvk; eviction disabled for it", "gvk", gvk) + continue + } + inf, err := c.informerCache.GetInformer(ctx, obj) + if err != nil { + log.Error(err, "failed to get informer for gvk; eviction disabled for it", "gvk", gvk) + continue + } + if _, err := inf.AddEventHandler(c.evictionHandler(gvk)); err != nil { + log.Error(err, "failed to add eviction event handler", "gvk", gvk) + } + } + + interval := max(c.ttl/4, minCleanupInterval) + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case now := <-ticker.C: + c.cleanupExpired(now) + } + } +} + +// NeedLeaderElection reports that the Overlay's Start lifecycle must run +// on every replica, not only the elected leader. The overlay is per-process +// state, so its eviction handlers and TTL cleanup have to run wherever the +// client is used, regardless of leader election. +func (c *Overlay) NeedLeaderElection() bool { + return false +} + +// evictionHandler returns an informer event handler that evicts overlay entries +// for the GVK when the real object is observed at a >= ResourceVersion. +func (c *Overlay) evictionHandler(gvk schema.GroupVersionKind) toolscachek8s.ResourceEventHandler { + evict := func(o any) { + obj, ok := o.(client.Object) + if !ok { + return + } + c.evictIfSeen(gvk, obj) + } + return toolscachek8s.ResourceEventHandlerFuncs{ + AddFunc: func(o any) { evict(o) }, + UpdateFunc: func(_, o any) { evict(o) }, + DeleteFunc: func(o any) { + if d, ok := o.(toolscachek8s.DeletedFinalStateUnknown); ok { + evict(d.Obj) + return + } + evict(o) + }, + } +} + +// newObjectForGVK builds an empty typed object for the GVK using the scheme. +func (c *Overlay) newObjectForGVK(gvk schema.GroupVersionKind) (client.Object, error) { + ro, err := c.scheme.New(gvk) + if err != nil { + return nil, err + } + obj, ok := ro.(client.Object) + if !ok { + return nil, fmt.Errorf("cache: object for gvk %s does not implement client.Object", gvk) + } + return obj, nil +} diff --git a/pkg/multicluster/client.go b/pkg/multicluster/client.go index 8b3a65a60..92cab3430 100644 --- a/pkg/multicluster/client.go +++ b/pkg/multicluster/client.go @@ -22,6 +22,21 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cluster" ) +// ClusterWrapper can be registered on the Client to transparently transform each +// cluster before it is stored for routing. WrapCluster receives the current +// cluster: raw for the first wrapper in the chain, or the previous wrapper's +// result for subsequent ones. It must return the (possibly wrapped) cluster to +// use for routing and is responsible for registering any lifecycle Runnables +// with the manager itself. Because the manager is not passed to WrapCluster, a +// wrapper that needs it must capture it at construction time (e.g. by storing it +// on the wrapper). Wrappers are applied in order for every home and remote +// cluster during InitFromConf. For remote clusters the original unwrapped +// cluster is always added to the manager separately so its informers start +// independently of wrapping. +type ClusterWrapper interface { + WrapCluster(cl cluster.Cluster) (cluster.Cluster, error) +} + // A remote cluster with routing labels used to match resources to clusters. type remoteCluster struct { cluster cluster.Cluster @@ -33,6 +48,12 @@ type Client struct { // when multiple clusters serve the same GVK. ResourceRouters map[schema.GroupVersionKind]ResourceRouter + // Wrappers are applied to every cluster (home and remotes) during InitFromConf. + // Each wrapper transforms the cluster before it is stored for routing. + // Applied in slice order; the raw inner cluster is always added to the manager + // so its informers start independently of wrapping. + Wrappers []ClusterWrapper + // The cluster in which cortex is deployed. HomeCluster cluster.Cluster // The REST config for the home cluster in which cortex is deployed. @@ -55,48 +76,6 @@ type Client struct { homeGVKs map[schema.GroupVersionKind]bool } -type ClientConfig struct { - // Apiserver configuration mapping GVKs to home or remote clusters. - // Every GVK used through the multicluster client must be listed - // in either Home or Remotes. Unknown GVKs will cause an error. - APIServers APIServersConfig `json:"apiservers"` -} - -// APIServersConfig separates resources into home and remote clusters. -type APIServersConfig struct { - // Resources managed in the cluster where cortex is deployed. - Home HomeConfig `json:"home"` - // Resources managed in remote clusters. - Remotes []RemoteConfig `json:"remotes,omitempty"` -} - -// HomeConfig lists GVKs that are managed in the home cluster. -type HomeConfig struct { - // The resource GVKs formatted as "//". - GVKs []string `json:"gvks"` -} - -// RemoteConfig maps multiple GVKs to a remote kubernetes apiserver with -// routing labels. It is assumed that the remote apiserver accepts the -// serviceaccount tokens issued by the local cluster. -type RemoteConfig struct { - // The remote kubernetes apiserver url, e.g. "https://my-apiserver:6443". - Host string `json:"host"` - // The root CA certificate to verify the remote apiserver. - // Ignored if InsecureSkipTLSVerify is true. - CACert string `json:"caCert,omitempty"` - // InsecureSkipTLSVerify disables verification of the remote apiserver's - // TLS certificate. Use this for apiservers whose CA certificate rotates - // frequently and does not chain to a stable root. Mutually exclusive - // with CACert: when true, CACert is ignored. - InsecureSkipTLSVerify bool `json:"insecureSkipTLSVerify,omitempty"` - // The resource GVKs this apiserver serves, formatted as "//". - GVKs []string `json:"gvks"` - // Labels used by ResourceRouters to match resources to this cluster - // for write operations (Create/Update/Delete/Patch). - Labels map[string]string `json:"labels,omitempty"` -} - // Helper function to initialize a new multicluster client during service startup, // using the conf module provided by cortex. func (c *Client) InitFromConf(ctx context.Context, mgr ctrl.Manager, conf ClientConfig) error { @@ -136,10 +115,23 @@ func (c *Client) InitFromConf(ctx context.Context, mgr ctrl.Manager, conf Client if err != nil { return err } + // Add the raw inner cluster so its informers/caches start. if err := mgr.Add(cl); err != nil { return err } } + // Apply wrappers to the home cluster. The manager already owns the home + // cluster's lifecycle, so we only apply wrappers (each wrapper registers its + // own Runnables) and must NOT re-Start the inner home cluster. + if c.HomeCluster != nil { + for _, w := range c.Wrappers { + wrapped, werr := w.WrapCluster(c.HomeCluster) + if werr != nil { + return werr + } + c.HomeCluster = wrapped + } + } return nil } @@ -153,6 +145,11 @@ func (c *Client) InitFromConf(ctx context.Context, mgr ctrl.Manager, conf Client // This can be used when the remote cluster accepts the home cluster's service // account tokens. See the kubernetes documentation on structured auth to // learn more about jwt-based authentication across clusters. +// AddRemote returns the raw inner cluster.Cluster (which the caller must add to +// the manager so its informers/caches start). Each registered Wrapper is +// responsible for adding its own lifecycle Runnables to mgr directly. +// The wrapped cluster is stored in remoteClusters so all routing goes through +// any per-cluster wrapper. func (c *Client) AddRemote(ctx context.Context, host, caCert string, insecureSkipTLSVerify bool, labels map[string]string, gvks ...schema.GroupVersionKind) (cluster.Cluster, error) { log := ctrl.LoggerFrom(ctx) homeRestConfig := *c.HomeRestConfig @@ -173,6 +170,17 @@ func (c *Client) AddRemote(ctx context.Context, host, caCert string, insecureSki if err != nil { return nil, err } + // Apply each registered wrapper in order. stored is the cluster placed in + // remoteClusters. Each wrapper is responsible for registering its own + // lifecycle Runnables with mgr directly. + stored := cl + for _, w := range c.Wrappers { + wrapped, werr := w.WrapCluster(stored) + if werr != nil { + return nil, werr + } + stored = wrapped + } c.remoteClustersMu.Lock() defer c.remoteClustersMu.Unlock() if c.remoteClusters == nil { @@ -181,10 +189,11 @@ func (c *Client) AddRemote(ctx context.Context, host, caCert string, insecureSki for _, gvk := range gvks { log.Info("adding remote cluster for resource", "gvk", gvk, "host", host, "labels", labels, "insecureSkipTLSVerify", insecureSkipTLSVerify) c.remoteClusters[gvk] = append(c.remoteClusters[gvk], remoteCluster{ - cluster: cl, + cluster: stored, labels: labels, }) } + // Return the raw inner cluster so the caller starts its informers/caches. return cl, nil } @@ -854,7 +863,7 @@ func (c *Client) IndexField(ctx context.Context, obj client.Object, list client. continue } indexed[ch] = true - if err := ch.IndexField(ctx, obj, field, extractValue); err != nil { + if err := cl.GetFieldIndexer().IndexField(ctx, obj, field, extractValue); err != nil { log.Error(err, "failed to register field index for cluster — objects from this cluster will be absent from index queries; restart required to recover", "field", field) continue } @@ -869,7 +878,7 @@ func (c *Client) IndexField(ctx context.Context, obj client.Object, list client. continue } indexed[ch] = true - if err := ch.IndexField(ctx, obj, field, extractValue); err != nil { + if err := cl.GetFieldIndexer().IndexField(ctx, obj, field, extractValue); err != nil { log.Error(err, "failed to register field index for cluster — objects from this cluster will be absent from index queries; restart required to recover", "field", field) continue } diff --git a/pkg/multicluster/client_test.go b/pkg/multicluster/client_test.go index 41fd3a6b9..29ea8e0ca 100644 --- a/pkg/multicluster/client_test.go +++ b/pkg/multicluster/client_test.go @@ -82,16 +82,28 @@ type fakeCluster struct { fakeClient client.Client fakeCache *fakeCache fakeRecorder events.EventRecorder + scheme *runtime.Scheme } func (f *fakeCluster) GetClient() client.Client { return f.fakeClient } +func (f *fakeCluster) GetScheme() *runtime.Scheme { + return f.scheme +} + func (f *fakeCluster) GetCache() cache.Cache { return f.fakeCache } +// GetFieldIndexer returns the fake cache, which implements client.FieldIndexer +// via its IndexField method. This mirrors production, where an unwrapped +// cluster's field indexer is its cache. +func (f *fakeCluster) GetFieldIndexer() client.FieldIndexer { + return f.fakeCache +} + func (f *fakeCluster) GetEventRecorder(_ string) events.EventRecorder { if f.fakeRecorder != nil { return f.fakeRecorder @@ -107,6 +119,7 @@ func newFakeCluster(scheme *runtime.Scheme, objs ...client.Object) *fakeCluster return &fakeCluster{ fakeClient: fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build(), fakeCache: &fakeCache{}, + scheme: scheme, } } @@ -115,6 +128,7 @@ func newFakeClusterWithCache(scheme *runtime.Scheme, fakeCache *fakeCache, objs return &fakeCluster{ fakeClient: fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build(), fakeCache: fakeCache, + scheme: scheme, } } diff --git a/pkg/multicluster/config.go b/pkg/multicluster/config.go new file mode 100644 index 000000000..359d225cb --- /dev/null +++ b/pkg/multicluster/config.go @@ -0,0 +1,46 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package multicluster + +type ClientConfig struct { + // Apiserver configuration mapping GVKs to home or remote clusters. + // Every GVK used through the multicluster client must be listed + // in either Home or Remotes. Unknown GVKs will cause an error. + APIServers APIServersConfig `json:"apiservers"` +} + +// APIServersConfig separates resources into home and remote clusters. +type APIServersConfig struct { + // Resources managed in the cluster where cortex is deployed. + Home HomeConfig `json:"home"` + // Resources managed in remote clusters. + Remotes []RemoteConfig `json:"remotes,omitempty"` +} + +// HomeConfig lists GVKs that are managed in the home cluster. +type HomeConfig struct { + // The resource GVKs formatted as "//". + GVKs []string `json:"gvks"` +} + +// RemoteConfig maps multiple GVKs to a remote kubernetes apiserver with +// routing labels. It is assumed that the remote apiserver accepts the +// serviceaccount tokens issued by the local cluster. +type RemoteConfig struct { + // The remote kubernetes apiserver url, e.g. "https://my-apiserver:6443". + Host string `json:"host"` + // The root CA certificate to verify the remote apiserver. + // Ignored if InsecureSkipTLSVerify is true. + CACert string `json:"caCert,omitempty"` + // InsecureSkipTLSVerify disables verification of the remote apiserver's + // TLS certificate. Use this for apiservers whose CA certificate rotates + // frequently and does not chain to a stable root. Mutually exclusive + // with CACert: when true, CACert is ignored. + InsecureSkipTLSVerify bool `json:"insecureSkipTLSVerify,omitempty"` + // The resource GVKs this apiserver serves, formatted as "//". + GVKs []string `json:"gvks"` + // Labels used by ResourceRouters to match resources to this cluster + // for write operations (Create/Update/Delete/Patch). + Labels map[string]string `json:"labels,omitempty"` +} diff --git a/postgres/Dockerfile b/postgres/Dockerfile index 24baed44b..f75e0d66f 100644 --- a/postgres/Dockerfile +++ b/postgres/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:trixie-slim@sha256:3a39a0592364683e6bab97937b72cad5a8fa6dcbbee90edb3bb48c7f8e94f258 +FROM debian:trixie-slim@sha256:d7e12182ce18b85b93007c1dedf31f2d29e01ccf3182cc4017c709b6259bc132 # explicitly set user/group IDs RUN set -eux; \ diff --git a/postgres/rebuild-trigger b/postgres/rebuild-trigger index 3fbb2fb47..417ba9d4a 100644 --- a/postgres/rebuild-trigger +++ b/postgres/rebuild-trigger @@ -1 +1 @@ -29312059711 +32936771649