diff --git a/CHANGELOG.md b/CHANGELOG.md index 50ab10b23..115214a35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,103 @@ # Changelog +## cortex 0.9.1 + +### Refactoring + +- **CR & Reservation controller decomposition** ([#1535](https://github.com/cobaltcore-dev/cortex/pull/1535)): The monolithic `Reconcile()` methods in both `CommittedResourceController` and `CommitmentReservationController` have been split into focused sub-functions (`handleRejectedGeneration()`, `reconcileEvicted()`, `reconcilePlaced()`, `reconcileUnplaced()`) for improved readability, testability, and maintainability. The echo-nudge mechanism (`isAlreadyReserving()` helper and `cortex.cloud/echo-nudge` annotation) has been removed — the controller now relies on the Reservation watch plus a 3 s fallback requeue. Spec-change re-placement now clears `Spec.TargetHost` to force re-placement through the eviction → unplaced flow instead of silently stamping a new `ObservedGeneration`. Roadmap TODOs (`#778`, `#781`, `#783`, `#785`–`#790`) are documented inline. + +### Fixes + +- **Flavor sort order: prefer fewest CPUs on memory tie** ([#1535](https://github.com/cobaltcore-dev/cortex/pull/1535)): The secondary sort for flavors within a group has been changed from VCPUs descending to VCPUs ascending when memory is equal. This minimizes CPU blocking on hosts when multiple flavors share the same memory tier. New test `TestSelectFlavor_SameMemory_PrefersFewestCPU` validates the tiebreak. + +### Housekeeping + +- **Tiltfile: stale Helm chart cleanup** ([#1535](https://github.com/cobaltcore-dev/cortex/pull/1535)): Added a `find … -delete` step that removes stale packaged Helm chart `.tgz` files before copying the new package, preventing version collisions during local development. + +### Dependency Updates + +| Dependency | Change | Note | +|---|---|---| +| PostgreSQL 18 | `18.6-1.pgdg13+2` → `18.7-1.pgdg13+2` | Dockerfile bump | +| `go.opentelemetry.io/otel/trace` | → `v1.47.0` | ⚠️ Security | +| `go.opentelemetry.io/otel/sdk` | → `v1.47.0` | ⚠️ Security | +| `github.com/sapcc/go-bits` | `13886743d3a7` → `2726a2c437f4` | | +| `kube-prometheus-stack` | `v93.2.2` → `v93.3.0` | Helm chart | +| `github.com/gophercloud/gophercloud/v2` | `v2.15.0` → `v2.16.0` | | +| `sigs.k8s.io/controller-runtime` | `v0.25.2` → `v0.25.3` | | + +### Rollout Notes + +- **Echo-nudge removal:** Stale reservations may take up to 3 s (requeue interval) to recover instead of being poked immediately via annotation patch. The Reservation watch plus fallback requeue covers this path; the nudge was a secondary mechanism. +- **Spec-change re-placement:** Clearing `TargetHost` on generation mismatch now triggers a two-cycle eviction → re-place flow. This reuses the existing eviction codepath and is covered by integration tests. +- **Flavor sort reversal:** Only affects *new* reservations; existing placements are unaffected. +- **Security dependencies — prioritize rollout:** OpenTelemetry `v1.47.0` patches security vulnerabilities. + +## cortex-postgres 0.6.33 + +- PostgreSQL `18.6-1.pgdg13+2` → `18.7-1.pgdg13+2`. +- Bump to align with cortex 0.9.1 release. + +## cortex-shim 0.1.39 + +- Bump to align with cortex 0.9.1 release. + +## bundles 0.0.115 + +- Bump to align with cortex 0.9.1 release. + +## cortex-placement-shim 0.1.39 + +- Bump to align with cortex 0.9.1 release. + +## Release PR [#1535](https://github.com/cobaltcore-dev/cortex/pull/1535) — 2026-11-16 + +### Bumped Versions + +| Package | Previous | New | +|---|---|---| +| cortex | 0.9.0 | 0.9.1 | +| cortex-postgres | 0.6.32 | 0.6.33 | +| cortex-shim | 0.1.38 | 0.1.39 | +| bundles | 0.0.114 | 0.0.115 | +| cortex-placement-shim | 0.1.38 | 0.1.39 | + +### Refactoring + +- **CR & Reservation controller decomposition** — The monolithic `Reconcile()` methods in `CommittedResourceController` and `CommitmentReservationController` have been split into focused sub-functions (`handleRejectedGeneration()`, `reconcileEvicted()`, `reconcilePlaced()`, `reconcileUnplaced()`). The echo-nudge mechanism (`isAlreadyReserving()` + `cortex.cloud/echo-nudge` annotation) has been removed in favor of Reservation watch + 3 s fallback requeue. Spec-change re-placement now clears `Spec.TargetHost` to force eviction → re-place instead of in-place `ObservedGeneration` stamping. + +### Bug Fixes + +- **Flavor sort order: prefer fewest CPUs on memory tie** — The secondary sort for flavors within a group changed from VCPUs descending to VCPUs ascending when memory is equal, minimizing CPU blocking on hosts. New test `TestSelectFlavor_SameMemory_PrefersFewestCPU` validates the behavior. + +### Chores + +- **Tiltfile: stale Helm chart cleanup** — Added `find … -delete` to remove stale `.tgz` chart files before copying the new package. + +### Dependency Updates + +| Dependency | Old | New | Note | +|---|---|---|---| +| PostgreSQL 18 | `18.6-1.pgdg13+2` | `18.7-1.pgdg13+2` | Dockerfile | +| `go.opentelemetry.io/otel/trace` | — | `v1.47.0` | ⚠️ Security | +| `go.opentelemetry.io/otel/sdk` | — | `v1.47.0` | ⚠️ Security | +| `github.com/sapcc/go-bits` | digest `13886743d3a7` | digest `2726a2c437f4` | | +| `kube-prometheus-stack` | `v93.2.2` | `v93.3.0` | Helm chart | +| `github.com/gophercloud/gophercloud/v2` | `v2.15.0` | `v2.16.0` | | +| `sigs.k8s.io/controller-runtime` | `v0.25.2` | `v0.25.3` | | + +### ⚠️ Operational Notes + +1. **No schema or RBAC changes:** This is a purely internal restructuring with zero new CRDs or API surface changes. A rolling upgrade is sufficient. +2. **Echo-nudge removal:** Stale reservations may take up to 3 s (requeue interval) to recover instead of the immediate annotation poke. The Reservation watch + fallback requeue covers the path. +3. **Spec-change re-placement:** When a placed reservation's `ObservedGeneration` falls behind `res.Generation`, `Spec.TargetHost` is now cleared to force re-placement through the eviction → unplaced flow. This matches the existing eviction codepath and is covered by integration tests. +4. **Flavor sort reversal:** The scheduler now prefers the flavor with fewest CPU cores when memory ties. Only affects new reservations; existing placements are unaffected. +5. **Security dependencies — prioritize rollout:** OpenTelemetry trace/SDK `v1.47.0` patches security vulnerabilities. + +**Contributors:** @mblos, @renovate[bot] + +**PR:** [#1535](https://github.com/cobaltcore-dev/cortex/pull/1535) | **Commits:** 7 | **Files changed:** 18 | **+314 / −308** + ## cortex 0.9.0 ### Features diff --git a/Tiltfile b/Tiltfile index 0c912021b..47b4dd672 100644 --- a/Tiltfile +++ b/Tiltfile @@ -205,6 +205,9 @@ for (bundle_chart_path, bundle_chart_name) in bundle_charts: local('sh helm/sync.sh ' + dep_chart_path) pkg_out = str(local('helm package ' + dep_chart_path)).strip() gen_tgz = os.path.basename(pkg_out) + # Always remove stale old versions of this chart (e.g. cortex-0.5.1.tgz when current is 0.9.0). + # Uses [0-9] prefix so cortex-[0-9]*.tgz does not accidentally match cortex-postgres-*.tgz. + local('find ' + bundle_chart_path + '/charts/ -maxdepth 1 -name "' + dep_chart_name + '-[0-9]*.tgz" ! -name "' + gen_tgz + '" -delete') # If the file isn't there yet, copy it over. if not os.path.exists(bundle_chart_path + '/charts/' + gen_tgz): print('Adding ' + dep_chart_name + ' to ' + bundle_chart_name) diff --git a/go.mod b/go.mod index 728fabfaa..b874eb5b8 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/go-gorp/gorp v2.2.0+incompatible github.com/go-logr/logr v1.4.4 github.com/google/uuid v1.6.0 - github.com/gophercloud/gophercloud/v2 v2.15.0 + github.com/gophercloud/gophercloud/v2 v2.16.0 github.com/ironcore-dev/ironcore v0.9.0 github.com/lib/pq v1.12.3 github.com/mattn/go-sqlite3 v1.14.53 @@ -16,7 +16,7 @@ require ( github.com/prometheus/client_golang v1.25.0 github.com/prometheus/client_model v0.6.3 github.com/sapcc/go-api-declarations v1.25.1 - github.com/sapcc/go-bits v0.0.0-20261105181628-13886743d3a7 + github.com/sapcc/go-bits v0.0.0-20261112131659-2726a2c437f4 go.uber.org/zap v1.28.0 go.xyrillian.de/gg v1.21.0 golang.org/x/sync v0.25.0 @@ -27,7 +27,7 @@ require ( k8s.io/apimachinery v0.37.2 k8s.io/apiserver v0.37.2 k8s.io/client-go v0.37.2 - sigs.k8s.io/controller-runtime v0.25.2 + sigs.k8s.io/controller-runtime v0.25.3 sigs.k8s.io/yaml v1.6.0 ) @@ -99,12 +99,13 @@ require ( github.com/ziutek/mymysql v1.5.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect - go.opentelemetry.io/otel v1.46.0 // indirect + go.opentelemetry.io/otel v1.47.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 // indirect - go.opentelemetry.io/otel/metric v1.46.0 // indirect - go.opentelemetry.io/otel/sdk v1.46.0 // indirect - go.opentelemetry.io/otel/trace v1.46.0 // indirect + go.opentelemetry.io/otel/log v1.47.0 // indirect + go.opentelemetry.io/otel/metric v1.47.0 // indirect + go.opentelemetry.io/otel/sdk v1.47.0 // indirect + go.opentelemetry.io/otel/trace v1.47.0 // indirect go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect @@ -114,7 +115,7 @@ require ( golang.org/x/net v0.62.0 // indirect golang.org/x/oauth2 v0.37.0 // indirect golang.org/x/sys v0.50.0 // indirect - golang.org/x/text v0.43.0 // indirect + golang.org/x/text v0.44.0 // indirect golang.org/x/time v0.16.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect diff --git a/go.sum b/go.sum index 53e7434f2..c3e037bbf 100644 --- a/go.sum +++ b/go.sum @@ -113,8 +113,8 @@ github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oX github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gophercloud/gophercloud/v2 v2.15.0 h1:4zLiLYTFraZMlJ77FH1Kzq7itjfVP+BIbWcCurCrgic= -github.com/gophercloud/gophercloud/v2 v2.15.0/go.mod h1:4fs5I9VH6Wg2LyocDL9xf0ASb8VD63tyLA8sgAX/69U= +github.com/gophercloud/gophercloud/v2 v2.16.0 h1:qhQKoU5mNX8hp4YVN5eks8Y6zFFlBAMYCu55Fc+tGkc= +github.com/gophercloud/gophercloud/v2 v2.16.0/go.mod h1:4fs5I9VH6Wg2LyocDL9xf0ASb8VD63tyLA8sgAX/69U= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gotestyourself/gotestyourself v2.2.0+incompatible h1:AQwinXlbQR2HvPjQZOmDhRqsv5mZf+Jb1RnSLxcqZcI= @@ -189,8 +189,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sapcc/go-api-declarations v1.25.1 h1:Lrbf8TZY/ARg0J9IrlrObqnXjrzSzmYKQd/L/a71Lsg= github.com/sapcc/go-api-declarations v1.25.1/go.mod h1:bJkK8wphzlmUmN5lxBv8BbfxUXMOkW+UiC10xBHXsFs= -github.com/sapcc/go-bits v0.0.0-20261105181628-13886743d3a7 h1:wMaeqAzroobJLeToye7ksGihdX8onvg0tEvU940QEJA= -github.com/sapcc/go-bits v0.0.0-20261105181628-13886743d3a7/go.mod h1:2Q+U9Heq6HLNRzSFz9ArClQknQSmK2qs5epOWZraxhc= +github.com/sapcc/go-bits v0.0.0-20261112131659-2726a2c437f4 h1:mggGIR4U1lrQcdVIoycMrURTjV7P82WdY28VdZZ7eO8= +github.com/sapcc/go-bits v0.0.0-20261112131659-2726a2c437f4/go.mod h1:5TyYcpATrqAaEaK6XNvvm84T76PgaTz4XFokRipne8w= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= @@ -212,20 +212,22 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= -go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= -go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= +go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 h1:w53CDeOA/Kurp7yRsegSr6pbbr759dOvJ+yNmWM6Hxs= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0/go.mod h1:BOmGMCbAtvcJiSJ+hLuhgPLdDbimnraSl8irz3iY8sY= -go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= -go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= -go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= -go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= -go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= -go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= -go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= -go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= +go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= +go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= +go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= +go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= +go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -245,8 +247,8 @@ go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBs go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= -golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= -golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= +golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= @@ -258,12 +260,12 @@ golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs= golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.48.0 h1:G6UU1eiSB8nl3DzeUGt7Mg+2SctI8tPS6Z5/mBt6cOk= golang.org/x/term v0.48.0/go.mod h1:BRDLHCZr2SUjIeBHHpRyQeC9xzSAVbzY2rvKgafnHf4= -golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= -golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo= +golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY= golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= -golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= -golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA= +golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs= +golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= @@ -309,8 +311,8 @@ k8s.io/utils v0.0.0-20261023123148-ca04ba58ecab h1:pjQrNLaF//ytZtWwCAHtW+pqwSKjt k8s.io/utils v0.0.0-20261023123148-ca04ba58ecab/go.mod h1:DdQ0BJxlQtuMlKDqdg7X5IMMQCXIk2b3d1/4M9OhUYo= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0 h1:/YpDJ4vReG7ZmzSpBGxduXgywWkJU9zHubgJG03MT+Y= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0/go.mod h1:tJo1aepTXyR+8Xs3sUsGBDk4Ub2AM5dPAPKJx0mpm5c= -sigs.k8s.io/controller-runtime v0.25.2 h1:bEkK3PVOIVK9X8QWLGVhJgmFc++47vfT6wakSzAOLsQ= -sigs.k8s.io/controller-runtime v0.25.2/go.mod h1:4QqLdT6z/L6Olj8JJCtvztid4/fnIiYsfaTFScegctc= +sigs.k8s.io/controller-runtime v0.25.3 h1:b4OcZSY/aKHY8JG+rBlQmQ1xLs574+ZBEjv1dtfLd80= +sigs.k8s.io/controller-runtime v0.25.3/go.mod h1:4QqLdT6z/L6Olj8JJCtvztid4/fnIiYsfaTFScegctc= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= diff --git a/helm/bundles/cortex-cinder/Chart.yaml b/helm/bundles/cortex-cinder/Chart.yaml index 2936e0c3f..6789f7702 100644 --- a/helm/bundles/cortex-cinder/Chart.yaml +++ b/helm/bundles/cortex-cinder/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-cinder description: A Helm chart deploying Cortex for Cinder. type: application -version: 0.0.114 +version: 0.0.115 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.32 + version: 0.6.33 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.0 + version: 0.9.1 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.0 + version: 0.9.1 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-crds/Chart.yaml b/helm/bundles/cortex-crds/Chart.yaml index 717f75e40..5a7816a15 100644 --- a/helm/bundles/cortex-crds/Chart.yaml +++ b/helm/bundles/cortex-crds/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-crds description: A Helm chart deploying Cortex CRDs. type: application -version: 0.0.114 +version: 0.0.115 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.0 + version: 0.9.1 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-ironcore/Chart.yaml b/helm/bundles/cortex-ironcore/Chart.yaml index cf19d5027..59310ed60 100644 --- a/helm/bundles/cortex-ironcore/Chart.yaml +++ b/helm/bundles/cortex-ironcore/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-ironcore description: A Helm chart deploying Cortex for IronCore. type: application -version: 0.0.114 +version: 0.0.115 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.0 + version: 0.9.1 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-manila/Chart.yaml b/helm/bundles/cortex-manila/Chart.yaml index 1dcb1372b..a80ece477 100644 --- a/helm/bundles/cortex-manila/Chart.yaml +++ b/helm/bundles/cortex-manila/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-manila description: A Helm chart deploying Cortex for Manila. type: application -version: 0.0.114 +version: 0.0.115 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.32 + version: 0.6.33 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.0 + version: 0.9.1 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.0 + version: 0.9.1 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-nova/Chart.yaml b/helm/bundles/cortex-nova/Chart.yaml index 11508e63f..fd9344e06 100644 --- a/helm/bundles/cortex-nova/Chart.yaml +++ b/helm/bundles/cortex-nova/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-nova description: A Helm chart deploying Cortex for Nova. type: application -version: 0.0.114 +version: 0.0.115 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.32 + version: 0.6.33 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.0 + version: 0.9.1 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.0 + version: 0.9.1 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-placement-shim/Chart.yaml b/helm/bundles/cortex-placement-shim/Chart.yaml index 4320dca09..19c85a10a 100644 --- a/helm/bundles/cortex-placement-shim/Chart.yaml +++ b/helm/bundles/cortex-placement-shim/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-placement-shim description: A Helm chart deploying the Cortex placement shim. type: application -version: 0.1.38 +version: 0.1.39 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-shim - name: cortex-shim repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.1.38 + version: 0.1.39 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case # of issues. See: https://github.com/sapcc/helm-charts/pkgs/container/helm-charts%2Fowner-info diff --git a/helm/bundles/cortex-pods/Chart.yaml b/helm/bundles/cortex-pods/Chart.yaml index e446b6ef6..7b47307dc 100644 --- a/helm/bundles/cortex-pods/Chart.yaml +++ b/helm/bundles/cortex-pods/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-pods description: A Helm chart deploying Cortex for Pods. type: application -version: 0.0.114 +version: 0.0.115 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.0 + version: 0.9.1 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/dev/cortex-prometheus-operator/Chart.yaml b/helm/dev/cortex-prometheus-operator/Chart.yaml index 80960b745..b66b7b7a1 100644 --- a/helm/dev/cortex-prometheus-operator/Chart.yaml +++ b/helm/dev/cortex-prometheus-operator/Chart.yaml @@ -10,4 +10,4 @@ dependencies: # CRDs of the prometheus operator, such as PrometheusRule, ServiceMonitor, etc. - name: kube-prometheus-stack repository: oci://ghcr.io/prometheus-community/charts - version: 93.2.0 + version: 93.3.0 diff --git a/helm/library/cortex-postgres/Chart.yaml b/helm/library/cortex-postgres/Chart.yaml index b31b61531..0331a948b 100644 --- a/helm/library/cortex-postgres/Chart.yaml +++ b/helm/library/cortex-postgres/Chart.yaml @@ -5,5 +5,5 @@ apiVersion: v2 name: cortex-postgres description: Postgres setup for Cortex. type: application -version: 0.6.32 -appVersion: "sha-9179b000" +version: 0.6.33 +appVersion: "sha-f69b58ee" diff --git a/helm/library/cortex-shim/Chart.yaml b/helm/library/cortex-shim/Chart.yaml index 69c3307b0..771e5fbfe 100644 --- a/helm/library/cortex-shim/Chart.yaml +++ b/helm/library/cortex-shim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex-shim description: A Helm chart to distribute cortex shims. type: application -version: 0.1.38 -appVersion: "sha-5ccd47a1" +version: 0.1.39 +appVersion: "sha-01a68db0" icon: "https://example.com/icon.png" dependencies: [] diff --git a/helm/library/cortex/Chart.yaml b/helm/library/cortex/Chart.yaml index fc92f4503..094a8b3c5 100644 --- a/helm/library/cortex/Chart.yaml +++ b/helm/library/cortex/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex description: A Helm chart to distribute cortex. type: application -version: 0.9.0 -appVersion: "sha-c7679c75" +version: 0.9.1 +appVersion: "sha-bac55e3a" icon: "https://example.com/icon.png" dependencies: [] diff --git a/internal/knowledge/extractor/plugins/compute/flavor_groups.go b/internal/knowledge/extractor/plugins/compute/flavor_groups.go index fdebd4404..5b9578cae 100644 --- a/internal/knowledge/extractor/plugins/compute/flavor_groups.go +++ b/internal/knowledge/extractor/plugins/compute/flavor_groups.go @@ -180,13 +180,14 @@ func (e *FlavorGroupExtractor) Extract() ([]plugins.Feature, error) { continue } - // Sort flavors by size descending (largest first), tie break by name for consistent ordering + // Sort flavors by memory descending (largest first); tie-break by fewest vCPUs to + // minimize CPU blocking when multiple flavors share the same memory size. sort.Slice(flavors, func(i, j int) bool { if flavors[i].MemoryMB != flavors[j].MemoryMB { return flavors[i].MemoryMB > flavors[j].MemoryMB } if flavors[i].VCPUs != flavors[j].VCPUs { - return flavors[i].VCPUs > flavors[j].VCPUs + return flavors[i].VCPUs < flavors[j].VCPUs } return flavors[i].Name < flavors[j].Name }) diff --git a/internal/knowledge/extractor/plugins/compute/flavor_groups_test.go b/internal/knowledge/extractor/plugins/compute/flavor_groups_test.go index 9b2b4dabd..97d304118 100644 --- a/internal/knowledge/extractor/plugins/compute/flavor_groups_test.go +++ b/internal/knowledge/extractor/plugins/compute/flavor_groups_test.go @@ -92,7 +92,7 @@ func TestFlavorGroupExtractor_Extract(t *testing.T) { Ephemeral: 5, ExtraSpecs: `{"capabilities:hypervisor_type":"CH","quota:hw_version":"ch"}`, }, - // Corner case: Same memory as gp_c8_m32_v2 but MORE VCPUs (should come first) + // Corner case: Same memory as gp_c8_m32_v2 but MORE VCPUs (should come last, after fewer-VCPUs flavor) &nova.Flavor{ ID: "8", Name: "gp_c12_m32_v2", @@ -178,15 +178,17 @@ func TestFlavorGroupExtractor_Extract(t *testing.T) { t.Errorf("expected largest flavor to have quota:hw_version=v2, got %s", v2Group.LargestFlavor.ExtraSpecs["quota:hw_version"]) } - // Verify smallest flavor in v2 group should be gp_c4_m16_ch is NOT in v2, so it's gp_c8_m32_v2 (lowest memory among v2 flavors) - if v2Group.SmallestFlavor.Name != "gp_c8_m32_v2" { - t.Errorf("expected smallest flavor to be gp_c8_m32_v2, got %s", v2Group.SmallestFlavor.Name) + // Verify smallest flavor in v2 group: at 32768 MB we have gp_c8_m32_v2 (8 VCPUs), + // gp_c12_m32_alt (12 VCPUs), and gp_c12_m32_v2 (12 VCPUs). With fewest-VCPUs-first + // tiebreak, gp_c8_m32_v2 sorts first and gp_c12_m32_v2 sorts last → smallest. + if v2Group.SmallestFlavor.Name != "gp_c12_m32_v2" { + t.Errorf("expected smallest flavor to be gp_c12_m32_v2, got %s", v2Group.SmallestFlavor.Name) } if v2Group.SmallestFlavor.MemoryMB != 32768 { t.Errorf("expected smallest flavor memory to be 32768 MB, got %d", v2Group.SmallestFlavor.MemoryMB) } - if v2Group.SmallestFlavor.VCPUs != 8 { - t.Errorf("expected smallest flavor VCPUs to be 8, got %d", v2Group.SmallestFlavor.VCPUs) + if v2Group.SmallestFlavor.VCPUs != 12 { + t.Errorf("expected smallest flavor VCPUs to be 12, got %d", v2Group.SmallestFlavor.VCPUs) } // Verify Cloud-Hypervisor group @@ -224,8 +226,8 @@ func TestFlavorGroupExtractor_Extract(t *testing.T) { fg.Name, current.Name, current.MemoryMB, next.Name, next.MemoryMB) } - // Secondary sort: if memory equal, VCPUs descending - if current.MemoryMB == next.MemoryMB && current.VCPUs < next.VCPUs { + // Secondary sort: if memory equal, VCPUs ascending (fewest VCPUs first) + if current.MemoryMB == next.MemoryMB && current.VCPUs > next.VCPUs { t.Errorf("Flavors in group %s with equal memory not sorted by VCPUs: %s (%d VCPUs) should come after %s (%d VCPUs)", fg.Name, current.Name, current.VCPUs, next.Name, next.VCPUs) } diff --git a/internal/scheduling/reservations/commitments/api/change_commitments.go b/internal/scheduling/reservations/commitments/api/change_commitments.go index ab6314851..c3c59edee 100644 --- a/internal/scheduling/reservations/commitments/api/change_commitments.go +++ b/internal/scheduling/reservations/commitments/api/change_commitments.go @@ -115,6 +115,8 @@ type crSnapshot struct { // HandleChangeCommitments implements POST /commitments/v1/change-commitments from the Limes LIQUID API. // It writes CommittedResource CRDs (one per commitment) and polls their status conditions until // the controller confirms or rejects each one. On any failure the whole batch is rolled back. +// TODO (#786): replace direct CR writes + changeMutex with a CommittedResourceChange (source=api); +// poll CRC terminal phase and map Accepted/Rejected/Superseded to HTTP response. Flag-gated switchover. // // See: https://github.com/sapcc/go-api-declarations/blob/main/liquid/commitment.go // See: https://pkg.go.dev/github.com/sapcc/go-api-declarations/liquid @@ -228,6 +230,8 @@ func (api *HTTPAPI) HandleChangeCommitments(w http.ResponseWriter, r *http.Reque } } +// TODO (#788): once #786 switchover is stable, remove: changeMutex, crSnapshot type, reverse-order +// rollbackCR, performDryRun probe CRs, and this direct-write path entirely. func (api *HTTPAPI) processCommitmentChanges(ctx context.Context, w http.ResponseWriter, logger logr.Logger, req liquid.CommitmentChangeRequest, resp *liquid.CommitmentChangeResponse) error { flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, api.client, nil) if err != nil { diff --git a/internal/scheduling/reservations/commitments/changes/reconciler.go b/internal/scheduling/reservations/commitments/changes/reconciler.go index ec2be9b82..d07284f73 100644 --- a/internal/scheduling/reservations/commitments/changes/reconciler.go +++ b/internal/scheduling/reservations/commitments/changes/reconciler.go @@ -301,6 +301,10 @@ func (r *CommittedResourceChangeReconciler) markError(ctx context.Context, tx *v } } +// TODO (#781): add a TTL/GC controller that deletes terminal CommittedResourceChanges after their +// retention window (Accepted ~1h, Rejected ~5d). Non-terminal CRCs must never be GC'd — deleting +// one loses the rollback journal and the #783 AZ active-marker. + // SetupWithManager registers the controller and configures watches. // TODO (#783): set MaxConcurrentReconciles: 1 via WithOptions(controller.Options{MaxConcurrentReconciles: 1}). // TODO (#783): add predicate on WatchesMulticluster() to re-trigger same-AZ non-terminal siblings on terminal transition. diff --git a/internal/scheduling/reservations/commitments/committed_resource_controller.go b/internal/scheduling/reservations/commitments/committed_resource_controller.go index a4e55ad04..06a9a65ec 100644 --- a/internal/scheduling/reservations/commitments/committed_resource_controller.go +++ b/internal/scheduling/reservations/commitments/committed_resource_controller.go @@ -7,7 +7,6 @@ import ( "context" "errors" "fmt" - "strconv" "time" "k8s.io/apimachinery/pkg/api/equality" @@ -126,12 +125,7 @@ func (r *CommittedResourceController) reconcilePending(ctx context.Context, logg // error would cause controller-runtime to ignore RequeueAfter and use exponential backoff. return ctrl.Result{RequeueAfter: delay}, nil } - // alreadyReserving is true when the CR condition already shows Reserving for this generation, - // meaning at least one reconcile cycle (~1s) has elapsed since we first entered the waiting state. - // We only nudge stale Reservation slots after that delay to give the normal echo path a chance - // to complete first, keeping logs quiet in the happy path. - alreadyReserving := isAlreadyReserving(cr) - allReady, anyFailed, readySlots, failReason, firstSlotReason, failedSlots, err := r.checkChildReservationStatus(ctx, cr, result.TotalSlots, alreadyReserving) + allReady, anyFailed, readySlots, failReason, firstSlotReason, failedSlots, err := r.checkChildReservationStatus(ctx, cr, result.TotalSlots) if err != nil { return ctrl.Result{}, err } @@ -158,14 +152,8 @@ func (r *CommittedResourceController) reconcilePending(ctx context.Context, logg // Reset the retry timer: applyReservationState just succeeded, so the watch suppression // gate should not fire while we wait for slots to become ready. // Fallback requeue: in case the watch event is missed, - // re-check after 1s rather than waiting for the full watch timeout. - // Use "still waiting" message after the first cycle so the condition is a visible hint - // that a nudge has been (or is about to be) sent to stale slots. - msg := "waiting for reservation placement" - if alreadyReserving { - msg = "still waiting for reservation placement" - } - if err := r.patchNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, msg, true); err != nil { + // re-check after 3s rather than waiting for the full watch timeout. + if err := r.patchNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, "waiting for reservation placement", true); err != nil { return ctrl.Result{}, err } return ctrl.Result{RequeueAfter: 3 * time.Second}, nil @@ -189,6 +177,8 @@ func (r *CommittedResourceController) reconcilePending(ctx context.Context, logg return ctrl.Result{}, nil } +// TODO (#790): after reservation allocation changes (#787/#790), react here with a global view: +// resize/merge/recreate reservation slots and remap PAYG VMs onto available capacity. func (r *CommittedResourceController) reconcileCommitted(ctx context.Context, logger logr.Logger, cr *v1alpha1.CommittedResource) (ctrl.Result, error) { logger.V(1).Info("reconciling committed resource", "generation", cr.Generation, @@ -203,46 +193,7 @@ func (r *CommittedResourceController) reconcileCommitted(ctx context.Context, lo // Without this guard the controller oscillates: apply bad spec → rollback → // Reservation watch re-enqueues → apply bad spec again → loop. if isRejectedForGeneration(cr) { - // Recovery: if the current spec matches the accepted spec, the rejection was - // spurious (e.g. transient host failure). Check if slots have since recovered. - // Only safe when specs match — if they differ, the rejection was genuine (bad new spec) - // and calling setAccepted would re-expose the current bad spec on the next reconcile. - if cr.Status.AcceptedSpec != nil && equality.Semantic.DeepEqual(cr.Spec, *cr.Status.AcceptedSpec) { - result, applyErr := r.applyReservationState(ctx, logger, cr) - if applyErr != nil { - logger.V(1).Info("spurious-rejection recovery: apply failed, will retry", "err", applyErr) - } else { - allReady, anyFailed, _, _, _, _, checkErr := r.checkChildReservationStatus(ctx, cr, result.TotalSlots, false) - if checkErr == nil && allReady && !anyFailed { - logger.Info("recovered from spurious rejection: all slots ready", "generation", cr.Generation) - initialGen := cr.Generation - if err := r.setAccepted(ctx, cr); err != nil { - return ctrl.Result{}, err - } - var latest v1alpha1.CommittedResource - if err := r.Get(ctx, client.ObjectKeyFromObject(cr), &latest); err != nil { - logger.V(1).Info("re-Get after accept failed, requeueing", "err", err) - return ctrl.Result{RequeueAfter: 1 * time.Second}, nil - } - if latest.Generation > initialGen { - logger.V(1).Info("generation advanced during reconcile, requeueing", - "processedGen", initialGen, - "latestGen", latest.Generation, - ) - return ctrl.Result{RequeueAfter: 1 * time.Second}, nil - } - return ctrl.Result{}, nil - } - if checkErr != nil { - logger.V(1).Info("spurious-rejection recovery: slot check failed, will retry", "err", checkErr) - } - } - // Slots exist but are not all ready yet. No need to re-apply the same accepted spec; - // requeue and wait for the reservation controller to complete placement. - return ctrl.Result{RequeueAfter: r.retryDelay(cr)}, nil - } - logger.V(1).Info("spec already rejected for current generation, maintaining rollback state", "generation", cr.Generation) - return ctrl.Result{}, r.rollbackToAccepted(ctx, logger, cr) + return r.handleRejectedGeneration(ctx, logger, cr) } // Spec errors are permanent regardless of AllowRejection — a bad spec won't fix itself. if _, err := FromCommittedResource(*cr); err != nil { @@ -267,8 +218,7 @@ func (r *CommittedResourceController) reconcileCommitted(ctx context.Context, lo // error would cause controller-runtime to ignore RequeueAfter and use exponential backoff. return ctrl.Result{RequeueAfter: delay}, nil } - alreadyReserving := isAlreadyReserving(cr) - allReady, anyFailed, readySlots, failReason, firstSlotReason, failedSlots, err := r.checkChildReservationStatus(ctx, cr, result.TotalSlots, alreadyReserving) + allReady, anyFailed, readySlots, failReason, firstSlotReason, failedSlots, err := r.checkChildReservationStatus(ctx, cr, result.TotalSlots) if err != nil { return ctrl.Result{}, err } @@ -289,17 +239,8 @@ func (r *CommittedResourceController) reconcileCommitted(ctx context.Context, lo } if !allReady { // Reservation controller hasn't processed all slots yet; Reservation watch will re-enqueue. - // Reset the retry timer: applyReservationState just succeeded, so the watch suppression - // gate should not fire while we wait for slots to become ready. - // Fallback requeue: in case the watch event is missed, - // re-check after 1s rather than waiting for the full watch timeout. - // Use "still waiting" message after the first cycle so the condition is a visible hint - // that a nudge has been (or is about to be) sent to stale slots. - msg := "waiting for reservation placement" - if alreadyReserving { - msg = "still waiting for reservation placement" - } - if err := r.patchNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, msg, true); err != nil { + // Fallback requeue: in case the watch event is missed, re-check after 3s. + if err := r.patchNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, "waiting for reservation placement", true); err != nil { return ctrl.Result{}, err } return ctrl.Result{RequeueAfter: 3 * time.Second}, nil @@ -326,10 +267,58 @@ func (r *CommittedResourceController) reconcileCommitted(ctx context.Context, lo return ctrl.Result{}, nil } +// handleRejectedGeneration handles the case where isRejectedForGeneration(cr) is true. +// If the current spec matches AcceptedSpec, attempts spurious-rejection recovery: applies the +// spec and checks child slots. On success, calls setAccepted. On failure or if slots are +// not yet ready, requeues. If specs differ, the rejection is genuine: maintains rollback state. +func (r *CommittedResourceController) handleRejectedGeneration(ctx context.Context, logger logr.Logger, cr *v1alpha1.CommittedResource) (ctrl.Result, error) { + // Recovery: if the current spec matches the accepted spec, the rejection was + // spurious (e.g. transient host failure). Check if slots have since recovered. + // Only safe when specs match — if they differ, the rejection was genuine (bad new spec) + // and calling setAccepted would re-expose the current bad spec on the next reconcile. + if cr.Status.AcceptedSpec != nil && equality.Semantic.DeepEqual(cr.Spec, *cr.Status.AcceptedSpec) { + result, applyErr := r.applyReservationState(ctx, logger, cr) + if applyErr != nil { + logger.V(1).Info("spurious-rejection recovery: apply failed, will retry", "err", applyErr) + } else { + allReady, anyFailed, _, _, _, _, checkErr := r.checkChildReservationStatus(ctx, cr, result.TotalSlots) + if checkErr == nil && allReady && !anyFailed { + logger.Info("recovered from spurious rejection: all slots ready", "generation", cr.Generation) + initialGen := cr.Generation + if err := r.setAccepted(ctx, cr); err != nil { + return ctrl.Result{}, err + } + var latest v1alpha1.CommittedResource + if err := r.Get(ctx, client.ObjectKeyFromObject(cr), &latest); err != nil { + logger.V(1).Info("re-Get after accept failed, requeueing", "err", err) + return ctrl.Result{RequeueAfter: 1 * time.Second}, nil + } + if latest.Generation > initialGen { + logger.V(1).Info("generation advanced during reconcile, requeueing", + "processedGen", initialGen, + "latestGen", latest.Generation, + ) + return ctrl.Result{RequeueAfter: 1 * time.Second}, nil + } + return ctrl.Result{}, nil + } + if checkErr != nil { + logger.V(1).Info("spurious-rejection recovery: slot check failed, will retry", "err", checkErr) + } + } + // Slots exist but are not all ready yet. No need to re-apply the same accepted spec; + // requeue and wait for the reservation controller to complete placement. + return ctrl.Result{RequeueAfter: r.retryDelay(cr)}, nil + } + logger.V(1).Info("spec already rejected for current generation, maintaining rollback state", "generation", cr.Generation) + return ctrl.Result{}, r.rollbackToAccepted(ctx, logger, cr) +} + // reconcileCoresHeadroom handles acceptance of CPU core commitments. // No Reservation CRDs are created; instead it reads the flavor group's total CPU capacity // from the FlavorGroupCapacity CRD and sums already-accepted CPU CRs for the same // (flavorGroup, AZ) to check whether sufficient headroom exists. +// TODO (#789) unify with main reservation path once resource type abstraction is in place func (r *CommittedResourceController) reconcileCoresHeadroom(ctx context.Context, logger logr.Logger, cr *v1alpha1.CommittedResource) (ctrl.Result, error) { if isRejectedForGeneration(cr) { logger.V(1).Info("spec already rejected for current generation", "generation", cr.Generation) @@ -482,7 +471,9 @@ func (r *CommittedResourceController) applyReservationState(ctx context.Context, // Returns allReady=false, anyFailed=false when some children have no condition yet (placement pending). // failReason includes a slot count when more than one slot failed. // failedSlots lists the names of all failing reservation slots for log context. -func (r *CommittedResourceController) checkChildReservationStatus(ctx context.Context, cr *v1alpha1.CommittedResource, expectedSlots int, shouldNudge bool) (allReady, anyFailed bool, readySlots int, failReason, firstFailMessage string, failedSlots []string, err error) { +// TODO (#789) the CRC reconciler (#779) gates AwaitingScheduling→Accepted on +// cond.ObservedGeneration == res.Generation; ensure this check stays consistent with that gate. +func (r *CommittedResourceController) checkChildReservationStatus(ctx context.Context, cr *v1alpha1.CommittedResource, expectedSlots int) (allReady, anyFailed bool, readySlots int, failReason, firstFailMessage string, failedSlots []string, err error) { var list v1alpha1.ReservationList if err := r.List(ctx, &list, client.MatchingLabels{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource}, @@ -517,30 +508,8 @@ func (r *CommittedResourceController) checkChildReservationStatus(ctx context.Co ) if cond == nil || cond.ObservedGeneration != res.Generation { // Reservation controller hasn't processed the current generation yet. - // Poke the Reservation to generate a fresh watch event so the - // Reservation controller wakes up. - // Use gen+ms as the annotation value so each nudge attempt is a distinct string: - // a same-value patch is a no-op on the API server (no watch event generated), and - // in a double-stale scenario the first nudge may itself be processed with a stale - // object, so periodic retries (every 3s via the fallback requeue) are needed. - // shouldNudge is false on the first cycle (CR condition not yet set for this - // generation) to give the normal path ~3s to complete before intervening. - if shouldNudge { - nudgeVal := strconv.FormatInt(cr.Generation, 10) + "-" + strconv.FormatInt(time.Now().UnixMilli(), 10) - ctrl.LoggerFrom(ctx).V(1).Info("nudging stale reservation: condition ObservedGeneration behind reservation generation", - "slot", res.Name, - "resGen", res.Generation, - "condObsGen", condObsGen, - ) - old := res.DeepCopy() - if res.Annotations == nil { - res.Annotations = map[string]string{} - } - res.Annotations["cortex.cloud/echo-nudge"] = nudgeVal - if pErr := r.Patch(ctx, &res, client.MergeFrom(old)); client.IgnoreNotFound(pErr) != nil { - ctrl.LoggerFrom(ctx).V(1).Info("failed to nudge stale reservation", "slot", res.Name, "err", pErr) - } - } + // The Reservation watch in SetupWithManager re-enqueues this CR when the + // reservation status changes; the RequeueAfter is the fallback. allReady = false continue } @@ -635,6 +604,7 @@ func DeleteChildReservations(ctx context.Context, k8sClient client.Client, cr *v // AcceptedSpec is a full snapshot of the spec at the last successful reconcile, so rollback always // targets the correct AZ, amount, project, domain — even when the current spec has been mutated. // If AcceptedSpec is nil (CR was never accepted), all child Reservations are deleted. +// TODO (#789) remove once status.acceptedSpec is dropped (after #785+#786 land; spec written only by committed CRCs) func (r *CommittedResourceController) rollbackToAccepted(ctx context.Context, logger logr.Logger, cr *v1alpha1.CommittedResource) error { if cr.Status.AcceptedSpec == nil { return r.deleteChildReservations(ctx, cr) @@ -680,18 +650,6 @@ func isRejectedForGeneration(cr *v1alpha1.CommittedResource) bool { cond.ObservedGeneration == cr.Generation } -// isAlreadyReserving returns true when the CR condition already shows Reserving for this -// generation, meaning at least one reconcile cycle (~1s) has elapsed since we first entered -// the waiting-for-slots state. Used to gate the echo-nudge so the first reconcile cycle is -// always silent, giving the normal Reservation echo path a chance to complete first. -func isAlreadyReserving(cr *v1alpha1.CommittedResource) bool { - cond := meta.FindStatusCondition(cr.Status.Conditions, v1alpha1.CommittedResourceConditionReady) - return cond != nil && - cond.Status == metav1.ConditionFalse && - cond.Reason == v1alpha1.CommittedResourceReasonReserving && - cond.ObservedGeneration == cr.Generation -} - // retryDelay computes an exponential backoff interval for the AllowRejection=false retry paths. // The exponent is derived from the time already spent in Reserving state: each doubling period // advances one step, giving the same base→2*base→4*base→… sequence as a counter would, but diff --git a/internal/scheduling/reservations/commitments/committed_resource_controller_test.go b/internal/scheduling/reservations/commitments/committed_resource_controller_test.go index 589f13b85..2c6f30c0e 100644 --- a/internal/scheduling/reservations/commitments/committed_resource_controller_test.go +++ b/internal/scheduling/reservations/commitments/committed_resource_controller_test.go @@ -1152,10 +1152,9 @@ func TestCommittedResourceController_BadSpec(t *testing.T) { // Tests: checkChildReservationStatus generation guard // ============================================================================ -// TestCommittedResourceController_NudgeDelay verifies that the echo-nudge is not sent on -// the first reconcile cycle (when the condition is not yet Reserving for this generation), -// but is sent on the second cycle — and that the condition message reflects the cycle. -func TestCommittedResourceController_NudgeDelay(t *testing.T) { +// TestCommittedResourceController_ReservingMessage verifies that successive reconcile cycles +// both show "waiting for reservation placement" (no echo-nudge annotation is ever written). +func TestCommittedResourceController_ReservingMessage(t *testing.T) { scheme := newCRTestScheme(t) cr := newTestCommittedResource("test-cr", v1alpha1.CommitmentStatusPending) cr.Spec.AllowRejection = false @@ -1170,7 +1169,7 @@ func TestCommittedResourceController_NudgeDelay(t *testing.T) { } return updated } - getChildAnnotation := func() string { + hasNudgeAnnotation := func() bool { t.Helper() var list v1alpha1.ReservationList if err := k8sClient.List(context.Background(), &list, client.MatchingLabels{ @@ -1181,42 +1180,27 @@ func TestCommittedResourceController_NudgeDelay(t *testing.T) { for _, res := range list.Items { if res.Spec.CommittedResourceReservation != nil && res.Spec.CommittedResourceReservation.CommitmentUUID == cr.Spec.CommitmentUUID { - return res.Annotations["cortex.cloud/echo-nudge"] + _, ok := res.Annotations["cortex.cloud/echo-nudge"] + return ok } } - return "" + return false } - // Cycle 1: no Reserving condition exists yet → isAlreadyReserving=false → no nudge, - // condition message = "waiting for reservation placement". - if _, err := controller.Reconcile(context.Background(), reconcileReq(cr.Name)); err != nil { - t.Fatalf("reconcile 1: %v", err) - } - cond1 := meta.FindStatusCondition(getCR().Status.Conditions, v1alpha1.CommittedResourceConditionReady) - if cond1 == nil || cond1.Reason != v1alpha1.CommittedResourceReasonReserving { - t.Fatalf("cycle 1: expected Reserving condition, got %+v", cond1) - } - if cond1.Message != "waiting for reservation placement" { - t.Errorf("cycle 1: message want %q, got %q", "waiting for reservation placement", cond1.Message) - } - if got := getChildAnnotation(); got != "" { - t.Errorf("cycle 1: echo-nudge annotation should not be set, got %q", got) - } - - // Cycle 2: Reserving condition is now set for this generation → isAlreadyReserving=true → - // nudge fires, condition message = "still waiting for reservation placement". - if _, err := controller.Reconcile(context.Background(), reconcileReq(cr.Name)); err != nil { - t.Fatalf("reconcile 2: %v", err) - } - cond2 := meta.FindStatusCondition(getCR().Status.Conditions, v1alpha1.CommittedResourceConditionReady) - if cond2 == nil || cond2.Reason != v1alpha1.CommittedResourceReasonReserving { - t.Fatalf("cycle 2: expected Reserving condition, got %+v", cond2) - } - if cond2.Message != "still waiting for reservation placement" { - t.Errorf("cycle 2: message want %q, got %q", "still waiting for reservation placement", cond2.Message) - } - if got := getChildAnnotation(); got == "" { - t.Errorf("cycle 2: echo-nudge annotation should be set") + for cycle := 1; cycle <= 2; cycle++ { + if _, err := controller.Reconcile(context.Background(), reconcileReq(cr.Name)); err != nil { + t.Fatalf("reconcile %d: %v", cycle, err) + } + cond := meta.FindStatusCondition(getCR().Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond == nil || cond.Reason != v1alpha1.CommittedResourceReasonReserving { + t.Fatalf("cycle %d: expected Reserving condition, got %+v", cycle, cond) + } + if cond.Message != "waiting for reservation placement" { + t.Errorf("cycle %d: message want %q, got %q", cycle, "waiting for reservation placement", cond.Message) + } + if hasNudgeAnnotation() { + t.Errorf("cycle %d: echo-nudge annotation must not be set", cycle) + } } } @@ -1228,42 +1212,26 @@ func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) { obsGen int64 condStatus metav1.ConditionStatus // "" = no condition set condMessage string - shouldNudge bool wantAllReady bool wantAnyFailed bool wantReadySlots int wantReason string wantFailedSlots []string - wantNudge bool }{ { name: "Ready=False at stale generation: treated as pending", obsGen: 1, condStatus: metav1.ConditionFalse, condMessage: "no hosts available", - shouldNudge: true, - wantAllReady: false, - wantAnyFailed: false, - wantReadySlots: 0, - wantNudge: true, - }, - { - name: "Ready=False at stale generation: no nudge on first cycle", - obsGen: 1, - condStatus: metav1.ConditionFalse, - condMessage: "no hosts available", - shouldNudge: false, wantAllReady: false, wantAnyFailed: false, wantReadySlots: 0, - wantNudge: false, }, { name: "Ready=False at current generation: is a current failure", obsGen: 2, condStatus: metav1.ConditionFalse, condMessage: "no hosts available", - shouldNudge: true, wantAllReady: false, wantAnyFailed: true, wantReadySlots: 0, @@ -1274,29 +1242,24 @@ func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) { name: "Ready=True at current generation: allReady", obsGen: 2, condStatus: metav1.ConditionTrue, - shouldNudge: true, wantAllReady: true, wantReadySlots: 1, }, { - name: "no condition yet at current generation: still pending", + name: "no condition yet: still pending", obsGen: 2, condStatus: "", // no condition - shouldNudge: true, wantAllReady: false, wantAnyFailed: false, wantReadySlots: 0, - wantNudge: true, }, { - name: "parentGen written to spec but echo not yet in status: nudge annotation set", - obsGen: 1, // obs behind cr.Generation=2; spec.parentGeneration already set to cr.Generation + name: "condition ObservedGeneration behind reservation generation: still pending", + obsGen: 1, // obs behind cr.Generation=2 condStatus: metav1.ConditionTrue, - shouldNudge: true, wantAllReady: false, wantAnyFailed: false, wantReadySlots: 0, - wantNudge: true, }, } @@ -1338,7 +1301,7 @@ func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) { } controller := &CommittedResourceController{Client: k8sClient, Scheme: scheme} - allReady, anyFailed, readySlots, reason, _, failedSlots, err := controller.checkChildReservationStatus(context.Background(), cr, 1, tt.shouldNudge) + allReady, anyFailed, readySlots, reason, _, failedSlots, err := controller.checkChildReservationStatus(context.Background(), cr, 1) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -1357,14 +1320,6 @@ func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) { if !slices.Equal(failedSlots, tt.wantFailedSlots) { t.Errorf("failedSlots: want %v, got %v", tt.wantFailedSlots, failedSlots) } - var updated v1alpha1.Reservation - if err := k8sClient.Get(context.Background(), client.ObjectKey{Name: child.Name}, &updated); err != nil { - t.Fatalf("get updated reservation: %v", err) - } - _, hasNudge := updated.Annotations["cortex.cloud/echo-nudge"] - if hasNudge != tt.wantNudge { - t.Errorf("echo-nudge annotation: want %v, got %v", tt.wantNudge, hasNudge) - } }) } } diff --git a/internal/scheduling/reservations/commitments/payg_candidates.go b/internal/scheduling/reservations/commitments/payg_candidates.go index 952fe3cab..49a96b607 100644 --- a/internal/scheduling/reservations/commitments/payg_candidates.go +++ b/internal/scheduling/reservations/commitments/payg_candidates.go @@ -84,6 +84,8 @@ func ScanAZForPaygCandidates( // buildAllocatedVMSet lists CR Reservations in az and returns the set of VM UUIDs present // in any Status.Allocation (confirmed placements). Filtering by AZ keeps the set small — // only slots relevant to the current scan are included. One cache scan shared across all HV filters. +// TODO (#790): replace full List scan with the idxReservationByAllocationVMUUID field index +// (field_index.go) for O(1) reverse lookup instead of scanning all reservations. func buildAllocatedVMSet(ctx context.Context, k8sClient client.Client, az string) (map[string]struct{}, error) { var resList v1alpha1.ReservationList if err := k8sClient.List(ctx, &resList, diff --git a/internal/scheduling/reservations/commitments/reservation_controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go index 6364ec5d8..6c31b0725 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller.go +++ b/internal/scheduling/reservations/commitments/reservation_controller.go @@ -66,10 +66,8 @@ type CommitmentReservationController struct { // move the current state of the cluster closer to the desired state. // Note: This controller only handles commitment reservations, as filtered by the predicate. func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - // Fetch the reservation object first to check for creator request ID. var res v1alpha1.Reservation if err := r.Get(ctx, req.NamespacedName, &res); err != nil { - // Ignore not-found errors, since they can't be fixed by an immediate requeue return ctrl.Result{}, client.IgnoreNotFound(err) } @@ -94,95 +92,46 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr }) patch := client.MergeFrom(old) if err := r.Status().Patch(ctx, &res, patch); err != nil { - // Ignore not-found errors during background deletion if client.IgnoreNotFound(err) != nil { logger.Error(err, "failed to patch reservation status") return ctrl.Result{}, err } - // Object was deleted, no need to continue return ctrl.Result{}, nil } - return ctrl.Result{}, nil // Don't need to requeue. + return ctrl.Result{}, nil } // Spec.TargetHost was cleared (e.g. oversubscription eviction) while the slot was confirmed — // revoke status so the slot re-enters the placement flow on the next reconcile. if res.Status.Host != "" && res.Spec.TargetHost == "" { - old := res.DeepCopy() - res.Status.Host = "" - if res.Status.CommittedResourceReservation != nil { - res.Status.CommittedResourceReservation.Allocation = nil - } - meta.SetStatusCondition(&res.Status.Conditions, metav1.Condition{ - Type: v1alpha1.ReservationConditionReady, - Status: metav1.ConditionFalse, - Reason: "PlacementRevoked", - Message: "target host was cleared; pending re-placement", - ObservedGeneration: res.Generation, - }) - if err := r.Status().Patch(ctx, &res, client.MergeFrom(old)); client.IgnoreNotFound(err) != nil { - return ctrl.Result{}, err - } - logger.Info("revoked ready status after placement eviction, slot re-enters placement flow", - "component", "oversubscription-check") - return ctrl.Result{}, nil + return r.reconcileEvicted(ctx, logger, &res) } - // Slot is placed and confirmed — verify allocations and keep ObservedGeneration current. + // Slot is placed and confirmed. + // If the spec changed since last confirmed placement (ObservedGeneration behind), clear + // TargetHost to trigger re-placement via reconcileEvicted → reconcileUnplaced on the next + // two cycles. This handles both benign changes (same host re-confirmed, new ObservedGeneration + // stamped) and significant changes (resources/flavorGroup — new host may be selected). if res.Spec.TargetHost != "" && res.Status.Host == res.Spec.TargetHost { - // Deprecated: remove when Spec.CommittedResourceReservation.Allocations is removed from the schema. - // One-time upgrade migration: blind-scheduler slots populated Spec.Allocations on the old schema. - // Seed AnnotationCandidateAllocation from the lexically-first Spec entry so the existing - // annotation recovery path re-confirms the VM on the next pass. - if res.Spec.CommittedResourceReservation != nil && - res.Annotations[v1alpha1.AnnotationCandidateAllocation] == "" && - len(res.Spec.CommittedResourceReservation.Allocations) > 0 { //nolint:staticcheck - keys := make([]string, 0, len(res.Spec.CommittedResourceReservation.Allocations)) //nolint:staticcheck - for k := range res.Spec.CommittedResourceReservation.Allocations { //nolint:staticcheck - keys = append(keys, k) - } - sort.Strings(keys) - vmUUID := keys[0] - logger.Info("migrating first Spec allocation to annotation (one-time upgrade path)", - "vm", vmUUID, "reservation", res.Name) - old := res.DeepCopy() - if res.Annotations == nil { - res.Annotations = make(map[string]string) - } - res.Annotations[v1alpha1.AnnotationCandidateAllocation] = vmUUID - if err := r.Patch(ctx, &res, client.MergeFrom(old)); client.IgnoreNotFound(err) != nil { - return ctrl.Result{}, err - } - } - - logger.V(1).Info("reservation is active, verifying allocations") - if err := r.reconcileAllocations(ctx, &res); err != nil { - logger.Error(err, "failed to reconcile allocations") - return ctrl.Result{}, err - } - // Refresh ObservedGeneration if the spec changed since the last reconcile - // (e.g. endTime extension or metadata update via syncReservationMetadata). cond := meta.FindStatusCondition(res.Status.Conditions, v1alpha1.ReservationConditionReady) - if cond == nil || cond.ObservedGeneration < res.Generation { + if cond != nil && cond.ObservedGeneration < res.Generation { + logger.Info("spec changed since last placement, clearing target host to trigger re-placement", + "host", res.Spec.TargetHost, + "lastObservedGen", cond.ObservedGeneration, + "currentGen", res.Generation, + ) old := res.DeepCopy() - meta.SetStatusCondition(&res.Status.Conditions, metav1.Condition{ - Type: v1alpha1.ReservationConditionReady, - Status: metav1.ConditionTrue, - Reason: "ReservationActive", - Message: "reservation is successfully scheduled", - ObservedGeneration: res.Generation, - }) - if err := r.Status().Patch(ctx, &res, client.MergeFrom(old)); client.IgnoreNotFound(err) != nil { + res.Spec.TargetHost = "" + if err := r.Patch(ctx, &res, client.MergeFrom(old)); client.IgnoreNotFound(err) != nil { return ctrl.Result{}, err } + return ctrl.Result{}, nil } - return ctrl.Result{RequeueAfter: r.Conf.RequeueIntervalActive.Duration}, nil + return r.reconcilePlaced(ctx, logger, &res) } - // Sync Spec values to Status fields for non-pre-allocated reservations - // This ensures the observed state reflects the desired state from Spec - // When TargetHost is set in Spec but not synced to Status, this means - // the scheduler found a host and we need to mark the reservation as ready. + // Spec.TargetHost was written by the scheduler but not yet reflected in Status. + // Sync Status.Host once and mark ready; the next reconcile runs allocation verification. if res.Spec.TargetHost != "" && res.Status.Host != res.Spec.TargetHost { old := res.DeepCopy() res.Status.Host = res.Spec.TargetHost @@ -195,19 +144,109 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr }) patch := client.MergeFrom(old) if err := r.Status().Patch(ctx, &res, patch); err != nil { - // Ignore not-found errors during background deletion if client.IgnoreNotFound(err) != nil { logger.Error(err, "failed to sync spec to status") return ctrl.Result{}, err } - // Object was deleted, no need to continue return ctrl.Result{}, nil } logger.V(1).Info("synced spec to status and marked ready", "host", res.Status.Host) - // Return and let next reconcile handle allocation verification return ctrl.Result{}, nil } + // No target host yet — run the scheduler to find a placement. + return r.reconcileUnplaced(ctx, logger, &res) +} + +// reconcileEvicted handles the case where Spec.TargetHost was cleared (e.g. oversubscription eviction) +// while the slot was already placed. Clears Status.Host and allocations so the slot re-enters +// the normal placement flow on the next reconcile. +func (r *CommitmentReservationController) reconcileEvicted(ctx context.Context, logger logr.Logger, res *v1alpha1.Reservation) (ctrl.Result, error) { + old := res.DeepCopy() + res.Status.Host = "" + if res.Status.CommittedResourceReservation != nil { + res.Status.CommittedResourceReservation.Allocation = nil + } + meta.SetStatusCondition(&res.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ReservationConditionReady, + Status: metav1.ConditionFalse, + Reason: "PlacementRevoked", + Message: "target host was cleared; pending re-placement", + ObservedGeneration: res.Generation, + }) + if err := r.Status().Patch(ctx, res, client.MergeFrom(old)); client.IgnoreNotFound(err) != nil { + return ctrl.Result{}, err + } + logger.Info("revoked ready status after placement eviction, slot re-enters placement flow", + "component", "oversubscription-check") + return ctrl.Result{}, nil +} + +// reconcilePlaced handles an active reservation: verifies that allocated VMs are still on the +// expected host and keeps ObservedGeneration current after benign spec updates (e.g. endTime extension). +// TODO (#787) watch VM CRD for allocation changes once VM CRD is ready — today this relies on +// the periodic HV-CRD watch + RequeueIntervalActive instead. +func (r *CommitmentReservationController) reconcilePlaced(ctx context.Context, logger logr.Logger, res *v1alpha1.Reservation) (ctrl.Result, error) { + // Deprecated: remove when Spec.CommittedResourceReservation.Allocations is removed from the schema. + // One-time upgrade migration: blind-scheduler slots populated Spec.Allocations on the old schema. + // Seed AnnotationCandidateAllocation from the lexically-first Spec entry so the existing + // annotation recovery path re-confirms the VM on the next pass. + if res.Spec.CommittedResourceReservation != nil && + res.Annotations[v1alpha1.AnnotationCandidateAllocation] == "" && + len(res.Spec.CommittedResourceReservation.Allocations) > 0 { //nolint:staticcheck + keys := make([]string, 0, len(res.Spec.CommittedResourceReservation.Allocations)) //nolint:staticcheck + for k := range res.Spec.CommittedResourceReservation.Allocations { //nolint:staticcheck + keys = append(keys, k) + } + sort.Strings(keys) + vmUUID := keys[0] + logger.Info("migrating first Spec allocation to annotation (one-time upgrade path)", + "vm", vmUUID, "reservation", res.Name) + old := res.DeepCopy() + if res.Annotations == nil { + res.Annotations = make(map[string]string) + } + res.Annotations[v1alpha1.AnnotationCandidateAllocation] = vmUUID + if err := r.Patch(ctx, res, client.MergeFrom(old)); client.IgnoreNotFound(err) != nil { + return ctrl.Result{}, err + } + } + + logger.V(1).Info("reservation is active, verifying allocations") + if err := r.reconcileAllocations(ctx, res); err != nil { + logger.Error(err, "failed to reconcile allocations") + return ctrl.Result{}, err + } + + // Refresh ObservedGeneration if the spec changed since the last reconcile + // (e.g. endTime extension or metadata update via syncReservationMetadata). + cond := meta.FindStatusCondition(res.Status.Conditions, v1alpha1.ReservationConditionReady) + if cond == nil || cond.ObservedGeneration < res.Generation { + old := res.DeepCopy() + meta.SetStatusCondition(&res.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ReservationConditionReady, + Status: metav1.ConditionTrue, + Reason: "ReservationActive", + Message: "reservation is successfully scheduled", + ObservedGeneration: res.Generation, + }) + if err := r.Status().Patch(ctx, res, client.MergeFrom(old)); client.IgnoreNotFound(err) != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{RequeueAfter: r.Conf.RequeueIntervalActive.Duration}, nil +} + +// reconcileUnplaced runs the in-process scheduler to find a host for an unplaced reservation slot, +// then writes the selected host to Spec.TargetHost. The next reconcile cycle detects +// Spec.TargetHost != Status.Host and syncs Status in the Reconcile dispatcher. +// TODO (#778) replace HV-CRD host scoring with VM CRD data when ready +func (r *CommitmentReservationController) reconcileUnplaced(ctx context.Context, logger logr.Logger, res *v1alpha1.Reservation) (ctrl.Result, error) { + resourceName := "" + if res.Spec.CommittedResourceReservation != nil { + resourceName = res.Spec.CommittedResourceReservation.ResourceName + } + // Get project ID from CommittedResourceReservation spec if available. projectID := "" if res.Spec.CommittedResourceReservation != nil { @@ -280,7 +319,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr ObservedGeneration: res.Generation, }) patch := client.MergeFrom(old) - if err := r.Status().Patch(ctx, &res, patch); err != nil { + if err := r.Status().Patch(ctx, res, patch); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } return ctrl.Result{RequeueAfter: r.Conf.RequeueIntervalRetry.Duration}, nil @@ -357,13 +396,11 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr ObservedGeneration: res.Generation, }) patch := client.MergeFrom(old) - if err := r.Status().Patch(ctx, &res, patch); err != nil { - // Ignore not-found errors during background deletion + if err := r.Status().Patch(ctx, res, patch); err != nil { if client.IgnoreNotFound(err) != nil { logger.Error(err, "failed to patch reservation status") return ctrl.Result{}, err } - // Object was deleted, no need to continue return ctrl.Result{}, nil } return ctrl.Result{}, fmt.Errorf("no hosts found for reservation %s (flavor %s)", res.Name, resourceName) @@ -377,18 +414,14 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr old := res.DeepCopy() res.Spec.TargetHost = host - if err := r.Patch(ctx, &res, client.MergeFrom(old)); err != nil { - // Ignore not-found errors during background deletion + if err := r.Patch(ctx, res, client.MergeFrom(old)); err != nil { if client.IgnoreNotFound(err) != nil { logger.Error(err, "failed to patch reservation spec") return ctrl.Result{}, err } - // Object was deleted, no need to continue return ctrl.Result{}, nil } - // The Spec patch will trigger a re-reconcile, which will sync Status in the - // "Sync Spec values to Status" section above return ctrl.Result{}, nil } @@ -397,6 +430,8 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr // host have their partial allocation recomputed; VMs that have disappeared are removed. // 2. Promote a PAYG candidate VM (AnnotationCandidateAllocation) into // status.allocation when it is confirmed active on the expected host. +// +// TODO (#778) replace HV-CRD VMSource with VM CRD once available func (r *CommitmentReservationController) reconcileAllocations(ctx context.Context, res *v1alpha1.Reservation) error { logger := LoggerFromContext(ctx) @@ -494,7 +529,7 @@ func (r *CommitmentReservationController) reconcileAllocations(ctx context.Conte continue } newStatusAllocations[vmUUID] = v1alpha1.CommittedResourceAllocation{ - // TODO replace with VM CRD ref once the VM CRD is strictly considered as being the mandatory gate + // TODO (#778) replace with VM CRD ref once VM CRD is the mandatory source of truth ObjectRef: corev1.ObjectReference{Name: vmUUID}, Resources: newAlloc, } @@ -807,6 +842,7 @@ func (r *CommitmentReservationController) SetupWithManager(mgr ctrl.Manager, mcl // assigned to that host. This replaces periodic polling for the established-VM // verification path — changes are detected in seconds rather than up to // RequeueIntervalActive. RequeueIntervalActive remains as a safety-net fallback. + // TODO (#787) add VM CRD watch for allocation changes once VM CRD is ready bldr, err = bldr.WatchesMulticluster( &hv1.Hypervisor{}, handler.EnqueueRequestsFromMapFunc(r.hypervisorToReservations), diff --git a/internal/scheduling/reservations/commitments/reservation_manager_test.go b/internal/scheduling/reservations/commitments/reservation_manager_test.go index 84b798cc4..51c241b0a 100644 --- a/internal/scheduling/reservations/commitments/reservation_manager_test.go +++ b/internal/scheduling/reservations/commitments/reservation_manager_test.go @@ -1177,9 +1177,10 @@ func variableRatioFlavorGroup() compute.FlavorGroupFeature { } } -func TestNewReservation_VariableRatioGroup_SelectsLargestByMemory(t *testing.T) { +func TestNewReservation_VariableRatioGroup_SelectsLargestByMemoryFewestCPU(t *testing.T) { // For GP (variable CPU:RAM ratio) groups, flavor selection is driven by memory - // descending, not by ratio. The largest flavor fitting the delta is always chosen. + // descending. Among flavors at the same memory level, the one with fewest vCPUs + // is preferred to minimize CPU blocking on the host. manager := &ReservationManager{} fg := variableRatioFlavorGroup() @@ -1208,7 +1209,7 @@ func TestNewReservation_VariableRatioGroup_SelectsLargestByMemory(t *testing.T) wantCores: 8, }, { - name: "delta equals c8_m16: picks c8_m16 (more vCPUs than c4_m8 at same memory)", + name: "delta equals c8_m16: picks c8_m16 (only 16 GiB flavor)", deltaMemoryMB: 16384, wantFlavor: "c8_m16", wantCores: 8, @@ -1245,6 +1246,39 @@ func TestNewReservation_VariableRatioGroup_SelectsLargestByMemory(t *testing.T) } } +// sameMemotyFlavorGroup returns a flavor group where two flavors share the same memory +// but differ in vCPU count, to exercise the fewest-vCPU tiebreak in selectFlavor. +func sameMemoryFlavorGroup() compute.FlavorGroupFeature { + minRatio := uint64(2048) // MiB/vCPU + maxRatio := uint64(4096) // MiB/vCPU + return compute.FlavorGroupFeature{ + Name: "gp-same-mem", + Flavors: []compute.FlavorInGroup{ + {Name: "c4_m16", VCPUs: 4, MemoryMB: 16384, DiskGB: 50}, // fewest vCPUs at 16 GiB + {Name: "c8_m16", VCPUs: 8, MemoryMB: 16384, DiskGB: 50}, // more vCPUs at 16 GiB + {Name: "c4_m8", VCPUs: 4, MemoryMB: 8192, DiskGB: 25}, + }, + SmallestFlavor: compute.FlavorInGroup{Name: "c4_m8", VCPUs: 4, MemoryMB: 8192, DiskGB: 25}, + LargestFlavor: compute.FlavorInGroup{Name: "c4_m16", VCPUs: 4, MemoryMB: 16384, DiskGB: 50}, + RamCoreRatioMin: &minRatio, + RamCoreRatioMax: &maxRatio, + } +} + +func TestSelectFlavor_SameMemory_PrefersFewestCPU(t *testing.T) { + // When two flavors share the same memory, selectFlavor must pick the one with + // fewer vCPUs to avoid over-blocking CPU capacity on the host. + fg := sameMemoryFlavorGroup() + + flavorInGroup, _ := selectFlavor(int64(16384)*1024*1024, fg) + if got := flavorInGroup.Name; got != "c4_m16" { + t.Errorf("want c4_m16 (fewest vCPUs at 16 GiB), got %s", got) + } + if got := flavorInGroup.VCPUs; got != 4 { + t.Errorf("want 4 vCPUs, got %d", got) + } +} + // ============================================================================ // Tests: selectFlavor // ============================================================================ diff --git a/internal/scheduling/reservations/commitments/syncer.go b/internal/scheduling/reservations/commitments/syncer.go index 1071b0bd5..b13339cd6 100644 --- a/internal/scheduling/reservations/commitments/syncer.go +++ b/internal/scheduling/reservations/commitments/syncer.go @@ -230,9 +230,10 @@ func (s *Syncer) getCommitmentStates(ctx context.Context, log logr.Logger, flavo } // SyncReservations fetches commitments from Limes and synchronizes Reservation CRDs. +// TODO (#785): route writes through CommittedResourceChange (source=syncer) instead of writing +// CR.spec directly. The per-project recency check (skip projects touched in the last ~10 min by +// an API CRC) moves to the CRC reconciler (#782). func (s *Syncer) SyncReservations(ctx context.Context) error { - // TODO handle concurrency with change API: consider creation time of reservations and status ready - // Create context with request ID for this sync execution runID := fmt.Sprintf("sync-%d", time.Now().Unix()) ctx = reservations.WithGlobalRequestID(ctx, runID) @@ -488,6 +489,10 @@ func (s *Syncer) SyncReservations(ctx context.Context) error { // time and can settle on hosts whose labels no longer match; pruning them lets the reconciler // re-create fresh slots. A zero max age or per-run cap disables pruning. Returns the number of // Reservation CRDs deleted. +// TODO (#790): this only deletes stale slots one by one. The target behaviour (#790) is for the +// CR controller to take a global view across all slots of a CR: collect unused/empty capacity, +// compute an optimal new slot layout, and recreate at the best size. That logic belongs in the +// CR controller (committed_resource_controller.go), not the syncer. func (s *Syncer) pruneAgedReservations(ctx context.Context, logger logr.Logger) (int, error) { maxAge := s.resourceConfig.StaleReservationMaxAge.Duration maxDeletes := s.resourceConfig.MaxReservationDeletesPerRun diff --git a/postgres/Dockerfile b/postgres/Dockerfile index 223438de8..a1da75089 100644 --- a/postgres/Dockerfile +++ b/postgres/Dockerfile @@ -73,7 +73,7 @@ RUN set -ex; \ ENV PG_MAJOR 18 ENV PATH $PATH:/usr/lib/postgresql/$PG_MAJOR/bin -ENV PG_VERSION 18.6-1.pgdg13+2 +ENV PG_VERSION 18.7-1.pgdg13+2 RUN set -ex; \ \