diff --git a/CHANGELOG.md b/CHANGELOG.md index 115214a35..9319b4b16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,76 @@ # Changelog +## cortex 0.9.2 + +### Fixes + +- **More stable E2E tests** ([#1538](https://github.com/cobaltcore-dev/cortex/pull/1538)): E2E subcommands now get a default zap logger initialized early in `main()`, preventing log output from being silently discarded before the flag-configured logger is set up. Flavor-group extraction now logs skip counters (`skippedParseErr`, `skippedNoHWVersion`, `skippedInvalidGroup`) and a summary line at extraction completion for easier diagnosis of data-quality issues in CI. Key success-path log lines in commitment E2E checks have been promoted from `V(1)` (debug) to `Info` level so they appear in standard CI output. A new panic on internal dry-run errors (`"internal error processing dry-run request"`) catches controller timeouts and misconfigurations immediately instead of falling through to the generic rejection handler. + +### Documentation + +- **Add missing plugin entries & fix stale option reference** ([#1533](https://github.com/cobaltcore-dev/cortex/pull/1533)): Added 3 missing plugin entries to nova smart-scheduling docs (`filter_failover_host`, `weigher_failover_host`, `kvm_cr_migration_slot`) and updated the filter count from "seventeen" to "nineteen". Fixed stale reference to `ignoredReservationTypes` → `AssumeEmptyHosts: true` in failover host reservations docs. + +### Dependency Updates + +| Dependency | Old | New | Note | +|---|---|---|---| +| `kube-prometheus-stack` | `v93.3.0` | `v93.3.1` | Helm chart | + +## cortex-postgres 0.6.34 + +- Bump to align with cortex 0.9.2 release. + +## cortex-shim 0.1.40 + +- Bump to align with cortex 0.9.2 release. + +## bundles 0.0.116 + +- Bump to align with cortex 0.9.2 release. + +## cortex-placement-shim 0.1.40 + +- Bump to align with cortex 0.9.2 release. + +## Release PR [#1540](https://github.com/cobaltcore-dev/cortex/pull/1540) — 2026-11-17 + +### Bumped Versions + +| Package | Previous | New | +|---|---|---| +| cortex | 0.9.1 | 0.9.2 | +| cortex-postgres | 0.6.33 | 0.6.34 | +| cortex-shim | 0.1.39 | 0.1.40 | +| bundles | 0.0.115 | 0.0.116 | +| cortex-placement-shim | 0.1.39 | 0.1.40 | + +### Bug Fixes + +- **More stable E2E tests** — Default zap logger for E2E subcommands prevents silently discarded log output. Flavor-group extraction now logs skip counters and a summary line. Commitment E2E check success-path logs promoted to `Info` level. New panic on internal dry-run errors catches controller timeouts immediately. + +### Documentation + +- **Missing plugin entries & stale option reference** — Added `filter_failover_host`, `weigher_failover_host`, `kvm_cr_migration_slot` to nova smart-scheduling docs (filter count "seventeen" → "nineteen"). Fixed `ignoredReservationTypes` → `AssumeEmptyHosts: true` in failover host reservations docs. + +### Chores + +- **Bump app version** — `sha-bac55e3a` → `sha-cdb4dfaa` in `helm/library/cortex/Chart.yaml`. + +### Dependency Updates + +| Dependency | Old | New | Note | +|---|---|---|---| +| `kube-prometheus-stack` | `v93.3.0` | `v93.3.1` | Helm chart | + +### ⚠️ Operational Notes + +1. **No schema or RBAC changes:** Rolling upgrade is sufficient. +2. **Dry-run panic:** E2E tests will now panic immediately on internal dry-run errors that were previously silently swallowed. If CI failures surface from this path, investigate controller timeouts or misconfigurations rather than reverting. + +**Contributors:** @mblos, @cortex-ai-agents[bot], @renovate[bot], @SoWieMarkus + +**PR:** [#1540](https://github.com/cobaltcore-dev/cortex/pull/1540) | **Commits:** 4 | **Files changed:** 7 | **+35 / −12** + ## cortex 0.9.1 ### Refactoring diff --git a/cmd/manager/main.go b/cmd/manager/main.go index f9aa8c9b0..fb7f8c0ad 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -136,6 +136,11 @@ func main() { SetOverrideUserAgent(mainConfig.UserAgent.Component, mainConfig.UserAgent.Version) sso.SetUserAgent(mainConfig.UserAgent.Component, mainConfig.UserAgent.Version) + // E2E subcommands return before the flag-configured logger is set up below. + // Set a default logger now so e2e log output is not silently discarded; + // the controller path will override it with the flag-configured zap instance. + ctrl.SetLogger(zap.New()) + // Custom entrypoint for scheduler e2e tests. // Usage: /main [json-override] // The optional json-override is merged on top of the ConfigMap config, e.g.: diff --git a/docs/02-external-scheduler-api/02-nova-smart-scheduling.md b/docs/02-external-scheduler-api/02-nova-smart-scheduling.md index a977063ad..adb5f5ec9 100644 --- a/docs/02-external-scheduler-api/02-nova-smart-scheduling.md +++ b/docs/02-external-scheduler-api/02-nova-smart-scheduling.md @@ -57,7 +57,7 @@ selection, so a KVM request never runs a VMware weigher and vice versa. ## The filters -Nova ships seventeen filters (registered under `internal/scheduling/nova/plugins/filters/`). Filters run +Nova ships nineteen filters (registered under `internal/scheduling/nova/plugins/filters/`). Filters run sequentially and each removes hosts that fail its rule: | Filter | Removes hosts that… | @@ -72,6 +72,7 @@ sequentially and each removes hosts that fail its rule: | `filter_allowed_projects` | are not allowed for the request's project | | `filter_external_customer` | violate external-customer isolation rules | | `filter_domain_pinning` | are not among a pinned domain's dedicated hypervisors | +| `filter_failover_host` | are reserved as failover landing zones (non-evacuation), or are not the selected failover target (evacuation) | | `filter_instance_group_affinity` | break a required instance-group affinity | | `filter_instance_group_anti_affinity` | break a required instance-group anti-affinity | | `filter_live_migratable` | cannot accept a live-migratable instance | @@ -95,7 +96,10 @@ They split by hypervisor family: - `kvm_failover_reservation_consolidation` — consolidate to preserve failover headroom. - `kvm_failover_evacuation` — bias placement during evacuation to keep failover capacity (see [Failover reservations](../03-reservations-and-inventory/03-failover-reservations.md)). +- `kvm_cr_migration_slot` — boost hosts that can accommodate a migrating VM's committed-resource reservation slot. - `kvm_instance_group_soft_affinity` — honor *soft* instance-group affinity as a preference. +- `weigher_failover_host` — lift the selected failover target host to the front during gang evacuation (see + [Failover host reservations](../03-reservations-and-inventory/04-failover-host-reservations.md)). **VMware weighers:** diff --git a/docs/03-reservations-and-inventory/04-failover-host-reservations.md b/docs/03-reservations-and-inventory/04-failover-host-reservations.md index 5ced35d17..461856c87 100644 --- a/docs/03-reservations-and-inventory/04-failover-host-reservations.md +++ b/docs/03-reservations-and-inventory/04-failover-host-reservations.md @@ -530,11 +530,11 @@ is held back so a *second*, concurrent host failure still has its own free landi it already consumed. Keeping `h3` and `h4` means that if the evacuation onto `h1` fails, Nova retries onto an ordinary host instead of being forced onto an unmatched reserved slot. -The filter also honors the **call-time** `ignoredReservationTypes` request option. This is what keeps the -**path finder probe** working: the probe asks the scheduler which hosts would accept a flavor, and it passes -`ignoredReservationTypes: [FailoverHostReservation]` precisely so a reserved host is *not* hidden from its own -partition. Because the filter obeys that option, the probe still discovers already-reserved hosts and the -partitioning does not fragment. +The filter also honors the **call-time** `AssumeEmptyHosts` request option. This is what keeps the +**path finder probe** working: the probe asks the scheduler which hosts would accept a flavor, and it sets +`AssumeEmptyHosts: true` so candidates are judged by total capacity with every reservation ignored — a +reserved host is *not* hidden from its own partition. Because the filter skips all failover-host blocking +under that option, the probe still discovers already-reserved hosts and the partitioning does not fragment. ### The weigher: lift the selected target to the front diff --git a/helm/bundles/cortex-cinder/Chart.yaml b/helm/bundles/cortex-cinder/Chart.yaml index 6789f7702..d86cc6977 100644 --- a/helm/bundles/cortex-cinder/Chart.yaml +++ b/helm/bundles/cortex-cinder/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-cinder description: A Helm chart deploying Cortex for Cinder. type: application -version: 0.0.115 +version: 0.0.116 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.33 + version: 0.6.34 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.1 + version: 0.9.2 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.1 + version: 0.9.2 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-crds/Chart.yaml b/helm/bundles/cortex-crds/Chart.yaml index 5a7816a15..a9a3f64f4 100644 --- a/helm/bundles/cortex-crds/Chart.yaml +++ b/helm/bundles/cortex-crds/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-crds description: A Helm chart deploying Cortex CRDs. type: application -version: 0.0.115 +version: 0.0.116 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.1 + version: 0.9.2 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-ironcore/Chart.yaml b/helm/bundles/cortex-ironcore/Chart.yaml index 59310ed60..9a8b966e1 100644 --- a/helm/bundles/cortex-ironcore/Chart.yaml +++ b/helm/bundles/cortex-ironcore/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-ironcore description: A Helm chart deploying Cortex for IronCore. type: application -version: 0.0.115 +version: 0.0.116 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.1 + version: 0.9.2 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-manila/Chart.yaml b/helm/bundles/cortex-manila/Chart.yaml index a80ece477..881473c78 100644 --- a/helm/bundles/cortex-manila/Chart.yaml +++ b/helm/bundles/cortex-manila/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-manila description: A Helm chart deploying Cortex for Manila. type: application -version: 0.0.115 +version: 0.0.116 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.33 + version: 0.6.34 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.1 + version: 0.9.2 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.1 + version: 0.9.2 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-nova/Chart.yaml b/helm/bundles/cortex-nova/Chart.yaml index fd9344e06..e03a1b61d 100644 --- a/helm/bundles/cortex-nova/Chart.yaml +++ b/helm/bundles/cortex-nova/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-nova description: A Helm chart deploying Cortex for Nova. type: application -version: 0.0.115 +version: 0.0.116 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.33 + version: 0.6.34 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.1 + version: 0.9.2 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.1 + version: 0.9.2 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-placement-shim/Chart.yaml b/helm/bundles/cortex-placement-shim/Chart.yaml index 19c85a10a..19e4f3628 100644 --- a/helm/bundles/cortex-placement-shim/Chart.yaml +++ b/helm/bundles/cortex-placement-shim/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-placement-shim description: A Helm chart deploying the Cortex placement shim. type: application -version: 0.1.39 +version: 0.1.40 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-shim - name: cortex-shim repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.1.39 + version: 0.1.40 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case # of issues. See: https://github.com/sapcc/helm-charts/pkgs/container/helm-charts%2Fowner-info diff --git a/helm/bundles/cortex-pods/Chart.yaml b/helm/bundles/cortex-pods/Chart.yaml index 7b47307dc..a51079a47 100644 --- a/helm/bundles/cortex-pods/Chart.yaml +++ b/helm/bundles/cortex-pods/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-pods description: A Helm chart deploying Cortex for Pods. type: application -version: 0.0.115 +version: 0.0.116 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.1 + version: 0.9.2 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/dev/cortex-prometheus-operator/Chart.yaml b/helm/dev/cortex-prometheus-operator/Chart.yaml index b66b7b7a1..4ce9443b8 100644 --- a/helm/dev/cortex-prometheus-operator/Chart.yaml +++ b/helm/dev/cortex-prometheus-operator/Chart.yaml @@ -10,4 +10,4 @@ dependencies: # CRDs of the prometheus operator, such as PrometheusRule, ServiceMonitor, etc. - name: kube-prometheus-stack repository: oci://ghcr.io/prometheus-community/charts - version: 93.3.0 + version: 93.3.1 diff --git a/helm/library/cortex-postgres/Chart.yaml b/helm/library/cortex-postgres/Chart.yaml index 0331a948b..61ef34521 100644 --- a/helm/library/cortex-postgres/Chart.yaml +++ b/helm/library/cortex-postgres/Chart.yaml @@ -5,5 +5,5 @@ apiVersion: v2 name: cortex-postgres description: Postgres setup for Cortex. type: application -version: 0.6.33 +version: 0.6.34 appVersion: "sha-f69b58ee" diff --git a/helm/library/cortex-shim/Chart.yaml b/helm/library/cortex-shim/Chart.yaml index 771e5fbfe..c20d13aff 100644 --- a/helm/library/cortex-shim/Chart.yaml +++ b/helm/library/cortex-shim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex-shim description: A Helm chart to distribute cortex shims. type: application -version: 0.1.39 +version: 0.1.40 appVersion: "sha-01a68db0" icon: "https://example.com/icon.png" dependencies: [] diff --git a/helm/library/cortex/Chart.yaml b/helm/library/cortex/Chart.yaml index 094a8b3c5..358724252 100644 --- a/helm/library/cortex/Chart.yaml +++ b/helm/library/cortex/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex description: A Helm chart to distribute cortex. type: application -version: 0.9.1 -appVersion: "sha-bac55e3a" +version: 0.9.2 +appVersion: "sha-cdb4dfaa" icon: "https://example.com/icon.png" dependencies: [] diff --git a/internal/knowledge/extractor/plugins/compute/flavor_groups.go b/internal/knowledge/extractor/plugins/compute/flavor_groups.go index 5b9578cae..9488d722d 100644 --- a/internal/knowledge/extractor/plugins/compute/flavor_groups.go +++ b/internal/knowledge/extractor/plugins/compute/flavor_groups.go @@ -139,6 +139,7 @@ func (e *FlavorGroupExtractor) Extract() ([]plugins.Feature, error) { // Group flavors by flavorGroupIdentifierName groupMap := make(map[string][]FlavorInGroup) + var skippedParseErr, skippedNoHWVersion int for _, row := range rows { // Parse extra_specs JSON @@ -146,6 +147,7 @@ func (e *FlavorGroupExtractor) Extract() ([]plugins.Feature, error) { if row.ExtraSpecs != "" { if err := json.Unmarshal([]byte(row.ExtraSpecs), &extraSpecs); err != nil { flavorGroupLog.V(1).Info("failed to parse extra_specs for flavor", "flavor", row.Name, "error", err) + skippedParseErr++ continue } } @@ -153,6 +155,7 @@ func (e *FlavorGroupExtractor) Extract() ([]plugins.Feature, error) { hwVersion, exists := extraSpecs["quota:hw_version"] if !exists || hwVersion == "" { flavorGroupLog.V(1).Info("flavor missing hw_version extra_spec", "flavor", row.Name) + skippedNoHWVersion++ continue } @@ -175,6 +178,7 @@ func (e *FlavorGroupExtractor) Extract() ([]plugins.Feature, error) { // Convert map to features features := make([]FlavorGroupFeature, 0, len(groupMap)) + var skippedInvalidGroup int for groupName, flavors := range groupMap { if len(flavors) == 0 { continue @@ -243,6 +247,7 @@ func (e *FlavorGroupExtractor) Extract() ([]plugins.Feature, error) { } if err := fg.Validate(); err != nil { flavorGroupLog.Error(err, "skipping flavor group with invalid data", "groupName", groupName) + skippedInvalidGroup++ continue } features = append(features, fg) @@ -253,6 +258,13 @@ func (e *FlavorGroupExtractor) Extract() ([]plugins.Feature, error) { return features[i].Name < features[j].Name }) + flavorGroupLog.Info("flavor group extraction complete", + "totalFlavors", len(rows), + "skippedParseError", skippedParseErr, + "skippedNoHWVersion", skippedNoHWVersion, + "skippedInvalidGroup", skippedInvalidGroup, + "groups", len(features)) + return e.Extracted(features) } diff --git a/internal/scheduling/reservations/commitments/e2e_checks.go b/internal/scheduling/reservations/commitments/e2e_checks.go index c03129774..bc5a6a6b1 100644 --- a/internal/scheduling/reservations/commitments/e2e_checks.go +++ b/internal/scheduling/reservations/commitments/e2e_checks.go @@ -225,7 +225,7 @@ func e2eRoundTripResource( "resource", resourceName, "reason", rejectionReason) return } - e2eLog.V(1).Info("round-trip check: commitment accepted", "resource", resourceName, "uuid", testUUID) + e2eLog.Info("round-trip check: commitment accepted", "resource", resourceName, "uuid", testUUID) // Register cleanup immediately so it runs even if the usage check panics. defer func() { @@ -428,7 +428,7 @@ func e2eBatchFlavorGroupResource( "resource", resourceName, "reason", reason) return } - e2eLog.V(1).Info("batch check: batch confirmed", "resource", resourceName, "uuidA", uuidA, "uuidB", uuidB) + e2eLog.Info("batch check: batch confirmed", "resource", resourceName, "uuidA", uuidA, "uuidB", uuidB) report := e2eFetchUsageReport(ctx, baseURL, az, projectID) e2eLogUsageReport(report, az, projectID) @@ -549,10 +549,12 @@ func e2eDryRunResource( rejectionReason := e2eSendChangeCommitments(ctx, baseURL, req, requestTimeout) switch { case rejectionReason == "": - e2eLog.V(1).Info("dry-run check: accepted", "resource", resourceName, "az", az) + e2eLog.Info("dry-run check: accepted", "resource", resourceName, "az", az) case strings.Contains(rejectionReason, "not sufficient capacity") || strings.Contains(rejectionReason, "insufficient CPU cores"): e2eLog.Info("dry-run check: capacity rejection (expected in constrained clusters)", "resource", resourceName, "az", az, "reason", rejectionReason) + case strings.Contains(rejectionReason, "internal error processing dry-run request"): + panic(fmt.Sprintf("dry-run check: internal error for resource %s in %s (likely controller timeout or misconfiguration): %s", resourceName, az, rejectionReason)) default: panic(fmt.Sprintf("dry-run check: unexpected rejection for resource %s in %s: %s", resourceName, az, rejectionReason)) } @@ -822,7 +824,7 @@ func e2eMarketplaceTransferResource( }, } - e2eLog.V(1).Info("transfer check: transferring commitment from projectA to projectB", + e2eLog.Info("transfer check: transferring commitment from projectA to projectB", "resource", resourceName, "uuid", testUUID, "projectA", projectA, "projectB", projectB, "az", az) if reason := e2eSendChangeCommitments(ctx, baseURL, transferReq, requestTimeout); reason != "" { if !strings.Contains(reason, "not sufficient capacity") && !strings.Contains(reason, "insufficient CPU cores") {