Skip to content

CKS: fix scaling failures for clusters without a default service offering #14138

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Pearl1594 wants to merge 3 commits into
base: 4.22
Choose a base branch
from
Draft

CKS: fix scaling failures for clusters without a default service offering #14138

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
21b76ea
CCKS: fix scaling failures for clusters without a default service off…
Pearl1594 Sep 10, 2026
779fa79
extract common code and check node-type offering if present during up…
Pearl1594 Sep 11, 2026
1549543
CKS: fix scale-down SSH firewall conflict with etcd nodes and skip of…
Pearl1594 Nov 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 62 additions & 22 deletions ...etes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1398,6 +1398,16 @@ protected void ensureResourceLimitsForScale(final KubernetesClusterVO cluster,
}
}

public static long getNodeCountForType(KubernetesClusterNodeType nodeType, KubernetesCluster kubernetesCluster) {
if (WORKER == nodeType) {
return kubernetesCluster.getNodeCount();
} else if (CONTROL == nodeType) {
return kubernetesCluster.getControlNodeCount();
} else if (ETCD == nodeType) {
return kubernetesCluster.getEtcdNodeCount();
}
return kubernetesCluster.getTotalNodeCount();
}

protected void validateServiceOfferingsForNodeTypesScale(Map<String, Long> map, Long defaultServiceOfferingId, KubernetesClusterVO kubernetesCluster, KubernetesSupportedVersion clusterVersion) {
for (String key : CLUSTER_NODES_TYPES_LIST) {
Expand All @@ -1408,11 +1418,10 @@ protected void validateServiceOfferingsForNodeTypesScale(Map<String, Long> map,
throw new InvalidParameterValueException("Failed to find service offering ID: " + serviceOfferingId);
}
checkServiceOfferingForNodesScale(serviceOffering, kubernetesCluster, clusterVersion);
Long nodeTypeOfferingId = getExistingServiceOfferingIdForNodeType(key, kubernetesCluster);
if (nodeTypeOfferingId == null) {
nodeTypeOfferingId = kubernetesCluster.getServiceOfferingId();
final ServiceOffering existingServiceOffering = getEffectiveServiceOfferingForNodeType(key, kubernetesCluster);
if (existingServiceOffering == null) {
continue;
}
final ServiceOffering existingServiceOffering = serviceOfferingDao.findById(nodeTypeOfferingId);
if (KubernetesCluster.State.Running.equals(kubernetesCluster.getState()) && (serviceOffering.getRamSize() < existingServiceOffering.getRamSize() ||
serviceOffering.getCpu() * serviceOffering.getSpeed() < existingServiceOffering.getCpu() * existingServiceOffering.getSpeed())) {
logAndThrow(Level.WARN, String.format("Kubernetes cluster cannot be scaled down for service offering. Service offering : %s offers lesser resources as compared to service offering : %s of Kubernetes cluster : %s",
Expand All @@ -1433,6 +1442,33 @@ private Long getExistingServiceOfferingIdForNodeType(String key, KubernetesClust
return kubernetesCluster.getServiceOfferingId();
}

// Resolves the offering backing a node type, falling back to the legacy cluster-wide offering; null if the cluster has no nodes of this type.
protected ServiceOffering getEffectiveServiceOfferingForNodeType(String key, KubernetesClusterVO kubernetesCluster) {
if (getNodeCountForType(KubernetesClusterNodeType.valueOf(key), kubernetesCluster) <= 0) {
return null;
}
Long offeringId = getExistingServiceOfferingIdForNodeType(key, kubernetesCluster);
if (offeringId == null) {
offeringId = kubernetesCluster.getServiceOfferingId();
}
if (offeringId == null) {
return null;
}
return serviceOfferingDao.findByIdIncludingRemoved(offeringId);
}

// Effective offerings for every node type actually present on the cluster.
protected List<ServiceOffering> getEffectiveServiceOfferingsForCluster(KubernetesClusterVO kubernetesCluster) {
List<ServiceOffering> offerings = new ArrayList<>();
for (String key : CLUSTER_NODES_TYPES_LIST) {
ServiceOffering offering = getEffectiveServiceOfferingForNodeType(key, kubernetesCluster);
if (offering != null) {
offerings.add(offering);
}
}
return offerings;
}

protected void checkServiceOfferingForNodesScale(ServiceOffering serviceOffering, KubernetesClusterVO kubernetesCluster, KubernetesSupportedVersion clusterVersion) {
if (serviceOffering.isDynamic()) {
throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for Kubernetes clusters. Kubernetes cluster : %s, service offering : %s", kubernetesCluster.getName(), serviceOffering.getName()));
Expand Down Expand Up @@ -1497,17 +1533,19 @@ private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesCluster
throw new InvalidParameterValueException(String.format("Invalid Kubernetes version associated with cluster : %s",
kubernetesCluster.getName()));
}
final ServiceOffering serviceOffering = serviceOfferingDao.findByIdIncludingRemoved(kubernetesCluster.getServiceOfferingId());
if (serviceOffering == null) {
final List<ServiceOffering> effectiveServiceOfferings = getEffectiveServiceOfferingsForCluster(kubernetesCluster);
if (CollectionUtils.isEmpty(effectiveServiceOfferings)) {
throw new CloudRuntimeException(String.format("Invalid service offering associated with Kubernetes cluster : %s", kubernetesCluster.getName()));
}
if (serviceOffering.getCpu() < upgradeVersion.getMinimumCpu()) {
throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be upgraded with Kubernetes version : %s which needs minimum %d vCPUs while associated service offering : %s offers only %d vCPUs",
kubernetesCluster.getName(), upgradeVersion.getName(), upgradeVersion.getMinimumCpu(), serviceOffering.getName(), serviceOffering.getCpu()));
}
if (serviceOffering.getRamSize() < upgradeVersion.getMinimumRamSize()) {
throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be upgraded with Kubernetes version : %s which needs minimum %d MB RAM while associated service offering : %s offers only %d MB RAM",
kubernetesCluster.getName(), upgradeVersion.getName(), upgradeVersion.getMinimumRamSize(), serviceOffering.getName(), serviceOffering.getRamSize()));
for (ServiceOffering serviceOffering : effectiveServiceOfferings) {
if (serviceOffering.getCpu() < upgradeVersion.getMinimumCpu()) {
throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be upgraded with Kubernetes version : %s which needs minimum %d vCPUs while associated service offering : %s offers only %d vCPUs",
kubernetesCluster.getName(), upgradeVersion.getName(), upgradeVersion.getMinimumCpu(), serviceOffering.getName(), serviceOffering.getCpu()));
}
if (serviceOffering.getRamSize() < upgradeVersion.getMinimumRamSize()) {
throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be upgraded with Kubernetes version : %s which needs minimum %d MB RAM while associated service offering : %s offers only %d MB RAM",
kubernetesCluster.getName(), upgradeVersion.getName(), upgradeVersion.getMinimumRamSize(), serviceOffering.getName(), serviceOffering.getRamSize()));
}
}
// Check upgradeVersion is either patch upgrade or immediate minor upgrade
try {
Expand Down Expand Up @@ -2196,18 +2234,20 @@ public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws Clou
protected Map<String, ServiceOffering> createNodeTypeToServiceOfferingMap(Map<String, Long> idsMapping,
Long serviceOfferingId, KubernetesClusterVO kubernetesCluster) {
Map<String, ServiceOffering> map = new HashMap<>();
if (MapUtils.isEmpty(idsMapping)) {
ServiceOfferingVO offering = serviceOfferingId != null ?
serviceOfferingDao.findById(serviceOfferingId) :
serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId());
map.put(DEFAULT.name(), offering);
if (MapUtils.isNotEmpty(idsMapping)) {
for (String key : CLUSTER_NODES_TYPES_LIST) {
if (idsMapping.containsKey(key)) {
map.put(key, serviceOfferingDao.findById(idsMapping.get(key)));
}
}
return map;
}
for (String key : CLUSTER_NODES_TYPES_LIST) {
if (!idsMapping.containsKey(key)) {
continue;
Long defaultOfferingId = serviceOfferingId != null ? serviceOfferingId : kubernetesCluster.getServiceOfferingId();
if (defaultOfferingId != null) {
ServiceOfferingVO offering = serviceOfferingDao.findById(defaultOfferingId);
if (offering != null) {
map.put(DEFAULT.name(), offering);
}
map.put(key, serviceOfferingDao.findById(idsMapping.get(key)));
}
return map;
}
Expand Down
16 changes: 14 additions & 2 deletions ...cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@
import com.cloud.hypervisor.Hypervisor;
import com.cloud.kubernetes.cluster.KubernetesCluster;
import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl;
import com.cloud.kubernetes.cluster.KubernetesClusterService;
import com.cloud.kubernetes.cluster.KubernetesClusterVO;
import com.cloud.network.IpAddress;
import com.cloud.network.Network;
Expand Down Expand Up @@ -543,12 +544,23 @@ protected FirewallRule removeApiFirewallRule(final IpAddress publicIp) {
return rule;
}

private boolean isNodeSshFirewallRule(final FirewallRuleVO firewallRule, final long networkId) {
Integer startPort = firewallRule.getSourcePortStart();
if (Objects.equals(startPort, CLUSTER_NODES_DEFAULT_START_SSH_PORT)) {
return true;
}
if (startPort != null && startPort >= KubernetesClusterService.KubernetesEtcdNodeStartPort.value()) {
return false;
}
PortForwardingRuleVO pfRule = portForwardingRulesDao.findByNetworkAndPorts(networkId, startPort, firewallRule.getSourcePortEnd());
return Objects.nonNull(pfRule) && pfRule.getDestinationPortStart() == DEFAULT_SSH_PORT;
}

protected FirewallRule removeSshFirewallRule(final IpAddress publicIp, final long networkId) {
FirewallRule rule = null;
List<FirewallRuleVO> firewallRules = firewallRulesDao.listByIpPurposeProtocolAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall, NetUtils.TCP_PROTO);
for (FirewallRuleVO firewallRule : firewallRules) {
PortForwardingRuleVO pfRule = portForwardingRulesDao.findByNetworkAndPorts(networkId, firewallRule.getSourcePortStart(), firewallRule.getSourcePortEnd());
if (Objects.equals(firewallRule.getSourcePortStart(), CLUSTER_NODES_DEFAULT_START_SSH_PORT) || (Objects.nonNull(pfRule) && pfRule.getDestinationPortStart() == DEFAULT_SSH_PORT) ) {
if (isNodeSshFirewallRule(firewallRule, networkId)) {
rule = firewallRule;
firewallService.revokeIngressFwRule(firewallRule.getId(), true);
logger.debug("The SSH firewall rule {} with the id {} was revoked", firewallRule.getName(), firewallRule.getId());
Expand Down
30 changes: 14 additions & 16 deletions ...rc/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@
cores = serviceOffering.getCpu() * totalClusterSize;
memory = serviceOffering.getRamSize() * totalClusterSize;
} else {
long nodeCount = getNodeCountForType(nodeType, kubernetesCluster);
long nodeCount = KubernetesClusterManagerImpl.getNodeCountForType(nodeType, kubernetesCluster);
Long existingOfferingId = getExistingOfferingIdForNodeType(nodeType, kubernetesCluster);
if (existingOfferingId == null) {
existingOfferingId = serviceOffering.getId();
Expand All @@ -243,17 +243,6 @@
return new Pair<>(cores, memory);
}

private long getNodeCountForType(KubernetesClusterNodeType nodeType, KubernetesCluster kubernetesCluster) {
if (WORKER == nodeType) {
return kubernetesCluster.getNodeCount();
} else if (CONTROL == nodeType) {
return kubernetesCluster.getControlNodeCount();
} else if (ETCD == nodeType) {
return kubernetesCluster.getEtcdNodeCount();
}
return kubernetesCluster.getTotalNodeCount();
}

protected Pair<Long, Long> calculateClusterNewCapacity(KubernetesCluster kubernetesCluster,
Pair<Long, Long> previousNodeTypeCapacity,
Pair<Long, Long> newNodeTypeCapacity) {
Expand Down Expand Up @@ -337,17 +326,19 @@
// Check capacity and transition state
final long newVmRequiredCount = clusterSize - originalClusterSize;
final ServiceOffering clusterServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId());
if (clusterServiceOffering == null) {
final ServiceOffering existingWorkerOffering = serviceOfferingDao.findById(kubernetesCluster.getWorkerNodeServiceOfferingId());
final ServiceOffering effectiveOffering = existingWorkerOffering != null ? existingWorkerOffering : clusterServiceOffering;
if (effectiveOffering == null) {
logTransitStateToFailedIfNeededAndThrow(Level.WARN, String.format("Scaling failed for Kubernetes cluster : %s, cluster service offering not found", kubernetesCluster.getName()));
}
if (newVmRequiredCount > 0) {
final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId());
VMTemplateVO clusterTemplate = templateDao.findById(kubernetesCluster.getTemplateId());
try {
if (originalState.equals(KubernetesCluster.State.Running)) {
plan(newVmRequiredCount, zone, clusterServiceOffering, kubernetesCluster.getDomainId(), kubernetesCluster.getAccountId(), clusterTemplate.getHypervisorType(), clusterTemplate.getArch());
plan(newVmRequiredCount, zone, effectiveOffering, kubernetesCluster.getDomainId(), kubernetesCluster.getAccountId(), clusterTemplate.getHypervisorType(), clusterTemplate.getArch());
} else {
plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, zone, clusterServiceOffering, kubernetesCluster.getDomainId(), kubernetesCluster.getAccountId(), clusterTemplate.getHypervisorType(), clusterTemplate.getArch());
plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, zone, effectiveOffering, kubernetesCluster.getDomainId(), kubernetesCluster.getAccountId(), clusterTemplate.getHypervisorType(), clusterTemplate.getArch());
}
} catch (InsufficientCapacityException e) {
logTransitStateToFailedIfNeededAndThrow(Level.WARN, String.format("Scaling failed for Kubernetes cluster : %s in zone : %s, insufficient capacity", kubernetesCluster.getName(), zone.getName()));
Expand All @@ -370,7 +361,7 @@
kubernetesCluster = updateKubernetesClusterEntryForNodeType(null, nodeType, serviceOffering, updateNodeOffering, updateClusterOffering);
return;
}
final long size = getNodeCountForType(nodeType, kubernetesCluster);
final long size = KubernetesClusterManagerImpl.getNodeCountForType(nodeType, kubernetesCluster);
List<KubernetesClusterVmMapVO> vmList = kubernetesClusterVmMapDao.listByClusterIdAndVmType(kubernetesCluster.getId(), nodeType);
final long tobeScaledVMCount = Math.min(vmList.size(), size);
for (long i = 0; i < tobeScaledVMCount; i++) {
Expand Down Expand Up @@ -572,6 +563,10 @@
if (!scaleClusterDefaultOffering && !serviceOfferingNodeTypeMap.containsKey(nodeType.name()) && newVMRequired == 0) {
continue;
}
if (!serviceOfferingNodeTypeMap.containsKey(nodeType.name()) && newVMRequired == 0 &&
KubernetesClusterManagerImpl.getNodeCountForType(nodeType, kubernetesCluster) <= 0) {
continue;
}

ServiceOffering existingServiceOffering = getExistingServiceOfferingForNodeType(nodeType, kubernetesCluster);
ServiceOffering scalingServiceOffering = serviceOfferingNodeTypeMap.getOrDefault(nodeType.name(), defaultServiceOffering);
Expand All @@ -584,7 +579,7 @@
if (autoScaled && isNodeOfferingScalingNeeded) {
scaleKubernetesClusterOffering(nodeType, scalingServiceOffering, updateNodeOffering, updateClusterOffering);
}
stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded);

Check warning on line 582 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Reduce the total number of break and continue statements in this loop to use at most one. 


See more on https://sonarcloud.io/project/issues?id=apache_cloudstack&issues=AaGpCTc3AWsW_xt9-UJ9&open=AaGpCTc3AWsW_xt9-UJ9&pullRequest=14138
return autoScaled;
}
final boolean clusterSizeScalingNeeded = isWorkerNode && clusterSize != null && clusterSize != originalClusterSize;
Expand All @@ -610,6 +605,9 @@
private ServiceOffering getExistingServiceOfferingForNodeType(KubernetesClusterNodeType nodeType, KubernetesCluster kubernetesCluster) {
Long existingOfferingId = getExistingOfferingIdForNodeType(nodeType, kubernetesCluster);
if (existingOfferingId == null) {
if (KubernetesClusterManagerImpl.getNodeCountForType(nodeType, kubernetesCluster) <= 0) {
return null;
}
logAndThrow(Level.ERROR, String.format("The Kubernetes cluster %s does not have a service offering set for node type %s",
kubernetesCluster.getName(), nodeType.name()));
}
Expand Down
Loading