From 6c9ae12a4ff9290d6c0a6696bd95f187cfcc3693 Mon Sep 17 00:00:00 2001 From: "dejing.liu" Date: Fri, 31 Jul 2026 15:29:39 +0800 Subject: [PATCH 1/2] [lb]: reject tcp ipvs on old zvr Fail tcp ipvs listener creation before the listener row is persisted when the target appliance zvr version is older than 5.5.38. This keeps unsupported routers from receiving an unusable listener payload and returns a clear API error. Test: mvn -pl :loadBalancer -am -DskipTests -DskipJacoco=true package Test: mvn -pl :virtualRouterProvider -am -DskipTests -DskipJacoco=true package Test: mvn -f test/pom.xml -DskipTests -DskipJacoco=true test-compile Test: live create tcp ipvs listener on 172.24.194.62 Resolves: ZSTAC-86989 Change-Id: I6e84e4339efa26258a9ce2572d33ab1004795492 --- .../service/lb/LoadBalancerBackend.java | 4 + .../network/service/lb/LoadBalancerBase.java | 28 ++- .../lb/VirtualRouterLoadBalancerBackend.java | 41 ++++ .../virtualrouter/vyos/VyosConstants.java | 1 + .../TcpIpvsLoadBalancerListenerApiCase.groovy | 10 + .../TcpIpvsZvrVersionValidationCase.groovy | 202 ++++++++++++++++++ .../CloudOperationsErrorCode.java | 2 + 7 files changed, 286 insertions(+), 2 deletions(-) create mode 100644 test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsZvrVersionValidationCase.groovy diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBackend.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBackend.java index 561226ca769..1df195bedfb 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBackend.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBackend.java @@ -21,6 +21,10 @@ public interface LoadBalancerBackend { void addListener(LoadBalancerStruct struct, LoadBalancerListenerInventory listener, Completion completion); + default void validateBeforeCreateListener(LoadBalancerVO lbVO, APICreateLoadBalancerListenerMsg msg, Completion completion) { + completion.success(); + } + void removeListener(LoadBalancerStruct struct, LoadBalancerListenerInventory listener, Completion completion); void destroyLoadBalancer(LoadBalancerStruct struct, Completion completion); diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java index 0e78b167c0f..94afa88e6d5 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java @@ -1811,9 +1811,33 @@ public void run(final SyncTaskChain chain) { return; } - createListener(msg, new NoErrorCompletion(chain) { + LoadBalancerBackend backend = getBackend(); + if (backend == null) { + createListener(msg, new NoErrorCompletion(chain) { + @Override + public void done() { + chain.next(); + } + }); + return; + } + + backend.validateBeforeCreateListener(self, msg, new Completion(chain) { @Override - public void done() { + public void success() { + createListener(msg, new NoErrorCompletion(chain) { + @Override + public void done() { + chain.next(); + } + }); + } + + @Override + public void fail(ErrorCode errorCode) { + APICreateLoadBalancerListenerEvent evt = new APICreateLoadBalancerListenerEvent(msg.getId()); + evt.setError(errorCode); + bus.publish(evt); chain.next(); } }); diff --git a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java index 72faff543a0..9e088712254 100755 --- a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java +++ b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java @@ -47,6 +47,7 @@ import org.zstack.network.service.lb.*; import org.zstack.network.service.vip.*; import org.zstack.network.service.virtualrouter.*; +import org.zstack.network.service.virtualrouter.vyos.VyosConstants; import org.zstack.network.service.virtualrouter.vyos.VyosGlobalConfig; import org.zstack.network.service.virtualrouter.VirtualRouterCommands.AgentCommand; import org.zstack.network.service.virtualrouter.VirtualRouterCommands.AgentResponse; @@ -1303,6 +1304,46 @@ public void handle(ErrorCode errCode, Map data) { }).start(); } + @Override + public void validateBeforeCreateListener(LoadBalancerVO lbVO, APICreateLoadBalancerListenerMsg msg, Completion completion) { + if (!isTcpIpvsListener(msg)) { + completion.success(); + return; + } + + VirtualRouterVmInventory vr = findVirtualRouterVm(lbVO.getUuid()); + if (vr == null) { + completion.success(); + return; + } + + ErrorCode errorCode = validateTcpIpvsZvrVersion(vr.getUuid()); + if (errorCode != null) { + completion.fail(errorCode); + return; + } + + completion.success(); + } + + private boolean isTcpIpvsListener(APICreateLoadBalancerListenerMsg msg) { + return LoadBalancerConstants.LB_PROTOCOL_TCP.equals(msg.getProtocol()) && + LoadBalancerConstants.DATA_PLANE_IPVS.equals(msg.getDataPlane()); + } + + private ErrorCode validateTcpIpvsZvrVersion(String vmUuid) { + VirtualRouterMetadataVO metadataVO = dbf.findByUuid(vmUuid, VirtualRouterMetadataVO.class); + String zvrVersion = metadataVO == null ? null : metadataVO.getZvrVersion(); + if (!VirtualRouterMetadataOperator.zvrVersionCheck(zvrVersion) || + new VersionComparator(zvrVersion).compare(VyosConstants.TCP_IPVS_MIN_ZVR_VERSION) < 0) { + return operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10190, + "target appliance vm[uuid:%s] zvr version [%s] does not support tcp ipvs listener, required >= %s", + vmUuid, zvrVersion == null ? "unknown" : zvrVersion, VyosConstants.TCP_IPVS_MIN_ZVR_VERSION); + } + + return null; + } + public void refreshCertsAndListeners(VirtualRouterVmInventory vr, Map certs, List listeners, diff --git a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConstants.java b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConstants.java index 93941c5dacc..8b88d644e2f 100755 --- a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConstants.java +++ b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConstants.java @@ -23,6 +23,7 @@ enum BootstrapInfoKey { String VYOS_VERSION_PATH = "ansible/zvr/version"; int VYOS_VERSION_LENGTH = 4; + String TCP_IPVS_MIN_ZVR_VERSION = "5.5.38.0"; String REPLACE_FIREWALL_WITH_IPTBALES = "SkipVyosIptables"; String HA_STATUS = "haStatus"; diff --git a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy index 70631a2af93..5947515e7cd 100644 --- a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy @@ -14,6 +14,8 @@ import org.zstack.network.service.lb.LoadBalancerSystemTags import org.zstack.network.service.lb.LoadBalancerVO import org.zstack.network.service.lb.LoadBalancerVO_ import org.zstack.network.service.portforwarding.PortForwardingConstant +import org.zstack.network.service.virtualrouter.VirtualRouterCommands +import org.zstack.network.service.virtualrouter.VirtualRouterConstant import org.zstack.network.service.virtualrouter.lb.VirtualRouterLoadBalancerBackend import org.zstack.network.service.virtualrouter.vyos.VyosConstants import org.zstack.network.service.virtualrouter.vyos.VyosGlobalConfig @@ -201,6 +203,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { @Override void test() { env.create { + installSupportedZvrVersionSimulator() prepareDedicatedLoadBalancer() installRefreshLbCapture() testTcpIpvsIpv6Validation() @@ -218,6 +221,13 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { } } + void installSupportedZvrVersionSimulator() { + env.afterSimulator(VirtualRouterConstant.VR_INIT) { VirtualRouterCommands.InitRsp rsp, HttpEntity e -> + rsp.zvrVersion = VyosConstants.TCP_IPVS_MIN_ZVR_VERSION + return rsp + } + } + void prepareDedicatedLoadBalancer() { L3NetworkInventory publicL3 = env.inventoryByName("publicL3") as L3NetworkInventory VipInventory vip = createVip { diff --git a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsZvrVersionValidationCase.groovy b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsZvrVersionValidationCase.groovy new file mode 100644 index 00000000000..d2f532059db --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsZvrVersionValidationCase.groovy @@ -0,0 +1,202 @@ +package org.zstack.test.integration.networkservice.provider.virtualrouter.loadbalancer + +import org.zstack.appliancevm.ApplianceVmConstant +import org.zstack.appliancevm.ApplianceVmHaStatus +import org.zstack.appliancevm.ApplianceVmStatus +import org.zstack.core.Platform +import org.zstack.core.db.Q +import org.zstack.core.db.DatabaseFacade +import org.zstack.header.identity.AccountConstant +import org.zstack.header.image.ImagePlatform +import org.zstack.header.network.service.NetworkServiceType +import org.zstack.header.vm.VmInstanceState +import org.zstack.network.service.lb.LoadBalancerConstants +import org.zstack.network.service.lb.LoadBalancerListenerVO +import org.zstack.network.service.lb.LoadBalancerListenerVO_ +import org.zstack.network.service.lb.LoadBalancerVO +import org.zstack.network.service.virtualrouter.VirtualRouterConstant +import org.zstack.network.service.virtualrouter.VirtualRouterMetadataVO +import org.zstack.network.service.virtualrouter.VirtualRouterVmVO +import org.zstack.network.service.virtualrouter.lb.VirtualRouterLoadBalancerRefVO +import org.zstack.network.service.virtualrouter.lb.VirtualRouterLoadBalancerRefVO_ +import org.zstack.network.service.virtualrouter.vyos.VyosConstants +import org.zstack.sdk.CreateLoadBalancerListenerAction +import org.zstack.sdk.L3NetworkInventory +import org.zstack.sdk.LoadBalancerInventory +import org.zstack.sdk.VipInventory +import org.zstack.test.integration.networkservice.provider.NetworkServiceProviderTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase + +class TcpIpvsZvrVersionValidationCase extends SubCase { + EnvSpec env + DatabaseFacade dbf + String syntheticVrUuid + + @Override + void setup() { + useSpring(NetworkServiceProviderTest.springSpec) + } + + @Override + void environment() { + env = env { + zone { + name = "zone" + + l2NoVlanNetwork { + name = "l2" + physicalInterface = "eth0" + + l3Network { + name = "guestL3" + service { + provider = VyosConstants.VYOS_ROUTER_PROVIDER_TYPE + types = [NetworkServiceType.DHCP.toString(), + NetworkServiceType.DNS.toString(), + NetworkServiceType.SNAT.toString(), + LoadBalancerConstants.LB_NETWORK_SERVICE_TYPE_STRING] + } + ip { + startIp = "10.2.226.10" + endIp = "10.2.226.200" + gateway = "10.2.226.1" + netmask = "255.255.255.0" + } + } + + l3Network { + name = "publicL3" + ip { + startIp = "172.24.3.10" + endIp = "172.24.3.200" + gateway = "172.24.3.1" + netmask = "255.255.255.0" + } + } + } + } + } + } + + @Override + void test() { + env.create { + dbf = bean(DatabaseFacade.class) + testTcpIpvsCreateRejectedByOldZvrVersion() + } + } + + @Override + void clean() { + if (syntheticVrUuid != null) { + VirtualRouterLoadBalancerRefVO ref = Q.New(VirtualRouterLoadBalancerRefVO.class) + .eq(VirtualRouterLoadBalancerRefVO_.virtualRouterVmUuid, syntheticVrUuid) + .find() + if (ref != null) { + dbf.remove(ref) + } + + VirtualRouterMetadataVO metadata = dbf.findByUuid(syntheticVrUuid, VirtualRouterMetadataVO.class) + if (metadata != null) { + dbf.remove(metadata) + } + + VirtualRouterVmVO vr = dbf.findByUuid(syntheticVrUuid, VirtualRouterVmVO.class) + if (vr != null) { + dbf.remove(vr) + } + } + env.delete() + } + + void testTcpIpvsCreateRejectedByOldZvrVersion() { + L3NetworkInventory publicL3 = env.inventoryByName("publicL3") as L3NetworkInventory + + VipInventory vip = createVip { + name = "tcp-ipvs-zvr-version-vip" + l3NetworkUuid = publicL3.uuid + } + + LoadBalancerInventory lb = createLoadBalancer { + name = "tcp-ipvs-zvr-version-lb" + vipUuid = vip.uuid + } + setLbProviderType(lb.uuid) + + String vrUuid = createVirtualRouterRef(lb.uuid) + setZvrVersion(vrUuid, "5.5.28.0") + + CreateLoadBalancerListenerAction action = createTcpIpvsListenerAction(lb.uuid, "tcp-ipvs-old-zvr-version", 19095) + CreateLoadBalancerListenerAction.Result result = action.call() + + assert result.error != null + assert result.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10190" + assert result.error.details.contains("does not support tcp ipvs listener") + assert !Q.New(LoadBalancerListenerVO.class) + .eq(LoadBalancerListenerVO_.name, action.name) + .isExists() + + setZvrVersion(vrUuid, VyosConstants.TCP_IPVS_MIN_ZVR_VERSION) + result = createTcpIpvsListenerAction(lb.uuid, "tcp-ipvs-supported-zvr-version", 19096).call() + assert result.error == null + } + + String createVirtualRouterRef(String lbUuid) { + String vrUuid = Platform.getUuid() + VirtualRouterVmVO vr = new VirtualRouterVmVO() + vr.uuid = vrUuid + vr.name = "tcp-ipvs-zvr-version-vr" + vr.type = ApplianceVmConstant.APPLIANCE_VM_TYPE + vr.applianceVmType = VirtualRouterConstant.VIRTUAL_ROUTER_VM_TYPE + vr.hypervisorType = "KVM" + vr.platform = ImagePlatform.Linux.toString() + vr.state = VmInstanceState.Running + vr.status = ApplianceVmStatus.Connected + vr.haStatus = ApplianceVmHaStatus.NoHa + vr.agentPort = 7272 + vr.internalId = 1L + vr.accountUuid = AccountConstant.INITIAL_SYSTEM_ADMIN_UUID + dbf.persist(vr) + syntheticVrUuid = vrUuid + + VirtualRouterLoadBalancerRefVO ref = new VirtualRouterLoadBalancerRefVO() + ref.loadBalancerUuid = lbUuid + ref.virtualRouterVmUuid = vrUuid + dbf.persist(ref) + + return vrUuid + } + + void setLbProviderType(String lbUuid) { + LoadBalancerVO lbVO = dbf.findByUuid(lbUuid, LoadBalancerVO.class) + lbVO.providerType = VyosConstants.VYOS_ROUTER_PROVIDER_TYPE + dbf.update(lbVO) + } + + void setZvrVersion(String vrUuid, String version) { + VirtualRouterMetadataVO metadata = dbf.findByUuid(vrUuid, VirtualRouterMetadataVO.class) + if (metadata == null) { + metadata = new VirtualRouterMetadataVO() + metadata.uuid = vrUuid + metadata.zvrVersion = version + dbf.persist(metadata) + } else { + metadata.zvrVersion = version + dbf.update(metadata) + } + } + + CreateLoadBalancerListenerAction createTcpIpvsListenerAction(String lbUuid, String name, int port) { + CreateLoadBalancerListenerAction action = new CreateLoadBalancerListenerAction() + action.name = name + action.loadBalancerUuid = lbUuid + action.protocol = LoadBalancerConstants.LB_PROTOCOL_TCP + action.loadBalancerPort = port + action.instancePort = 8080 + action.dataPlane = LoadBalancerConstants.DATA_PLANE_IPVS + action.forwardMode = LoadBalancerConstants.FORWARD_MODE_FULL_NAT + action.sessionId = adminSession() + return action + } +} diff --git a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java index 68ab7087c6c..d42a9f392aa 100644 --- a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java +++ b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java @@ -13917,6 +13917,8 @@ public class CloudOperationsErrorCode { public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10189 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10189"; + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10190 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10190"; + public static final String ORG_ZSTACK_IPSEC_10000 = "ORG_ZSTACK_IPSEC_10000"; public static final String ORG_ZSTACK_IPSEC_10001 = "ORG_ZSTACK_IPSEC_10001"; From debf5a694af08e5c9acc4561e432bc3acc56cc7d Mon Sep 17 00:00:00 2001 From: "dejing.liu" Date: Fri, 31 Jul 2026 19:25:06 +0800 Subject: [PATCH 2/2] [lb]: reject unsupported ipvs parameters Reject HAProxy-only connectionIdleTimeout, nbprocess, and enabled proxy protocol settings for TCP IPVS listeners. Do not generate HAProxy defaults for new TCP IPVS listeners, and validate generic system tag create or update operations so they cannot bypass the listener API checks. Keep HAProxy listener behavior intact. Test: loadBalancer compile and Groovy test-compile Resolves: ZSTAC-86990 Change-Id: Ife46ca83696fdbfdb5988e45c91f4c485a7f1523 --- .../lb/LoadBalancerApiInterceptor.java | 73 ++++++-- .../service/lb/LoadBalancerManagerImpl.java | 27 +++ .../TcpIpvsLoadBalancerListenerApiCase.groovy | 177 ++++++++++++++++++ .../CloudOperationsErrorCode.java | 6 + 4 files changed, 271 insertions(+), 12 deletions(-) diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java index ff0f7c34896..19089f9f256 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java @@ -815,6 +815,33 @@ private boolean hasHttpHealthCheckParameters(APIChangeLoadBalancerListenerMsg ms msg.getHealthCheckHttpCode() != null; } + private boolean hasEnabledTcpProxyProtocol(String tcpProxyProtocol) { + return !StringUtils.isEmpty(tcpProxyProtocol) && + !DisableLbSupportTcpProxyProtocol.equals(tcpProxyProtocol); + } + + private void validateTcpIpvsDoesNotUseTcpProxyProtocol(String protocol, String dataPlane, String tcpProxyProtocol) { + if (isTcpIpvsListener(protocol, dataPlane) && hasEnabledTcpProxyProtocol(tcpProxyProtocol)) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10191, "tcp ipvs listener doesn't support tcpProxyProtocol")); + } + } + + private void validateTcpIpvsDoesNotUseConnectionIdleTimeout(String protocol, String dataPlane, + Integer connectionIdleTimeout) { + if (isTcpIpvsListener(protocol, dataPlane) && connectionIdleTimeout != null) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10192, "tcp ipvs listener doesn't support connectionIdleTimeout")); + } + } + + private void validateTcpIpvsDoesNotUseNbprocess(String protocol, String dataPlane, Integer nbprocess) { + if (isTcpIpvsListener(protocol, dataPlane) && nbprocess != null) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10193, "tcp ipvs listener doesn't support nbprocess")); + } + } + private String getHealthCheckProtocolFromTarget(String healthCheckTarget) { if (healthCheckTarget == null) { return null; @@ -970,6 +997,18 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { throw new ApiMessageInterceptionException( operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10189, "tcp ipvs listener doesn't support healthCheckTimeout")); } + if (isTcpIpvsListener(msg.getProtocol(), dataPlane) && hasTag(msg, LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT)) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10192, "tcp ipvs listener doesn't support connectionIdleTimeout")); + } + if (isTcpIpvsListener(msg.getProtocol(), dataPlane) && hasTag(msg, LoadBalancerSystemTags.NUMBER_OF_PROCESS)) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10193, "tcp ipvs listener doesn't support nbprocess")); + } + if (isTcpIpvsListener(msg.getProtocol(), dataPlane) && hasTag(msg, LoadBalancerSystemTags.TCP_PROXYPROTOCOL)) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10191, "tcp ipvs listener doesn't support tcpProxyProtocol")); + } if (isHealthCheckProtocolNotSupportedByListenerProtocol(msg.getProtocol(), dataPlane, msg.getHealthCheckProtocol())) { throw new ApiMessageInterceptionException( @@ -1011,12 +1050,14 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { validateAcl(msg.getAclUuids(),new ArrayList<>(), msg.getLoadBalancerUuid()); } - insertTagIfNotExisting( - msg, LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT, - LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT.instantiateTag( - map(e(LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT_TOKEN, LoadBalancerGlobalConfig.CONNECTION_IDLE_TIMEOUT.value(Long.class))) - ) - ); + if (!isTcpIpvsListener(msg.getProtocol(), dataPlane)) { + insertTagIfNotExisting( + msg, LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT, + LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT.instantiateTag( + map(e(LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT_TOKEN, LoadBalancerGlobalConfig.CONNECTION_IDLE_TIMEOUT.value(Long.class))) + ) + ); + } insertTagIfNotExisting( msg, LoadBalancerSystemTags.HEALTHY_THRESHOLD, @@ -1076,12 +1117,14 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { ) ); - insertTagIfNotExisting( - msg, LoadBalancerSystemTags.NUMBER_OF_PROCESS, - LoadBalancerSystemTags.NUMBER_OF_PROCESS.instantiateTag( - map(e(LoadBalancerSystemTags.NUMBER_OF_PROCESS_TOKEN, LoadBalancerGlobalConfig.NUMBER_OF_PROCESS.value())) - ) - ); + if (!isTcpIpvsListener(msg.getProtocol(), dataPlane)) { + insertTagIfNotExisting( + msg, LoadBalancerSystemTags.NUMBER_OF_PROCESS, + LoadBalancerSystemTags.NUMBER_OF_PROCESS.instantiateTag( + map(e(LoadBalancerSystemTags.NUMBER_OF_PROCESS_TOKEN, LoadBalancerGlobalConfig.NUMBER_OF_PROCESS.value())) + ) + ); + } if (LoadBalancerConstants.LB_PROTOCOL_HTTP.equals(msg.getProtocol()) || LoadBalancerConstants.LB_PROTOCOL_HTTPS.equals(msg.getProtocol())) { @@ -1381,6 +1424,7 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { throw new ApiMessageInterceptionException( argerr(ORG_ZSTACK_NETWORK_SERVICE_LB_10093, "cloud not create the loadbalancer listener, because only support tcp proxy protocol %s", LbSupportTcpProxyProtocol)); } + validateTcpIpvsDoesNotUseTcpProxyProtocol(msg.getProtocol(), dataPlane, msg.getTcpProxyProtocol()); if (!msg.getTcpProxyProtocol().equals(DisableLbSupportTcpProxyProtocol)) { insertTagIfNotExisting( @@ -1620,6 +1664,10 @@ private void validate(APIChangeLoadBalancerListenerMsg msg) { eq(LoadBalancerListenerVO_.uuid,msg.getLoadBalancerListenerUuid()).find(); String dataPlane = getListenerDataPlane(msg.getLoadBalancerListenerUuid()); + validateTcpIpvsDoesNotUseConnectionIdleTimeout(listenerVO.getProtocol(), dataPlane, + msg.getConnectionIdleTimeout()); + validateTcpIpvsDoesNotUseNbprocess(listenerVO.getProtocol(), dataPlane, msg.getNbprocess()); + if (msg.getSecurityPolicyType() != null) { if (!listenerVO.getProtocol().equals(LB_PROTOCOL_HTTPS)) { throw new ApiMessageInterceptionException(operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10118, "the listener with protocol [%s] doesn't support select security policy", listenerVO.getProtocol(), msg.getHealthCheckProtocol())); @@ -1697,6 +1745,7 @@ private void validate(APIChangeLoadBalancerListenerMsg msg) { throw new ApiMessageInterceptionException( argerr(ORG_ZSTACK_NETWORK_SERVICE_LB_10125, "cloud not change the loadbalancer listener, because only support tcp proxy protocol %s", LbSupportTcpProxyProtocol)); } + validateTcpIpvsDoesNotUseTcpProxyProtocol(listenerVO.getProtocol(), dataPlane, msg.getTcpProxyProtocol()); } if (!CollectionUtils.isEmpty(msg.getHttpCompressAlgos())) { diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerManagerImpl.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerManagerImpl.java index fe6c99a0826..6a1883ce741 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerManagerImpl.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerManagerImpl.java @@ -502,6 +502,21 @@ public void validateGlobalConfig(String category, String name, String oldValue, } + private boolean isTcpIpvsListener(String listenerUuid) { + LoadBalancerListenerVO listener = Q.New(LoadBalancerListenerVO.class) + .eq(LoadBalancerListenerVO_.uuid, listenerUuid) + .find(); + return listener != null && LoadBalancerConstants.LB_PROTOCOL_TCP.equals(listener.getProtocol()) && + LoadBalancerConstants.DATA_PLANE_IPVS.equals(listener.getDataPlane()); + } + + private void rejectTcpIpvsSystemTag(String listenerUuid, String errorCode, String parameter) { + if (isTcpIpvsListener(listenerUuid)) { + throw new OperationFailureException( + operr(errorCode, "tcp ipvs listener doesn't support %s", parameter)); + } + } + private void prepareSystemTags() { AbstractSystemTagOperationJudger judger = new AbstractSystemTagOperationJudger() { @Override @@ -554,6 +569,14 @@ public void validateSystemTag(String resourceUuid, Class resourceType, String sy } }); + LoadBalancerSystemTags.TCP_PROXYPROTOCOL.installValidator(new SystemTagValidator() { + @Override + public void validateSystemTag(String resourceUuid, Class resourceType, String systemTag) { + rejectTcpIpvsSystemTag(resourceUuid, ORG_ZSTACK_NETWORK_SERVICE_LB_10191, + "tcpProxyProtocol"); + } + }); + LoadBalancerSystemTags.HEALTH_PARAMETER.installLifeCycleListener(new SystemTagLifeCycleListener() { @Override public void tagCreated(SystemTagInventory tag) { @@ -661,6 +684,8 @@ public void validateSystemTag(String resourceUuid, Class resourceType, String sy LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT.installValidator(new SystemTagValidator() { @Override public void validateSystemTag(String resourceUuid, Class resourceType, String systemTag) { + rejectTcpIpvsSystemTag(resourceUuid, ORG_ZSTACK_NETWORK_SERVICE_LB_10192, + "connectionIdleTimeout"); String s = LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT.getTokenByTag(systemTag, LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT_TOKEN); @@ -703,6 +728,8 @@ public void validateSystemTag(String resourceUuid, Class resourceType, String sy LoadBalancerSystemTags.NUMBER_OF_PROCESS.installValidator(new SystemTagValidator() { @Override public void validateSystemTag(String resourceUuid, Class resourceType, String systemTag) { + rejectTcpIpvsSystemTag(resourceUuid, ORG_ZSTACK_NETWORK_SERVICE_LB_10193, + "nbprocess"); String s = LoadBalancerSystemTags.NUMBER_OF_PROCESS.getTokenByTag(systemTag, LoadBalancerSystemTags.NUMBER_OF_PROCESS_TOKEN); diff --git a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy index 5947515e7cd..19bd9c275e2 100644 --- a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy @@ -4,6 +4,8 @@ import org.zstack.appliancevm.ApplianceVmVO import org.zstack.appliancevm.ApplianceVmVO_ import org.zstack.core.db.Q import org.zstack.header.network.service.NetworkServiceType +import org.zstack.header.tag.SystemTagVO +import org.zstack.header.tag.SystemTagVO_ import org.zstack.network.service.eip.EipConstant import org.zstack.network.service.lb.LoadBalancerAclStatus import org.zstack.network.service.lb.LoadBalancerAclType @@ -28,11 +30,14 @@ import org.zstack.sdk.ChangeLoadBalancerListenerAction import org.zstack.sdk.ApiResult import org.zstack.sdk.CreateLoadBalancerListenerAction import org.zstack.sdk.CreateLoadBalancerListenerResult +import org.zstack.sdk.CreateSystemTagAction +import org.zstack.sdk.CreateSystemTagsAction import org.zstack.sdk.L3NetworkInventory import org.zstack.sdk.LoadBalancerInventory import org.zstack.sdk.LoadBalancerListenerInventory import org.zstack.sdk.LoadBalancerServerGroupInventory import org.zstack.sdk.QueryLoadBalancerListenerAction +import org.zstack.sdk.UpdateSystemTagAction import org.zstack.sdk.VipInventory import org.zstack.sdk.VmInstanceInventory import org.zstack.sdk.ZSClient @@ -214,7 +219,11 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { testTcpIpvsDefaultForwardMode() testTcpIpvsForwardModeCannotBeChanged() testTcpIpvsCreateValidation() + testTcpIpvsTcpProxyProtocolValidation() testTcpIpvsHealthCheckParameterValidation() + testTcpIpvsUnsupportedHaproxyParameterValidation() + testTcpIpvsSystemTagApiValidation() + testTcpHaproxyParameterRegression() testTcpHaproxyBackendRefreshPayload() testUdpHaproxyBackendRefreshPayload() testTcpIpvsDedicatedListenerBackendRefreshPayload() @@ -551,6 +560,47 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assert result.error.details.contains("forwardMode is only supported when dataPlane is ipvs") } + void testTcpIpvsTcpProxyProtocolValidation() { + ["v1", "v2"].eachWithIndex { String protocol, int index -> + int port = 11260 + index + CreateLoadBalancerListenerAction.Result result = assertCreateTcpIpvsListenerError(port) { CreateLoadBalancerListenerAction action -> + action.tcpProxyProtocol = protocol + } + assertUnsupportedTcpProxyProtocolError(result.error) + assert !Q.New(LoadBalancerListenerVO.class) + .eq(LoadBalancerListenerVO_.name, "tcp-ipvs-error-${port}".toString()) + .isExists() + } + + LoadBalancerListenerInventory disabledOnCreate = createLoadBalancerListener { + delegate.name = "tcp-ipvs-disabled-tcp-proxy-protocol" + delegate.loadBalancerUuid = lb.uuid + delegate.protocol = LoadBalancerConstants.LB_PROTOCOL_TCP + delegate.loadBalancerPort = 11262 + delegate.instancePort = 8080 + delegate.dataPlane = LoadBalancerConstants.DATA_PLANE_IPVS + delegate.forwardMode = LoadBalancerConstants.FORWARD_MODE_FULL_NAT + delegate.tcpProxyProtocol = LoadBalancerConstants.DisableLbSupportTcpProxyProtocol + } + assert disabledOnCreate.uuid != null + assert !LoadBalancerSystemTags.TCP_PROXYPROTOCOL.hasTag(disabledOnCreate.uuid, LoadBalancerListenerVO.class) + + LoadBalancerListenerInventory listener = createTcpIpvsListener( + "tcp-ipvs-change-tcp-proxy-protocol", 11263, LoadBalancerConstants.BALANCE_ALGORITHM_ROUND_ROBIN) + ["v1", "v2"].each { String protocol -> + ChangeLoadBalancerListenerAction.Result result = assertChangeListenerError(listener.uuid) { ChangeLoadBalancerListenerAction action -> + action.tcpProxyProtocol = protocol + } + assertUnsupportedTcpProxyProtocolError(result.error) + } + assert !LoadBalancerSystemTags.TCP_PROXYPROTOCOL.hasTag(listener.uuid, LoadBalancerListenerVO.class) + + assertChangeListenerSuccess(listener.uuid) { ChangeLoadBalancerListenerAction action -> + action.tcpProxyProtocol = LoadBalancerConstants.DisableLbSupportTcpProxyProtocol + } + assert !LoadBalancerSystemTags.TCP_PROXYPROTOCOL.hasTag(listener.uuid, LoadBalancerListenerVO.class) + } + void testTcpIpvsHealthCheckParameterValidation() { [ { CreateLoadBalancerListenerAction action -> action.healthCheckMethod = "GET" }, @@ -686,6 +736,123 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assert noneSpecificTargetResult.error.details.contains("health check protocol none only supports default target") } + void testTcpIpvsUnsupportedHaproxyParameterValidation() { + [ + ["connectionIdleTimeout::77", "ORG_ZSTACK_NETWORK_SERVICE_LB_10192", "connectionIdleTimeout"], + ["Nbprocess::2", "ORG_ZSTACK_NETWORK_SERVICE_LB_10193", "nbprocess"], + ["tcpProxyProtocol::v2", "ORG_ZSTACK_NETWORK_SERVICE_LB_10191", "tcpProxyProtocol"], + ["tcpProxyProtocol::disable", "ORG_ZSTACK_NETWORK_SERVICE_LB_10191", "tcpProxyProtocol"] + ].eachWithIndex { List param, int index -> + int port = 11270 + index + CreateLoadBalancerListenerAction.Result result = assertCreateTcpIpvsListenerError(port) { CreateLoadBalancerListenerAction action -> + action.systemTags = [param[0]] + } + assertUnsupportedTcpIpvsParameterError(result.error, param[1] as String, param[2] as String) + assert !Q.New(LoadBalancerListenerVO.class) + .eq(LoadBalancerListenerVO_.name, "tcp-ipvs-invalid-health-check-${port}".toString()) + .isExists() + } + + LoadBalancerListenerInventory listener = createTcpIpvsListener( + "tcp-ipvs-unsupported-haproxy-parameters", 11274, + LoadBalancerConstants.BALANCE_ALGORITHM_ROUND_ROBIN) + List tags = Q.New(SystemTagVO.class) + .select(SystemTagVO_.tag) + .eq(SystemTagVO_.resourceUuid, listener.uuid) + .listValues() + assert !tags.any { it.startsWith("connectionIdleTimeout::") } + assert !tags.any { it.startsWith("Nbprocess::") } + + ChangeLoadBalancerListenerAction.Result idleTimeoutResult = assertChangeListenerError(listener.uuid) { ChangeLoadBalancerListenerAction action -> + action.connectionIdleTimeout = 77 + } + assertUnsupportedTcpIpvsParameterError(idleTimeoutResult.error, + "ORG_ZSTACK_NETWORK_SERVICE_LB_10192", "connectionIdleTimeout") + + ChangeLoadBalancerListenerAction.Result nbprocessResult = assertChangeListenerError(listener.uuid) { ChangeLoadBalancerListenerAction action -> + action.nbprocess = 2 + } + assertUnsupportedTcpIpvsParameterError(nbprocessResult.error, + "ORG_ZSTACK_NETWORK_SERVICE_LB_10193", "nbprocess") + } + + void testTcpIpvsSystemTagApiValidation() { + LoadBalancerListenerInventory listener = createTcpIpvsListener( + "tcp-ipvs-system-tag-validation", 11275, + LoadBalancerConstants.BALANCE_ALGORITHM_ROUND_ROBIN) + + [ + ["connectionIdleTimeout::77", "ORG_ZSTACK_NETWORK_SERVICE_LB_10192", "connectionIdleTimeout"], + ["Nbprocess::2", "ORG_ZSTACK_NETWORK_SERVICE_LB_10193", "nbprocess"], + ["tcpProxyProtocol::v2", "ORG_ZSTACK_NETWORK_SERVICE_LB_10191", "tcpProxyProtocol"] + ].each { List param -> + CreateSystemTagAction action = new CreateSystemTagAction() + action.resourceType = LoadBalancerListenerVO.class.simpleName + action.resourceUuid = listener.uuid + action.tag = param[0] + action.sessionId = adminSession() + + CreateSystemTagAction.Result result = action.call() + assert result.error != null + assertUnsupportedTcpIpvsParameterError(result.error, param[1] as String, param[2] as String) + } + + CreateSystemTagsAction batchAction = new CreateSystemTagsAction() + batchAction.resourceType = LoadBalancerListenerVO.class.simpleName + batchAction.resourceUuid = listener.uuid + batchAction.tags = ["connectionIdleTimeout::88"] + batchAction.sessionId = adminSession() + CreateSystemTagsAction.Result batchResult = batchAction.call() + assert batchResult.error != null + assertUnsupportedTcpIpvsParameterError(batchResult.error, + "ORG_ZSTACK_NETWORK_SERVICE_LB_10192", "connectionIdleTimeout") + + SystemTagVO maxConnectionTag = Q.New(SystemTagVO.class) + .eq(SystemTagVO_.resourceUuid, listener.uuid) + .like(SystemTagVO_.tag, "maxConnection::%") + .find() + assert maxConnectionTag != null + + UpdateSystemTagAction updateAction = new UpdateSystemTagAction() + updateAction.uuid = maxConnectionTag.uuid + updateAction.tag = "Nbprocess::3" + updateAction.sessionId = adminSession() + UpdateSystemTagAction.Result updateResult = updateAction.call() + assert updateResult.error != null + assertUnsupportedTcpIpvsParameterError(updateResult.error, + "ORG_ZSTACK_NETWORK_SERVICE_LB_10193", "nbprocess") + + assert Q.New(SystemTagVO.class) + .eq(SystemTagVO_.uuid, maxConnectionTag.uuid) + .like(SystemTagVO_.tag, "maxConnection::%") + .isExists() + } + + void testTcpHaproxyParameterRegression() { + LoadBalancerListenerInventory listener = createLoadBalancerListener { + delegate.name = "tcp-haproxy-parameter-regression" + delegate.loadBalancerUuid = lb.uuid + delegate.protocol = LoadBalancerConstants.LB_PROTOCOL_TCP + delegate.loadBalancerPort = 11276 + delegate.instancePort = 8080 + delegate.systemTags = [ + "connectionIdleTimeout::77", + "Nbprocess::2", + "tcpProxyProtocol::v2" + ] + } + + assert LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT.hasTag(listener.uuid, LoadBalancerListenerVO.class) + assert LoadBalancerSystemTags.NUMBER_OF_PROCESS.hasTag(listener.uuid, LoadBalancerListenerVO.class) + assert LoadBalancerSystemTags.TCP_PROXYPROTOCOL.hasTag(listener.uuid, LoadBalancerListenerVO.class) + + assertChangeListenerSuccess(listener.uuid) { ChangeLoadBalancerListenerAction action -> + action.connectionIdleTimeout = 88 + action.nbprocess = 3 + action.tcpProxyProtocol = "v1" + } + } + void testTcpHaproxyBackendRefreshPayload() { LoadBalancerListenerInventory listener = createLoadBalancerListener { delegate.name = "tcp-haproxy-refresh" @@ -1163,6 +1330,16 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assert error.details.contains("tcp ipvs listener doesn't support healthCheckTimeout") } + void assertUnsupportedTcpProxyProtocolError(def error) { + assert error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10191" + assert error.details.contains("tcp ipvs listener doesn't support tcpProxyProtocol") + } + + void assertUnsupportedTcpIpvsParameterError(def error, String errorCode, String parameter) { + assert error.globalErrorCode == errorCode + assert error.details.contains("tcp ipvs listener doesn't support ${parameter}".toString()) + } + String getApiResultString(ApiResult result) { def field = ApiResult.class.getDeclaredField("resultString") field.setAccessible(true) diff --git a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java index d42a9f392aa..75c3e5685b6 100644 --- a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java +++ b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java @@ -13919,6 +13919,12 @@ public class CloudOperationsErrorCode { public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10190 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10190"; + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10191 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10191"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10192 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10192"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10193 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10193"; + public static final String ORG_ZSTACK_IPSEC_10000 = "ORG_ZSTACK_IPSEC_10000"; public static final String ORG_ZSTACK_IPSEC_10001 = "ORG_ZSTACK_IPSEC_10001";