diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java index ff0f7c34896..19089f9f256 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java @@ -815,6 +815,33 @@ private boolean hasHttpHealthCheckParameters(APIChangeLoadBalancerListenerMsg ms msg.getHealthCheckHttpCode() != null; } + private boolean hasEnabledTcpProxyProtocol(String tcpProxyProtocol) { + return !StringUtils.isEmpty(tcpProxyProtocol) && + !DisableLbSupportTcpProxyProtocol.equals(tcpProxyProtocol); + } + + private void validateTcpIpvsDoesNotUseTcpProxyProtocol(String protocol, String dataPlane, String tcpProxyProtocol) { + if (isTcpIpvsListener(protocol, dataPlane) && hasEnabledTcpProxyProtocol(tcpProxyProtocol)) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10191, "tcp ipvs listener doesn't support tcpProxyProtocol")); + } + } + + private void validateTcpIpvsDoesNotUseConnectionIdleTimeout(String protocol, String dataPlane, + Integer connectionIdleTimeout) { + if (isTcpIpvsListener(protocol, dataPlane) && connectionIdleTimeout != null) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10192, "tcp ipvs listener doesn't support connectionIdleTimeout")); + } + } + + private void validateTcpIpvsDoesNotUseNbprocess(String protocol, String dataPlane, Integer nbprocess) { + if (isTcpIpvsListener(protocol, dataPlane) && nbprocess != null) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10193, "tcp ipvs listener doesn't support nbprocess")); + } + } + private String getHealthCheckProtocolFromTarget(String healthCheckTarget) { if (healthCheckTarget == null) { return null; @@ -970,6 +997,18 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { throw new ApiMessageInterceptionException( operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10189, "tcp ipvs listener doesn't support healthCheckTimeout")); } + if (isTcpIpvsListener(msg.getProtocol(), dataPlane) && hasTag(msg, LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT)) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10192, "tcp ipvs listener doesn't support connectionIdleTimeout")); + } + if (isTcpIpvsListener(msg.getProtocol(), dataPlane) && hasTag(msg, LoadBalancerSystemTags.NUMBER_OF_PROCESS)) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10193, "tcp ipvs listener doesn't support nbprocess")); + } + if (isTcpIpvsListener(msg.getProtocol(), dataPlane) && hasTag(msg, LoadBalancerSystemTags.TCP_PROXYPROTOCOL)) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10191, "tcp ipvs listener doesn't support tcpProxyProtocol")); + } if (isHealthCheckProtocolNotSupportedByListenerProtocol(msg.getProtocol(), dataPlane, msg.getHealthCheckProtocol())) { throw new ApiMessageInterceptionException( @@ -1011,12 +1050,14 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { validateAcl(msg.getAclUuids(),new ArrayList<>(), msg.getLoadBalancerUuid()); } - insertTagIfNotExisting( - msg, LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT, - LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT.instantiateTag( - map(e(LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT_TOKEN, LoadBalancerGlobalConfig.CONNECTION_IDLE_TIMEOUT.value(Long.class))) - ) - ); + if (!isTcpIpvsListener(msg.getProtocol(), dataPlane)) { + insertTagIfNotExisting( + msg, LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT, + LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT.instantiateTag( + map(e(LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT_TOKEN, LoadBalancerGlobalConfig.CONNECTION_IDLE_TIMEOUT.value(Long.class))) + ) + ); + } insertTagIfNotExisting( msg, LoadBalancerSystemTags.HEALTHY_THRESHOLD, @@ -1076,12 +1117,14 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { ) ); - insertTagIfNotExisting( - msg, LoadBalancerSystemTags.NUMBER_OF_PROCESS, - LoadBalancerSystemTags.NUMBER_OF_PROCESS.instantiateTag( - map(e(LoadBalancerSystemTags.NUMBER_OF_PROCESS_TOKEN, LoadBalancerGlobalConfig.NUMBER_OF_PROCESS.value())) - ) - ); + if (!isTcpIpvsListener(msg.getProtocol(), dataPlane)) { + insertTagIfNotExisting( + msg, LoadBalancerSystemTags.NUMBER_OF_PROCESS, + LoadBalancerSystemTags.NUMBER_OF_PROCESS.instantiateTag( + map(e(LoadBalancerSystemTags.NUMBER_OF_PROCESS_TOKEN, LoadBalancerGlobalConfig.NUMBER_OF_PROCESS.value())) + ) + ); + } if (LoadBalancerConstants.LB_PROTOCOL_HTTP.equals(msg.getProtocol()) || LoadBalancerConstants.LB_PROTOCOL_HTTPS.equals(msg.getProtocol())) { @@ -1381,6 +1424,7 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { throw new ApiMessageInterceptionException( argerr(ORG_ZSTACK_NETWORK_SERVICE_LB_10093, "cloud not create the loadbalancer listener, because only support tcp proxy protocol %s", LbSupportTcpProxyProtocol)); } + validateTcpIpvsDoesNotUseTcpProxyProtocol(msg.getProtocol(), dataPlane, msg.getTcpProxyProtocol()); if (!msg.getTcpProxyProtocol().equals(DisableLbSupportTcpProxyProtocol)) { insertTagIfNotExisting( @@ -1620,6 +1664,10 @@ private void validate(APIChangeLoadBalancerListenerMsg msg) { eq(LoadBalancerListenerVO_.uuid,msg.getLoadBalancerListenerUuid()).find(); String dataPlane = getListenerDataPlane(msg.getLoadBalancerListenerUuid()); + validateTcpIpvsDoesNotUseConnectionIdleTimeout(listenerVO.getProtocol(), dataPlane, + msg.getConnectionIdleTimeout()); + validateTcpIpvsDoesNotUseNbprocess(listenerVO.getProtocol(), dataPlane, msg.getNbprocess()); + if (msg.getSecurityPolicyType() != null) { if (!listenerVO.getProtocol().equals(LB_PROTOCOL_HTTPS)) { throw new ApiMessageInterceptionException(operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10118, "the listener with protocol [%s] doesn't support select security policy", listenerVO.getProtocol(), msg.getHealthCheckProtocol())); @@ -1697,6 +1745,7 @@ private void validate(APIChangeLoadBalancerListenerMsg msg) { throw new ApiMessageInterceptionException( argerr(ORG_ZSTACK_NETWORK_SERVICE_LB_10125, "cloud not change the loadbalancer listener, because only support tcp proxy protocol %s", LbSupportTcpProxyProtocol)); } + validateTcpIpvsDoesNotUseTcpProxyProtocol(listenerVO.getProtocol(), dataPlane, msg.getTcpProxyProtocol()); } if (!CollectionUtils.isEmpty(msg.getHttpCompressAlgos())) { diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBackend.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBackend.java index 561226ca769..1df195bedfb 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBackend.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBackend.java @@ -21,6 +21,10 @@ public interface LoadBalancerBackend { void addListener(LoadBalancerStruct struct, LoadBalancerListenerInventory listener, Completion completion); + default void validateBeforeCreateListener(LoadBalancerVO lbVO, APICreateLoadBalancerListenerMsg msg, Completion completion) { + completion.success(); + } + void removeListener(LoadBalancerStruct struct, LoadBalancerListenerInventory listener, Completion completion); void destroyLoadBalancer(LoadBalancerStruct struct, Completion completion); diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java index 0e78b167c0f..94afa88e6d5 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java @@ -1811,9 +1811,33 @@ public void run(final SyncTaskChain chain) { return; } - createListener(msg, new NoErrorCompletion(chain) { + LoadBalancerBackend backend = getBackend(); + if (backend == null) { + createListener(msg, new NoErrorCompletion(chain) { + @Override + public void done() { + chain.next(); + } + }); + return; + } + + backend.validateBeforeCreateListener(self, msg, new Completion(chain) { @Override - public void done() { + public void success() { + createListener(msg, new NoErrorCompletion(chain) { + @Override + public void done() { + chain.next(); + } + }); + } + + @Override + public void fail(ErrorCode errorCode) { + APICreateLoadBalancerListenerEvent evt = new APICreateLoadBalancerListenerEvent(msg.getId()); + evt.setError(errorCode); + bus.publish(evt); chain.next(); } }); diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerManagerImpl.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerManagerImpl.java index fe6c99a0826..6a1883ce741 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerManagerImpl.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerManagerImpl.java @@ -502,6 +502,21 @@ public void validateGlobalConfig(String category, String name, String oldValue, } + private boolean isTcpIpvsListener(String listenerUuid) { + LoadBalancerListenerVO listener = Q.New(LoadBalancerListenerVO.class) + .eq(LoadBalancerListenerVO_.uuid, listenerUuid) + .find(); + return listener != null && LoadBalancerConstants.LB_PROTOCOL_TCP.equals(listener.getProtocol()) && + LoadBalancerConstants.DATA_PLANE_IPVS.equals(listener.getDataPlane()); + } + + private void rejectTcpIpvsSystemTag(String listenerUuid, String errorCode, String parameter) { + if (isTcpIpvsListener(listenerUuid)) { + throw new OperationFailureException( + operr(errorCode, "tcp ipvs listener doesn't support %s", parameter)); + } + } + private void prepareSystemTags() { AbstractSystemTagOperationJudger judger = new AbstractSystemTagOperationJudger() { @Override @@ -554,6 +569,14 @@ public void validateSystemTag(String resourceUuid, Class resourceType, String sy } }); + LoadBalancerSystemTags.TCP_PROXYPROTOCOL.installValidator(new SystemTagValidator() { + @Override + public void validateSystemTag(String resourceUuid, Class resourceType, String systemTag) { + rejectTcpIpvsSystemTag(resourceUuid, ORG_ZSTACK_NETWORK_SERVICE_LB_10191, + "tcpProxyProtocol"); + } + }); + LoadBalancerSystemTags.HEALTH_PARAMETER.installLifeCycleListener(new SystemTagLifeCycleListener() { @Override public void tagCreated(SystemTagInventory tag) { @@ -661,6 +684,8 @@ public void validateSystemTag(String resourceUuid, Class resourceType, String sy LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT.installValidator(new SystemTagValidator() { @Override public void validateSystemTag(String resourceUuid, Class resourceType, String systemTag) { + rejectTcpIpvsSystemTag(resourceUuid, ORG_ZSTACK_NETWORK_SERVICE_LB_10192, + "connectionIdleTimeout"); String s = LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT.getTokenByTag(systemTag, LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT_TOKEN); @@ -703,6 +728,8 @@ public void validateSystemTag(String resourceUuid, Class resourceType, String sy LoadBalancerSystemTags.NUMBER_OF_PROCESS.installValidator(new SystemTagValidator() { @Override public void validateSystemTag(String resourceUuid, Class resourceType, String systemTag) { + rejectTcpIpvsSystemTag(resourceUuid, ORG_ZSTACK_NETWORK_SERVICE_LB_10193, + "nbprocess"); String s = LoadBalancerSystemTags.NUMBER_OF_PROCESS.getTokenByTag(systemTag, LoadBalancerSystemTags.NUMBER_OF_PROCESS_TOKEN); diff --git a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java index 72faff543a0..9e088712254 100755 --- a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java +++ b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java @@ -47,6 +47,7 @@ import org.zstack.network.service.lb.*; import org.zstack.network.service.vip.*; import org.zstack.network.service.virtualrouter.*; +import org.zstack.network.service.virtualrouter.vyos.VyosConstants; import org.zstack.network.service.virtualrouter.vyos.VyosGlobalConfig; import org.zstack.network.service.virtualrouter.VirtualRouterCommands.AgentCommand; import org.zstack.network.service.virtualrouter.VirtualRouterCommands.AgentResponse; @@ -1303,6 +1304,46 @@ public void handle(ErrorCode errCode, Map data) { }).start(); } + @Override + public void validateBeforeCreateListener(LoadBalancerVO lbVO, APICreateLoadBalancerListenerMsg msg, Completion completion) { + if (!isTcpIpvsListener(msg)) { + completion.success(); + return; + } + + VirtualRouterVmInventory vr = findVirtualRouterVm(lbVO.getUuid()); + if (vr == null) { + completion.success(); + return; + } + + ErrorCode errorCode = validateTcpIpvsZvrVersion(vr.getUuid()); + if (errorCode != null) { + completion.fail(errorCode); + return; + } + + completion.success(); + } + + private boolean isTcpIpvsListener(APICreateLoadBalancerListenerMsg msg) { + return LoadBalancerConstants.LB_PROTOCOL_TCP.equals(msg.getProtocol()) && + LoadBalancerConstants.DATA_PLANE_IPVS.equals(msg.getDataPlane()); + } + + private ErrorCode validateTcpIpvsZvrVersion(String vmUuid) { + VirtualRouterMetadataVO metadataVO = dbf.findByUuid(vmUuid, VirtualRouterMetadataVO.class); + String zvrVersion = metadataVO == null ? null : metadataVO.getZvrVersion(); + if (!VirtualRouterMetadataOperator.zvrVersionCheck(zvrVersion) || + new VersionComparator(zvrVersion).compare(VyosConstants.TCP_IPVS_MIN_ZVR_VERSION) < 0) { + return operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10190, + "target appliance vm[uuid:%s] zvr version [%s] does not support tcp ipvs listener, required >= %s", + vmUuid, zvrVersion == null ? "unknown" : zvrVersion, VyosConstants.TCP_IPVS_MIN_ZVR_VERSION); + } + + return null; + } + public void refreshCertsAndListeners(VirtualRouterVmInventory vr, Map certs, List listeners, diff --git a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConstants.java b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConstants.java index 93941c5dacc..8b88d644e2f 100755 --- a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConstants.java +++ b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConstants.java @@ -23,6 +23,7 @@ enum BootstrapInfoKey { String VYOS_VERSION_PATH = "ansible/zvr/version"; int VYOS_VERSION_LENGTH = 4; + String TCP_IPVS_MIN_ZVR_VERSION = "5.5.38.0"; String REPLACE_FIREWALL_WITH_IPTBALES = "SkipVyosIptables"; String HA_STATUS = "haStatus"; diff --git a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy index 70631a2af93..19bd9c275e2 100644 --- a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy @@ -4,6 +4,8 @@ import org.zstack.appliancevm.ApplianceVmVO import org.zstack.appliancevm.ApplianceVmVO_ import org.zstack.core.db.Q import org.zstack.header.network.service.NetworkServiceType +import org.zstack.header.tag.SystemTagVO +import org.zstack.header.tag.SystemTagVO_ import org.zstack.network.service.eip.EipConstant import org.zstack.network.service.lb.LoadBalancerAclStatus import org.zstack.network.service.lb.LoadBalancerAclType @@ -14,6 +16,8 @@ import org.zstack.network.service.lb.LoadBalancerSystemTags import org.zstack.network.service.lb.LoadBalancerVO import org.zstack.network.service.lb.LoadBalancerVO_ import org.zstack.network.service.portforwarding.PortForwardingConstant +import org.zstack.network.service.virtualrouter.VirtualRouterCommands +import org.zstack.network.service.virtualrouter.VirtualRouterConstant import org.zstack.network.service.virtualrouter.lb.VirtualRouterLoadBalancerBackend import org.zstack.network.service.virtualrouter.vyos.VyosConstants import org.zstack.network.service.virtualrouter.vyos.VyosGlobalConfig @@ -26,11 +30,14 @@ import org.zstack.sdk.ChangeLoadBalancerListenerAction import org.zstack.sdk.ApiResult import org.zstack.sdk.CreateLoadBalancerListenerAction import org.zstack.sdk.CreateLoadBalancerListenerResult +import org.zstack.sdk.CreateSystemTagAction +import org.zstack.sdk.CreateSystemTagsAction import org.zstack.sdk.L3NetworkInventory import org.zstack.sdk.LoadBalancerInventory import org.zstack.sdk.LoadBalancerListenerInventory import org.zstack.sdk.LoadBalancerServerGroupInventory import org.zstack.sdk.QueryLoadBalancerListenerAction +import org.zstack.sdk.UpdateSystemTagAction import org.zstack.sdk.VipInventory import org.zstack.sdk.VmInstanceInventory import org.zstack.sdk.ZSClient @@ -201,6 +208,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { @Override void test() { env.create { + installSupportedZvrVersionSimulator() prepareDedicatedLoadBalancer() installRefreshLbCapture() testTcpIpvsIpv6Validation() @@ -211,13 +219,24 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { testTcpIpvsDefaultForwardMode() testTcpIpvsForwardModeCannotBeChanged() testTcpIpvsCreateValidation() + testTcpIpvsTcpProxyProtocolValidation() testTcpIpvsHealthCheckParameterValidation() + testTcpIpvsUnsupportedHaproxyParameterValidation() + testTcpIpvsSystemTagApiValidation() + testTcpHaproxyParameterRegression() testTcpHaproxyBackendRefreshPayload() testUdpHaproxyBackendRefreshPayload() testTcpIpvsDedicatedListenerBackendRefreshPayload() } } + void installSupportedZvrVersionSimulator() { + env.afterSimulator(VirtualRouterConstant.VR_INIT) { VirtualRouterCommands.InitRsp rsp, HttpEntity e -> + rsp.zvrVersion = VyosConstants.TCP_IPVS_MIN_ZVR_VERSION + return rsp + } + } + void prepareDedicatedLoadBalancer() { L3NetworkInventory publicL3 = env.inventoryByName("publicL3") as L3NetworkInventory VipInventory vip = createVip { @@ -541,6 +560,47 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assert result.error.details.contains("forwardMode is only supported when dataPlane is ipvs") } + void testTcpIpvsTcpProxyProtocolValidation() { + ["v1", "v2"].eachWithIndex { String protocol, int index -> + int port = 11260 + index + CreateLoadBalancerListenerAction.Result result = assertCreateTcpIpvsListenerError(port) { CreateLoadBalancerListenerAction action -> + action.tcpProxyProtocol = protocol + } + assertUnsupportedTcpProxyProtocolError(result.error) + assert !Q.New(LoadBalancerListenerVO.class) + .eq(LoadBalancerListenerVO_.name, "tcp-ipvs-error-${port}".toString()) + .isExists() + } + + LoadBalancerListenerInventory disabledOnCreate = createLoadBalancerListener { + delegate.name = "tcp-ipvs-disabled-tcp-proxy-protocol" + delegate.loadBalancerUuid = lb.uuid + delegate.protocol = LoadBalancerConstants.LB_PROTOCOL_TCP + delegate.loadBalancerPort = 11262 + delegate.instancePort = 8080 + delegate.dataPlane = LoadBalancerConstants.DATA_PLANE_IPVS + delegate.forwardMode = LoadBalancerConstants.FORWARD_MODE_FULL_NAT + delegate.tcpProxyProtocol = LoadBalancerConstants.DisableLbSupportTcpProxyProtocol + } + assert disabledOnCreate.uuid != null + assert !LoadBalancerSystemTags.TCP_PROXYPROTOCOL.hasTag(disabledOnCreate.uuid, LoadBalancerListenerVO.class) + + LoadBalancerListenerInventory listener = createTcpIpvsListener( + "tcp-ipvs-change-tcp-proxy-protocol", 11263, LoadBalancerConstants.BALANCE_ALGORITHM_ROUND_ROBIN) + ["v1", "v2"].each { String protocol -> + ChangeLoadBalancerListenerAction.Result result = assertChangeListenerError(listener.uuid) { ChangeLoadBalancerListenerAction action -> + action.tcpProxyProtocol = protocol + } + assertUnsupportedTcpProxyProtocolError(result.error) + } + assert !LoadBalancerSystemTags.TCP_PROXYPROTOCOL.hasTag(listener.uuid, LoadBalancerListenerVO.class) + + assertChangeListenerSuccess(listener.uuid) { ChangeLoadBalancerListenerAction action -> + action.tcpProxyProtocol = LoadBalancerConstants.DisableLbSupportTcpProxyProtocol + } + assert !LoadBalancerSystemTags.TCP_PROXYPROTOCOL.hasTag(listener.uuid, LoadBalancerListenerVO.class) + } + void testTcpIpvsHealthCheckParameterValidation() { [ { CreateLoadBalancerListenerAction action -> action.healthCheckMethod = "GET" }, @@ -676,6 +736,123 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assert noneSpecificTargetResult.error.details.contains("health check protocol none only supports default target") } + void testTcpIpvsUnsupportedHaproxyParameterValidation() { + [ + ["connectionIdleTimeout::77", "ORG_ZSTACK_NETWORK_SERVICE_LB_10192", "connectionIdleTimeout"], + ["Nbprocess::2", "ORG_ZSTACK_NETWORK_SERVICE_LB_10193", "nbprocess"], + ["tcpProxyProtocol::v2", "ORG_ZSTACK_NETWORK_SERVICE_LB_10191", "tcpProxyProtocol"], + ["tcpProxyProtocol::disable", "ORG_ZSTACK_NETWORK_SERVICE_LB_10191", "tcpProxyProtocol"] + ].eachWithIndex { List param, int index -> + int port = 11270 + index + CreateLoadBalancerListenerAction.Result result = assertCreateTcpIpvsListenerError(port) { CreateLoadBalancerListenerAction action -> + action.systemTags = [param[0]] + } + assertUnsupportedTcpIpvsParameterError(result.error, param[1] as String, param[2] as String) + assert !Q.New(LoadBalancerListenerVO.class) + .eq(LoadBalancerListenerVO_.name, "tcp-ipvs-invalid-health-check-${port}".toString()) + .isExists() + } + + LoadBalancerListenerInventory listener = createTcpIpvsListener( + "tcp-ipvs-unsupported-haproxy-parameters", 11274, + LoadBalancerConstants.BALANCE_ALGORITHM_ROUND_ROBIN) + List tags = Q.New(SystemTagVO.class) + .select(SystemTagVO_.tag) + .eq(SystemTagVO_.resourceUuid, listener.uuid) + .listValues() + assert !tags.any { it.startsWith("connectionIdleTimeout::") } + assert !tags.any { it.startsWith("Nbprocess::") } + + ChangeLoadBalancerListenerAction.Result idleTimeoutResult = assertChangeListenerError(listener.uuid) { ChangeLoadBalancerListenerAction action -> + action.connectionIdleTimeout = 77 + } + assertUnsupportedTcpIpvsParameterError(idleTimeoutResult.error, + "ORG_ZSTACK_NETWORK_SERVICE_LB_10192", "connectionIdleTimeout") + + ChangeLoadBalancerListenerAction.Result nbprocessResult = assertChangeListenerError(listener.uuid) { ChangeLoadBalancerListenerAction action -> + action.nbprocess = 2 + } + assertUnsupportedTcpIpvsParameterError(nbprocessResult.error, + "ORG_ZSTACK_NETWORK_SERVICE_LB_10193", "nbprocess") + } + + void testTcpIpvsSystemTagApiValidation() { + LoadBalancerListenerInventory listener = createTcpIpvsListener( + "tcp-ipvs-system-tag-validation", 11275, + LoadBalancerConstants.BALANCE_ALGORITHM_ROUND_ROBIN) + + [ + ["connectionIdleTimeout::77", "ORG_ZSTACK_NETWORK_SERVICE_LB_10192", "connectionIdleTimeout"], + ["Nbprocess::2", "ORG_ZSTACK_NETWORK_SERVICE_LB_10193", "nbprocess"], + ["tcpProxyProtocol::v2", "ORG_ZSTACK_NETWORK_SERVICE_LB_10191", "tcpProxyProtocol"] + ].each { List param -> + CreateSystemTagAction action = new CreateSystemTagAction() + action.resourceType = LoadBalancerListenerVO.class.simpleName + action.resourceUuid = listener.uuid + action.tag = param[0] + action.sessionId = adminSession() + + CreateSystemTagAction.Result result = action.call() + assert result.error != null + assertUnsupportedTcpIpvsParameterError(result.error, param[1] as String, param[2] as String) + } + + CreateSystemTagsAction batchAction = new CreateSystemTagsAction() + batchAction.resourceType = LoadBalancerListenerVO.class.simpleName + batchAction.resourceUuid = listener.uuid + batchAction.tags = ["connectionIdleTimeout::88"] + batchAction.sessionId = adminSession() + CreateSystemTagsAction.Result batchResult = batchAction.call() + assert batchResult.error != null + assertUnsupportedTcpIpvsParameterError(batchResult.error, + "ORG_ZSTACK_NETWORK_SERVICE_LB_10192", "connectionIdleTimeout") + + SystemTagVO maxConnectionTag = Q.New(SystemTagVO.class) + .eq(SystemTagVO_.resourceUuid, listener.uuid) + .like(SystemTagVO_.tag, "maxConnection::%") + .find() + assert maxConnectionTag != null + + UpdateSystemTagAction updateAction = new UpdateSystemTagAction() + updateAction.uuid = maxConnectionTag.uuid + updateAction.tag = "Nbprocess::3" + updateAction.sessionId = adminSession() + UpdateSystemTagAction.Result updateResult = updateAction.call() + assert updateResult.error != null + assertUnsupportedTcpIpvsParameterError(updateResult.error, + "ORG_ZSTACK_NETWORK_SERVICE_LB_10193", "nbprocess") + + assert Q.New(SystemTagVO.class) + .eq(SystemTagVO_.uuid, maxConnectionTag.uuid) + .like(SystemTagVO_.tag, "maxConnection::%") + .isExists() + } + + void testTcpHaproxyParameterRegression() { + LoadBalancerListenerInventory listener = createLoadBalancerListener { + delegate.name = "tcp-haproxy-parameter-regression" + delegate.loadBalancerUuid = lb.uuid + delegate.protocol = LoadBalancerConstants.LB_PROTOCOL_TCP + delegate.loadBalancerPort = 11276 + delegate.instancePort = 8080 + delegate.systemTags = [ + "connectionIdleTimeout::77", + "Nbprocess::2", + "tcpProxyProtocol::v2" + ] + } + + assert LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT.hasTag(listener.uuid, LoadBalancerListenerVO.class) + assert LoadBalancerSystemTags.NUMBER_OF_PROCESS.hasTag(listener.uuid, LoadBalancerListenerVO.class) + assert LoadBalancerSystemTags.TCP_PROXYPROTOCOL.hasTag(listener.uuid, LoadBalancerListenerVO.class) + + assertChangeListenerSuccess(listener.uuid) { ChangeLoadBalancerListenerAction action -> + action.connectionIdleTimeout = 88 + action.nbprocess = 3 + action.tcpProxyProtocol = "v1" + } + } + void testTcpHaproxyBackendRefreshPayload() { LoadBalancerListenerInventory listener = createLoadBalancerListener { delegate.name = "tcp-haproxy-refresh" @@ -1153,6 +1330,16 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assert error.details.contains("tcp ipvs listener doesn't support healthCheckTimeout") } + void assertUnsupportedTcpProxyProtocolError(def error) { + assert error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10191" + assert error.details.contains("tcp ipvs listener doesn't support tcpProxyProtocol") + } + + void assertUnsupportedTcpIpvsParameterError(def error, String errorCode, String parameter) { + assert error.globalErrorCode == errorCode + assert error.details.contains("tcp ipvs listener doesn't support ${parameter}".toString()) + } + String getApiResultString(ApiResult result) { def field = ApiResult.class.getDeclaredField("resultString") field.setAccessible(true) diff --git a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsZvrVersionValidationCase.groovy b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsZvrVersionValidationCase.groovy new file mode 100644 index 00000000000..d2f532059db --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsZvrVersionValidationCase.groovy @@ -0,0 +1,202 @@ +package org.zstack.test.integration.networkservice.provider.virtualrouter.loadbalancer + +import org.zstack.appliancevm.ApplianceVmConstant +import org.zstack.appliancevm.ApplianceVmHaStatus +import org.zstack.appliancevm.ApplianceVmStatus +import org.zstack.core.Platform +import org.zstack.core.db.Q +import org.zstack.core.db.DatabaseFacade +import org.zstack.header.identity.AccountConstant +import org.zstack.header.image.ImagePlatform +import org.zstack.header.network.service.NetworkServiceType +import org.zstack.header.vm.VmInstanceState +import org.zstack.network.service.lb.LoadBalancerConstants +import org.zstack.network.service.lb.LoadBalancerListenerVO +import org.zstack.network.service.lb.LoadBalancerListenerVO_ +import org.zstack.network.service.lb.LoadBalancerVO +import org.zstack.network.service.virtualrouter.VirtualRouterConstant +import org.zstack.network.service.virtualrouter.VirtualRouterMetadataVO +import org.zstack.network.service.virtualrouter.VirtualRouterVmVO +import org.zstack.network.service.virtualrouter.lb.VirtualRouterLoadBalancerRefVO +import org.zstack.network.service.virtualrouter.lb.VirtualRouterLoadBalancerRefVO_ +import org.zstack.network.service.virtualrouter.vyos.VyosConstants +import org.zstack.sdk.CreateLoadBalancerListenerAction +import org.zstack.sdk.L3NetworkInventory +import org.zstack.sdk.LoadBalancerInventory +import org.zstack.sdk.VipInventory +import org.zstack.test.integration.networkservice.provider.NetworkServiceProviderTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase + +class TcpIpvsZvrVersionValidationCase extends SubCase { + EnvSpec env + DatabaseFacade dbf + String syntheticVrUuid + + @Override + void setup() { + useSpring(NetworkServiceProviderTest.springSpec) + } + + @Override + void environment() { + env = env { + zone { + name = "zone" + + l2NoVlanNetwork { + name = "l2" + physicalInterface = "eth0" + + l3Network { + name = "guestL3" + service { + provider = VyosConstants.VYOS_ROUTER_PROVIDER_TYPE + types = [NetworkServiceType.DHCP.toString(), + NetworkServiceType.DNS.toString(), + NetworkServiceType.SNAT.toString(), + LoadBalancerConstants.LB_NETWORK_SERVICE_TYPE_STRING] + } + ip { + startIp = "10.2.226.10" + endIp = "10.2.226.200" + gateway = "10.2.226.1" + netmask = "255.255.255.0" + } + } + + l3Network { + name = "publicL3" + ip { + startIp = "172.24.3.10" + endIp = "172.24.3.200" + gateway = "172.24.3.1" + netmask = "255.255.255.0" + } + } + } + } + } + } + + @Override + void test() { + env.create { + dbf = bean(DatabaseFacade.class) + testTcpIpvsCreateRejectedByOldZvrVersion() + } + } + + @Override + void clean() { + if (syntheticVrUuid != null) { + VirtualRouterLoadBalancerRefVO ref = Q.New(VirtualRouterLoadBalancerRefVO.class) + .eq(VirtualRouterLoadBalancerRefVO_.virtualRouterVmUuid, syntheticVrUuid) + .find() + if (ref != null) { + dbf.remove(ref) + } + + VirtualRouterMetadataVO metadata = dbf.findByUuid(syntheticVrUuid, VirtualRouterMetadataVO.class) + if (metadata != null) { + dbf.remove(metadata) + } + + VirtualRouterVmVO vr = dbf.findByUuid(syntheticVrUuid, VirtualRouterVmVO.class) + if (vr != null) { + dbf.remove(vr) + } + } + env.delete() + } + + void testTcpIpvsCreateRejectedByOldZvrVersion() { + L3NetworkInventory publicL3 = env.inventoryByName("publicL3") as L3NetworkInventory + + VipInventory vip = createVip { + name = "tcp-ipvs-zvr-version-vip" + l3NetworkUuid = publicL3.uuid + } + + LoadBalancerInventory lb = createLoadBalancer { + name = "tcp-ipvs-zvr-version-lb" + vipUuid = vip.uuid + } + setLbProviderType(lb.uuid) + + String vrUuid = createVirtualRouterRef(lb.uuid) + setZvrVersion(vrUuid, "5.5.28.0") + + CreateLoadBalancerListenerAction action = createTcpIpvsListenerAction(lb.uuid, "tcp-ipvs-old-zvr-version", 19095) + CreateLoadBalancerListenerAction.Result result = action.call() + + assert result.error != null + assert result.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10190" + assert result.error.details.contains("does not support tcp ipvs listener") + assert !Q.New(LoadBalancerListenerVO.class) + .eq(LoadBalancerListenerVO_.name, action.name) + .isExists() + + setZvrVersion(vrUuid, VyosConstants.TCP_IPVS_MIN_ZVR_VERSION) + result = createTcpIpvsListenerAction(lb.uuid, "tcp-ipvs-supported-zvr-version", 19096).call() + assert result.error == null + } + + String createVirtualRouterRef(String lbUuid) { + String vrUuid = Platform.getUuid() + VirtualRouterVmVO vr = new VirtualRouterVmVO() + vr.uuid = vrUuid + vr.name = "tcp-ipvs-zvr-version-vr" + vr.type = ApplianceVmConstant.APPLIANCE_VM_TYPE + vr.applianceVmType = VirtualRouterConstant.VIRTUAL_ROUTER_VM_TYPE + vr.hypervisorType = "KVM" + vr.platform = ImagePlatform.Linux.toString() + vr.state = VmInstanceState.Running + vr.status = ApplianceVmStatus.Connected + vr.haStatus = ApplianceVmHaStatus.NoHa + vr.agentPort = 7272 + vr.internalId = 1L + vr.accountUuid = AccountConstant.INITIAL_SYSTEM_ADMIN_UUID + dbf.persist(vr) + syntheticVrUuid = vrUuid + + VirtualRouterLoadBalancerRefVO ref = new VirtualRouterLoadBalancerRefVO() + ref.loadBalancerUuid = lbUuid + ref.virtualRouterVmUuid = vrUuid + dbf.persist(ref) + + return vrUuid + } + + void setLbProviderType(String lbUuid) { + LoadBalancerVO lbVO = dbf.findByUuid(lbUuid, LoadBalancerVO.class) + lbVO.providerType = VyosConstants.VYOS_ROUTER_PROVIDER_TYPE + dbf.update(lbVO) + } + + void setZvrVersion(String vrUuid, String version) { + VirtualRouterMetadataVO metadata = dbf.findByUuid(vrUuid, VirtualRouterMetadataVO.class) + if (metadata == null) { + metadata = new VirtualRouterMetadataVO() + metadata.uuid = vrUuid + metadata.zvrVersion = version + dbf.persist(metadata) + } else { + metadata.zvrVersion = version + dbf.update(metadata) + } + } + + CreateLoadBalancerListenerAction createTcpIpvsListenerAction(String lbUuid, String name, int port) { + CreateLoadBalancerListenerAction action = new CreateLoadBalancerListenerAction() + action.name = name + action.loadBalancerUuid = lbUuid + action.protocol = LoadBalancerConstants.LB_PROTOCOL_TCP + action.loadBalancerPort = port + action.instancePort = 8080 + action.dataPlane = LoadBalancerConstants.DATA_PLANE_IPVS + action.forwardMode = LoadBalancerConstants.FORWARD_MODE_FULL_NAT + action.sessionId = adminSession() + return action + } +} diff --git a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java index 68ab7087c6c..75c3e5685b6 100644 --- a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java +++ b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java @@ -13917,6 +13917,14 @@ public class CloudOperationsErrorCode { public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10189 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10189"; + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10190 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10190"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10191 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10191"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10192 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10192"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10193 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10193"; + public static final String ORG_ZSTACK_IPSEC_10000 = "ORG_ZSTACK_IPSEC_10000"; public static final String ORG_ZSTACK_IPSEC_10001 = "ORG_ZSTACK_IPSEC_10001";